def extract_claims(remark, company, brand, question=None):
"""Extract and clarify claims from a remark in the sustainability report.
"""
# references aren't meaningful outside the page
remark = SEE_RE.sub('', remark).strip()
for claim in [remark]: # TODO: split into sentences if appropriate
claim = remark
if BAD_CLAIM_RE.match(claim):
continue
claim = clarify_claim(claim, CLAIM_CLARIFICATIONS)
claim = ltrim_sentence(claim, [company, brand])
claim = claim.strip()
if claim:
yield claim
def scrape_campaign():
log.info("Main page")
soup = scrape_soup(URL)
# campaign record
cn = {"url": URL, "goal": GOAL}
cn["campaign"], cn["author"] = soup.title.text.split("|")
# remove double spaces
cn["copyright"] = scrape_copyright(soup)
cn["facebook_url"] = scrape_facebook_url(soup)
cn["twitter_handle"] = scrape_twitter_handle(soup)
# get year
cn["date"] = INT_RE.search(soup.select("div.content h2")[0].text).group()
for a in soup.findAll("a"):
if a.text.strip() == "Donate":
cn["donate_url"] = urljoin(URL, a["href"])
break
if "donate_url" not in cn:
raise ValueError("Donate URL not found")
yield "campaign", cn
rating_divs = soup.select("div#corank div.row")
if not rating_divs:
raise ValueError("ratings not found")
for div in rating_divs:
c = {}
r = {"company": c}
company_a = div.select("a.coname")[0]
company = company_a.text
c["company"] = company
teaser = div.select("span.teaser")[0].text
r["categories"] = CATEGORIES_SEP.split(CATEGORIES_RE.match(teaser).group(1))
for rank_class, judgment in RANK_CLASS_TO_JUDGMENT.items():
if div.select("span.rank." + rank_class):
r["judgment"] = judgment
break
else:
raise ValueError("rating for {} not found".format(r["company"]))
r["score"] = int(INT_RE.search(div.select("div.col_score")[0].text).group())
r["categories"] = [CATEGORY]
# fetch details
company_id = company_a["href"].split("#")[-1]
query = dict(action="getcompany", companyid=company_id)
# use POST to get details JSON
log.info("Details for {}".format(company))
details = scrape_json(DETAILS_URL, data=urlencode(query))
details = details[0][0] # wrapped in lists. why?
c["url"] = details["ext_url"]
# TODO: details['message'] might be useful too. It's a message
# that participants are supposed to send to the company:
# "Thank you for the leadership you have shown in working to..."
yield "rating", r
detail_soup = BeautifulSoup(details["detail"])
claim_lis = detail_soup.select("li")
# First two bullet points are categories and a description
# of the company's ranking (reversed for Nokia)
# Last bullet point is what the company can do to improve its score.
claim_lis = claim_lis[2:-1]
for i, claim_li in enumerate(claim_lis):
claim = claim_li.text
judgment = claim_to_judgment(claim)
claim = clarify_claim(claim, CLAIM_CLARIFICATIONS)
yield "claim", dict(company=company, claim=claim, judgment=judgment)
