def judge_claim(claim): """Process claim text, and handle some special cases.""" judgment = claim_to_judgment(claim) if judgment != 0 and CLAIM_NOT_GOOD_RE.match(claim): judgment = -1 if judgment == 1 and '%' in claim and '100%' not in claim: judgment = 0 return judgment
def scrape_company(url, known_brands): soup = scrape_soup(url) company = strip_company( soup.select('#company_score_company_title h1')[0].text) c = dict(company=company) # rating score = int(soup.select('#company_score_score')[0].text.strip()) r = dict(company=company, score=score, max_score=MAX_SCORE) status_path = soup.select('#company_score_status img')[0]['src'] r['description'], r['judgment'] = scrape_description_and_judgment( status_path) r['url'] = url # icon icon_as = soup.select('#company_score_company_icon a') if icon_as: icon_a = icon_as[0] c['url'] = icon_a['href'] c['logo_url'] = urljoin(url, icon_a.img['src']) # sector for a in soup.select('#breadcrumbs a'): if 'sectors' in a['href']: c['category'] = a.text break # ignore "Industry Innovators" category # match up brands to logos brands = sorted(known_brands[company]) sb2b = dict((smunch(b), b) for b in brands) for img in soup.select('span.brand_icon img'): logo_url = urljoin(url, img['src']) sb = smunch(img['alt']) sb = SMUNCHED_BRAND_CORRECTIONS.get(sb, sb) if sb in sb2b: brand = sb2b[sb] yield 'brand', dict( company=company, brand=brand, logo_url=logo_url) else: log.warn(u'No matching brand for {} ({}: {})'.format( repr(img['alt']), company, u', '.join(brands))) # match twitter handles to company/brand sc = smunch(company) sbs = sorted(sb2b) sb2th = {} twitter_handles = [ a.text.strip() for a in soup.select('#company_score_action_right a')] def match_twitter_handle(th): if th in IGNORE_TWITTER_HANDLES: return sth = smunch(th[1:]) for i in range(len(sth), 1, -1): if (not th.endswith('Brand') and sth[:i] == sc[:i] and 'twitter_handle' not in c): c['twitter_handle'] = th return for sb in sbs: if sth[:i] == sb[:i] and sb not in sb2th: sb2th[sb] = th return else: if 'twitter_handle' not in c: c['twitter_handle'] = th else: log.warn(u'No matching brand/company for {} ({}: {})'.format( repr(th), company, u', '.join(brands))) for th in twitter_handles: match_twitter_handle(th) for sb, th in sb2th.iteritems(): brand = sb2b[sb] yield 'brand', dict(company=company, brand=brand, twitter_handle=th) yield 'company', c yield 'rating', r # parse claims for b in soup.find(id='company_score').parent.select('b'): m = SUBSCORE_RE.match(b.text) if m and isinstance(b.next_sibling, unicode): # used this for debugging #area = m.group(1) area_score = int(m.group(2)) area_max_score = int(m.group(3)) raw_claim = b.next_sibling if NO_SPLIT_CLAIM_RE.match(raw_claim): claims = [raw_claim] else: claims = list(split_into_sentences(raw_claim)) for claim in claims: # strip company name off claim claim = ltrim_sentence(claim, [company, 'the company']) judgment = claim_to_judgment(claim) # if score is low, maybe it's not so positive after all if judgment == 1 and area_score / area_max_score < 0.5: judgment == 0 yield 'claim', dict(company=company, claim=claim, judgment=judgment)
def scrape_campaign(): log.info("Main page") soup = scrape_soup(URL) # campaign record cn = {"url": URL, "goal": GOAL} cn["campaign"], cn["author"] = soup.title.text.split("|") # remove double spaces cn["copyright"] = scrape_copyright(soup) cn["facebook_url"] = scrape_facebook_url(soup) cn["twitter_handle"] = scrape_twitter_handle(soup) # get year cn["date"] = INT_RE.search(soup.select("div.content h2")[0].text).group() for a in soup.findAll("a"): if a.text.strip() == "Donate": cn["donate_url"] = urljoin(URL, a["href"]) break if "donate_url" not in cn: raise ValueError("Donate URL not found") yield "campaign", cn rating_divs = soup.select("div#corank div.row") if not rating_divs: raise ValueError("ratings not found") for div in rating_divs: c = {} r = {"company": c} company_a = div.select("a.coname")[0] company = company_a.text c["company"] = company teaser = div.select("span.teaser")[0].text r["categories"] = CATEGORIES_SEP.split(CATEGORIES_RE.match(teaser).group(1)) for rank_class, judgment in RANK_CLASS_TO_JUDGMENT.items(): if div.select("span.rank." + rank_class): r["judgment"] = judgment break else: raise ValueError("rating for {} not found".format(r["company"])) r["score"] = int(INT_RE.search(div.select("div.col_score")[0].text).group()) r["categories"] = [CATEGORY] # fetch details company_id = company_a["href"].split("#")[-1] query = dict(action="getcompany", companyid=company_id) # use POST to get details JSON log.info("Details for {}".format(company)) details = scrape_json(DETAILS_URL, data=urlencode(query)) details = details[0][0] # wrapped in lists. why? c["url"] = details["ext_url"] # TODO: details['message'] might be useful too. It's a message # that participants are supposed to send to the company: # "Thank you for the leadership you have shown in working to..." yield "rating", r detail_soup = BeautifulSoup(details["detail"]) claim_lis = detail_soup.select("li") # First two bullet points are categories and a description # of the company's ranking (reversed for Nokia) # Last bullet point is what the company can do to improve its score. claim_lis = claim_lis[2:-1] for i, claim_li in enumerate(claim_lis): claim = claim_li.text judgment = claim_to_judgment(claim) claim = clarify_claim(claim, CLAIM_CLARIFICATIONS) yield "claim", dict(company=company, claim=claim, judgment=judgment)