def fetch_lobbying(td_metadata): td_id = td_metadata['id'] type = td_metadata['type'] out = {} if type == 'organization' or type == 'industry': out['top_issues'] = api.org.issues(td_id)[:5] out['expenditures'] = float(td_metadata['totals']['-1']['non_firm_spending']) + float(td_metadata['totals']['-1']['firm_income']) out['is_lobbyist'] = False if td_metadata['metadata']['lobbying_firm']: out['top_issues'] = api.org.registrant_issues(td_id)[:5] out['is_lobbying_firm'] = True out['clients'] = [{ 'name': standardize_name(client['client_name'], 'organization'), 'slug': slugify(standardize_name(client['client_name'], 'organization')), 'count': client['count'], 'id': client['client_entity'], 'type': 'organization', } for client in api.org.registrant_clients(td_id)][:5] else: out['top_issues'] = api.org.issues(td_id)[:5] out['is_lobbying_firm'] = False out['clients'] = None elif type == 'individual': out['is_lobbyist'] = bool(td_metadata['lobbying_years']) out['is_lobbying_firm'] = None if out['is_lobbyist']: out['top_issues'] = api.org.issues(td_id)[:5] out['clients'] = [{ 'name': standardize_name(client['client_name'], 'organization'), 'slug': slugify(standardize_name(client['client_name'], 'organization')), 'count': client['count'], 'id': client['client_entity'], 'type': 'organization', } for client in api.indiv.clients(td_id)][:5] else: out['top_issues'] = None out['clients'] = None elif type == 'politician': out['is_lobbyist'] = None out['is_lobbying_firm'] = None out['top_issues'] = None out['expenditures'] = None out['is_lobbying_firm'] = None return out
def fetch_finance(td_metadata): td_id = td_metadata['id'] type = td_metadata['type'] out = {} if type == 'organization' or type == 'industry': recipient_breakdown = api.org.party_breakdown(td_id) out['recipient_breakdown'] = {'dem': float(recipient_breakdown.get('Democrats', [0, 0])[1]), 'rep': float(recipient_breakdown.get('Republicans', [0, 0])[1]), 'other': float(recipient_breakdown.get('Other', [0, 0])[1])} out['contributor_type_breakdown'] = None out['contributor_local_breakdown'] = None out['top_industries'] = None # for totals in TD, the 'contributor' and 'recipient' totals seem backwards, so I'm just going to call everything 'contribution' here out['contribution_total'] = td_metadata['totals']['-1']['contributor_amount'] elif type == 'individual': recipient_breakdown = api.indiv.party_breakdown(td_id) out['recipient_breakdown'] = {'dem': float(recipient_breakdown.get('Democrats', [0, 0])[1]), 'rep': float(recipient_breakdown.get('Republicans', [0, 0])[1]), 'other': float(recipient_breakdown.get('Other', [0, 0])[1])} out['contributor_type_breakdown'] = None out['contributor_local_breakdown'] = None out['top_industries'] = None out['contribution_total'] = td_metadata['totals']['-1']['contributor_amount'] elif type == 'politician': out['recipient_breakdown'] = None contributor_type_breakdown = api.pol.contributor_type_breakdown(td_id) out['contributor_type_breakdown'] = {'individual': float(contributor_type_breakdown.get('Individuals', [0, 0])[1]), 'pac': float(contributor_type_breakdown.get('PACs', [0, 0])[1])} contributor_local_breakdown = api.pol.local_breakdown(td_id) out['contributor_local_breakdown'] = {'in_state': float(contributor_local_breakdown.get('in-state', [0, 0])[1]), 'out_of_state': float(contributor_local_breakdown.get('out-of-state', [0, 0])[1])} top_industries = api.pol.industries(td_id) out['top_industries'] = [{ 'amount': float(s['amount']), 'id': s['id'], 'name': standardize_name(s['name'], 'industry'), 'slug': slugify(standardize_name(s['name'], 'industry')), 'type': 'industry', } for s in top_industries if s['should_show_entity']][:5] out['contribution_total'] = td_metadata['totals']['-1']['recipient_amount'] return out
def sender_info(request): # api key test if not (hasattr(request, 'apikey') and request.apikey.status == 'A'): return HttpResponse("Authorization Required", status=401) name = request.REQUEST.get('name', '').strip() email = request.REQUEST.get('email', '').strip() organization = None out = {} organization = '' org_info = None if email: parts = email.split("@") if len(parts) > 1: domain = parts[1] # if it's a US TLD, just use the actual purchased domain name (not subdomains) for a match if re.match(r'.*\.(com|net|org)$', domain): domain = '.'.join(domain.split('.')[-2:]) orgs = lookup_domain(domain) if len(orgs) > 1: orgs = sorted(orgs, key=lambda org: Levenshtein.ratio(domain, org['name'].lower()), reverse=True) if orgs: organization = orgs[0]['name'] matches = matching.match(organization) if matches: org_info = get_entity_data(matches.keys()[0]) results = None lat, lon = (None, None) geoip = ip_lookup(request.META['REMOTE_ADDR']) if geoip is not None: lat, lon = geoip if not lat or not lon: # hard-code DC's info for now so that it still works, since our API can't deal with not having geo data lat = '38.895112' lon = '-77.036366' if name and ' ' in name: results = api._get_url_json('contributions/contributor_geo.json', parse_json=True, query=name, lat=lat, lon=lon) sender_info = [] if results: for result in results: loc = result['contributor_location'].split(', ') if len(loc) > 1: city = loc[0].split('-')[0] state = loc[1].replace(' MSA', '').split('-')[0] else: sloc = result['contributor_location'].split(' ') state = sloc[0] city = string.capwords(' '.join(sloc[1:])) sender_info.append({ 'name': standardize_name(result['contributor_name'], "individual"), 'city': city, 'state': state, 'total': float(result['amount_total']), 'dem_total': float(result['amount_democrat']), 'rep_total': float(result['amount_republican']), 'other_total': float(result['amount_total']) - (float(result['amount_democrat']) + float(result['amount_republican'])), 'count': result['count'], 'url': base64.b64encode(urllib.urlencode({'contributor_ft': name, 'msa_ft': result['contributor_location']})) }) out = { 'name': name, 'email': email, 'sender_info': sender_info, 'url': base64.b64encode(urllib.urlencode({'contributor_ft': name})), 'organization': organization, 'org_info': org_info } if 'callback' in request.GET: return HttpResponse('%s(%s)' % (request.GET['callback'], json.dumps(out)), 'text/javascript') else: return HttpResponse(json.dumps(out), mimetype="application/json")
def sender_info(request): # api key test if not (hasattr(request, 'apikey') and request.apikey.status == 'A'): return HttpResponse("Authorization Required", status=401) name = request.REQUEST.get('name', '').strip() email = request.REQUEST.get('email', '').strip() organization = None out = {} organization = '' org_info = None if email: parts = email.split("@") if len(parts) > 1: domain = parts[1] # if it's a US TLD, just use the actual purchased domain name (not subdomains) for a match if re.match(r'.*\.(com|net|org)$', domain): domain = '.'.join(domain.split('.')[-2:]) orgs = lookup_domain(domain) if len(orgs) > 1: orgs = sorted(orgs, key=lambda org: Levenshtein.ratio( domain, org['name'].lower()), reverse=True) if orgs: organization = orgs[0]['name'] matches = matching.match(organization) if matches: org_info = get_entity_data(matches.keys()[0]) results = None lat, lon = (None, None) geoip = ip_lookup(request.META['REMOTE_ADDR']) if geoip is not None: lat, lon = geoip if not lat or not lon: # hard-code DC's info for now so that it still works, since our API can't deal with not having geo data lat = '38.895112' lon = '-77.036366' if name and ' ' in name: results = api._get_url_json('contributions/contributor_geo.json', parse_json=True, query=name, lat=lat, lon=lon) sender_info = [] if results: for result in results: loc = result['contributor_location'].split(', ') if len(loc) > 1: city = loc[0].split('-')[0] state = loc[1].replace(' MSA', '').split('-')[0] else: sloc = result['contributor_location'].split(' ') state = sloc[0] city = string.capwords(' '.join(sloc[1:])) sender_info.append({ 'name': standardize_name(result['contributor_name'], "individual"), 'city': city, 'state': state, 'total': float(result['amount_total']), 'dem_total': float(result['amount_democrat']), 'rep_total': float(result['amount_republican']), 'other_total': float(result['amount_total']) - (float(result['amount_democrat']) + float(result['amount_republican'])), 'count': result['count'], 'url': base64.b64encode( urllib.urlencode({ 'contributor_ft': name, 'msa_ft': result['contributor_location'] })) }) out = { 'name': name, 'email': email, 'sender_info': sender_info, 'url': base64.b64encode(urllib.urlencode({'contributor_ft': name})), 'organization': organization, 'org_info': org_info } if 'callback' in request.GET: return HttpResponse( '%s(%s)' % (request.GET['callback'], json.dumps(out)), 'text/javascript') else: return HttpResponse(json.dumps(out), mimetype="application/json")
def generate_entity_data(td_id, skip_frequent=False): td_metadata = api.entities.metadata(td_id) if not bool(td_metadata['totals']): return None struct = { 'id': td_metadata['id'], 'name': str(standardize_name(td_metadata['name'], td_metadata['type'])), 'raw_name': td_metadata['name'], 'type': td_metadata['type'], 'bioguide_id': td_metadata['metadata'].get('bioguide_id', None), 'seat': None, 'seat_label': None, 'held_seat': None, 'affiliated_organizations': None } struct['slug'] = slugify(struct['name']) if struct['type'] == 'politician': # politician name poli_name = PoliticianNameCleaver(td_metadata['name']).parse().plus_metadata(td_metadata['metadata']['party'], td_metadata['metadata']['state']) struct['name'] = str(poli_name) # politician office info years = sorted([key for key in td_metadata['metadata'].keys() if is_int(key)]) wins = [td_metadata['metadata'][year] for year in years if td_metadata['metadata'][year]['seat_result'] == 'W'] if wins: struct['seat'] = wins[-1]['seat'] struct['held_seat'] = True else: losses = [td_metadata['metadata'][year] for year in years if td_metadata['metadata'][year]['seat_result'] == 'L'] if losses: struct['seat'] = losses[-1]['seat'] struct['held_seat'] = False elif years: struct['seat'] = td_metadata['metadata'][years[-1]]['seat'] struct['held_seat'] = False if struct['seat'] and struct['seat'] in seat_labels: struct['seat_label'] = seat_labels[struct['seat']] elif struct['type'] == 'individual': if 'affiliated_organizations' in td_metadata['metadata'] and type(td_metadata['metadata']['affiliated_organizations']) == list and len(td_metadata['metadata']['affiliated_organizations']) > 0: struct['affiliated_organizations'] = [{ 'type': org['type'], 'name': org['name'], 'id': org['id'], 'slug': slugify(standardize_name(org['name'], org['type'])) } for org in td_metadata['metadata']['affiliated_organizations']] crp_ids = filter(lambda x: x['namespace'] == 'urn:crp:recipient', td_metadata['external_ids']) if crp_ids: struct['crp_id'] = crp_ids[0]['id'] else: struct['crp_id'] = None struct['campaign_finance'] = fetch_finance(td_metadata) struct['lobbying'] = fetch_lobbying(td_metadata) struct['upcoming_fundraisers'] = fetch_pt(td_metadata) if not skip_frequent else None return struct
def fetch_lobbying(td_metadata): td_id = td_metadata['id'] type = td_metadata['type'] out = {} if type == 'organization' or type == 'industry': out['top_issues'] = api.org.issues(td_id)[:5] out['expenditures'] = float( td_metadata['totals']['-1']['non_firm_spending']) + float( td_metadata['totals']['-1']['firm_income']) out['is_lobbyist'] = False if td_metadata['metadata']['lobbying_firm']: out['top_issues'] = api.org.registrant_issues(td_id)[:5] out['is_lobbying_firm'] = True out['clients'] = [{ 'name': standardize_name(client['client_name'], 'organization'), 'slug': slugify(standardize_name(client['client_name'], 'organization')), 'count': client['count'], 'id': client['client_entity'], 'type': 'organization', } for client in api.org.registrant_clients(td_id)][:5] else: out['top_issues'] = api.org.issues(td_id)[:5] out['is_lobbying_firm'] = False out['clients'] = None elif type == 'individual': out['is_lobbyist'] = bool(td_metadata['lobbying_years']) out['is_lobbying_firm'] = None if out['is_lobbyist']: out['top_issues'] = api.org.issues(td_id)[:5] out['clients'] = [{ 'name': standardize_name(client['client_name'], 'organization'), 'slug': slugify(standardize_name(client['client_name'], 'organization')), 'count': client['count'], 'id': client['client_entity'], 'type': 'organization', } for client in api.indiv.clients(td_id)][:5] else: out['top_issues'] = None out['clients'] = None elif type == 'politician': out['is_lobbyist'] = None out['is_lobbying_firm'] = None out['top_issues'] = None out['expenditures'] = None out['is_lobbying_firm'] = None return out
def generate_entity_data(td_id, skip_frequent=False): td_metadata = api.entities.metadata(td_id) if not bool(td_metadata['totals']): return None struct = { 'id': td_metadata['id'], 'name': str(standardize_name(td_metadata['name'], td_metadata['type'])), 'raw_name': td_metadata['name'], 'type': td_metadata['type'], 'bioguide_id': td_metadata['metadata'].get('bioguide_id', None), 'seat': None, 'seat_label': None, 'held_seat': None, 'affiliated_organizations': None } struct['slug'] = slugify(struct['name']) if struct['type'] == 'politician': # politician name poli_name = PoliticianNameCleaver( td_metadata['name']).parse().plus_metadata( td_metadata['metadata']['party'], td_metadata['metadata']['state']) struct['name'] = str(poli_name) # politician office info years = sorted( [key for key in td_metadata['metadata'].keys() if is_int(key)]) wins = [ td_metadata['metadata'][year] for year in years if td_metadata['metadata'][year]['seat_result'] == 'W' ] if wins: struct['seat'] = wins[-1]['seat'] struct['held_seat'] = True else: losses = [ td_metadata['metadata'][year] for year in years if td_metadata['metadata'][year]['seat_result'] == 'L' ] if losses: struct['seat'] = losses[-1]['seat'] struct['held_seat'] = False elif years: struct['seat'] = td_metadata['metadata'][years[-1]]['seat'] struct['held_seat'] = False if struct['seat'] and struct['seat'] in seat_labels: struct['seat_label'] = seat_labels[struct['seat']] elif struct['type'] == 'individual': if 'affiliated_organizations' in td_metadata['metadata'] and type( td_metadata['metadata'] ['affiliated_organizations']) == list and len( td_metadata['metadata']['affiliated_organizations']) > 0: struct['affiliated_organizations'] = [{ 'type': org['type'], 'name': org['name'], 'id': org['id'], 'slug': slugify(standardize_name(org['name'], org['type'])) } for org in td_metadata['metadata']['affiliated_organizations']] crp_ids = filter(lambda x: x['namespace'] == 'urn:crp:recipient', td_metadata['external_ids']) if crp_ids: struct['crp_id'] = crp_ids[0]['id'] else: struct['crp_id'] = None struct['campaign_finance'] = fetch_finance(td_metadata) struct['lobbying'] = fetch_lobbying(td_metadata) struct['upcoming_fundraisers'] = fetch_pt( td_metadata) if not skip_frequent else None return struct
def fetch_finance(td_metadata): td_id = td_metadata['id'] type = td_metadata['type'] out = {} if type == 'organization' or type == 'industry': recipient_breakdown = api.org.party_breakdown(td_id) out['recipient_breakdown'] = { 'dem': float(recipient_breakdown.get('Democrats', [0, 0])[1]), 'rep': float(recipient_breakdown.get('Republicans', [0, 0])[1]), 'other': float(recipient_breakdown.get('Other', [0, 0])[1]) } out['contributor_type_breakdown'] = None out['contributor_local_breakdown'] = None out['top_industries'] = None # for totals in TD, the 'contributor' and 'recipient' totals seem backwards, so I'm just going to call everything 'contribution' here out['contribution_total'] = td_metadata['totals']['-1'][ 'contributor_amount'] elif type == 'individual': recipient_breakdown = api.indiv.party_breakdown(td_id) out['recipient_breakdown'] = { 'dem': float(recipient_breakdown.get('Democrats', [0, 0])[1]), 'rep': float(recipient_breakdown.get('Republicans', [0, 0])[1]), 'other': float(recipient_breakdown.get('Other', [0, 0])[1]) } out['contributor_type_breakdown'] = None out['contributor_local_breakdown'] = None out['top_industries'] = None out['contribution_total'] = td_metadata['totals']['-1'][ 'contributor_amount'] elif type == 'politician': out['recipient_breakdown'] = None contributor_type_breakdown = api.pol.contributor_type_breakdown(td_id) out['contributor_type_breakdown'] = { 'individual': float(contributor_type_breakdown.get('Individuals', [0, 0])[1]), 'pac': float(contributor_type_breakdown.get('PACs', [0, 0])[1]) } contributor_local_breakdown = api.pol.local_breakdown(td_id) out['contributor_local_breakdown'] = { 'in_state': float(contributor_local_breakdown.get('in-state', [0, 0])[1]), 'out_of_state': float(contributor_local_breakdown.get('out-of-state', [0, 0])[1]) } top_industries = api.pol.industries(td_id) out['top_industries'] = [{ 'amount': float(s['amount']), 'id': s['id'], 'name': standardize_name(s['name'], 'industry'), 'slug': slugify(standardize_name(s['name'], 'industry')), 'type': 'industry', } for s in top_industries if s['should_show_entity']][:5] out['contribution_total'] = td_metadata['totals']['-1'][ 'recipient_amount'] return out