def fetch_lobbying(td_metadata):
td_id = td_metadata['id']
type = td_metadata['type']
out = {}
if type == 'organization' or type == 'industry':
out['top_issues'] = api.org.issues(td_id)[:5]
out['expenditures'] = float(td_metadata['totals']['-1']['non_firm_spending']) + float(td_metadata['totals']['-1']['firm_income'])
out['is_lobbyist'] = False
if td_metadata['metadata']['lobbying_firm']:
out['top_issues'] = api.org.registrant_issues(td_id)[:5]
out['is_lobbying_firm'] = True
out['clients'] = [{
'name': standardize_name(client['client_name'], 'organization'),
'slug': slugify(standardize_name(client['client_name'], 'organization')),
'count': client['count'],
'id': client['client_entity'],
'type': 'organization',
} for client in api.org.registrant_clients(td_id)][:5]
else:
out['top_issues'] = api.org.issues(td_id)[:5]
out['is_lobbying_firm'] = False
out['clients'] = None
elif type == 'individual':
out['is_lobbyist'] = bool(td_metadata['lobbying_years'])
out['is_lobbying_firm'] = None
if out['is_lobbyist']:
out['top_issues'] = api.org.issues(td_id)[:5]
out['clients'] = [{
'name': standardize_name(client['client_name'], 'organization'),
'slug': slugify(standardize_name(client['client_name'], 'organization')),
'count': client['count'],
'id': client['client_entity'],
'type': 'organization',
} for client in api.indiv.clients(td_id)][:5]
else:
out['top_issues'] = None
out['clients'] = None
elif type == 'politician':
out['is_lobbyist'] = None
out['is_lobbying_firm'] = None
out['top_issues'] = None
out['expenditures'] = None
out['is_lobbying_firm'] = None
return out
def fetch_finance(td_metadata):
td_id = td_metadata['id']
type = td_metadata['type']
out = {}
if type == 'organization' or type == 'industry':
recipient_breakdown = api.org.party_breakdown(td_id)
out['recipient_breakdown'] = {'dem': float(recipient_breakdown.get('Democrats', [0, 0])[1]), 'rep': float(recipient_breakdown.get('Republicans', [0, 0])[1]), 'other': float(recipient_breakdown.get('Other', [0, 0])[1])}
out['contributor_type_breakdown'] = None
out['contributor_local_breakdown'] = None
out['top_industries'] = None
# for totals in TD, the 'contributor' and 'recipient' totals seem backwards, so I'm just going to call everything 'contribution' here
out['contribution_total'] = td_metadata['totals']['-1']['contributor_amount']
elif type == 'individual':
recipient_breakdown = api.indiv.party_breakdown(td_id)
out['recipient_breakdown'] = {'dem': float(recipient_breakdown.get('Democrats', [0, 0])[1]), 'rep': float(recipient_breakdown.get('Republicans', [0, 0])[1]), 'other': float(recipient_breakdown.get('Other', [0, 0])[1])}
out['contributor_type_breakdown'] = None
out['contributor_local_breakdown'] = None
out['top_industries'] = None
out['contribution_total'] = td_metadata['totals']['-1']['contributor_amount']
elif type == 'politician':
out['recipient_breakdown'] = None
contributor_type_breakdown = api.pol.contributor_type_breakdown(td_id)
out['contributor_type_breakdown'] = {'individual': float(contributor_type_breakdown.get('Individuals', [0, 0])[1]), 'pac': float(contributor_type_breakdown.get('PACs', [0, 0])[1])}
contributor_local_breakdown = api.pol.local_breakdown(td_id)
out['contributor_local_breakdown'] = {'in_state': float(contributor_local_breakdown.get('in-state', [0, 0])[1]), 'out_of_state': float(contributor_local_breakdown.get('out-of-state', [0, 0])[1])}
top_industries = api.pol.industries(td_id)
out['top_industries'] = [{
'amount': float(s['amount']),
'id': s['id'],
'name': standardize_name(s['name'], 'industry'),
'slug': slugify(standardize_name(s['name'], 'industry')),
'type': 'industry',
} for s in top_industries if s['should_show_entity']][:5]
out['contribution_total'] = td_metadata['totals']['-1']['recipient_amount']
return out
def sender_info(request):
# api key test
if not (hasattr(request, 'apikey') and request.apikey.status == 'A'):
return HttpResponse("Authorization Required", status=401)
name = request.REQUEST.get('name', '').strip()
email = request.REQUEST.get('email', '').strip()
organization = None
out = {}
organization = ''
org_info = None
if email:
parts = email.split("@")
if len(parts) > 1:
domain = parts[1]
# if it's a US TLD, just use the actual purchased domain name (not subdomains) for a match
if re.match(r'.*\.(com|net|org)$', domain):
domain = '.'.join(domain.split('.')[-2:])
orgs = lookup_domain(domain)
if len(orgs) > 1:
orgs = sorted(orgs, key=lambda org: Levenshtein.ratio(domain, org['name'].lower()), reverse=True)
if orgs:
organization = orgs[0]['name']
matches = matching.match(organization)
if matches:
org_info = get_entity_data(matches.keys()[0])
results = None
lat, lon = (None, None)
geoip = ip_lookup(request.META['REMOTE_ADDR'])
if geoip is not None:
lat, lon = geoip
if not lat or not lon:
# hard-code DC's info for now so that it still works, since our API can't deal with not having geo data
lat = '38.895112'
lon = '-77.036366'
if name and ' ' in name:
results = api._get_url_json('contributions/contributor_geo.json', parse_json=True, query=name, lat=lat, lon=lon)
sender_info = []
if results:
for result in results:
loc = result['contributor_location'].split(', ')
if len(loc) > 1:
city = loc[0].split('-')[0]
state = loc[1].replace(' MSA', '').split('-')[0]
else:
sloc = result['contributor_location'].split(' ')
state = sloc[0]
city = string.capwords(' '.join(sloc[1:]))
sender_info.append({
'name': standardize_name(result['contributor_name'], "individual"),
'city': city,
'state': state,
'total': float(result['amount_total']),
'dem_total': float(result['amount_democrat']),
'rep_total': float(result['amount_republican']),
'other_total': float(result['amount_total']) - (float(result['amount_democrat']) + float(result['amount_republican'])),
'count': result['count'],
'url': base64.b64encode(urllib.urlencode({'contributor_ft': name, 'msa_ft': result['contributor_location']}))
})
out = {
'name': name,
'email': email,
'sender_info': sender_info,
'url': base64.b64encode(urllib.urlencode({'contributor_ft': name})),
'organization': organization,
'org_info': org_info
}
if 'callback' in request.GET:
return HttpResponse('%s(%s)' % (request.GET['callback'], json.dumps(out)), 'text/javascript')
else:
return HttpResponse(json.dumps(out), mimetype="application/json")
def sender_info(request):
# api key test
if not (hasattr(request, 'apikey') and request.apikey.status == 'A'):
return HttpResponse("Authorization Required", status=401)
name = request.REQUEST.get('name', '').strip()
email = request.REQUEST.get('email', '').strip()
organization = None
out = {}
organization = ''
org_info = None
if email:
parts = email.split("@")
if len(parts) > 1:
domain = parts[1]
# if it's a US TLD, just use the actual purchased domain name (not subdomains) for a match
if re.match(r'.*\.(com|net|org)$', domain):
domain = '.'.join(domain.split('.')[-2:])
orgs = lookup_domain(domain)
if len(orgs) > 1:
orgs = sorted(orgs,
key=lambda org: Levenshtein.ratio(
domain, org['name'].lower()),
reverse=True)
if orgs:
organization = orgs[0]['name']
matches = matching.match(organization)
if matches:
org_info = get_entity_data(matches.keys()[0])
results = None
lat, lon = (None, None)
geoip = ip_lookup(request.META['REMOTE_ADDR'])
if geoip is not None:
lat, lon = geoip
if not lat or not lon:
# hard-code DC's info for now so that it still works, since our API can't deal with not having geo data
lat = '38.895112'
lon = '-77.036366'
if name and ' ' in name:
results = api._get_url_json('contributions/contributor_geo.json',
parse_json=True,
query=name,
lat=lat,
lon=lon)
sender_info = []
if results:
for result in results:
loc = result['contributor_location'].split(', ')
if len(loc) > 1:
city = loc[0].split('-')[0]
state = loc[1].replace(' MSA', '').split('-')[0]
else:
sloc = result['contributor_location'].split(' ')
state = sloc[0]
city = string.capwords(' '.join(sloc[1:]))
sender_info.append({
'name':
standardize_name(result['contributor_name'], "individual"),
'city':
city,
'state':
state,
'total':
float(result['amount_total']),
'dem_total':
float(result['amount_democrat']),
'rep_total':
float(result['amount_republican']),
'other_total':
float(result['amount_total']) -
(float(result['amount_democrat']) +
float(result['amount_republican'])),
'count':
result['count'],
'url':
base64.b64encode(
urllib.urlencode({
'contributor_ft': name,
'msa_ft': result['contributor_location']
}))
})
out = {
'name': name,
'email': email,
'sender_info': sender_info,
'url': base64.b64encode(urllib.urlencode({'contributor_ft': name})),
'organization': organization,
'org_info': org_info
}
if 'callback' in request.GET:
return HttpResponse(
'%s(%s)' % (request.GET['callback'], json.dumps(out)),
'text/javascript')
else:
return HttpResponse(json.dumps(out), mimetype="application/json")
def generate_entity_data(td_id, skip_frequent=False):
td_metadata = api.entities.metadata(td_id)
if not bool(td_metadata['totals']):
return None
struct = {
'id': td_metadata['id'],
'name': str(standardize_name(td_metadata['name'], td_metadata['type'])),
'raw_name': td_metadata['name'],
'type': td_metadata['type'],
'bioguide_id': td_metadata['metadata'].get('bioguide_id', None),
'seat': None,
'seat_label': None,
'held_seat': None,
'affiliated_organizations': None
}
struct['slug'] = slugify(struct['name'])
if struct['type'] == 'politician':
# politician name
poli_name = PoliticianNameCleaver(td_metadata['name']).parse().plus_metadata(td_metadata['metadata']['party'], td_metadata['metadata']['state'])
struct['name'] = str(poli_name)
# politician office info
years = sorted([key for key in td_metadata['metadata'].keys() if is_int(key)])
wins = [td_metadata['metadata'][year] for year in years if td_metadata['metadata'][year]['seat_result'] == 'W']
if wins:
struct['seat'] = wins[-1]['seat']
struct['held_seat'] = True
else:
losses = [td_metadata['metadata'][year] for year in years if td_metadata['metadata'][year]['seat_result'] == 'L']
if losses:
struct['seat'] = losses[-1]['seat']
struct['held_seat'] = False
elif years:
struct['seat'] = td_metadata['metadata'][years[-1]]['seat']
struct['held_seat'] = False
if struct['seat'] and struct['seat'] in seat_labels:
struct['seat_label'] = seat_labels[struct['seat']]
elif struct['type'] == 'individual':
if 'affiliated_organizations' in td_metadata['metadata'] and type(td_metadata['metadata']['affiliated_organizations']) == list and len(td_metadata['metadata']['affiliated_organizations']) > 0:
struct['affiliated_organizations'] = [{
'type': org['type'],
'name': org['name'],
'id': org['id'],
'slug': slugify(standardize_name(org['name'], org['type']))
} for org in td_metadata['metadata']['affiliated_organizations']]
crp_ids = filter(lambda x: x['namespace'] == 'urn:crp:recipient', td_metadata['external_ids'])
if crp_ids:
struct['crp_id'] = crp_ids[0]['id']
else:
struct['crp_id'] = None
struct['campaign_finance'] = fetch_finance(td_metadata)
struct['lobbying'] = fetch_lobbying(td_metadata)
struct['upcoming_fundraisers'] = fetch_pt(td_metadata) if not skip_frequent else None
return struct
def fetch_lobbying(td_metadata):
td_id = td_metadata['id']
type = td_metadata['type']
out = {}
if type == 'organization' or type == 'industry':
out['top_issues'] = api.org.issues(td_id)[:5]
out['expenditures'] = float(
td_metadata['totals']['-1']['non_firm_spending']) + float(
td_metadata['totals']['-1']['firm_income'])
out['is_lobbyist'] = False
if td_metadata['metadata']['lobbying_firm']:
out['top_issues'] = api.org.registrant_issues(td_id)[:5]
out['is_lobbying_firm'] = True
out['clients'] = [{
'name':
standardize_name(client['client_name'], 'organization'),
'slug':
slugify(standardize_name(client['client_name'],
'organization')),
'count':
client['count'],
'id':
client['client_entity'],
'type':
'organization',
} for client in api.org.registrant_clients(td_id)][:5]
else:
out['top_issues'] = api.org.issues(td_id)[:5]
out['is_lobbying_firm'] = False
out['clients'] = None
elif type == 'individual':
out['is_lobbyist'] = bool(td_metadata['lobbying_years'])
out['is_lobbying_firm'] = None
if out['is_lobbyist']:
out['top_issues'] = api.org.issues(td_id)[:5]
out['clients'] = [{
'name':
standardize_name(client['client_name'], 'organization'),
'slug':
slugify(standardize_name(client['client_name'],
'organization')),
'count':
client['count'],
'id':
client['client_entity'],
'type':
'organization',
} for client in api.indiv.clients(td_id)][:5]
else:
out['top_issues'] = None
out['clients'] = None
elif type == 'politician':
out['is_lobbyist'] = None
out['is_lobbying_firm'] = None
out['top_issues'] = None
out['expenditures'] = None
out['is_lobbying_firm'] = None
return out
def generate_entity_data(td_id, skip_frequent=False):
td_metadata = api.entities.metadata(td_id)
if not bool(td_metadata['totals']):
return None
struct = {
'id': td_metadata['id'],
'name': str(standardize_name(td_metadata['name'],
td_metadata['type'])),
'raw_name': td_metadata['name'],
'type': td_metadata['type'],
'bioguide_id': td_metadata['metadata'].get('bioguide_id', None),
'seat': None,
'seat_label': None,
'held_seat': None,
'affiliated_organizations': None
}
struct['slug'] = slugify(struct['name'])
if struct['type'] == 'politician':
# politician name
poli_name = PoliticianNameCleaver(
td_metadata['name']).parse().plus_metadata(
td_metadata['metadata']['party'],
td_metadata['metadata']['state'])
struct['name'] = str(poli_name)
# politician office info
years = sorted(
[key for key in td_metadata['metadata'].keys() if is_int(key)])
wins = [
td_metadata['metadata'][year] for year in years
if td_metadata['metadata'][year]['seat_result'] == 'W'
]
if wins:
struct['seat'] = wins[-1]['seat']
struct['held_seat'] = True
else:
losses = [
td_metadata['metadata'][year] for year in years
if td_metadata['metadata'][year]['seat_result'] == 'L'
]
if losses:
struct['seat'] = losses[-1]['seat']
struct['held_seat'] = False
elif years:
struct['seat'] = td_metadata['metadata'][years[-1]]['seat']
struct['held_seat'] = False
if struct['seat'] and struct['seat'] in seat_labels:
struct['seat_label'] = seat_labels[struct['seat']]
elif struct['type'] == 'individual':
if 'affiliated_organizations' in td_metadata['metadata'] and type(
td_metadata['metadata']
['affiliated_organizations']) == list and len(
td_metadata['metadata']['affiliated_organizations']) > 0:
struct['affiliated_organizations'] = [{
'type':
org['type'],
'name':
org['name'],
'id':
org['id'],
'slug':
slugify(standardize_name(org['name'], org['type']))
} for org in td_metadata['metadata']['affiliated_organizations']]
crp_ids = filter(lambda x: x['namespace'] == 'urn:crp:recipient',
td_metadata['external_ids'])
if crp_ids:
struct['crp_id'] = crp_ids[0]['id']
else:
struct['crp_id'] = None
struct['campaign_finance'] = fetch_finance(td_metadata)
struct['lobbying'] = fetch_lobbying(td_metadata)
struct['upcoming_fundraisers'] = fetch_pt(
td_metadata) if not skip_frequent else None
return struct
def fetch_finance(td_metadata):
td_id = td_metadata['id']
type = td_metadata['type']
out = {}
if type == 'organization' or type == 'industry':
recipient_breakdown = api.org.party_breakdown(td_id)
out['recipient_breakdown'] = {
'dem': float(recipient_breakdown.get('Democrats', [0, 0])[1]),
'rep': float(recipient_breakdown.get('Republicans', [0, 0])[1]),
'other': float(recipient_breakdown.get('Other', [0, 0])[1])
}
out['contributor_type_breakdown'] = None
out['contributor_local_breakdown'] = None
out['top_industries'] = None
# for totals in TD, the 'contributor' and 'recipient' totals seem backwards, so I'm just going to call everything 'contribution' here
out['contribution_total'] = td_metadata['totals']['-1'][
'contributor_amount']
elif type == 'individual':
recipient_breakdown = api.indiv.party_breakdown(td_id)
out['recipient_breakdown'] = {
'dem': float(recipient_breakdown.get('Democrats', [0, 0])[1]),
'rep': float(recipient_breakdown.get('Republicans', [0, 0])[1]),
'other': float(recipient_breakdown.get('Other', [0, 0])[1])
}
out['contributor_type_breakdown'] = None
out['contributor_local_breakdown'] = None
out['top_industries'] = None
out['contribution_total'] = td_metadata['totals']['-1'][
'contributor_amount']
elif type == 'politician':
out['recipient_breakdown'] = None
contributor_type_breakdown = api.pol.contributor_type_breakdown(td_id)
out['contributor_type_breakdown'] = {
'individual':
float(contributor_type_breakdown.get('Individuals', [0, 0])[1]),
'pac':
float(contributor_type_breakdown.get('PACs', [0, 0])[1])
}
contributor_local_breakdown = api.pol.local_breakdown(td_id)
out['contributor_local_breakdown'] = {
'in_state':
float(contributor_local_breakdown.get('in-state', [0, 0])[1]),
'out_of_state':
float(contributor_local_breakdown.get('out-of-state', [0, 0])[1])
}
top_industries = api.pol.industries(td_id)
out['top_industries'] = [{
'amount':
float(s['amount']),
'id':
s['id'],
'name':
standardize_name(s['name'], 'industry'),
'slug':
slugify(standardize_name(s['name'], 'industry')),
'type':
'industry',
} for s in top_industries if s['should_show_entity']][:5]
out['contribution_total'] = td_metadata['totals']['-1'][
'recipient_amount']
return out
