def load_votesmart():
# Candidates from votesmart
for district, cands in votesmart.candidates():
district = tools.fix_district_name(district)
for pol in cands:
vs_id = pol['candidateId']
wd = get_wd_id(vs_id)
if not wd: continue
polid = wd['watchdog_id']
pol_cand = filter_dict(cand_mapping, pol)
if not db.select('politician', where='id=$polid', vars=locals()):
db.insert('politician',
seqname=False,
id=polid,
**unidecode(
filter_dict(schema.Politician.columns.keys(),
pol_cand)))
else:
# @@ Should probably check that we really want to do this, but
# it apears as though the data has two entries for current
# members (the second having more info filled out).
db.update('politician',
where='id=$polid',
vars=locals(),
**unidecode(
filter_dict(schema.Politician.columns.keys(),
pol_cand)))
if not db.select(
'congress',
where="politician_id=$polid AND congress_num='-1'",
vars=locals()):
db.insert('congress',
seqname=False,
congress_num='-1',
politician_id=polid,
district_id=district,
party=pol_cand['party'])
# Bios from votesmart
for vs_id, p in votesmart.bios():
pol = p['candidate']
if pol['gender']:
pol['gender'] = pol['gender'][0]
if pol['education']:
pol['education'] = pol['education'].replace('\r\n', '\n')
wd = get_wd_id(vs_id)
if not wd: continue
polid = wd['watchdog_id']
pol_people = filter_dict(schema.Politician.columns.keys(),
filter_dict(bios_mapping, pol))
if db.select('politician', where='votesmartid=$vs_id', vars=locals()):
db.update('politician',
where='votesmartid=$vs_id',
vars=locals(),
**unidecode(pol_people))
def load_votesmart():
# Candidates from votesmart
for district, cands in votesmart.candidates():
district=tools.fix_district_name(district)
for pol in cands:
vs_id=pol['candidateId']
wd = get_wd_id(vs_id)
if not wd: continue
polid = wd['watchdog_id']
pol_cand = filter_dict(cand_mapping, pol)
if not db.select('politician',
where='id=$polid', vars=locals()):
db.insert('politician',
seqname=False,
id=polid,
**unidecode(filter_dict(schema.Politician.columns.keys(),
pol_cand)))
else:
# @@ Should probably check that we really want to do this, but
# it apears as though the data has two entries for current
# members (the second having more info filled out).
db.update('politician', where='id=$polid', vars=locals(),
**unidecode(filter_dict(schema.Politician.columns.keys(),
pol_cand)))
if not db.select('congress',
where="politician_id=$polid AND congress_num='-1'",
vars=locals()):
db.insert('congress', seqname=False, congress_num='-1',
politician_id=polid, district_id=district,
party=pol_cand['party'])
# Bios from votesmart
for vs_id, p in votesmart.bios():
pol = p['candidate']
if pol['gender']:
pol['gender']=pol['gender'][0]
if pol['education']:
pol['education'] = pol['education'].replace('\r\n', '\n')
wd = get_wd_id(vs_id)
if not wd: continue
polid = wd['watchdog_id']
pol_people = filter_dict(schema.Politician.columns.keys(),
filter_dict(bios_mapping, pol))
if db.select('politician', where='votesmartid=$vs_id',vars=locals()):
db.update('politician', where='votesmartid=$vs_id',
vars=locals(), **unidecode(pol_people))
def combine():
watchdog_map = {}
govtrack_map = {}
print "Processing govtrack.us basics."
for pol in govtrack.parse_basics():
wd = get_wd_id(pol.id)
if not wd: continue
current_member = wd.get('current_member')
watchdog_id = wd['watchdog_id']
govtrack_map[
pol.id] = watchdog_map[watchdog_id] = newpol = web.storage()
newpol['current_member'] = current_member
for k, v in mapping.iteritems():
if k in pol: newpol[v] = pol[k]
print "Processing govtrack.us stats."
for pol in govtrack.parse_stats(
['enacted', 'introduced', 'cosponsor', 'speeches']):
if pol.id not in govtrack_map:
continue
else:
newpol = govtrack_map[pol.id]
if pol.get('SponsorEnacted'):
newpol.n_bills_introduced = int(pol.NumSponsor)
newpol.n_bills_enacted = int(pol.SponsorEnacted)
if pol.get('SponsorIntroduced'):
newpol.n_bills_debated = int(pol.NumSponsor) - int(
pol.SponsorIntroduced)
if pol.get('NumCosponsor'):
newpol.n_bills_cosponsored = int(pol.NumCosponsor)
if pol.get('Speeches'):
newpol.n_speeches = int(pol.Speeches)
newpol.words_per_speech = int(pol.WordsPerSpeech)
return watchdog_map
def combine():
watchdog_map = {}
govtrack_map = {}
print "Processing govtrack.us basics."
for pol in govtrack.parse_basics():
wd = get_wd_id(pol.id)
if not wd:
continue
current_member = wd.get("current_member")
watchdog_id = wd["watchdog_id"]
govtrack_map[pol.id] = watchdog_map[watchdog_id] = newpol = web.storage()
newpol["current_member"] = current_member
for k, v in mapping.iteritems():
if k in pol:
newpol[v] = pol[k]
print "Processing govtrack.us stats."
for pol in govtrack.parse_stats(["enacted", "introduced", "cosponsor", "speeches"]):
if pol.id not in govtrack_map:
continue
else:
newpol = govtrack_map[pol.id]
if pol.get("SponsorEnacted"):
newpol.n_bills_introduced = int(pol.NumSponsor)
newpol.n_bills_enacted = int(pol.SponsorEnacted)
if pol.get("SponsorIntroduced"):
newpol.n_bills_debated = int(pol.NumSponsor) - int(pol.SponsorIntroduced)
if pol.get("NumCosponsor"):
newpol.n_bills_cosponsored = int(pol.NumCosponsor)
if pol.get("Speeches"):
newpol.n_speeches = int(pol.Speeches)
newpol.words_per_speech = int(pol.WordsPerSpeech)
return watchdog_map