def generate_ids():
wd_to_gt = load_wd_mapping()
# Govtrack
for pol in govtrack.parse_basics():
current_member = False
collision = False
watchdog_id = tools.getWatchdogID(pol.get('represents'),pol.lastname)
if watchdog_id:
current_member = True
# pol.represents should always be the same as current_member, if we
# remove the origional politician.json file we can use that
# instead.
assert(pol.represents)
else:
try:
assert(not pol.get('represents'))
except:
print "no watchdog id for", web.safestr(pol.name), web.safestr(pol.represents)
continue
watchdog_id = gen_pol_id(pol)
if watchdog_id in wd_to_gt and \
wd_to_gt[watchdog_id]['govtrack_id'] != pol.id:
collision = True
if (not collision) or current_member:
if watchdog_id not in wd_to_gt:
wd_to_gt[watchdog_id] = {}
wd_to_gt[watchdog_id]['govtrack_id'] = pol.id
if collision:
wd_to_gt[watchdog_id]['collision'] = True
if current_member:
wd_to_gt[watchdog_id]['current_member'] = True
# Votesmart
for district, cands in votesmart.candidates():
district=tools.fix_district_name(district)
for pol in cands:
watchdog_id = tools.getWatchdogID(district, pol['lastName'])
if not watchdog_id:
watchdog_id = gen_pol_id(pol)
vsid=pol['candidateId']
#TODO: Could use some more checking to be sure we are 1. adding the
# correct votesmart id to the correct watchdog_id (eg. in the
# case that there was a collision in processing the govtrack
# data). And 2. aren't creating a new watchdog_id when there
# was already one for this person.
if watchdog_id not in wd_to_gt:
wd_to_gt[watchdog_id] = {}
wd_to_gt[watchdog_id]['votesmart_id'] = vsid
return wd_to_gt
def main():
for pol in voteview.parse():
if not pol.get('district_id'): continue #@@
if not tools.districtp(pol.district_id) and pol.district_id.endswith('01'):
pol.district_id = pol.district_id.split('-')[0] + '-' + '00'
watchdog_id = tools.getWatchdogID(pol.district_id, pol.last_name)
if watchdog_id:
db.update('politician', where='id=$watchdog_id', vars=locals(),
icpsrid = pol.icpsr_id,
nominate = pol.dim1,
predictability = 1 - (pol.n_errs / float(pol.n_votes)))
def main():
for dist, canl in cans.iteritems():
dist=dist.replace("-SEN1","").replace("-SEN2","") ## JT: Hack
for can in canl:
wid = tools.getWatchdogID(dist,can['lastName'])
if wid:
bio = bios[can['candidateId']]['candidate']
db.update('politician', where='id = $wid', vars=locals(),
votesmartid=can['candidateId'],
nickname=can['nickName'],
birthplace=bio['birthPlace'],
education=bio['education'].replace('\r\n', '\n')
)
def main():
for pol in voteview.parse():
if not pol.get('district_id'): continue #@@
if not tools.districtp(
pol.district_id) and pol.district_id.endswith('01'):
pol.district_id = pol.district_id.split('-')[0] + '-' + '00'
watchdog_id = tools.getWatchdogID(pol.district_id, pol.last_name)
if watchdog_id:
db.update('politician',
where='id=$watchdog_id',
vars=locals(),
icpsrid=pol.icpsr_id,
nominate=pol.dim1,
predictability=1 - (pol.n_errs / float(pol.n_votes)))
def combine():
watchdog_map = {}
govtrack_map = {}
for pol in govtrack.parse_basics():
watchdog_id = tools.getWatchdogID(pol.get('represents'),pol.lastname)
if watchdog_id:
govtrack_map[pol.id] = watchdog_map[watchdog_id] = newpol = web.storage()
else:
continue
for k, v in mapping.iteritems():
if k in pol: newpol[v] = pol[k]
for pol in govtrack.parse_stats([
'enacted', 'introduced', 'cosponsor', 'speeches']):
if pol.id not in govtrack_map:
continue
else:
newpol = govtrack_map[pol.id]
if pol.get('SponsorEnacted'):
newpol.n_bills_introduced = int(pol.NumSponsor)
newpol.n_bills_enacted = int(pol.SponsorEnacted)
if pol.get('SponsorIntroduced'):
newpol.n_bills_debated = int(pol.NumSponsor) - int(pol.SponsorIntroduced)
if pol.get('NumCosponsor'):
newpol.n_bills_cosponsored = int(pol.NumCosponsor)
if pol.get('Speeches'):
newpol.n_speeches = int(pol.Speeches)
newpol.words_per_speech = int(pol.WordsPerSpeech)
return watchdog_map
'gender': 'gender',
'id': 'govtrackid',
'lastname': 'lastname',
'middlename': 'middlename',
'osid': 'opensecretsid',
'party': 'party',
'religion': 'religion',
'represents': 'district',
'url': 'officeurl'
}
watchdog_map = {}
govtrack_map = {}
for pol in govtrack.parse_basics():
watchdog_id = tools.getWatchdogID(pol.get('represents'),pol.lastname)
if watchdog_id:
govtrack_map[pol.id] = watchdog_map[watchdog_id] = newpol = web.storage()
else:
continue
for k, v in mapping.iteritems():
if k in pol: newpol[v] = pol[k]
for pol in govtrack.parse_stats(['enacted', 'introduced', 'cosponsor',
'speeches']):
if pol.id not in govtrack_map:
continue
else:
newpol = govtrack_map[pol.id]
"""
import voteview
"""
import simplejson
from parse import voteview
import tools
out = {}
for pol in voteview.parse():
if pol.congress != 110 or not pol.get('district_id'): continue #@@
if not tools.districtp(pol.district_id) and pol.district_id.endswith('01'):
pol.district_id = pol.district_id.split('-')[0] + '-' + '00'
watchdog_id = tools.getWatchdogID(pol.district_id,pol.last_name)
if watchdog_id:
out[watchdog_id] = {
'icpsrid': pol.icpsr_id,
'nominate': pol.dim1,
'predictability': 1 - (pol.n_errs / float(pol.n_votes))
}
if __name__ == "__main__":
print simplejson.dumps(out, indent=2, sort_keys=True)
