def load_votesmart(): # Candidates from votesmart for district, cands in votesmart.candidates(): district = tools.fix_district_name(district) for pol in cands: vs_id = pol['candidateId'] wd = get_wd_id(vs_id) if not wd: continue polid = wd['watchdog_id'] pol_cand = filter_dict(cand_mapping, pol) if not db.select('politician', where='id=$polid', vars=locals()): db.insert('politician', seqname=False, id=polid, **unidecode( filter_dict(schema.Politician.columns.keys(), pol_cand))) else: # @@ Should probably check that we really want to do this, but # it apears as though the data has two entries for current # members (the second having more info filled out). db.update('politician', where='id=$polid', vars=locals(), **unidecode( filter_dict(schema.Politician.columns.keys(), pol_cand))) if not db.select( 'congress', where="politician_id=$polid AND congress_num='-1'", vars=locals()): db.insert('congress', seqname=False, congress_num='-1', politician_id=polid, district_id=district, party=pol_cand['party']) # Bios from votesmart for vs_id, p in votesmart.bios(): pol = p['candidate'] if pol['gender']: pol['gender'] = pol['gender'][0] if pol['education']: pol['education'] = pol['education'].replace('\r\n', '\n') wd = get_wd_id(vs_id) if not wd: continue polid = wd['watchdog_id'] pol_people = filter_dict(schema.Politician.columns.keys(), filter_dict(bios_mapping, pol)) if db.select('politician', where='votesmartid=$vs_id', vars=locals()): db.update('politician', where='votesmartid=$vs_id', vars=locals(), **unidecode(pol_people))
def generate_ids(): wd_to_gt = load_wd_mapping() # Govtrack for pol in govtrack.parse_basics(): current_member = False collision = False watchdog_id = tools.getWatchdogID(pol.get('represents'),pol.lastname) if watchdog_id: current_member = True # pol.represents should always be the same as current_member, if we # remove the origional politician.json file we can use that # instead. assert(pol.represents) else: try: assert(not pol.get('represents')) except: print "no watchdog id for", web.safestr(pol.name), web.safestr(pol.represents) continue watchdog_id = gen_pol_id(pol) if watchdog_id in wd_to_gt and \ wd_to_gt[watchdog_id]['govtrack_id'] != pol.id: collision = True if (not collision) or current_member: if watchdog_id not in wd_to_gt: wd_to_gt[watchdog_id] = {} wd_to_gt[watchdog_id]['govtrack_id'] = pol.id if collision: wd_to_gt[watchdog_id]['collision'] = True if current_member: wd_to_gt[watchdog_id]['current_member'] = True # Votesmart for district, cands in votesmart.candidates(): district=tools.fix_district_name(district) for pol in cands: watchdog_id = tools.getWatchdogID(district, pol['lastName']) if not watchdog_id: watchdog_id = gen_pol_id(pol) vsid=pol['candidateId'] #TODO: Could use some more checking to be sure we are 1. adding the # correct votesmart id to the correct watchdog_id (eg. in the # case that there was a collision in processing the govtrack # data). And 2. aren't creating a new watchdog_id when there # was already one for this person. if watchdog_id not in wd_to_gt: wd_to_gt[watchdog_id] = {} wd_to_gt[watchdog_id]['votesmart_id'] = vsid return wd_to_gt
def load_votesmart(): # Candidates from votesmart for district, cands in votesmart.candidates(): district=tools.fix_district_name(district) for pol in cands: vs_id=pol['candidateId'] wd = get_wd_id(vs_id) if not wd: continue polid = wd['watchdog_id'] pol_cand = filter_dict(cand_mapping, pol) if not db.select('politician', where='id=$polid', vars=locals()): db.insert('politician', seqname=False, id=polid, **unidecode(filter_dict(schema.Politician.columns.keys(), pol_cand))) else: # @@ Should probably check that we really want to do this, but # it apears as though the data has two entries for current # members (the second having more info filled out). db.update('politician', where='id=$polid', vars=locals(), **unidecode(filter_dict(schema.Politician.columns.keys(), pol_cand))) if not db.select('congress', where="politician_id=$polid AND congress_num='-1'", vars=locals()): db.insert('congress', seqname=False, congress_num='-1', politician_id=polid, district_id=district, party=pol_cand['party']) # Bios from votesmart for vs_id, p in votesmart.bios(): pol = p['candidate'] if pol['gender']: pol['gender']=pol['gender'][0] if pol['education']: pol['education'] = pol['education'].replace('\r\n', '\n') wd = get_wd_id(vs_id) if not wd: continue polid = wd['watchdog_id'] pol_people = filter_dict(schema.Politician.columns.keys(), filter_dict(bios_mapping, pol)) if db.select('politician', where='votesmartid=$vs_id',vars=locals()): db.update('politician', where='votesmartid=$vs_id', vars=locals(), **unidecode(pol_people))