def generate_ids(): wd_to_gt = load_wd_mapping() # Govtrack for pol in govtrack.parse_basics(): current_member = False collision = False watchdog_id = tools.getWatchdogID(pol.get('represents'),pol.lastname) if watchdog_id: current_member = True # pol.represents should always be the same as current_member, if we # remove the origional politician.json file we can use that # instead. assert(pol.represents) else: try: assert(not pol.get('represents')) except: print "no watchdog id for", web.safestr(pol.name), web.safestr(pol.represents) continue watchdog_id = gen_pol_id(pol) if watchdog_id in wd_to_gt and \ wd_to_gt[watchdog_id]['govtrack_id'] != pol.id: collision = True if (not collision) or current_member: if watchdog_id not in wd_to_gt: wd_to_gt[watchdog_id] = {} wd_to_gt[watchdog_id]['govtrack_id'] = pol.id if collision: wd_to_gt[watchdog_id]['collision'] = True if current_member: wd_to_gt[watchdog_id]['current_member'] = True # Votesmart for district, cands in votesmart.candidates(): district=tools.fix_district_name(district) for pol in cands: watchdog_id = tools.getWatchdogID(district, pol['lastName']) if not watchdog_id: watchdog_id = gen_pol_id(pol) vsid=pol['candidateId'] #TODO: Could use some more checking to be sure we are 1. adding the # correct votesmart id to the correct watchdog_id (eg. in the # case that there was a collision in processing the govtrack # data). And 2. aren't creating a new watchdog_id when there # was already one for this person. if watchdog_id not in wd_to_gt: wd_to_gt[watchdog_id] = {} wd_to_gt[watchdog_id]['votesmart_id'] = vsid return wd_to_gt
def combine(): watchdog_map = {} govtrack_map = {} print "Processing govtrack.us basics." for pol in govtrack.parse_basics(): wd = get_wd_id(pol.id) if not wd: continue current_member = wd.get('current_member') watchdog_id = wd['watchdog_id'] govtrack_map[ pol.id] = watchdog_map[watchdog_id] = newpol = web.storage() newpol['current_member'] = current_member for k, v in mapping.iteritems(): if k in pol: newpol[v] = pol[k] print "Processing govtrack.us stats." for pol in govtrack.parse_stats( ['enacted', 'introduced', 'cosponsor', 'speeches']): if pol.id not in govtrack_map: continue else: newpol = govtrack_map[pol.id] if pol.get('SponsorEnacted'): newpol.n_bills_introduced = int(pol.NumSponsor) newpol.n_bills_enacted = int(pol.SponsorEnacted) if pol.get('SponsorIntroduced'): newpol.n_bills_debated = int(pol.NumSponsor) - int( pol.SponsorIntroduced) if pol.get('NumCosponsor'): newpol.n_bills_cosponsored = int(pol.NumCosponsor) if pol.get('Speeches'): newpol.n_speeches = int(pol.Speeches) newpol.words_per_speech = int(pol.WordsPerSpeech) return watchdog_map
def combine(): watchdog_map = {} govtrack_map = {} print "Processing govtrack.us basics." for pol in govtrack.parse_basics(): wd = get_wd_id(pol.id) if not wd: continue current_member = wd.get("current_member") watchdog_id = wd["watchdog_id"] govtrack_map[pol.id] = watchdog_map[watchdog_id] = newpol = web.storage() newpol["current_member"] = current_member for k, v in mapping.iteritems(): if k in pol: newpol[v] = pol[k] print "Processing govtrack.us stats." for pol in govtrack.parse_stats(["enacted", "introduced", "cosponsor", "speeches"]): if pol.id not in govtrack_map: continue else: newpol = govtrack_map[pol.id] if pol.get("SponsorEnacted"): newpol.n_bills_introduced = int(pol.NumSponsor) newpol.n_bills_enacted = int(pol.SponsorEnacted) if pol.get("SponsorIntroduced"): newpol.n_bills_debated = int(pol.NumSponsor) - int(pol.SponsorIntroduced) if pol.get("NumCosponsor"): newpol.n_bills_cosponsored = int(pol.NumCosponsor) if pol.get("Speeches"): newpol.n_speeches = int(pol.Speeches) newpol.words_per_speech = int(pol.WordsPerSpeech) return watchdog_map
def combine(): watchdog_map = {} govtrack_map = {} for pol in govtrack.parse_basics(): watchdog_id = tools.getWatchdogID(pol.get('represents'),pol.lastname) if watchdog_id: govtrack_map[pol.id] = watchdog_map[watchdog_id] = newpol = web.storage() else: continue for k, v in mapping.iteritems(): if k in pol: newpol[v] = pol[k] for pol in govtrack.parse_stats([ 'enacted', 'introduced', 'cosponsor', 'speeches']): if pol.id not in govtrack_map: continue else: newpol = govtrack_map[pol.id] if pol.get('SponsorEnacted'): newpol.n_bills_introduced = int(pol.NumSponsor) newpol.n_bills_enacted = int(pol.SponsorEnacted) if pol.get('SponsorIntroduced'): newpol.n_bills_debated = int(pol.NumSponsor) - int(pol.SponsorIntroduced) if pol.get('NumCosponsor'): newpol.n_bills_cosponsored = int(pol.NumCosponsor) if pol.get('Speeches'): newpol.n_speeches = int(pol.Speeches) newpol.words_per_speech = int(pol.WordsPerSpeech) return watchdog_map
'firstname': 'firstname', 'gender': 'gender', 'id': 'govtrackid', 'lastname': 'lastname', 'middlename': 'middlename', 'osid': 'opensecretsid', 'party': 'party', 'religion': 'religion', 'represents': 'district', 'url': 'officeurl' } watchdog_map = {} govtrack_map = {} for pol in govtrack.parse_basics(): watchdog_id = tools.getWatchdogID(pol.get('represents'),pol.lastname) if watchdog_id: govtrack_map[pol.id] = watchdog_map[watchdog_id] = newpol = web.storage() else: continue for k, v in mapping.iteritems(): if k in pol: newpol[v] = pol[k] for pol in govtrack.parse_stats(['enacted', 'introduced', 'cosponsor', 'speeches']): if pol.id not in govtrack_map: continue else: newpol = govtrack_map[pol.id]