示例#1
0
def load_votesmart():
    # Candidates from votesmart
    for district, cands in votesmart.candidates():
        district = tools.fix_district_name(district)
        for pol in cands:
            vs_id = pol['candidateId']
            wd = get_wd_id(vs_id)
            if not wd: continue
            polid = wd['watchdog_id']

            pol_cand = filter_dict(cand_mapping, pol)
            if not db.select('politician', where='id=$polid', vars=locals()):
                db.insert('politician',
                          seqname=False,
                          id=polid,
                          **unidecode(
                              filter_dict(schema.Politician.columns.keys(),
                                          pol_cand)))
            else:
                # @@ Should probably check that we really want to do this, but
                # it apears as though the data has two entries for current
                # members (the second having more info filled out).
                db.update('politician',
                          where='id=$polid',
                          vars=locals(),
                          **unidecode(
                              filter_dict(schema.Politician.columns.keys(),
                                          pol_cand)))

            if not db.select(
                    'congress',
                    where="politician_id=$polid AND congress_num='-1'",
                    vars=locals()):
                db.insert('congress',
                          seqname=False,
                          congress_num='-1',
                          politician_id=polid,
                          district_id=district,
                          party=pol_cand['party'])

    # Bios from votesmart
    for vs_id, p in votesmart.bios():
        pol = p['candidate']
        if pol['gender']:
            pol['gender'] = pol['gender'][0]
        if pol['education']:
            pol['education'] = pol['education'].replace('\r\n', '\n')
        wd = get_wd_id(vs_id)
        if not wd: continue
        polid = wd['watchdog_id']
        pol_people = filter_dict(schema.Politician.columns.keys(),
                                 filter_dict(bios_mapping, pol))
        if db.select('politician', where='votesmartid=$vs_id', vars=locals()):
            db.update('politician',
                      where='votesmartid=$vs_id',
                      vars=locals(),
                      **unidecode(pol_people))
示例#2
0
def generate_ids():
    wd_to_gt = load_wd_mapping()
    # Govtrack
    for pol in govtrack.parse_basics():
        current_member = False
        collision = False
        watchdog_id = tools.getWatchdogID(pol.get('represents'),pol.lastname)
        if watchdog_id:
            current_member = True
            # pol.represents should always be the same as current_member, if we
            # remove the origional politician.json file we can use that
            # instead.
            assert(pol.represents)
        else:
            try:
                assert(not pol.get('represents'))
            except:
                print "no watchdog id for", web.safestr(pol.name), web.safestr(pol.represents)
                continue
            watchdog_id = gen_pol_id(pol)
            if watchdog_id in wd_to_gt and \
                    wd_to_gt[watchdog_id]['govtrack_id'] != pol.id: 
                collision = True
        if (not collision) or current_member:
            if watchdog_id not in wd_to_gt: 
                wd_to_gt[watchdog_id] = {}
            wd_to_gt[watchdog_id]['govtrack_id'] = pol.id
        if collision: 
            wd_to_gt[watchdog_id]['collision'] = True
        if current_member: 
            wd_to_gt[watchdog_id]['current_member'] = True
    # Votesmart
    for district, cands in votesmart.candidates():
        district=tools.fix_district_name(district)
        for pol in cands:
            watchdog_id = tools.getWatchdogID(district, pol['lastName'])
            if not watchdog_id:
                watchdog_id = gen_pol_id(pol)
            vsid=pol['candidateId']
            #TODO: Could use some more checking to be sure we are 1. adding the
            #      correct votesmart id to the correct watchdog_id (eg. in the
            #      case that there was a collision in processing the govtrack
            #      data). And 2. aren't creating a new watchdog_id when there
            #      was already one for this person.
            if watchdog_id not in wd_to_gt:
                wd_to_gt[watchdog_id] = {}
            wd_to_gt[watchdog_id]['votesmart_id'] = vsid
    return wd_to_gt
示例#3
0
def load_votesmart():
    # Candidates from votesmart
    for district, cands in votesmart.candidates():
        district=tools.fix_district_name(district)
        for pol in cands:
            vs_id=pol['candidateId']
            wd = get_wd_id(vs_id)
            if not wd: continue
            polid = wd['watchdog_id']

            pol_cand = filter_dict(cand_mapping, pol)
            if not db.select('politician', 
                    where='id=$polid', vars=locals()):
                db.insert('politician', 
                        seqname=False, 
                        id=polid, 
                        **unidecode(filter_dict(schema.Politician.columns.keys(),
                            pol_cand)))
            else:
                # @@ Should probably check that we really want to do this, but
                # it apears as though the data has two entries for current
                # members (the second having more info filled out).
                db.update('politician', where='id=$polid', vars=locals(),
                        **unidecode(filter_dict(schema.Politician.columns.keys(),
                            pol_cand)))

            if not db.select('congress',
                    where="politician_id=$polid AND congress_num='-1'", 
                    vars=locals()):
                db.insert('congress', seqname=False, congress_num='-1',
                        politician_id=polid, district_id=district,
                        party=pol_cand['party'])

    # Bios from votesmart
    for vs_id, p in votesmart.bios():
        pol = p['candidate']
        if pol['gender']:
            pol['gender']=pol['gender'][0]
        if pol['education']:
            pol['education'] = pol['education'].replace('\r\n', '\n')
        wd = get_wd_id(vs_id)
        if not wd: continue
        polid = wd['watchdog_id']
        pol_people = filter_dict(schema.Politician.columns.keys(),
                filter_dict(bios_mapping, pol))
        if db.select('politician', where='votesmartid=$vs_id',vars=locals()):
            db.update('politician', where='votesmartid=$vs_id', 
                    vars=locals(), **unidecode(pol_people))