def loadroll(fn): roll = web.storage() roll.id = fn.split('/')[-1].split('.')[0] vote = xmltramp.load(fn) if vote['bill':]: b = vote.bill roll.bill_id = 'us/%s/%s%s' % (b('session'), b('type'), b('number')) else: roll.bill_id = None roll.type = str(vote.type) roll.question = str(vote.question) roll.required = str(vote.required) roll.result = str(vote.result) try: db.insert('roll', seqname=False, **roll) except IntegrityError: if not db.update('roll', where="id=" + web.sqlquote(roll.id), bill_id=roll.bill_id): print "\nMissing bill:", roll.bill_id raise NotDone with db.transaction(): db.delete('vote', where="roll_id=$roll.id", vars=locals()) for voter in vote['voter':]: rep = govtrackp(voter('id')) if rep: db.insert('vote', seqname=False, politician_id=rep, roll_id=roll.id, vote=fixvote(voter('vote'))) else: pass #@@!--check again after load_everyone
def loadbill(fn, maplightid=None): bill = xmltramp.load(fn) d = bill2dict(bill) d.maplightid = maplightid try: bill_id = d.id db.insert('bill', seqname=False, **d) except IntegrityError: bill_id = d.pop('id') db.update('bill', where="id=" + web.sqlquote(bill_id), **d) positions = {} for vote in bill.actions['vote':]: if not vote().get('roll'): continue rolldoc = '/us/%s/rolls/%s%s-%s.xml' % ( d.session, vote('where'), vote('datetime')[:4], vote('roll')) roll = xmltramp.load(GOVTRACK_CRAWL + rolldoc) for voter in roll['voter':]: positions[govtrackp(voter('id'))] = fixvote(voter('vote')) if None in positions: del positions[None] with db.transaction(): db.delete('position', where='bill_id=$bill_id', vars=locals()) for p, v in positions.iteritems(): db.insert('position', seqname=False, bill_id=bill_id, politician_id=p, vote=v)
def loadbill(fn, maplightid=None): bill = xmltramp.load(fn) d = bill2dict(bill) if maplightid: d["maplightid"] = maplightid db.insert("bill", seqname=False, **d) print "\r %-25s" % d["id"], sys.stdout.flush() done = [] for vote in bill.actions["vote":]: if not vote().get("roll"): continue if vote("where") in done: continue # don't count veto overrides done.append(vote("where")) votedoc = "%s/rolls/%s%s-%s.xml" % (d["session"], vote("where"), vote("datetime")[:4], vote("roll")) vote = xmltramp.load("../data/crawl/govtrack/us/" + votedoc) yeas = 0 neas = 0 for voter in vote["voter":]: if fixvote(voter("vote")) == 1: yeas += 1 elif fixvote(voter("vote")) == -1: neas += 1 rep = govtrackp(voter("id")) if rep: # UGLY HACK: if a politician (bob_menendez for instance) voted # for the same bill in both chambers of congress the insert # fails. if not db.select("vote", where="bill_id=$d['id'] AND politician_id=$rep", vars=locals()): db.insert("vote", seqname=False, politician_id=rep, bill_id=d["id"], vote=fixvote(voter("vote"))) else: print print "Updating:", votedoc, rep, d["id"], fixvote(voter("vote")) db.update( "vote", where="bill_id=$d['id'] AND politician_id=$rep", vote=fixvote(voter("vote")), vars=locals(), ) db.update("bill", where="id = $d['id']", yeas=yeas, neas=neas, vars=locals())
def bill2dict(bill): d = {} d["id"] = "us/%s/%s%s" % (bill("session"), bill("type"), bill("number")) d["session"] = bill("session") d["type"] = bill("type") d["number"] = bill("number") d["introduced"] = bill.introduced("datetime") titles = [unicode(x) for x in bill.titles["title":] if x("type") == "short"] if not titles: titles = [unicode(x) for x in bill.titles["title":]] d["title"] = titles[0] summaries = [unicode(x) for x in bill.titles["title":] if x("type") == "official"] if not summaries: summaries = [unicode(x) for x in bill.titles["title":]] d["summary"] = summaries[0] d["sponsor"] = govtrackp(bill.sponsor().get("id")) return d
def bill2dict(bill): d = web.storage() d.id = 'us/%s/%s%s' % (bill('session'), bill('type'), bill('number')) d.session = bill('session') d.type = bill('type') d.number = bill('number') d.introduced = bill.introduced('datetime') titles = [unicode(x) for x in bill.titles['title':] if x('type') == 'short'] if not titles: titles = [unicode(x) for x in bill.titles['title':]] d.title = titles[0] summaries = [unicode(x) for x in bill.titles['title':] if x('type') == 'official'] if not summaries: summaries = [unicode(x) for x in bill.titles['title':]] d.summary = summaries[0] d.sponsor_id = govtrackp(bill.sponsor().get('id')) return d
def bill2dict(bill): d = web.storage() d.id = 'us/%s/%s%s' % (bill('session'), bill('type'), bill('number')) d.session = bill('session') d.type = bill('type') d.number = bill('number') d.introduced = bill.introduced('datetime') titles = [ unicode(x) for x in bill.titles['title':] if x('type') == 'short' ] if not titles: titles = [unicode(x) for x in bill.titles['title':]] d.title = titles[0] summaries = [ unicode(x) for x in bill.titles['title':] if x('type') == 'official' ] if not summaries: summaries = [unicode(x) for x in bill.titles['title':]] d.summary = summaries[0] d.sponsor_id = govtrackp(bill.sponsor().get('id')) return d