def loadroll(fn):
roll = web.storage()
roll.id = fn.split('/')[-1].split('.')[0]
vote = xmltramp.load(fn)
if vote['bill':]:
b = vote.bill
roll.bill_id = 'us/%s/%s%s' % (b('session'), b('type'), b('number'))
else:
roll.bill_id = None
roll.type = str(vote.type)
roll.question = str(vote.question)
roll.required = str(vote.required)
roll.result = str(vote.result)
try:
db.insert('roll', seqname=False, **roll)
except IntegrityError:
if not db.update('roll', where="id=" + web.sqlquote(roll.id), bill_id=roll.bill_id):
print "\nMissing bill:", roll.bill_id
raise NotDone
with db.transaction():
db.delete('vote', where="roll_id=$roll.id", vars=locals())
for voter in vote['voter':]:
rep = govtrackp(voter('id'))
if rep:
db.insert('vote', seqname=False,
politician_id=rep, roll_id=roll.id, vote=fixvote(voter('vote')))
else:
pass #@@!--check again after load_everyone
def loadbill(fn, maplightid=None):
bill = xmltramp.load(fn)
d = bill2dict(bill)
d.maplightid = maplightid
try:
bill_id = d.id
db.insert('bill', seqname=False, **d)
except IntegrityError:
bill_id = d.pop('id')
db.update('bill', where="id=" + web.sqlquote(bill_id), **d)
positions = {}
for vote in bill.actions['vote':]:
if not vote().get('roll'): continue
rolldoc = '/us/%s/rolls/%s%s-%s.xml' % (
d.session, vote('where'), vote('datetime')[:4], vote('roll'))
roll = xmltramp.load(GOVTRACK_CRAWL + rolldoc)
for voter in roll['voter':]:
positions[govtrackp(voter('id'))] = fixvote(voter('vote'))
if None in positions: del positions[None]
with db.transaction():
db.delete('position', where='bill_id=$bill_id', vars=locals())
for p, v in positions.iteritems():
db.insert('position',
seqname=False,
bill_id=bill_id,
politician_id=p,
vote=v)
def loadbill(fn, maplightid=None):
bill = xmltramp.load(fn)
d = bill2dict(bill)
d.maplightid = maplightid
try:
bill_id = d.id
db.insert('bill', seqname=False, **d)
except IntegrityError:
bill_id = d.pop('id')
db.update('bill', where="id=" + web.sqlquote(bill_id), **d)
positions = {}
for vote in bill.actions['vote':]:
if not vote().get('roll'): continue
rolldoc = '/us/%s/rolls/%s%s-%s.xml' % (
d.session, vote('where'), vote('datetime')[:4], vote('roll'))
roll = xmltramp.load(GOVTRACK_CRAWL + rolldoc)
for voter in roll['voter':]:
positions[govtrackp(voter('id'))] = fixvote(voter('vote'))
if None in positions: del positions[None]
with db.transaction():
db.delete('position', where='bill_id=$bill_id', vars=locals())
for p, v in positions.iteritems():
db.insert('position', seqname=False,
bill_id=bill_id, politician_id=p, vote=v)
def loadroll(fn):
roll = web.storage()
roll.id = fn.split('/')[-1].split('.')[0]
vote = xmltramp.load(fn)
if vote['bill':]:
b = vote.bill
roll.bill_id = 'us/%s/%s%s' % (b('session'), b('type'), b('number'))
else:
roll.bill_id = None
roll.type = str(vote.type)
roll.question = str(vote.question)
roll.required = str(vote.required)
roll.result = str(vote.result)
try:
db.insert('roll', seqname=False, **roll)
except IntegrityError:
if not db.update('roll',
where="id=" + web.sqlquote(roll.id),
bill_id=roll.bill_id):
print "\nMissing bill:", roll.bill_id
raise NotDone
with db.transaction():
db.delete('vote', where="roll_id=$roll.id", vars=locals())
for voter in vote['voter':]:
rep = govtrackp(voter('id'))
if rep:
db.insert('vote',
seqname=False,
politician_id=rep,
roll_id=roll.id,
vote=fixvote(voter('vote')))
else:
pass #@@!--check again after load_everyone
def loadbill(fn, maplightid=None):
bill = xmltramp.load(fn)
d = bill2dict(bill)
if maplightid:
d["maplightid"] = maplightid
db.insert("bill", seqname=False, **d)
print "\r %-25s" % d["id"],
sys.stdout.flush()
done = []
for vote in bill.actions["vote":]:
if not vote().get("roll"):
continue
if vote("where") in done:
continue # don't count veto overrides
done.append(vote("where"))
votedoc = "%s/rolls/%s%s-%s.xml" % (d["session"], vote("where"), vote("datetime")[:4], vote("roll"))
vote = xmltramp.load("../data/crawl/govtrack/us/" + votedoc)
yeas = 0
neas = 0
for voter in vote["voter":]:
if fixvote(voter("vote")) == 1:
yeas += 1
elif fixvote(voter("vote")) == -1:
neas += 1
rep = govtrackp(voter("id"))
if rep:
# UGLY HACK: if a politician (bob_menendez for instance) voted
# for the same bill in both chambers of congress the insert
# fails.
if not db.select("vote", where="bill_id=$d['id'] AND politician_id=$rep", vars=locals()):
db.insert("vote", seqname=False, politician_id=rep, bill_id=d["id"], vote=fixvote(voter("vote")))
else:
print
print "Updating:", votedoc, rep, d["id"], fixvote(voter("vote"))
db.update(
"vote",
where="bill_id=$d['id'] AND politician_id=$rep",
vote=fixvote(voter("vote")),
vars=locals(),
)
db.update("bill", where="id = $d['id']", yeas=yeas, neas=neas, vars=locals())
def bill2dict(bill):
d = {}
d["id"] = "us/%s/%s%s" % (bill("session"), bill("type"), bill("number"))
d["session"] = bill("session")
d["type"] = bill("type")
d["number"] = bill("number")
d["introduced"] = bill.introduced("datetime")
titles = [unicode(x) for x in bill.titles["title":] if x("type") == "short"]
if not titles:
titles = [unicode(x) for x in bill.titles["title":]]
d["title"] = titles[0]
summaries = [unicode(x) for x in bill.titles["title":] if x("type") == "official"]
if not summaries:
summaries = [unicode(x) for x in bill.titles["title":]]
d["summary"] = summaries[0]
d["sponsor"] = govtrackp(bill.sponsor().get("id"))
return d
def bill2dict(bill):
d = web.storage()
d.id = 'us/%s/%s%s' % (bill('session'), bill('type'), bill('number'))
d.session = bill('session')
d.type = bill('type')
d.number = bill('number')
d.introduced = bill.introduced('datetime')
titles = [unicode(x) for x in bill.titles['title':]
if x('type') == 'short']
if not titles:
titles = [unicode(x) for x in bill.titles['title':]]
d.title = titles[0]
summaries = [unicode(x) for x in bill.titles['title':]
if x('type') == 'official']
if not summaries:
summaries = [unicode(x) for x in bill.titles['title':]]
d.summary = summaries[0]
d.sponsor_id = govtrackp(bill.sponsor().get('id'))
return d
def bill2dict(bill):
d = web.storage()
d.id = 'us/%s/%s%s' % (bill('session'), bill('type'), bill('number'))
d.session = bill('session')
d.type = bill('type')
d.number = bill('number')
d.introduced = bill.introduced('datetime')
titles = [
unicode(x) for x in bill.titles['title':] if x('type') == 'short'
]
if not titles:
titles = [unicode(x) for x in bill.titles['title':]]
d.title = titles[0]
summaries = [
unicode(x) for x in bill.titles['title':] if x('type') == 'official'
]
if not summaries:
summaries = [unicode(x) for x in bill.titles['title':]]
d.summary = summaries[0]
d.sponsor_id = govtrackp(bill.sponsor().get('id'))
return d
