def add_vote(vote_option, voter):
if vote_option == "Present, Giving Live Pair": vote_option = "Present"
vote["votes"].setdefault(vote_option, []).append(voter)
# In the 101st Congress, 1st session (1989), votes 133 through 136 lack lis_member_id nodes.
if voter != "VP" and voter["id"] == "":
logging.warn("Missing lis_member_id in %s, falling back to name lookup for %s" % (vote["vote_id"], voter["last_name"]))
voter["id"] = utils.lookup_legislator(vote["congress"], "sen", voter["last_name"], voter["state"], voter["party"], vote["date"], "lis")
if voter["id"] == None:
raise Exception("Could not find ID for %s (%s-%s)" % (voter["last_name"], voter["state"], voter["party"]))
def add_vote(vote_option, voter):
if vote_option == "Present, Giving Live Pair": vote_option = "Present"
vote["votes"].setdefault(vote_option, []).append(voter)
# In the 101st Congress, 1st session (1989), votes 133 through 136 lack lis_member_id nodes.
if voter != "VP" and voter["id"] == "":
voter["id"] = utils.lookup_legislator(vote["congress"], "sen", voter["last_name"], voter["state"], voter["party"], vote["date"], "lis")
if voter["id"] == None:
logging.error("[%s] Missing lis_member_id and name lookup failed for %s" % (vote["vote_id"], voter["last_name"]))
raise Exception("Could not find ID for %s (%s-%s)" % (voter["last_name"], voter["state"], voter["party"]))
else:
logging.info("[%s] Missing lis_member_id, falling back to name lookup for %s" % (vote["vote_id"], voter["last_name"]))
def parse_house_vote(dom, vote):
def parse_date(d):
d = d.strip()
if " " in d:
return datetime.datetime.strptime(d, "%d-%b-%Y %I:%M %p")
else: # some votes have no times?
print vote
return datetime.datetime.strptime(d, "%d-%b-%Y")
vote["date"] = parse_date(str(dom.xpath("string(vote-metadata/action-date)")) + " " + str(dom.xpath("string(vote-metadata/action-time)")))
vote["question"] = unicode(dom.xpath("string(vote-metadata/vote-question)"))
vote["type"] = unicode(dom.xpath("string(vote-metadata/vote-question)"))
vote["type"] = normalize_vote_type(vote["type"])
vote["category"] = get_vote_category(vote["question"])
vote["subject"] = unicode(dom.xpath("string(vote-metadata/vote-desc)"))
if not vote["subject"]: del vote["subject"]
vote_types = { "YEA-AND-NAY": "1/2", "2/3 YEA-AND-NAY": "2/3", "3/5 YEA-AND-NAY": "3/5", "1/2": "1/2", "2/3" : "2/3", "QUORUM": "QUORUM", "RECORDED VOTE" : "1/2", "2/3 RECORDED VOTE": "2/3", "3/5 RECORDED VOTE": "3/5" }
vote["requires"] = vote_types.get(str(dom.xpath("string(vote-metadata/vote-type)")), "unknown")
vote["result_text"] = unicode(dom.xpath("string(vote-metadata/vote-result)"))
vote["result"] = unicode(dom.xpath("string(vote-metadata/vote-result)"))
bill_num = unicode(dom.xpath("string(vote-metadata/legis-num)"))
if bill_num not in ("", "QUORUM", "JOURNAL", "MOTION", "ADJOURN") and not re.match(r"QUORUM \d+$", bill_num):
bill_types = { "S": "s", "S CON RES": "sconres", "S J RES": "sjres", "S RES": "sres", "H R": "hr", "H CON RES": "hconres", "H J RES": "hjres", "H RES": "hres" }
try:
bill_type, bill_number = bill_num.rsplit(" ", 1)
vote["bill"] = {
"congress": vote["congress"],
"type": bill_types[bill_type],
"number": int(bill_number)
}
except ValueError: # rsplit failed, i.e. there is no space in the legis-num field
raise Exception("Unhandled bill number in the legis-num field")
if str(dom.xpath("string(vote-metadata/amendment-num)")):
vote["amendment"] = {
"type": "h-bill",
"number": int(str(dom.xpath("string(vote-metadata/amendment-num)"))),
"author": unicode(dom.xpath("string(vote-metadata/amendment-author)")),
}
# Assemble a complete question from the vote type, amendment, and bill number.
if "amendment" in vote and "bill" in vote:
vote["question"] += ": Amendment %s to %s" % (vote["amendment"]["number"], unicode(dom.xpath("string(vote-metadata/legis-num)")))
elif "amendment" in vote:
vote["question"] += ": Amendment %s to [unknown bill]" % vote["amendment"]["number"]
elif "bill" in vote:
vote["question"] += ": " + unicode(dom.xpath("string(vote-metadata/legis-num)"))
if "subject" in vote: vote["question"] += " " + vote["subject"]
elif "subject" in vote:
vote["question"] += ": " + vote["subject"]
# Count up the votes.
vote["votes"] = { } # by vote type
def add_vote(vote_option, voter):
vote["votes"].setdefault(vote_option, []).append(voter)
# Ensure the options are noted, even if no one votes that way.
if unicode(dom.xpath("string(vote-metadata/vote-question)")) == "Election of the Speaker":
for n in dom.xpath('vote-metadata/vote-totals/totals-by-candidate/candidate'):
vote["votes"][n.text] = []
elif unicode(dom.xpath("string(vote-metadata/vote-question)")) == "Call of the House":
for n in dom.xpath('vote-metadata/vote-totals/totals-by-candidate/candidate'):
vote["votes"][n.text] = []
elif "YEA-AND-NAY" in dom.xpath('string(vote-metadata/vote-type)'):
vote["votes"]['Yea'] = []
vote["votes"]['Nay'] = []
vote["votes"]['Present'] = []
vote["votes"]['Not Voting'] = []
else:
vote["votes"]['Aye'] = []
vote["votes"]['No'] = []
vote["votes"]['Present'] = []
vote["votes"]['Not Voting'] = []
for member in dom.xpath("vote-data/recorded-vote"):
display_name = unicode(member.xpath("string(legislator)"))
state = str(member.xpath("string(legislator/@state)"))
party = str(member.xpath("string(legislator/@party)"))
vote_cast = str(member.xpath("string(vote)"))
bioguideid = str(member.xpath("string(legislator/@name-id)"))
add_vote(vote_cast, {
"id": bioguideid,
"state": state,
"party": party,
"display_name": display_name,
})
# Through the 107th Congress and sporadically in more recent data, the bioguide field
# is not present. Look up the Members' bioguide IDs by name/state/party/date. This works
# reasonably well, but there are many gaps. When there's a gap, it raises an exception
# and the file is not saved.
#
# Take into account that the vote may list both a "Smith" and a "Smith, John". Resolve
# "Smith" by process of elimination, i.e. he must not be whoever "Smith, John" resolved
# to. To do that, process the voters from longest specified display name to shortest.
#
# One example of a sporadic case is 108th Congress, 2nd session (2004), votes 405 through
# 544, where G.K. Butterfield's bioguide ID is 000000. It should have been B001251.
# See https://github.com/unitedstates/congress/issues/46.
seen_ids = set()
all_voters = sum(vote["votes"].values(), [])
all_voters.sort(key = lambda v : len(v["display_name"]), reverse=True) # process longer names first
for v in all_voters:
if v["id"] not in ("", "0000000"): continue
if vote["congress"] > 107:
logging.warn("Missing bioguide ID in %s, falling back to name lookup for %s" % (vote["vote_id"], v["display_name"]))
# get the last name without the state abbreviation in parenthesis, if it is present
display_name = v["display_name"]
ss = " (%s)" % v["state"]
if display_name.endswith(ss): display_name = display_name[:-len(ss)]
# look up ID
v["id"] = utils.lookup_legislator(vote["congress"], "rep", display_name, v["state"], v["party"], vote["date"], "bioguide", exclude=seen_ids)
if v["id"] == None:
raise Exception("No bioguide ID for %s (%s-%s)" % (display_name, v["state"], v["party"]))
else:
seen_ids.add(v["id"])
def parse_house_vote(dom, vote):
def parse_date(d):
d = d.strip()
if " " in d:
return datetime.datetime.strptime(d, "%d-%b-%Y %I:%M %p")
else: # some votes have no times?
print vote
return datetime.datetime.strptime(d, "%d-%b-%Y")
vote["date"] = parse_date(
str(dom.xpath("string(vote-metadata/action-date)")) + " " +
str(dom.xpath("string(vote-metadata/action-time)")))
vote["question"] = unicode(
dom.xpath("string(vote-metadata/vote-question)"))
vote["type"] = unicode(dom.xpath("string(vote-metadata/vote-question)"))
vote["type"] = normalize_vote_type(vote["type"])
if unicode(dom.xpath("string(vote-metadata/vote-desc)")).startswith(
"Impeaching "):
vote["category"] = "impeachment"
else:
vote["category"] = get_vote_category(vote["question"])
vote["subject"] = unicode(dom.xpath("string(vote-metadata/vote-desc)"))
if not vote["subject"]:
del vote["subject"]
vote_types = {
"YEA-AND-NAY": "1/2",
"2/3 YEA-AND-NAY": "2/3",
"3/5 YEA-AND-NAY": "3/5",
"1/2": "1/2",
"2/3": "2/3",
"QUORUM": "QUORUM",
"RECORDED VOTE": "1/2",
"2/3 RECORDED VOTE": "2/3",
"3/5 RECORDED VOTE": "3/5"
}
vote["requires"] = vote_types.get(
str(dom.xpath("string(vote-metadata/vote-type)")), "unknown")
vote["result_text"] = unicode(
dom.xpath("string(vote-metadata/vote-result)"))
vote["result"] = unicode(dom.xpath("string(vote-metadata/vote-result)"))
bill_num = unicode(dom.xpath("string(vote-metadata/legis-num)"))
if bill_num not in ("", "QUORUM", "JOURNAL", "MOTION",
"ADJOURN") and not re.match(r"QUORUM \d+$", bill_num):
bill_types = {
"S": "s",
"S CON RES": "sconres",
"S J RES": "sjres",
"S RES": "sres",
"H R": "hr",
"H CON RES": "hconres",
"H J RES": "hjres",
"H RES": "hres"
}
try:
bill_type, bill_number = bill_num.rsplit(" ", 1)
vote["bill"] = {
"congress": vote["congress"],
"type": bill_types[bill_type],
"number": int(bill_number)
}
except ValueError: # rsplit failed, i.e. there is no space in the legis-num field
raise Exception("Unhandled bill number in the legis-num field")
if str(dom.xpath("string(vote-metadata/amendment-num)")):
vote["amendment"] = {
"type": "h-bill",
"number":
int(str(dom.xpath("string(vote-metadata/amendment-num)"))),
"author":
unicode(dom.xpath("string(vote-metadata/amendment-author)")),
}
# Assemble a complete question from the vote type, amendment, and bill number.
if "amendment" in vote and "bill" in vote:
vote["question"] += ": Amendment %s to %s" % (
vote["amendment"]["number"],
unicode(dom.xpath("string(vote-metadata/legis-num)")))
elif "amendment" in vote:
vote["question"] += ": Amendment %s to [unknown bill]" % vote[
"amendment"]["number"]
elif "bill" in vote:
vote["question"] += ": " + unicode(
dom.xpath("string(vote-metadata/legis-num)"))
if "subject" in vote:
vote["question"] += " " + vote["subject"]
elif "subject" in vote:
vote["question"] += ": " + vote["subject"]
# Count up the votes.
vote["votes"] = {} # by vote type
def add_vote(vote_option, voter):
vote["votes"].setdefault(vote_option, []).append(voter)
# Ensure the options are noted, even if no one votes that way.
if unicode(dom.xpath("string(vote-metadata/vote-question)")
) == "Election of the Speaker":
for n in dom.xpath(
'vote-metadata/vote-totals/totals-by-candidate/candidate'):
vote["votes"][n.text] = []
elif unicode(dom.xpath(
"string(vote-metadata/vote-question)")) == "Call of the House":
for n in dom.xpath(
'vote-metadata/vote-totals/totals-by-candidate/candidate'):
vote["votes"][n.text] = []
elif "YEA-AND-NAY" in dom.xpath('string(vote-metadata/vote-type)'):
vote["votes"]['Yea'] = []
vote["votes"]['Nay'] = []
vote["votes"]['Present'] = []
vote["votes"]['Not Voting'] = []
else:
vote["votes"]['Aye'] = []
vote["votes"]['No'] = []
vote["votes"]['Present'] = []
vote["votes"]['Not Voting'] = []
for member in dom.xpath("vote-data/recorded-vote"):
display_name = unicode(member.xpath("string(legislator)"))
state = str(member.xpath("string(legislator/@state)"))
party = str(member.xpath("string(legislator/@party)"))
vote_cast = str(member.xpath("string(vote)"))
bioguideid = str(member.xpath("string(legislator/@name-id)"))
add_vote(
vote_cast, {
"id": bioguideid,
"state": state,
"party": party,
"display_name": display_name,
})
# Through the 107th Congress and sporadically in more recent data, the bioguide field
# is not present. Look up the Members' bioguide IDs by name/state/party/date. This works
# reasonably well, but there are many gaps. When there's a gap, it raises an exception
# and the file is not saved.
#
# Take into account that the vote may list both a "Smith" and a "Smith, John". Resolve
# "Smith" by process of elimination, i.e. he must not be whoever "Smith, John" resolved
# to. To do that, process the voters from longest specified display name to shortest.
#
# One example of a sporadic case is 108th Congress, 2nd session (2004), votes 405 through
# 544, where G.K. Butterfield's bioguide ID is 000000. It should have been B001251.
# See https://github.com/unitedstates/congress/issues/46.
seen_ids = set()
all_voters = sum(vote["votes"].values(), [])
all_voters.sort(key=lambda v: len(v["display_name"]),
reverse=True) # process longer names first
for v in all_voters:
if v["id"] not in ("", "0000000"):
continue
# here are wierd cases from h610-103.1993 that confound our name lookup since it has the wrong state abbr
if v["state"] == "XX":
for st in ("PR", "AS", "GU", "VI", "DC"):
if v["display_name"].endswith(" (%s)" % st):
v["state"] = st
# get the last name without the state abbreviation in parenthesis, if it is present
display_name = v["display_name"].strip()
ss = " (%s)" % v["state"]
if display_name.endswith(ss):
display_name = display_name[:-len(ss)].strip()
# wrong party in upstream data
if vote["vote_id"] == "h2-106.1999" and display_name == "Hastert":
v["id"] = "H000323"
continue
# dead man recorded as Not Voting (he died the day before, so none of our roles match the vote date)
if vote["vote_id"] == "h306-106.1999" and display_name == "Brown" and v[
"state"] == "CA":
v["id"] = "B000918"
continue
# look up ID
v["id"] = utils.lookup_legislator(vote["congress"],
"rep",
display_name,
v["state"],
v["party"],
vote["date"],
"bioguide",
exclude=seen_ids)
if v["id"] == None:
logging.error(
"[%s] Missing bioguide ID and name lookup failed for %s (%s-%s on %s)"
% (vote["vote_id"], display_name, v["state"], v["party"],
vote["date"]))
raise Exception("No bioguide ID for %s (%s-%s)" %
(display_name, v["state"], v["party"]))
else:
if vote["congress"] > 107:
logging.warn(
"[%s] Used name lookup for %s because bioguide ID was missing."
% (vote["vote_id"], v["display_name"]))
seen_ids.add(v["id"])
