def output_vote(vote, options, id_type=None):
logging.info("[%s] Writing to disk..." % vote['vote_id'])
# output JSON - so easy!
utils.write(json.dumps(vote,
sort_keys=True,
indent=2,
default=utils.format_datetime),
output_for_vote(vote["vote_id"], "json"),
options=options)
# What kind of IDs are we passed for Members of Congress?
# For current data, we infer from the chamber. For historical data from voteview,
# we're passed the type in id_type, which is set to "bioguide".
if not id_type:
id_type = ("bioguide" if vote["chamber"] == "h" else "lis")
# output XML
root = etree.Element("roll")
root.set("where", "house" if vote['chamber'] == "h" else "senate")
root.set("session", str(vote["congress"]))
root.set("year", str(vote["date"].year))
root.set("roll", str(vote["number"]))
if "voteview" in vote["source_url"]:
root.set("source", "keithpoole")
else:
root.set("source",
"house.gov" if vote["chamber"] == "h" else "senate.gov")
root.set("datetime", utils.format_datetime(vote['date']))
root.set("updated", utils.format_datetime(vote['updated_at']))
def get_votes(option):
return len(vote["votes"].get(option, []))
root.set("aye", str(get_votes("Yea") + get_votes("Aye")))
root.set("nay", str(get_votes("Nay") + get_votes("No")))
root.set("nv", str(get_votes("Not Voting")))
root.set("present", str(get_votes("Present")))
utils.make_node(root, "category", vote["category"])
utils.make_node(root, "type", vote["type"])
utils.make_node(root, "question", vote["question"])
utils.make_node(root, "required", vote["requires"])
utils.make_node(root, "result", vote["result"])
if vote.get("bill"):
govtrack_type_codes = {
'hr': 'h',
's': 's',
'hres': 'hr',
'sres': 'sr',
'hjres': 'hj',
'sjres': 'sj',
'hconres': 'hc',
'sconres': 'sc'
}
utils.make_node(root,
"bill",
None,
session=str(vote["bill"]["congress"]),
type=govtrack_type_codes[vote["bill"]["type"]],
number=str(vote["bill"]["number"]))
if "amendment" in vote:
n = utils.make_node(root, "amendment", None)
if vote["amendment"]["type"] == "s":
n.set("ref", "regular")
n.set("session", str(vote["congress"]))
n.set("number", "s" + str(vote["amendment"]["number"]))
elif vote["amendment"]["type"] == "h-bill":
n.set("ref", "bill-serial")
n.set("session", str(vote["congress"]))
n.set("number", str(vote["amendment"]["number"]))
# well-known keys for certain vote types: +/-/P/0
option_keys = {
"Aye": "+",
"Yea": "+",
"Nay": "-",
"No": "-",
"Present": "P",
"Not Voting": "0",
"Guilty": "+",
"Not Guilty": "-"
}
# preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty
# and handling other options like people's names for votes for the Speaker.
option_sort_order = ('Aye', 'Yea', 'Guilty', 'No', 'Nay', 'Not Guilty',
'OTHER', 'Present', 'Not Voting')
options_list = sorted(
vote["votes"].keys(),
key=lambda o: option_sort_order.index(o)
if o in option_sort_order else option_sort_order.index("OTHER"))
for option in options_list:
if option not in option_keys:
option_keys[option] = option
utils.make_node(root, "option", option, key=option_keys[option])
for option in options_list:
for v in vote["votes"][option]:
n = utils.make_node(root, "voter", None)
if v == "VP":
n.set("id", "0")
n.set("VP", "1")
elif not options.get("govtrack", False):
n.set("id", str(v["id"]))
else:
n.set("id", str(utils.get_govtrack_person_id(id_type,
v["id"])))
n.set("vote", option_keys[option])
n.set("value", option)
if v != "VP":
n.set("state", v["state"])
if v.get("voteview_votecode_extra") is not None:
n.set("voteview_votecode_extra",
v["voteview_votecode_extra"])
xmloutput = etree.tostring(root, pretty_print=True, encoding="utf8")
# mimick two hard line breaks in GovTrack's legacy output to ease running diffs
xmloutput = re.sub('(source=".*?") ', r"\1\n ", xmloutput)
xmloutput = re.sub('(updated=".*?") ', r"\1\n ", xmloutput)
utils.write(xmloutput,
output_for_vote(vote['vote_id'], "xml"),
options=options)
def output_vote(vote, options):
logging.info("[%s] Writing to disk..." % vote['vote_id'])
# output JSON - so easy!
utils.write(
json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime),
output_for_vote(vote["vote_id"], "json"),
)
# output XML
root = etree.Element("roll")
root.set("where", "house" if vote['chamber'] == "h" else "senate")
root.set("session", str(vote["congress"]))
root.set("year", str(vote["date"].year))
root.set("roll", str(vote["number"]))
root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov")
root.set("datetime", utils.format_datetime(vote['date']))
root.set("updated", utils.format_datetime(vote['updated_at']))
def get_votes(option): return len(vote["votes"].get(option, []))
root.set("aye", str(get_votes("Yea") + get_votes("Aye")))
root.set("nay", str(get_votes("Nay") + get_votes("No")))
root.set("nv", str(get_votes("Not Voting")))
root.set("present", str(get_votes("Present")))
utils.make_node(root, "category", vote["category"])
utils.make_node(root, "type", vote["type"])
utils.make_node(root, "question", vote["question"])
utils.make_node(root, "required", vote["requires"])
utils.make_node(root, "result", vote["result"])
if "bill" in vote:
govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' }
utils.make_node(root, "bill", None, session=str(vote["bill"]["congress"]), type=govtrack_type_codes[vote["bill"]["type"]], number=str(vote["bill"]["number"]))
if "amendment" in vote:
n = utils.make_node(root, "amendment", None)
if vote["amendment"]["type"] == "s":
n.set("ref", "regular")
n.set("session", str(vote["congress"]))
n.set("number", "s" + str(vote["amendment"]["number"]))
elif vote["amendment"]["type"] == "h-bill":
n.set("ref", "bill-serial")
n.set("session", str(vote["congress"]))
n.set("number", str(vote["amendment"]["number"]))
# well-known keys for certain vote types: +/-/P/0
option_keys = { "Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0" }
# preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty
# and handling other options like people's names for votes for the Speaker.
option_sort_order = ('Aye', 'Yea', 'Guilty', 'No', 'Nay', 'Not Guilty', 'OTHER', 'Present', 'Not Voting')
options_list = sorted(vote["votes"].keys(), key = lambda o : option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER") )
for option in options_list:
if option not in option_keys: option_keys[option] = option
utils.make_node(root, "option", option, key=option_keys[option])
for option in options_list:
for v in vote["votes"][option]:
n = utils.make_node(root, "voter", None)
if v == "VP":
n.set("id", "0")
n.set("VP", "1")
elif not options.get("govtrack", False):
n.set("id", str(v["id"]))
else:
n.set("id", str(utils.get_govtrack_person_id("bioguide" if vote["chamber"] == "h" else "lis", v["id"])))
n.set("vote", option_keys[option])
n.set("value", option)
if v != "VP":
n.set("state", v["state"])
xmloutput = etree.tostring(root, pretty_print=True, encoding="utf8")
# mimick two hard line breaks in GovTrack's legacy output to ease running diffs
xmloutput = re.sub('(source=".*?") ', r"\1\n ", xmloutput)
xmloutput = re.sub('(updated=".*?") ', r"\1\n ", xmloutput)
utils.write(
xmloutput,
output_for_vote(vote['vote_id'], "xml")
)
def output_vote(vote, options):
logging.info("[%s] Writing to disk..." % vote["vote_id"])
# output JSON - so easy!
utils.write(
json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime),
output_for_vote(vote["vote_id"], "json"),
)
# output XML
root = etree.Element("roll")
root.set("where", "house" if vote["chamber"] == "h" else "senate")
root.set("session", str(vote["congress"]))
root.set("year", str(vote["date"].year))
root.set("roll", str(vote["number"]))
root.set("datetime", utils.format_datetime(vote["date"]))
root.set("updated", utils.format_datetime(vote["updated_at"]))
def get_votes(option):
return len(vote["votes"].get(option, []))
root.set("aye", str(get_votes("Yea") + get_votes("Aye")))
root.set("nay", str(get_votes("Nay") + get_votes("No")))
root.set("present", str(get_votes("Present")))
root.set("nv", str(get_votes("Not Voting")))
root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov")
utils.make_node(root, "category", vote["category"])
utils.make_node(root, "type", vote["type"])
utils.make_node(root, "question", vote["question"])
utils.make_node(root, "required", vote["requires"])
utils.make_node(root, "result", vote["result"])
if "bill" in vote:
govtrack_type_codes = {
"hr": "h",
"s": "s",
"hres": "hr",
"sres": "sr",
"hjres": "hj",
"sjres": "sj",
"hconres": "hc",
"sconres": "sc",
}
utils.make_node(
root,
"bill",
None,
session=str(vote["bill"]["congress"]),
type=govtrack_type_codes[vote["bill"]["type"]],
number=str(vote["bill"]["number"]),
)
if "amendment" in vote:
if vote["amendment"]["type"] == "s":
utils.make_node(root, "amendment", None, ref="regular", number="s" + str(vote["amendment"]["number"]))
elif vote["amendment"]["type"] == "h-bill":
utils.make_node(root, "amendment", None, ref="bill-serial", number=str(vote["amendment"]["number"]))
# well-known keys for certain vote types: +/-/P/0
option_keys = {"Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0"}
# preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty
# and handling other options like people's names for votes for the Speaker.
option_sort_order = ("Aye", "Yea", "Guilty", "No", "Nay", "Not Guilty", "OTHER", "Present", "Not Voting")
options_list = sorted(
vote["votes"].keys(),
key=lambda o: option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER"),
)
for option in options_list:
if option not in option_keys:
option_keys[option] = option
utils.make_node(root, "option", option, key=option_keys[option])
for option in options_list:
for v in vote["votes"][option]:
attrs = {"vote": option_keys[option], "value": option}
if v == "VP":
attrs["id"] = "0"
attrs["VP"] = "1"
elif not options.get("govtrack", False):
attrs["id"] = str(v["id"])
attrs["state"] = v["state"]
else:
attrs["id"] = str(
utils.get_govtrack_person_id("bioguide" if vote["chamber"] == "h" else "lis", v["id"])
)
attrs["state"] = v["state"]
utils.make_node(root, "voter", None, **attrs)
utils.write(etree.tostring(root, pretty_print=True), output_for_vote(vote["vote_id"], "xml"))
def output_amendment(amdt, options):
logging.info("[%s] Writing to disk..." % amdt['amendment_id'])
# output JSON - so easy!
utils.write(
json.dumps(amdt,
sort_keys=True,
indent=2,
default=utils.format_datetime),
output_for_amdt(amdt['amendment_id'], "json"))
# output XML
govtrack_type_codes = {
'hr': 'h',
's': 's',
'hres': 'hr',
'sres': 'sr',
'hjres': 'hj',
'sjres': 'sj',
'hconres': 'hc',
'sconres': 'sc'
}
root = etree.Element("amendment")
root.set("session", amdt['congress'])
root.set("chamber", amdt['amendment_type'][0])
root.set("number", str(amdt['number']))
root.set("updated", utils.format_datetime(amdt['updated_at']))
make_node = utils.make_node
if amdt.get("amends_bill", None):
make_node(root,
"amends",
None,
type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]],
number=str(amdt["amends_bill"]["number"]),
sequence=str(amdt["house_number"]) if amdt.get(
"house_number", None) else "")
elif amdt.get("amends_treaty", None):
make_node(root,
"amends",
None,
type="treaty",
number=str(amdt["amends_treaty"]["number"]))
make_node(root, "status", amdt['status'], datetime=amdt['status_at'])
if amdt['sponsor'] and amdt['sponsor']['type'] == 'person':
v = amdt['sponsor']['thomas_id']
if not options.get("govtrack", False):
make_node(root, "sponsor", None, thomas_id=v)
else:
v = str(utils.get_govtrack_person_id('thomas', v))
make_node(root, "sponsor", None, id=v)
elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee':
make_node(root, "sponsor", None, committee=amdt['sponsor']['name'])
else:
make_node(root, "sponsor", None)
make_node(root, "offered", None, datetime=amdt['introduced_at'])
make_node(root, "description",
amdt["description"] if amdt["description"] else amdt["purpose"])
if amdt["description"]: make_node(root, "purpose", amdt["purpose"])
actions = make_node(root, "actions", None)
for action in amdt['actions']:
a = make_node(
actions,
action['type'] if action['type'] in ("vote", ) else "action",
None,
datetime=action['acted_at'])
if action['type'] == 'vote':
a.set("how", action["how"])
a.set("result", action["result"])
if action.get("roll") != None: a.set("roll", str(action["roll"]))
if action.get('text'): make_node(a, "text", action['text'])
if action.get('in_committee'):
make_node(a, "committee", None, name=action['in_committee'])
for cr in action['references']:
make_node(a,
"reference",
None,
ref=cr['reference'],
label=cr['type'])
utils.write(etree.tostring(root, pretty_print=True),
output_for_amdt(amdt['amendment_id'], "xml"))
def output_amendment(amdt, options):
logging.info("[%s] Writing to disk..." % amdt['amendment_id'])
# output JSON - so easy!
utils.write(
json.dumps(amdt, sort_keys=True, indent=2, default=utils.format_datetime),
output_for_amdt(amdt['amendment_id'], "json")
)
# output XML
govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' }
root = etree.Element("amendment")
root.set("session", amdt['congress'])
root.set("chamber", amdt['amendment_type'][0])
root.set("number", amdt['number'])
root.set("updated", utils.format_datetime(amdt['updated_at']))
make_node = utils.make_node
make_node(root, "amends", None,
type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]],
number=str(amdt["amends_bill"]["number"]),
sequence=str(int(amdt["house_number"][1:])) if amdt["house_number"] else "") # chop off A from the house_number
make_node(root, "status", amdt['status'], datetime=amdt['status_at'])
if amdt['sponsor'] and amdt['sponsor']['type'] == 'person':
v = amdt['sponsor']['thomas_id']
if not options.get("govtrack", False):
make_node(root, "sponsor", None, thomas_id=v)
else:
v = str(utils.get_govtrack_person_id('thomas', v))
make_node(root, "sponsor", None, id=v)
elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee':
make_node(root, "sponsor", None, committee=amdt['sponsor']['name'])
else:
make_node(root, "sponsor", None)
make_node(root, "offered", None, datetime=amdt['offered_at'] if amdt['offered_at'] else amdt['submitted_at'])
if amdt["title"]: make_node(root, "title", amdt["title"])
make_node(root, "description", amdt["description"] if amdt["description"] else amdt["purpose"])
make_node(root, "purpose", amdt["purpose"])
actions = make_node(root, "actions", None)
for action in amdt['actions']:
a = make_node(actions,
action['type'] if action['type'] in ("vote",) else "action",
None,
datetime=action['acted_at'])
if action['type'] == 'vote':
a.set("how", action["how"])
a.set("result", action["result"])
if action.get("roll") != None: a.set("roll", str(action["roll"]))
if action.get('text'): make_node(a, "text", action['text'])
if action.get('in_committee'): make_node(a, "committee", None, name=action['in_committee'])
for cr in action['references']:
make_node(a, "reference", None, ref=cr['reference'], label=cr['type'])
utils.write(
etree.tostring(root, pretty_print=True),
output_for_amdt(amdt['amendment_id'], "xml")
)
def output_vote(vote, options, id_type=None):
logging.info("[%s] Writing to disk..." % vote["vote_id"])
# output JSON - so easy!
utils.write(
json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime),
output_for_vote(vote["vote_id"], "json"),
options=options,
)
# What kind of IDs are we passed for Members of Congress?
# For current data, we infer from the chamber. For historical data from voteview,
# we're passed the type in id_type, which is set to "bioguide".
if not id_type:
id_type = "bioguide" if vote["chamber"] == "h" else "lis"
# output XML
root = etree.Element("roll")
root.set("where", "house" if vote["chamber"] == "h" else "senate")
root.set("session", str(vote["congress"]))
root.set("year", str(vote["date"].year))
root.set("roll", str(vote["number"]))
if "voteview" in vote["source_url"]:
root.set("source", "keithpoole")
else:
root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov")
root.set("datetime", utils.format_datetime(vote["date"]))
root.set("updated", utils.format_datetime(vote["updated_at"]))
def get_votes(option):
return len(vote["votes"].get(option, []))
root.set("aye", str(get_votes("Yea") + get_votes("Aye")))
root.set("nay", str(get_votes("Nay") + get_votes("No")))
root.set("nv", str(get_votes("Not Voting")))
root.set("present", str(get_votes("Present")))
utils.make_node(root, "category", vote["category"])
utils.make_node(root, "type", vote["type"])
utils.make_node(root, "question", vote["question"])
utils.make_node(root, "required", vote["requires"])
utils.make_node(root, "result", vote["result"])
if vote.get("bill"):
govtrack_type_codes = {
"hr": "h",
"s": "s",
"hres": "hr",
"sres": "sr",
"hjres": "hj",
"sjres": "sj",
"hconres": "hc",
"sconres": "sc",
}
utils.make_node(
root,
"bill",
None,
session=str(vote["bill"]["congress"]),
type=govtrack_type_codes[vote["bill"]["type"]],
number=str(vote["bill"]["number"]),
)
if "amendment" in vote:
n = utils.make_node(root, "amendment", None)
if vote["amendment"]["type"] == "s":
n.set("ref", "regular")
n.set("session", str(vote["congress"]))
n.set("number", "s" + str(vote["amendment"]["number"]))
elif vote["amendment"]["type"] == "h-bill":
n.set("ref", "bill-serial")
n.set("session", str(vote["congress"]))
n.set("number", str(vote["amendment"]["number"]))
# well-known keys for certain vote types: +/-/P/0
option_keys = {
"Aye": "+",
"Yea": "+",
"Nay": "-",
"No": "-",
"Present": "P",
"Not Voting": "0",
"Guilty": "+",
"Not Guilty": "-",
}
# preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty
# and handling other options like people's names for votes for the Speaker.
option_sort_order = ("Aye", "Yea", "Guilty", "No", "Nay", "Not Guilty", "OTHER", "Present", "Not Voting")
options_list = sorted(
vote["votes"].keys(),
key=lambda o: option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER"),
)
for option in options_list:
if option not in option_keys:
option_keys[option] = option
utils.make_node(root, "option", option, key=option_keys[option])
for option in options_list:
for v in vote["votes"][option]:
n = utils.make_node(root, "voter", None)
if v == "VP":
n.set("id", "0")
n.set("VP", "1")
elif not options.get("govtrack", False):
n.set("id", str(v["id"]))
else:
n.set("id", str(utils.get_govtrack_person_id(id_type, v["id"])))
n.set("vote", option_keys[option])
n.set("value", option)
if v != "VP":
n.set("state", v["state"])
if v.get("voteview_votecode_extra") is not None:
n.set("voteview_votecode_extra", v["voteview_votecode_extra"])
xmloutput = etree.tostring(root, pretty_print=True, encoding="utf8")
# mimick two hard line breaks in GovTrack's legacy output to ease running diffs
xmloutput = re.sub('(source=".*?") ', r"\1\n ", xmloutput)
xmloutput = re.sub('(updated=".*?") ', r"\1\n ", xmloutput)
utils.write(xmloutput, output_for_vote(vote["vote_id"], "xml"), options=options)
def output_vote(vote, options):
logging.info("[%s] Writing to disk..." % vote['vote_id'])
# output JSON - so easy!
utils.write(
json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime),
output_for_vote(vote["vote_id"], "json"),
)
# output XML
root = etree.Element("roll")
root.set("where", "house" if vote['chamber'] == "h" else "senate")
root.set("session", str(vote["congress"]))
root.set("year", str(vote["date"].year))
root.set("roll", str(vote["number"]))
root.set("datetime", utils.format_datetime(vote['date']))
root.set("updated", utils.format_datetime(vote['updated_at']))
def get_votes(option): return len(vote["votes"].get(option, []))
root.set("aye", str(get_votes("Yea") + get_votes("Aye")))
root.set("nay", str(get_votes("Nay") + get_votes("No")))
root.set("present", str(get_votes("Present")))
root.set("nv", str(get_votes("Not Voting")))
root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov")
utils.make_node(root, "category", vote["category"])
utils.make_node(root, "type", vote["type"])
utils.make_node(root, "question", vote["question"])
utils.make_node(root, "required", vote["requires"])
utils.make_node(root, "result", vote["result"])
if "bill" in vote:
govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' }
utils.make_node(root, "bill", None, session=str(vote["bill"]["congress"]), type=govtrack_type_codes[vote["bill"]["type"]], number=str(vote["bill"]["number"]))
if "amendment" in vote:
if vote["amendment"]["type"] == "s":
utils.make_node(root, "amendment", None, ref="regular", number="s" + str(vote["amendment"]["number"]))
elif vote["amendment"]["type"] == "h-bill":
utils.make_node(root, "amendment", None, ref="bill-serial", number=str(vote["amendment"]["number"]))
# well-known keys for certain vote types: +/-/P/0
option_keys = { "Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0" }
# preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty
# and handling other options like people's names for votes for the Speaker.
option_sort_order = ('Aye', 'Yea', 'Guilty', 'No', 'Nay', 'Not Guilty', 'OTHER', 'Present', 'Not Voting')
options_list = sorted(vote["votes"].keys(), key = lambda o : option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER") )
for option in options_list:
if option not in option_keys: option_keys[option] = option
utils.make_node(root, "option", option, key=option_keys[option])
for option in options_list:
for v in vote["votes"][option]:
attrs = { "vote": option_keys[option], "value": option }
if v == "VP":
attrs["id"] = "0"
attrs["VP"] = "1"
elif not options.get("govtrack", False):
attrs["id"] = str(v["id"])
attrs["state"] = v["state"]
else:
attrs["id"] = str(utils.get_govtrack_person_id("bioguide" if vote["chamber"] == "h" else "lis", v["id"]))
attrs["state"] = v["state"]
utils.make_node(root, "voter", None, **attrs)
utils.write(
etree.tostring(root, pretty_print=True),
output_for_vote(vote['vote_id'], "xml")
)
