def output_vote(vote, options, id_type=None): logging.info("[%s] Writing to disk..." % vote['vote_id']) # output JSON - so easy! utils.write(json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime), output_for_vote(vote["vote_id"], "json"), options=options) # What kind of IDs are we passed for Members of Congress? # For current data, we infer from the chamber. For historical data from voteview, # we're passed the type in id_type, which is set to "bioguide". if not id_type: id_type = ("bioguide" if vote["chamber"] == "h" else "lis") # output XML root = etree.Element("roll") root.set("where", "house" if vote['chamber'] == "h" else "senate") root.set("session", str(vote["congress"])) root.set("year", str(vote["date"].year)) root.set("roll", str(vote["number"])) if "voteview" in vote["source_url"]: root.set("source", "keithpoole") else: root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov") root.set("datetime", utils.format_datetime(vote['date'])) root.set("updated", utils.format_datetime(vote['updated_at'])) def get_votes(option): return len(vote["votes"].get(option, [])) root.set("aye", str(get_votes("Yea") + get_votes("Aye"))) root.set("nay", str(get_votes("Nay") + get_votes("No"))) root.set("nv", str(get_votes("Not Voting"))) root.set("present", str(get_votes("Present"))) utils.make_node(root, "category", vote["category"]) utils.make_node(root, "type", vote["type"]) utils.make_node(root, "question", vote["question"]) utils.make_node(root, "required", vote["requires"]) utils.make_node(root, "result", vote["result"]) if vote.get("bill"): govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' } utils.make_node(root, "bill", None, session=str(vote["bill"]["congress"]), type=govtrack_type_codes[vote["bill"]["type"]], number=str(vote["bill"]["number"])) if "amendment" in vote: n = utils.make_node(root, "amendment", None) if vote["amendment"]["type"] == "s": n.set("ref", "regular") n.set("session", str(vote["congress"])) n.set("number", "s" + str(vote["amendment"]["number"])) elif vote["amendment"]["type"] == "h-bill": n.set("ref", "bill-serial") n.set("session", str(vote["congress"])) n.set("number", str(vote["amendment"]["number"])) # well-known keys for certain vote types: +/-/P/0 option_keys = { "Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0", "Guilty": "+", "Not Guilty": "-" } # preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty # and handling other options like people's names for votes for the Speaker. option_sort_order = ('Aye', 'Yea', 'Guilty', 'No', 'Nay', 'Not Guilty', 'OTHER', 'Present', 'Not Voting') options_list = sorted( vote["votes"].keys(), key=lambda o: option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER")) for option in options_list: if option not in option_keys: option_keys[option] = option utils.make_node(root, "option", option, key=option_keys[option]) for option in options_list: for v in vote["votes"][option]: n = utils.make_node(root, "voter", None) if v == "VP": n.set("id", "0") n.set("VP", "1") elif not options.get("govtrack", False): n.set("id", str(v["id"])) else: n.set("id", str(utils.get_govtrack_person_id(id_type, v["id"]))) n.set("vote", option_keys[option]) n.set("value", option) if v != "VP": n.set("state", v["state"]) if v.get("voteview_votecode_extra") is not None: n.set("voteview_votecode_extra", v["voteview_votecode_extra"]) xmloutput = etree.tostring(root, pretty_print=True, encoding="utf8") # mimick two hard line breaks in GovTrack's legacy output to ease running diffs xmloutput = re.sub('(source=".*?") ', r"\1\n ", xmloutput) xmloutput = re.sub('(updated=".*?") ', r"\1\n ", xmloutput) utils.write(xmloutput, output_for_vote(vote['vote_id'], "xml"), options=options)
def output_vote(vote, options): logging.info("[%s] Writing to disk..." % vote['vote_id']) # output JSON - so easy! utils.write( json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime), output_for_vote(vote["vote_id"], "json"), ) # output XML root = etree.Element("roll") root.set("where", "house" if vote['chamber'] == "h" else "senate") root.set("session", str(vote["congress"])) root.set("year", str(vote["date"].year)) root.set("roll", str(vote["number"])) root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov") root.set("datetime", utils.format_datetime(vote['date'])) root.set("updated", utils.format_datetime(vote['updated_at'])) def get_votes(option): return len(vote["votes"].get(option, [])) root.set("aye", str(get_votes("Yea") + get_votes("Aye"))) root.set("nay", str(get_votes("Nay") + get_votes("No"))) root.set("nv", str(get_votes("Not Voting"))) root.set("present", str(get_votes("Present"))) utils.make_node(root, "category", vote["category"]) utils.make_node(root, "type", vote["type"]) utils.make_node(root, "question", vote["question"]) utils.make_node(root, "required", vote["requires"]) utils.make_node(root, "result", vote["result"]) if "bill" in vote: govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' } utils.make_node(root, "bill", None, session=str(vote["bill"]["congress"]), type=govtrack_type_codes[vote["bill"]["type"]], number=str(vote["bill"]["number"])) if "amendment" in vote: n = utils.make_node(root, "amendment", None) if vote["amendment"]["type"] == "s": n.set("ref", "regular") n.set("session", str(vote["congress"])) n.set("number", "s" + str(vote["amendment"]["number"])) elif vote["amendment"]["type"] == "h-bill": n.set("ref", "bill-serial") n.set("session", str(vote["congress"])) n.set("number", str(vote["amendment"]["number"])) # well-known keys for certain vote types: +/-/P/0 option_keys = { "Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0" } # preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty # and handling other options like people's names for votes for the Speaker. option_sort_order = ('Aye', 'Yea', 'Guilty', 'No', 'Nay', 'Not Guilty', 'OTHER', 'Present', 'Not Voting') options_list = sorted(vote["votes"].keys(), key = lambda o : option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER") ) for option in options_list: if option not in option_keys: option_keys[option] = option utils.make_node(root, "option", option, key=option_keys[option]) for option in options_list: for v in vote["votes"][option]: n = utils.make_node(root, "voter", None) if v == "VP": n.set("id", "0") n.set("VP", "1") elif not options.get("govtrack", False): n.set("id", str(v["id"])) else: n.set("id", str(utils.get_govtrack_person_id("bioguide" if vote["chamber"] == "h" else "lis", v["id"]))) n.set("vote", option_keys[option]) n.set("value", option) if v != "VP": n.set("state", v["state"]) xmloutput = etree.tostring(root, pretty_print=True, encoding="utf8") # mimick two hard line breaks in GovTrack's legacy output to ease running diffs xmloutput = re.sub('(source=".*?") ', r"\1\n ", xmloutput) xmloutput = re.sub('(updated=".*?") ', r"\1\n ", xmloutput) utils.write( xmloutput, output_for_vote(vote['vote_id'], "xml") )
def output_vote(vote, options): logging.info("[%s] Writing to disk..." % vote["vote_id"]) # output JSON - so easy! utils.write( json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime), output_for_vote(vote["vote_id"], "json"), ) # output XML root = etree.Element("roll") root.set("where", "house" if vote["chamber"] == "h" else "senate") root.set("session", str(vote["congress"])) root.set("year", str(vote["date"].year)) root.set("roll", str(vote["number"])) root.set("datetime", utils.format_datetime(vote["date"])) root.set("updated", utils.format_datetime(vote["updated_at"])) def get_votes(option): return len(vote["votes"].get(option, [])) root.set("aye", str(get_votes("Yea") + get_votes("Aye"))) root.set("nay", str(get_votes("Nay") + get_votes("No"))) root.set("present", str(get_votes("Present"))) root.set("nv", str(get_votes("Not Voting"))) root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov") utils.make_node(root, "category", vote["category"]) utils.make_node(root, "type", vote["type"]) utils.make_node(root, "question", vote["question"]) utils.make_node(root, "required", vote["requires"]) utils.make_node(root, "result", vote["result"]) if "bill" in vote: govtrack_type_codes = { "hr": "h", "s": "s", "hres": "hr", "sres": "sr", "hjres": "hj", "sjres": "sj", "hconres": "hc", "sconres": "sc", } utils.make_node( root, "bill", None, session=str(vote["bill"]["congress"]), type=govtrack_type_codes[vote["bill"]["type"]], number=str(vote["bill"]["number"]), ) if "amendment" in vote: if vote["amendment"]["type"] == "s": utils.make_node(root, "amendment", None, ref="regular", number="s" + str(vote["amendment"]["number"])) elif vote["amendment"]["type"] == "h-bill": utils.make_node(root, "amendment", None, ref="bill-serial", number=str(vote["amendment"]["number"])) # well-known keys for certain vote types: +/-/P/0 option_keys = {"Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0"} # preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty # and handling other options like people's names for votes for the Speaker. option_sort_order = ("Aye", "Yea", "Guilty", "No", "Nay", "Not Guilty", "OTHER", "Present", "Not Voting") options_list = sorted( vote["votes"].keys(), key=lambda o: option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER"), ) for option in options_list: if option not in option_keys: option_keys[option] = option utils.make_node(root, "option", option, key=option_keys[option]) for option in options_list: for v in vote["votes"][option]: attrs = {"vote": option_keys[option], "value": option} if v == "VP": attrs["id"] = "0" attrs["VP"] = "1" elif not options.get("govtrack", False): attrs["id"] = str(v["id"]) attrs["state"] = v["state"] else: attrs["id"] = str( utils.get_govtrack_person_id("bioguide" if vote["chamber"] == "h" else "lis", v["id"]) ) attrs["state"] = v["state"] utils.make_node(root, "voter", None, **attrs) utils.write(etree.tostring(root, pretty_print=True), output_for_vote(vote["vote_id"], "xml"))
def output_amendment(amdt, options): logging.info("[%s] Writing to disk..." % amdt['amendment_id']) # output JSON - so easy! utils.write( json.dumps(amdt, sort_keys=True, indent=2, default=utils.format_datetime), output_for_amdt(amdt['amendment_id'], "json")) # output XML govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' } root = etree.Element("amendment") root.set("session", amdt['congress']) root.set("chamber", amdt['amendment_type'][0]) root.set("number", str(amdt['number'])) root.set("updated", utils.format_datetime(amdt['updated_at'])) make_node = utils.make_node if amdt.get("amends_bill", None): make_node(root, "amends", None, type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]], number=str(amdt["amends_bill"]["number"]), sequence=str(amdt["house_number"]) if amdt.get( "house_number", None) else "") elif amdt.get("amends_treaty", None): make_node(root, "amends", None, type="treaty", number=str(amdt["amends_treaty"]["number"])) make_node(root, "status", amdt['status'], datetime=amdt['status_at']) if amdt['sponsor'] and amdt['sponsor']['type'] == 'person': v = amdt['sponsor']['thomas_id'] if not options.get("govtrack", False): make_node(root, "sponsor", None, thomas_id=v) else: v = str(utils.get_govtrack_person_id('thomas', v)) make_node(root, "sponsor", None, id=v) elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee': make_node(root, "sponsor", None, committee=amdt['sponsor']['name']) else: make_node(root, "sponsor", None) make_node(root, "offered", None, datetime=amdt['introduced_at']) make_node(root, "description", amdt["description"] if amdt["description"] else amdt["purpose"]) if amdt["description"]: make_node(root, "purpose", amdt["purpose"]) actions = make_node(root, "actions", None) for action in amdt['actions']: a = make_node( actions, action['type'] if action['type'] in ("vote", ) else "action", None, datetime=action['acted_at']) if action['type'] == 'vote': a.set("how", action["how"]) a.set("result", action["result"]) if action.get("roll") != None: a.set("roll", str(action["roll"])) if action.get('text'): make_node(a, "text", action['text']) if action.get('in_committee'): make_node(a, "committee", None, name=action['in_committee']) for cr in action['references']: make_node(a, "reference", None, ref=cr['reference'], label=cr['type']) utils.write(etree.tostring(root, pretty_print=True), output_for_amdt(amdt['amendment_id'], "xml"))
def output_amendment(amdt, options): logging.info("[%s] Writing to disk..." % amdt['amendment_id']) # output JSON - so easy! utils.write( json.dumps(amdt, sort_keys=True, indent=2, default=utils.format_datetime), output_for_amdt(amdt['amendment_id'], "json") ) # output XML govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' } root = etree.Element("amendment") root.set("session", amdt['congress']) root.set("chamber", amdt['amendment_type'][0]) root.set("number", amdt['number']) root.set("updated", utils.format_datetime(amdt['updated_at'])) make_node = utils.make_node make_node(root, "amends", None, type=govtrack_type_codes[amdt["amends_bill"]["bill_type"]], number=str(amdt["amends_bill"]["number"]), sequence=str(int(amdt["house_number"][1:])) if amdt["house_number"] else "") # chop off A from the house_number make_node(root, "status", amdt['status'], datetime=amdt['status_at']) if amdt['sponsor'] and amdt['sponsor']['type'] == 'person': v = amdt['sponsor']['thomas_id'] if not options.get("govtrack", False): make_node(root, "sponsor", None, thomas_id=v) else: v = str(utils.get_govtrack_person_id('thomas', v)) make_node(root, "sponsor", None, id=v) elif amdt['sponsor'] and amdt['sponsor']['type'] == 'committee': make_node(root, "sponsor", None, committee=amdt['sponsor']['name']) else: make_node(root, "sponsor", None) make_node(root, "offered", None, datetime=amdt['offered_at'] if amdt['offered_at'] else amdt['submitted_at']) if amdt["title"]: make_node(root, "title", amdt["title"]) make_node(root, "description", amdt["description"] if amdt["description"] else amdt["purpose"]) make_node(root, "purpose", amdt["purpose"]) actions = make_node(root, "actions", None) for action in amdt['actions']: a = make_node(actions, action['type'] if action['type'] in ("vote",) else "action", None, datetime=action['acted_at']) if action['type'] == 'vote': a.set("how", action["how"]) a.set("result", action["result"]) if action.get("roll") != None: a.set("roll", str(action["roll"])) if action.get('text'): make_node(a, "text", action['text']) if action.get('in_committee'): make_node(a, "committee", None, name=action['in_committee']) for cr in action['references']: make_node(a, "reference", None, ref=cr['reference'], label=cr['type']) utils.write( etree.tostring(root, pretty_print=True), output_for_amdt(amdt['amendment_id'], "xml") )
def output_vote(vote, options, id_type=None): logging.info("[%s] Writing to disk..." % vote["vote_id"]) # output JSON - so easy! utils.write( json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime), output_for_vote(vote["vote_id"], "json"), options=options, ) # What kind of IDs are we passed for Members of Congress? # For current data, we infer from the chamber. For historical data from voteview, # we're passed the type in id_type, which is set to "bioguide". if not id_type: id_type = "bioguide" if vote["chamber"] == "h" else "lis" # output XML root = etree.Element("roll") root.set("where", "house" if vote["chamber"] == "h" else "senate") root.set("session", str(vote["congress"])) root.set("year", str(vote["date"].year)) root.set("roll", str(vote["number"])) if "voteview" in vote["source_url"]: root.set("source", "keithpoole") else: root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov") root.set("datetime", utils.format_datetime(vote["date"])) root.set("updated", utils.format_datetime(vote["updated_at"])) def get_votes(option): return len(vote["votes"].get(option, [])) root.set("aye", str(get_votes("Yea") + get_votes("Aye"))) root.set("nay", str(get_votes("Nay") + get_votes("No"))) root.set("nv", str(get_votes("Not Voting"))) root.set("present", str(get_votes("Present"))) utils.make_node(root, "category", vote["category"]) utils.make_node(root, "type", vote["type"]) utils.make_node(root, "question", vote["question"]) utils.make_node(root, "required", vote["requires"]) utils.make_node(root, "result", vote["result"]) if vote.get("bill"): govtrack_type_codes = { "hr": "h", "s": "s", "hres": "hr", "sres": "sr", "hjres": "hj", "sjres": "sj", "hconres": "hc", "sconres": "sc", } utils.make_node( root, "bill", None, session=str(vote["bill"]["congress"]), type=govtrack_type_codes[vote["bill"]["type"]], number=str(vote["bill"]["number"]), ) if "amendment" in vote: n = utils.make_node(root, "amendment", None) if vote["amendment"]["type"] == "s": n.set("ref", "regular") n.set("session", str(vote["congress"])) n.set("number", "s" + str(vote["amendment"]["number"])) elif vote["amendment"]["type"] == "h-bill": n.set("ref", "bill-serial") n.set("session", str(vote["congress"])) n.set("number", str(vote["amendment"]["number"])) # well-known keys for certain vote types: +/-/P/0 option_keys = { "Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0", "Guilty": "+", "Not Guilty": "-", } # preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty # and handling other options like people's names for votes for the Speaker. option_sort_order = ("Aye", "Yea", "Guilty", "No", "Nay", "Not Guilty", "OTHER", "Present", "Not Voting") options_list = sorted( vote["votes"].keys(), key=lambda o: option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER"), ) for option in options_list: if option not in option_keys: option_keys[option] = option utils.make_node(root, "option", option, key=option_keys[option]) for option in options_list: for v in vote["votes"][option]: n = utils.make_node(root, "voter", None) if v == "VP": n.set("id", "0") n.set("VP", "1") elif not options.get("govtrack", False): n.set("id", str(v["id"])) else: n.set("id", str(utils.get_govtrack_person_id(id_type, v["id"]))) n.set("vote", option_keys[option]) n.set("value", option) if v != "VP": n.set("state", v["state"]) if v.get("voteview_votecode_extra") is not None: n.set("voteview_votecode_extra", v["voteview_votecode_extra"]) xmloutput = etree.tostring(root, pretty_print=True, encoding="utf8") # mimick two hard line breaks in GovTrack's legacy output to ease running diffs xmloutput = re.sub('(source=".*?") ', r"\1\n ", xmloutput) xmloutput = re.sub('(updated=".*?") ', r"\1\n ", xmloutput) utils.write(xmloutput, output_for_vote(vote["vote_id"], "xml"), options=options)
def output_vote(vote, options): logging.info("[%s] Writing to disk..." % vote['vote_id']) # output JSON - so easy! utils.write( json.dumps(vote, sort_keys=True, indent=2, default=utils.format_datetime), output_for_vote(vote["vote_id"], "json"), ) # output XML root = etree.Element("roll") root.set("where", "house" if vote['chamber'] == "h" else "senate") root.set("session", str(vote["congress"])) root.set("year", str(vote["date"].year)) root.set("roll", str(vote["number"])) root.set("datetime", utils.format_datetime(vote['date'])) root.set("updated", utils.format_datetime(vote['updated_at'])) def get_votes(option): return len(vote["votes"].get(option, [])) root.set("aye", str(get_votes("Yea") + get_votes("Aye"))) root.set("nay", str(get_votes("Nay") + get_votes("No"))) root.set("present", str(get_votes("Present"))) root.set("nv", str(get_votes("Not Voting"))) root.set("source", "house.gov" if vote["chamber"] == "h" else "senate.gov") utils.make_node(root, "category", vote["category"]) utils.make_node(root, "type", vote["type"]) utils.make_node(root, "question", vote["question"]) utils.make_node(root, "required", vote["requires"]) utils.make_node(root, "result", vote["result"]) if "bill" in vote: govtrack_type_codes = { 'hr': 'h', 's': 's', 'hres': 'hr', 'sres': 'sr', 'hjres': 'hj', 'sjres': 'sj', 'hconres': 'hc', 'sconres': 'sc' } utils.make_node(root, "bill", None, session=str(vote["bill"]["congress"]), type=govtrack_type_codes[vote["bill"]["type"]], number=str(vote["bill"]["number"])) if "amendment" in vote: if vote["amendment"]["type"] == "s": utils.make_node(root, "amendment", None, ref="regular", number="s" + str(vote["amendment"]["number"])) elif vote["amendment"]["type"] == "h-bill": utils.make_node(root, "amendment", None, ref="bill-serial", number=str(vote["amendment"]["number"])) # well-known keys for certain vote types: +/-/P/0 option_keys = { "Aye": "+", "Yea": "+", "Nay": "-", "No": "-", "Present": "P", "Not Voting": "0" } # preferred order of output: ayes, nays, present, then not voting, and similarly for guilty/not-guilty # and handling other options like people's names for votes for the Speaker. option_sort_order = ('Aye', 'Yea', 'Guilty', 'No', 'Nay', 'Not Guilty', 'OTHER', 'Present', 'Not Voting') options_list = sorted(vote["votes"].keys(), key = lambda o : option_sort_order.index(o) if o in option_sort_order else option_sort_order.index("OTHER") ) for option in options_list: if option not in option_keys: option_keys[option] = option utils.make_node(root, "option", option, key=option_keys[option]) for option in options_list: for v in vote["votes"][option]: attrs = { "vote": option_keys[option], "value": option } if v == "VP": attrs["id"] = "0" attrs["VP"] = "1" elif not options.get("govtrack", False): attrs["id"] = str(v["id"]) attrs["state"] = v["state"] else: attrs["id"] = str(utils.get_govtrack_person_id("bioguide" if vote["chamber"] == "h" else "lis", v["id"])) attrs["state"] = v["state"] utils.make_node(root, "voter", None, **attrs) utils.write( etree.tostring(root, pretty_print=True), output_for_vote(vote['vote_id'], "xml") )