def vote_ids_for_senate(congress, session_year, options): session_num = int(session_year) - utils.get_congress_first_year(int(congress)) + 1 vote_ids = [] url = "http://www.senate.gov/legislative/LIS/roll_call_lists/vote_menu_%s_%d.xml" % (congress, session_num) page = utils.download( url, "%s/votes/%s/pages/senate.xml" % (congress, session_year), utils.merge(options, {'binary': True}) ) if not page or b"Requested Page Not Found (404)" in page: logging.error("Couldn't download Senate vote XML index %s, skipping" % url) return None dom = etree.fromstring(page) # Sanity checks. if int(congress) != int(dom.xpath("congress")[0].text): logging.error("Senate vote XML returns the wrong Congress: %s" % dom.xpath("congress")[0].text) return None if int(session_year) != int(dom.xpath("congress_year")[0].text): logging.error("Senate vote XML returns the wrong session: %s" % dom.xpath("congress_year")[0].text) return None # Get vote list. for vote in dom.xpath("//vote"): num = int(vote.xpath("vote_number")[0].text) vote_id = "s" + str(num) + "-" + str(congress) + "." + session_year if not should_process(vote_id, options): continue vote_ids.append(vote_id) return vote_ids
def vote_ids_for_senate(congress, session_year, options): session_num = int(session_year) - utils.get_congress_first_year(int(congress)) + 1 vote_ids = [] page = utils.download( "http://www.senate.gov/legislative/LIS/roll_call_lists/vote_menu_%s_%d.xml" % (congress, session_num), "%s/votes/%s/pages/senate.xml" % (congress, session_year), utils.merge(options, {'binary': True}) ) if not page: logging.error("Couldn't download Senate vote XML index, aborting") return None dom = etree.fromstring(page) # Sanity checks. if int(congress) != int(dom.xpath("congress")[0].text): logging.error("Senate vote XML returns the wrong Congress: %s" % dom.xpath("congress")[0].text) return None if int(session_year) != int(dom.xpath("congress_year")[0].text): logging.error("Senate vote XML returns the wrong session: %s" % dom.xpath("congress_year")[0].text) return None # Get vote list. for vote in dom.xpath("//vote"): num = int(vote.xpath("vote_number")[0].text) vote_id = "s" + str(num) + "-" + str(congress) + "." + session_year if not should_process(vote_id, options): continue vote_ids.append(vote_id) return vote_ids
def fetch_vote(vote_id, options): logging.info("\n[%s] Fetching..." % vote_id) vote_chamber, vote_number, vote_congress, vote_session_year = utils.split_vote_id(vote_id) if vote_chamber == "h": url = "http://clerk.house.gov/evs/%s/roll%03d.xml" % (vote_session_year, int(vote_number)) else: session_num = int(vote_session_year) - utils.get_congress_first_year(int(vote_congress)) + 1 url = "http://www.senate.gov/legislative/LIS/roll_call_votes/vote%d%d/vote_%d_%d_%05d.xml" % (int(vote_congress), session_num, int(vote_congress), session_num, int(vote_number)) # fetch vote XML page body = utils.download( url, "%s/votes/%s/%s%s/%s%s.xml" % (vote_congress, vote_session_year, vote_chamber, vote_number, vote_chamber, vote_number), utils.merge(options, {'binary': True}), ) if not body: return {'saved': False, 'ok': False, 'reason': "failed to download"} if options.get("download_only", False): return {'saved': False, 'ok': True, 'reason': "requested download only"} if "This vote was vacated" in body: # Vacated votes: 2011-484, 2012-327, ... # Remove file, since it may previously have existed with data. for f in (output_for_vote(vote_id, "json"), output_for_vote(vote_id, "xml")): if os.path.exists(f): os.unlink(f) return {'saved': False, 'ok': True, 'reason': "vote was vacated"} dom = etree.fromstring(body) vote = { 'vote_id': vote_id, 'chamber': vote_chamber, 'congress': int(vote_congress), 'session': vote_session_year, 'number': int(vote_number), 'updated_at': datetime.datetime.fromtimestamp(time.time()), 'source_url': url, } # do the heavy lifting if vote_chamber == "h": parse_house_vote(dom, vote) elif vote_chamber == "s": parse_senate_vote(dom, vote) # output and return output_vote(vote, options) return {'ok': True, 'saved': True}
def vote_ids_for_senate(congress, session_year, options): session_num = int(session_year) - utils.get_congress_first_year(int(congress)) + 1 vote_ids = [] page = utils.download( "http://www.senate.gov/legislative/LIS/roll_call_lists/vote_menu_%s_%d.xml" % (congress, session_num), "%s/votes/%s/pages/senate.xml" % (congress, session_year), utils.merge(options, {'binary': True}) ) if not page: logging.error("Couldn't download Senate vote XML index, aborting") return None dom = etree.fromstring(page) for vote in dom.xpath("//vote"): num = int(vote.xpath("vote_number")[0].text) vote_id = "s" + str(num) + "-" + str(congress) + "." + session_year if not should_process(vote_id, options): continue vote_ids.append(vote_id) return vote_ids