def run(options): amdt_id = options.get('amendment_id', None) search_state = { } if amdt_id: amdt_type, number, congress = utils.split_bill_id(amdt_id) to_fetch = [amdt_id] else: congress = options.get('congress', utils.current_congress()) to_fetch = bill_ids_for(congress, utils.merge(options, {'amendments': True}), bill_states=search_state) if not to_fetch: if options.get("fast", False): logging.warn("No amendments changed.") else: logging.error("Error figuring out which amendments to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None logging.warn("Going to fetch %i amendments from congress #%s" % (len(to_fetch), congress)) saved_amendments = utils.process_set(to_fetch, fetch_amendment, options) save_bill_search_state(saved_amendments, search_state)
def run(options): bill_id = options.get('bill_id', None) bill_version_id = options.get('bill_version_id', None) # using a specific bill or version overrides the congress flag/default if bill_id: bill_type, number, congress = utils.split_bill_id(bill_id) elif bill_version_id: bill_type, number, congress, version_code = utils.split_bill_version_id(bill_version_id) else: congress = options.get('congress', utils.current_congress()) if bill_version_id: to_fetch = [bill_version_id] else: to_fetch = bill_version_ids_for(congress, options) if not to_fetch: logging.error("Error figuring out which bills to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] logging.warn("Going to fetch %i bill versions for congress #%s" % (len(to_fetch), congress)) saved_versions = utils.process_set(to_fetch, fetch_version, options)
def run(options): nomination_id = options.get('nomination_id', None) if nomination_id: nomination_type, number, congress = utils.split_nomination_id( nomination_id) to_fetch = [nomination_id] else: congress = options.get('congress', utils.current_congress()) to_fetch = nomination_ids_for(congress, options) if not to_fetch: if options.get("fast", False): logging.warn("No nominations changed.") else: logging.error( "Error figuring out which nominations to download, aborting." ) return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] logging.warn("Going to fetch %i nominations from congress #%s" % (len(to_fetch), congress)) saved_nominations = utils.process_set(to_fetch, nomination_info.fetch_nomination, options)
def run(options): bill_id = options.get('bill_id', None) search_state = {} if bill_id: bill_type, number, congress = utils.split_bill_id(bill_id) to_fetch = [bill_id] else: congress = options.get('congress', utils.current_congress()) to_fetch = bill_ids_for(congress, options, bill_states=search_state) if not to_fetch: if options.get("fast", False): logging.warn("No bills changed.") else: logging.error( "Error figuring out which bills to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] logging.warn("Going to fetch %i bills from congress #%s" % (len(to_fetch), congress)) saved_bills = utils.process_set(to_fetch, bill_info.fetch_bill, options) save_bill_search_state(saved_bills, search_state)
def run(options): bill_id = options.get('bill_id', None) search_state = {} if bill_id: bill_type, number, congress = utils.split_bill_id(bill_id) to_fetch = [bill_id] else: congress = options.get('congress', utils.current_congress()) to_fetch = bill_ids_for(congress, options, bill_states=search_state) if not to_fetch: if options.get("fast", False): logging.warn("No bills changed.") else: logging.error("Error figuring out which bills to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] logging.warn("Going to fetch %i bills from congress #%s" % (len(to_fetch), congress)) saved_bills = utils.process_set(to_fetch, bill_info.fetch_bill, options) save_bill_search_state(saved_bills, search_state)
def run(options): vote_id = options.get('vote_id', None) if vote_id: vote_chamber, vote_number, congress, session_year = utils.split_vote_id(vote_id) to_fetch = [vote_id] else: congress = options.get('congress', utils.current_congress()) session_year = options.get('session', str(datetime.datetime.now().year)) to_fetch = vote_ids_for_house(congress, session_year, options) + vote_ids_for_senate(congress, session_year, options) if not to_fetch: if not options.get("fast", False): logging.error("Error figuring out which votes to download, aborting.") else: logging.error("No new or recent votes.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None logging.warn("Going to fetch %i votes from congress #%s session %s" % (len(to_fetch), congress, session_year)) utils.process_set(to_fetch, vote_info.fetch_vote, options)
def run(options): bill_version_id = options.get("bill_version_id", None) if bill_version_id: bill_type, bill_number, congress, version_code = utils.split_bill_version_id( bill_version_id) bill_id = utils.build_bill_id(bill_type, bill_number, congress) else: version_code = None bill_id = options.get("bill_id", None) if bill_id: bill_type, bill_number, congress = utils.split_bill_id(bill_id) else: bill_type = bill_number = None congress = options.get("congress", utils.current_congress()) force = options.get("force", False) to_fetch = bill_version_ids_for(congress, bill_type, bill_number, version_code, force) if not to_fetch: return None saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
def run(options): amendment_id = options.get('amendment_id', None) bill_id = options.get('bill_id', None) search_state = {} if amendment_id: amendment_type, number, congress = utils.split_bill_id(amendment_id) to_fetch = [amendment_id] elif bill_id: # first, crawl the bill bill_type, number, congress = utils.split_bill_id(bill_id) bill_status = fetch_bill(bill_id, options) if bill_status['ok']: bill = json.loads(utils.read(output_for_bill(bill_id, "json"))) to_fetch = [x["amendment_id"] for x in bill["amendments"]] else: logging.error("Couldn't download information for that bill.") return None else: congress = options.get('congress', utils.current_congress()) to_fetch = bill_ids_for(congress, utils.merge(options, {'amendments': True}), bill_states=search_state) if not to_fetch: if options.get("fast", False): logging.warn("No amendments changed.") else: logging.error( "Error figuring out which amendments to download, aborting." ) return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None logging.warn("Going to fetch %i amendments from congress #%s" % (len(to_fetch), congress)) saved_amendments = utils.process_set(to_fetch, fetch_amendment, options) # keep record of the last state of all these amendments, for later fast-searching save_bill_search_state(saved_amendments, search_state)
def run(options): vote_id = options.get('vote_id', None) if vote_id: vote_chamber, vote_number, congress, session_year = utils.split_vote_id( vote_id) to_fetch = [vote_id] else: congress = options.get('congress', None) if congress: session_year = options.get('session', None) if not session_year: logging.error( "If you provide a --congress, provide a --session year.") return None else: congress = utils.current_congress() session_year = options.get('session', str(datetime.datetime.now().year)) chamber = options.get('chamber', None) if chamber == "house": to_fetch = vote_ids_for_house(congress, session_year, options) elif chamber == "senate": to_fetch = vote_ids_for_senate(congress, session_year, options) else: to_fetch = vote_ids_for_house(congress, session_year, options) + vote_ids_for_senate( congress, session_year, options) if not to_fetch: if not options.get("fast", False): logging.error( "Error figuring out which votes to download, aborting.") else: logging.warn("No new or recent votes.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None logging.warn("Going to fetch %i votes from congress #%s session %s" % (len(to_fetch), congress, session_year)) utils.process_set(to_fetch, vote_info.fetch_vote, options)
def run(options): congress = options.get("congress", None) congress = int(congress) if congress else utils.current_congress() chamber = options.get('chamber', None) congress_dates = get_congress_dates(options) # download the vote data now if chamber and chamber in ["h", "s"]: votes = get_votes(chamber, congress, options, congress_dates) else: votes = get_votes("h", congress, options, congress_dates) + get_votes("s", congress, options, congress_dates) utils.process_set(votes, put_vote, options)
def run(options): congress = options.get("congress", None) congress = int(congress) if congress else utils.current_congress() chamber = options.get('chamber', None) # we're going to need to map votes to sessions because in modern history the numbering resets by session session_dates = list(csv.DictReader(StringIO.StringIO(utils.download("http://www.govtrack.us/data/us/sessions.tsv").encode("utf8")), delimiter="\t")) # download the vote data now if chamber and chamber in [ "h", "s" ]: votes = get_votes(chamber, congress, options, session_dates) else: votes = get_votes("h", congress, options, session_dates) + get_votes("s", congress, options, session_dates) utils.process_set(votes, put_vote, options)
def run(options): congress = options.get("congress", None) congress = int(congress) if congress else utils.current_congress() chamber = options.get('chamber', None) congress_dates = get_congress_dates(options) # download the vote data now if chamber and chamber in ["h", "s"]: votes = get_votes(chamber, congress, options, congress_dates) else: votes = get_votes("h", congress, options, congress_dates) + get_votes( "s", congress, options, congress_dates) utils.process_set(votes, put_vote, options)
def run(options): bill_id = options.get('bill_id', None) if bill_id: bill_type, number, congress = utils.split_bill_id(bill_id) to_fetch = [bill_id] else: congress = options.get('congress', utils.current_congress()) to_fetch = bill_ids_for(congress, options) if not to_fetch: logging.error("Error figuring out which bills to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None print "Going to fetch %i bills from congress #%s" % (len(to_fetch), congress) errors = [] saved = [] skips = [] for bill_id in to_fetch: try: results = bill_info.fetch_bill(bill_id, options) except Exception, e: if options.get('raise', False): raise else: errors.append((bill_id, e)) continue if results.get('ok', False): if results.get('saved', False): saved.append(bill_id) logging.info("[%s] Updated bill" % bill_id) else: skips.append(bill_id) logging.error("[%s] Skipping bill: %s" % (bill_id, results['reason'])) else: errors.append((bill_id, results)) logging.error("[%s] Error: %s" % (bill_id, results['reason']))
def run(options): amendment_id = options.get('amendment_id', None) bill_id = options.get('bill_id', None) search_state = { } if amendment_id: amendment_type, number, congress = utils.split_bill_id(amendment_id) to_fetch = [amendment_id] elif bill_id: # first, crawl the bill bill_type, number, congress = utils.split_bill_id(bill_id) bill_status = fetch_bill(bill_id, options) if bill_status['ok']: bill = json.loads(utils.read(output_for_bill(bill_id, "json"))) to_fetch = [x["amendment_id"] for x in bill["amendments"]] else: logging.error("Couldn't download information for that bill.") return None else: congress = options.get('congress', utils.current_congress()) to_fetch = bill_ids_for(congress, utils.merge(options, {'amendments': True}), bill_states=search_state) if not to_fetch: if options.get("fast", False): logging.warn("No amendments changed.") else: logging.error("Error figuring out which amendments to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None logging.warn("Going to fetch %i amendments from congress #%s" % (len(to_fetch), congress)) saved_amendments = utils.process_set(to_fetch, fetch_amendment, options) # keep record of the last state of all these amendments, for later fast-searching save_bill_search_state(saved_amendments, search_state)
def run(options): vote_id = options.get('vote_id', None) if vote_id: vote_chamber, vote_number, congress, session_year = utils.split_vote_id(vote_id) to_fetch = [vote_id] else: congress = options.get('congress', None) if congress: session_year = options.get('session', None) if not session_year: logging.error("If you provide a --congress, provide a --session year.") return None else: congress = utils.current_congress() session_year = options.get('session', str(utils.current_legislative_year())) chamber = options.get('chamber', None) if chamber == "house": to_fetch = vote_ids_for_house(congress, session_year, options) elif chamber == "senate": to_fetch = vote_ids_for_senate(congress, session_year, options) else: to_fetch = (vote_ids_for_house(congress, session_year, options) or []) + (vote_ids_for_senate(congress, session_year, options) or []) if not to_fetch: if not options.get("fast", False): logging.error("Error figuring out which votes to download, aborting.") else: logging.warn("No new or recent votes.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None logging.warn("Going to fetch %i votes from congress #%s session %s" % (len(to_fetch), congress, session_year)) utils.process_set(to_fetch, vote_info.fetch_vote, options)
def run(options): bill_id = options.get('bill_id', None) bill_version_id = options.get('bill_version_id', None) # using a specific bill or version overrides the congress flag/default if bill_id: bill_type, number, congress = utils.split_bill_id(bill_id) elif bill_version_id: bill_type, number, congress, version_code = utils.split_bill_version_id(bill_version_id) else: congress = options.get('congress', utils.current_congress()) if bill_version_id: to_fetch = [bill_version_id] else: to_fetch = bill_version_ids_for(congress, options) if not to_fetch: logging.error("Error figuring out which bills to download, aborting.") return None saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
def run(options): bill_id = options.get('bill_id', None) bill_version_id = options.get('bill_version_id', None) # using a specific bill or version overrides the congress flag/default if bill_id: bill_type, number, congress = utils.split_bill_id(bill_id) elif bill_version_id: bill_type, number, congress, version_code = utils.split_bill_version_id( bill_version_id) else: congress = options.get('congress', utils.current_congress()) if bill_version_id: to_fetch = [bill_version_id] else: to_fetch = bill_version_ids_for(congress, options) if not to_fetch: logging.error( "Error figuring out which bills to download, aborting.") return None saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
def run(options): nomination_id = options.get('nomination_id', None) if nomination_id: nomination_type, number, congress = utils.split_nomination_id(nomination_id) to_fetch = [nomination_id] else: congress = options.get('congress', utils.current_congress()) to_fetch = nomination_ids_for(congress, options) if not to_fetch: if options.get("fast", False): logging.warn("No nominations changed.") else: logging.error("Error figuring out which nominations to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] logging.warn("Going to fetch %i nominations from congress #%s" % (len(to_fetch), congress)) saved_nominations = utils.process_set(to_fetch, nomination_info.fetch_nomination, options)
def run(options): bill_version_id = options.get("bill_version_id", None) if bill_version_id: bill_type, bill_number, congress, version_code = utils.split_bill_version_id(bill_version_id) bill_id = utils.build_bill_id(bill_type, bill_number, congress) else: version_code = None bill_id = options.get("bill_id", None) if bill_id: bill_type, bill_number, congress = utils.split_bill_id(bill_id) else: bill_type = bill_number = None congress = options.get("congress", utils.current_congress()) force = options.get("force", False) to_fetch = bill_version_ids_for(congress, bill_type, bill_number, version_code, force) if not to_fetch: return None saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)