def run(options): vote_id = options.get('vote_id', None) if vote_id: vote_chamber, vote_number, congress, session_year = utils.split_vote_id(vote_id) to_fetch = [vote_id] else: congress = options.get('congress', utils.current_congress()) session_year = options.get('session', str(datetime.datetime.now().year)) to_fetch = vote_ids_for_house(congress, session_year, options) + vote_ids_for_senate(congress, session_year, options) if not to_fetch: if not options.get("fast", False): logging.error("Error figuring out which votes to download, aborting.") else: logging.error("No new or recent votes.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None logging.warn("Going to fetch %i votes from congress #%s session %s" % (len(to_fetch), congress, session_year)) utils.process_set(to_fetch, vote_info.fetch_vote, options)
def run(options): root_dir = utils.data_dir() + '/fdsys/STATUTE' if "volume" in options: to_fetch = glob.glob(root_dir + "/*/STATUTE-" + str(int(options["volume"]))) elif "volumes" in options: start, end = options["volumes"].split("-") to_fetch = [] for v in xrange(int(start), int(end) + 1): to_fetch.extend(glob.glob(root_dir + "/*/STATUTE-" + str(v))) elif "year" in options: to_fetch = glob.glob(root_dir + "/" + str(int(options["year"])) + "/STATUTE-*") elif "years" in options: start, end = options["years"].split("-") to_fetch = [] for y in xrange(int(start), int(end) + 1): to_fetch.extend(glob.glob(root_dir + "/" + str(y) + "/STATUTE-*")) else: to_fetch = sorted(glob.glob(root_dir + "/*/STATUTE-*")) logging.warn("Going to process %i volumes" % len(to_fetch)) utils.process_set(to_fetch, proc_statute_volume, options)
def run(options): vote_id = options.get('vote_id', None) if vote_id: vote_chamber, vote_number, congress, session_year = utils.split_vote_id( vote_id) to_fetch = [vote_id] else: congress = options.get('congress', None) if congress: session_year = options.get('session', None) if not session_year: logging.error( "If you provide a --congress, provide a --session year.") return None else: congress = utils.current_congress() session_year = options.get('session', str(datetime.datetime.now().year)) chamber = options.get('chamber', None) if chamber == "house": to_fetch = vote_ids_for_house(congress, session_year, options) elif chamber == "senate": to_fetch = vote_ids_for_senate(congress, session_year, options) else: to_fetch = vote_ids_for_house(congress, session_year, options) + vote_ids_for_senate( congress, session_year, options) if not to_fetch: if not options.get("fast", False): logging.error( "Error figuring out which votes to download, aborting.") else: logging.warn("No new or recent votes.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None logging.warn("Going to fetch %i votes from congress #%s session %s" % (len(to_fetch), congress, session_year)) utils.process_set(to_fetch, vote_info.fetch_vote, options)
def run(options): congress = options.get("congress", None) congress = int(congress) if congress else utils.current_congress() chamber = options.get('chamber', None) congress_dates = get_congress_dates(options) # download the vote data now if chamber and chamber in ["h", "s"]: votes = get_votes(chamber, congress, options, congress_dates) else: votes = get_votes("h", congress, options, congress_dates) + get_votes("s", congress, options, congress_dates) utils.process_set(votes, put_vote, options)
def run(options): bill_id = options.get('bill_id', None) bill_version_id = options.get('bill_version_id', None) # using a specific bill or version overrides the congress flag/default if bill_id: bill_type, number, congress = utils.split_bill_id(bill_id) elif bill_version_id: bill_type, number, congress, version_code = utils.split_bill_version_id(bill_version_id) else: congress = options.get('congress', utils.current_congress()) if bill_version_id: to_fetch = [bill_version_id] else: to_fetch = bill_version_ids_for(congress, options) if not to_fetch: logging.error("Error figuring out which bills to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] logging.warn("Going to fetch %i bill versions for congress #%s" % (len(to_fetch), congress)) saved_versions = utils.process_set(to_fetch, fetch_version, options)
def run(options): congress = options.get("congress", None) congress = int(congress) if congress else utils.current_congress() chamber = options.get('chamber', None) # we're going to need to map votes to sessions because in modern history the numbering resets by session session_dates = list(csv.DictReader(StringIO.StringIO(utils.download("http://www.govtrack.us/data/us/sessions.tsv").encode("utf8")), delimiter="\t")) # download the vote data now if chamber and chamber in [ "h", "s" ]: votes = get_votes(chamber, congress, options, session_dates) else: votes = get_votes("h", congress, options, session_dates) + get_votes("s", congress, options, session_dates) utils.process_set(votes, put_vote, options)
def run(options): nomination_id = options.get('nomination_id', None) if nomination_id: nomination_type, number, congress = utils.split_nomination_id( nomination_id) to_fetch = [nomination_id] else: congress = options.get('congress', utils.current_congress()) to_fetch = nomination_ids_for(congress, options) if not to_fetch: if options.get("fast", False): logging.warn("No nominations changed.") else: logging.error( "Error figuring out which nominations to download, aborting." ) return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] logging.warn("Going to fetch %i nominations from congress #%s" % (len(to_fetch), congress)) saved_nominations = utils.process_set(to_fetch, nomination_info.fetch_nomination, options)
def run(options): congress = options.get("congress", None) congress = int(congress) if congress else utils.current_congress() chamber = options.get('chamber', None) congress_dates = get_congress_dates(options) # download the vote data now if chamber and chamber in ["h", "s"]: votes = get_votes(chamber, congress, options, congress_dates) else: votes = get_votes("h", congress, options, congress_dates) + get_votes( "s", congress, options, congress_dates) utils.process_set(votes, put_vote, options)
def run(options): bill_version_id = options.get("bill_version_id", None) if bill_version_id: bill_type, bill_number, congress, version_code = utils.split_bill_version_id( bill_version_id) bill_id = utils.build_bill_id(bill_type, bill_number, congress) else: version_code = None bill_id = options.get("bill_id", None) if bill_id: bill_type, bill_number, congress = utils.split_bill_id(bill_id) else: bill_type = bill_number = None congress = options.get("congress", utils.current_congress()) force = options.get("force", False) to_fetch = bill_version_ids_for(congress, bill_type, bill_number, version_code, force) if not to_fetch: return None saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
def run(options): bill_id = options.get('bill_id', None) search_state = {} if bill_id: bill_type, number, congress = utils.split_bill_id(bill_id) to_fetch = [bill_id] else: congress = options.get('congress', utils.current_congress()) to_fetch = bill_ids_for(congress, options, bill_states=search_state) if not to_fetch: if options.get("fast", False): logging.warn("No bills changed.") else: logging.error( "Error figuring out which bills to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] logging.warn("Going to fetch %i bills from congress #%s" % (len(to_fetch), congress)) saved_bills = utils.process_set(to_fetch, bill_info.fetch_bill, options) save_bill_search_state(saved_bills, search_state)
def run(options): amdt_id = options.get('amendment_id', None) search_state = { } if amdt_id: amdt_type, number, congress = utils.split_bill_id(amdt_id) to_fetch = [amdt_id] else: congress = options.get('congress', utils.current_congress()) to_fetch = bill_ids_for(congress, utils.merge(options, {'amendments': True}), bill_states=search_state) if not to_fetch: if options.get("fast", False): logging.warn("No amendments changed.") else: logging.error("Error figuring out which amendments to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None logging.warn("Going to fetch %i amendments from congress #%s" % (len(to_fetch), congress)) saved_amendments = utils.process_set(to_fetch, fetch_amendment, options) save_bill_search_state(saved_amendments, search_state)
def run(options): bill_id = options.get('bill_id', None) search_state = {} if bill_id: bill_type, number, congress = utils.split_bill_id(bill_id) to_fetch = [bill_id] else: congress = options.get('congress', utils.current_congress()) to_fetch = bill_ids_for(congress, options, bill_states=search_state) if not to_fetch: if options.get("fast", False): logging.warn("No bills changed.") else: logging.error("Error figuring out which bills to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] logging.warn("Going to fetch %i bills from congress #%s" % (len(to_fetch), congress)) saved_bills = utils.process_set(to_fetch, bill_info.fetch_bill, options) save_bill_search_state(saved_bills, search_state)
def run(options): bill_id = options.get('bill_id', None) if bill_id: bill_type, number, congress = utils.split_bill_id(bill_id) to_fetch = [bill_id] else: to_fetch = get_bills_to_process(options) if not to_fetch: logging.warn("No bills changed.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] utils.process_set(to_fetch, process_bill, options)
def run(options): vote_id = options.get('vote_id', None) if vote_id: vote_chamber, vote_number, congress, session_year = utils.split_vote_id(vote_id) to_fetch = [vote_id] else: congress = options.get('congress', None) if congress: session_year = options.get('session', None) if not session_year: logging.error("If you provide a --congress, provide a --session year.") return None else: congress = utils.current_congress() session_year = options.get('session', str(utils.current_legislative_year())) chamber = options.get('chamber', None) if chamber == "house": to_fetch = vote_ids_for_house(congress, session_year, options) elif chamber == "senate": to_fetch = vote_ids_for_senate(congress, session_year, options) else: to_fetch = (vote_ids_for_house(congress, session_year, options) or []) + (vote_ids_for_senate(congress, session_year, options) or []) if not to_fetch: if not options.get("fast", False): logging.error("Error figuring out which votes to download, aborting.") else: logging.warn("No new or recent votes.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None logging.warn("Going to fetch %i votes from congress #%s session %s" % (len(to_fetch), congress, session_year)) utils.process_set(to_fetch, vote_info.fetch_vote, options)
def run(options): amendment_id = options.get('amendment_id', None) bill_id = options.get('bill_id', None) search_state = {} if amendment_id: amendment_type, number, congress = utils.split_bill_id(amendment_id) to_fetch = [amendment_id] elif bill_id: # first, crawl the bill bill_type, number, congress = utils.split_bill_id(bill_id) bill_status = fetch_bill(bill_id, options) if bill_status['ok']: bill = json.loads(utils.read(output_for_bill(bill_id, "json"))) to_fetch = [x["amendment_id"] for x in bill["amendments"]] else: logging.error("Couldn't download information for that bill.") return None else: congress = options.get('congress', utils.current_congress()) to_fetch = bill_ids_for(congress, utils.merge(options, {'amendments': True}), bill_states=search_state) if not to_fetch: if options.get("fast", False): logging.warn("No amendments changed.") else: logging.error( "Error figuring out which amendments to download, aborting." ) return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None logging.warn("Going to fetch %i amendments from congress #%s" % (len(to_fetch), congress)) saved_amendments = utils.process_set(to_fetch, fetch_amendment, options) # keep record of the last state of all these amendments, for later fast-searching save_bill_search_state(saved_amendments, search_state)
def run(options): root_dir = utils.data_dir() + '/fdsys/STATUTE' if "volume" in options: to_fetch = glob.glob(root_dir + "/*/STATUTE-" + str(int(options["volume"]))) elif "volumes" in options: start, end = options["volumes"].split("-") to_fetch = [] for v in xrange(int(start), int(end)+1): to_fetch.extend(glob.glob(root_dir + "/*/STATUTE-" + str(v))) elif "year" in options: to_fetch = glob.glob(root_dir + "/" + str(int(options["year"])) + "/STATUTE-*") elif "years" in options: start, end = options["years"].split("-") to_fetch = [] for y in xrange(int(start), int(end)+1): to_fetch.extend(glob.glob(root_dir + "/" + str(y) + "/STATUTE-*")) else: to_fetch = sorted(glob.glob(root_dir + "/*/STATUTE-*")) logging.warn("Going to process %i volumes" % len(to_fetch)) utils.process_set(to_fetch, proc_statute_volume, options)
def run(options): bill_id = options.get('bill_id', None) processor_func = process_bill if options.get("reparse_actions"): # Overrid default behavior. processor_func = reparse_actions if bill_id: to_fetch = bill_id.split(",") else: to_fetch = get_bills_to_process(options) if not to_fetch: logging.warn("No bills changed.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] utils.process_set(to_fetch, processor_func, options)
def run(options): amendment_id = options.get('amendment_id', None) bill_id = options.get('bill_id', None) search_state = { } if amendment_id: amendment_type, number, congress = utils.split_bill_id(amendment_id) to_fetch = [amendment_id] elif bill_id: # first, crawl the bill bill_type, number, congress = utils.split_bill_id(bill_id) bill_status = fetch_bill(bill_id, options) if bill_status['ok']: bill = json.loads(utils.read(output_for_bill(bill_id, "json"))) to_fetch = [x["amendment_id"] for x in bill["amendments"]] else: logging.error("Couldn't download information for that bill.") return None else: congress = options.get('congress', utils.current_congress()) to_fetch = bill_ids_for(congress, utils.merge(options, {'amendments': True}), bill_states=search_state) if not to_fetch: if options.get("fast", False): logging.warn("No amendments changed.") else: logging.error("Error figuring out which amendments to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] if options.get('pages_only', False): return None logging.warn("Going to fetch %i amendments from congress #%s" % (len(to_fetch), congress)) saved_amendments = utils.process_set(to_fetch, fetch_amendment, options) # keep record of the last state of all these amendments, for later fast-searching save_bill_search_state(saved_amendments, search_state)
def run(options): bill_id = options.get('bill_id', None) bill_version_id = options.get('bill_version_id', None) # using a specific bill or version overrides the congress flag/default if bill_id: bill_type, number, congress = utils.split_bill_id(bill_id) elif bill_version_id: bill_type, number, congress, version_code = utils.split_bill_version_id(bill_version_id) else: congress = options.get('congress', utils.current_congress()) if bill_version_id: to_fetch = [bill_version_id] else: to_fetch = bill_version_ids_for(congress, options) if not to_fetch: logging.error("Error figuring out which bills to download, aborting.") return None saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
def run(options): bill_id = options.get('bill_id', None) bill_version_id = options.get('bill_version_id', None) # using a specific bill or version overrides the congress flag/default if bill_id: bill_type, number, congress = utils.split_bill_id(bill_id) elif bill_version_id: bill_type, number, congress, version_code = utils.split_bill_version_id( bill_version_id) else: congress = options.get('congress', utils.current_congress()) if bill_version_id: to_fetch = [bill_version_id] else: to_fetch = bill_version_ids_for(congress, options) if not to_fetch: logging.error( "Error figuring out which bills to download, aborting.") return None saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
def run(options): nomination_id = options.get('nomination_id', None) if nomination_id: nomination_type, number, congress = utils.split_nomination_id(nomination_id) to_fetch = [nomination_id] else: congress = options.get('congress', utils.current_congress()) to_fetch = nomination_ids_for(congress, options) if not to_fetch: if options.get("fast", False): logging.warn("No nominations changed.") else: logging.error("Error figuring out which nominations to download, aborting.") return None limit = options.get('limit', None) if limit: to_fetch = to_fetch[:int(limit)] logging.warn("Going to fetch %i nominations from congress #%s" % (len(to_fetch), congress)) saved_nominations = utils.process_set(to_fetch, nomination_info.fetch_nomination, options)
def run(options): bill_version_id = options.get("bill_version_id", None) if bill_version_id: bill_type, bill_number, congress, version_code = utils.split_bill_version_id(bill_version_id) bill_id = utils.build_bill_id(bill_type, bill_number, congress) else: version_code = None bill_id = options.get("bill_id", None) if bill_id: bill_type, bill_number, congress = utils.split_bill_id(bill_id) else: bill_type = bill_number = None congress = options.get("congress", utils.current_congress()) force = options.get("force", False) to_fetch = bill_version_ids_for(congress, bill_type, bill_number, version_code, force) if not to_fetch: return None saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)