def run(options):
vote_id = options.get('vote_id', None)
if vote_id:
vote_chamber, vote_number, congress, session_year = utils.split_vote_id(vote_id)
to_fetch = [vote_id]
else:
congress = options.get('congress', utils.current_congress())
session_year = options.get('session', str(datetime.datetime.now().year))
to_fetch = vote_ids_for_house(congress, session_year, options) + vote_ids_for_senate(congress, session_year, options)
if not to_fetch:
if not options.get("fast", False):
logging.error("Error figuring out which votes to download, aborting.")
else:
logging.error("No new or recent votes.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
if options.get('pages_only', False):
return None
logging.warn("Going to fetch %i votes from congress #%s session %s" % (len(to_fetch), congress, session_year))
utils.process_set(to_fetch, vote_info.fetch_vote, options)
def run(options):
root_dir = utils.data_dir() + '/fdsys/STATUTE'
if "volume" in options:
to_fetch = glob.glob(root_dir + "/*/STATUTE-" +
str(int(options["volume"])))
elif "volumes" in options:
start, end = options["volumes"].split("-")
to_fetch = []
for v in xrange(int(start), int(end) + 1):
to_fetch.extend(glob.glob(root_dir + "/*/STATUTE-" + str(v)))
elif "year" in options:
to_fetch = glob.glob(root_dir + "/" + str(int(options["year"])) +
"/STATUTE-*")
elif "years" in options:
start, end = options["years"].split("-")
to_fetch = []
for y in xrange(int(start), int(end) + 1):
to_fetch.extend(glob.glob(root_dir + "/" + str(y) + "/STATUTE-*"))
else:
to_fetch = sorted(glob.glob(root_dir + "/*/STATUTE-*"))
logging.warn("Going to process %i volumes" % len(to_fetch))
utils.process_set(to_fetch, proc_statute_volume, options)
def run(options):
vote_id = options.get('vote_id', None)
if vote_id:
vote_chamber, vote_number, congress, session_year = utils.split_vote_id(
vote_id)
to_fetch = [vote_id]
else:
congress = options.get('congress', None)
if congress:
session_year = options.get('session', None)
if not session_year:
logging.error(
"If you provide a --congress, provide a --session year.")
return None
else:
congress = utils.current_congress()
session_year = options.get('session',
str(datetime.datetime.now().year))
chamber = options.get('chamber', None)
if chamber == "house":
to_fetch = vote_ids_for_house(congress, session_year, options)
elif chamber == "senate":
to_fetch = vote_ids_for_senate(congress, session_year, options)
else:
to_fetch = vote_ids_for_house(congress, session_year,
options) + vote_ids_for_senate(
congress, session_year, options)
if not to_fetch:
if not options.get("fast", False):
logging.error(
"Error figuring out which votes to download, aborting.")
else:
logging.warn("No new or recent votes.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
if options.get('pages_only', False):
return None
logging.warn("Going to fetch %i votes from congress #%s session %s" %
(len(to_fetch), congress, session_year))
utils.process_set(to_fetch, vote_info.fetch_vote, options)
def run(options):
congress = options.get("congress", None)
congress = int(congress) if congress else utils.current_congress()
chamber = options.get('chamber', None)
congress_dates = get_congress_dates(options)
# download the vote data now
if chamber and chamber in ["h", "s"]:
votes = get_votes(chamber, congress, options, congress_dates)
else:
votes = get_votes("h", congress, options, congress_dates) + get_votes("s", congress, options, congress_dates)
utils.process_set(votes, put_vote, options)
def run(options):
bill_id = options.get('bill_id', None)
bill_version_id = options.get('bill_version_id', None)
# using a specific bill or version overrides the congress flag/default
if bill_id:
bill_type, number, congress = utils.split_bill_id(bill_id)
elif bill_version_id:
bill_type, number, congress, version_code = utils.split_bill_version_id(bill_version_id)
else:
congress = options.get('congress', utils.current_congress())
if bill_version_id:
to_fetch = [bill_version_id]
else:
to_fetch = bill_version_ids_for(congress, options)
if not to_fetch:
logging.error("Error figuring out which bills to download, aborting.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
logging.warn("Going to fetch %i bill versions for congress #%s" % (len(to_fetch), congress))
saved_versions = utils.process_set(to_fetch, fetch_version, options)
def run(options):
congress = options.get("congress", None)
congress = int(congress) if congress else utils.current_congress()
chamber = options.get('chamber', None)
# we're going to need to map votes to sessions because in modern history the numbering resets by session
session_dates = list(csv.DictReader(StringIO.StringIO(utils.download("http://www.govtrack.us/data/us/sessions.tsv").encode("utf8")), delimiter="\t"))
# download the vote data now
if chamber and chamber in [ "h", "s" ]:
votes = get_votes(chamber, congress, options, session_dates)
else:
votes = get_votes("h", congress, options, session_dates) + get_votes("s", congress, options, session_dates)
utils.process_set(votes, put_vote, options)
def run(options):
congress = options.get("congress", None)
congress = int(congress) if congress else utils.current_congress()
chamber = options.get('chamber', None)
# we're going to need to map votes to sessions because in modern history the numbering resets by session
session_dates = list(csv.DictReader(StringIO.StringIO(utils.download("http://www.govtrack.us/data/us/sessions.tsv").encode("utf8")), delimiter="\t"))
# download the vote data now
if chamber and chamber in [ "h", "s" ]:
votes = get_votes(chamber, congress, options, session_dates)
else:
votes = get_votes("h", congress, options, session_dates) + get_votes("s", congress, options, session_dates)
utils.process_set(votes, put_vote, options)
def run(options):
nomination_id = options.get('nomination_id', None)
if nomination_id:
nomination_type, number, congress = utils.split_nomination_id(
nomination_id)
to_fetch = [nomination_id]
else:
congress = options.get('congress', utils.current_congress())
to_fetch = nomination_ids_for(congress, options)
if not to_fetch:
if options.get("fast", False):
logging.warn("No nominations changed.")
else:
logging.error(
"Error figuring out which nominations to download, aborting."
)
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
logging.warn("Going to fetch %i nominations from congress #%s" %
(len(to_fetch), congress))
saved_nominations = utils.process_set(to_fetch,
nomination_info.fetch_nomination,
options)
def run(options):
congress = options.get("congress", None)
congress = int(congress) if congress else utils.current_congress()
chamber = options.get('chamber', None)
congress_dates = get_congress_dates(options)
# download the vote data now
if chamber and chamber in ["h", "s"]:
votes = get_votes(chamber, congress, options, congress_dates)
else:
votes = get_votes("h", congress, options, congress_dates) + get_votes(
"s", congress, options, congress_dates)
utils.process_set(votes, put_vote, options)
def run(options):
bill_version_id = options.get("bill_version_id", None)
if bill_version_id:
bill_type, bill_number, congress, version_code = utils.split_bill_version_id(
bill_version_id)
bill_id = utils.build_bill_id(bill_type, bill_number, congress)
else:
version_code = None
bill_id = options.get("bill_id", None)
if bill_id:
bill_type, bill_number, congress = utils.split_bill_id(bill_id)
else:
bill_type = bill_number = None
congress = options.get("congress", utils.current_congress())
force = options.get("force", False)
to_fetch = bill_version_ids_for(congress, bill_type, bill_number,
version_code, force)
if not to_fetch:
return None
saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
def run(options):
bill_id = options.get('bill_id', None)
bill_version_id = options.get('bill_version_id', None)
# using a specific bill or version overrides the congress flag/default
if bill_id:
bill_type, number, congress = utils.split_bill_id(bill_id)
elif bill_version_id:
bill_type, number, congress, version_code = utils.split_bill_version_id(bill_version_id)
else:
congress = options.get('congress', utils.current_congress())
if bill_version_id:
to_fetch = [bill_version_id]
else:
to_fetch = bill_version_ids_for(congress, options)
if not to_fetch:
logging.error("Error figuring out which bills to download, aborting.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
logging.warn("Going to fetch %i bill versions for congress #%s" % (len(to_fetch), congress))
saved_versions = utils.process_set(to_fetch, fetch_version, options)
def run(options):
bill_id = options.get('bill_id', None)
search_state = {}
if bill_id:
bill_type, number, congress = utils.split_bill_id(bill_id)
to_fetch = [bill_id]
else:
congress = options.get('congress', utils.current_congress())
to_fetch = bill_ids_for(congress, options, bill_states=search_state)
if not to_fetch:
if options.get("fast", False):
logging.warn("No bills changed.")
else:
logging.error(
"Error figuring out which bills to download, aborting.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
logging.warn("Going to fetch %i bills from congress #%s" %
(len(to_fetch), congress))
saved_bills = utils.process_set(to_fetch, bill_info.fetch_bill, options)
save_bill_search_state(saved_bills, search_state)
def run(options):
amdt_id = options.get('amendment_id', None)
search_state = { }
if amdt_id:
amdt_type, number, congress = utils.split_bill_id(amdt_id)
to_fetch = [amdt_id]
else:
congress = options.get('congress', utils.current_congress())
to_fetch = bill_ids_for(congress, utils.merge(options, {'amendments': True}), bill_states=search_state)
if not to_fetch:
if options.get("fast", False):
logging.warn("No amendments changed.")
else:
logging.error("Error figuring out which amendments to download, aborting.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
if options.get('pages_only', False):
return None
logging.warn("Going to fetch %i amendments from congress #%s" % (len(to_fetch), congress))
saved_amendments = utils.process_set(to_fetch, fetch_amendment, options)
save_bill_search_state(saved_amendments, search_state)
def run(options):
bill_id = options.get('bill_id', None)
search_state = {}
if bill_id:
bill_type, number, congress = utils.split_bill_id(bill_id)
to_fetch = [bill_id]
else:
congress = options.get('congress', utils.current_congress())
to_fetch = bill_ids_for(congress, options, bill_states=search_state)
if not to_fetch:
if options.get("fast", False):
logging.warn("No bills changed.")
else:
logging.error("Error figuring out which bills to download, aborting.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
logging.warn("Going to fetch %i bills from congress #%s" % (len(to_fetch), congress))
saved_bills = utils.process_set(to_fetch, bill_info.fetch_bill, options)
save_bill_search_state(saved_bills, search_state)
def run(options):
bill_id = options.get('bill_id', None)
if bill_id:
bill_type, number, congress = utils.split_bill_id(bill_id)
to_fetch = [bill_id]
else:
to_fetch = get_bills_to_process(options)
if not to_fetch:
logging.warn("No bills changed.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
utils.process_set(to_fetch, process_bill, options)
def run(options):
bill_id = options.get('bill_id', None)
if bill_id:
bill_type, number, congress = utils.split_bill_id(bill_id)
to_fetch = [bill_id]
else:
to_fetch = get_bills_to_process(options)
if not to_fetch:
logging.warn("No bills changed.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
utils.process_set(to_fetch, process_bill, options)
def run(options):
vote_id = options.get('vote_id', None)
if vote_id:
vote_chamber, vote_number, congress, session_year = utils.split_vote_id(vote_id)
to_fetch = [vote_id]
else:
congress = options.get('congress', None)
if congress:
session_year = options.get('session', None)
if not session_year:
logging.error("If you provide a --congress, provide a --session year.")
return None
else:
congress = utils.current_congress()
session_year = options.get('session', str(utils.current_legislative_year()))
chamber = options.get('chamber', None)
if chamber == "house":
to_fetch = vote_ids_for_house(congress, session_year, options)
elif chamber == "senate":
to_fetch = vote_ids_for_senate(congress, session_year, options)
else:
to_fetch = (vote_ids_for_house(congress, session_year, options) or []) + (vote_ids_for_senate(congress, session_year, options) or [])
if not to_fetch:
if not options.get("fast", False):
logging.error("Error figuring out which votes to download, aborting.")
else:
logging.warn("No new or recent votes.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
if options.get('pages_only', False):
return None
logging.warn("Going to fetch %i votes from congress #%s session %s" % (len(to_fetch), congress, session_year))
utils.process_set(to_fetch, vote_info.fetch_vote, options)
def run(options):
amendment_id = options.get('amendment_id', None)
bill_id = options.get('bill_id', None)
search_state = {}
if amendment_id:
amendment_type, number, congress = utils.split_bill_id(amendment_id)
to_fetch = [amendment_id]
elif bill_id:
# first, crawl the bill
bill_type, number, congress = utils.split_bill_id(bill_id)
bill_status = fetch_bill(bill_id, options)
if bill_status['ok']:
bill = json.loads(utils.read(output_for_bill(bill_id, "json")))
to_fetch = [x["amendment_id"] for x in bill["amendments"]]
else:
logging.error("Couldn't download information for that bill.")
return None
else:
congress = options.get('congress', utils.current_congress())
to_fetch = bill_ids_for(congress,
utils.merge(options, {'amendments': True}),
bill_states=search_state)
if not to_fetch:
if options.get("fast", False):
logging.warn("No amendments changed.")
else:
logging.error(
"Error figuring out which amendments to download, aborting."
)
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
if options.get('pages_only', False):
return None
logging.warn("Going to fetch %i amendments from congress #%s" %
(len(to_fetch), congress))
saved_amendments = utils.process_set(to_fetch, fetch_amendment, options)
# keep record of the last state of all these amendments, for later fast-searching
save_bill_search_state(saved_amendments, search_state)
def run(options):
root_dir = utils.data_dir() + '/fdsys/STATUTE'
if "volume" in options:
to_fetch = glob.glob(root_dir + "/*/STATUTE-" + str(int(options["volume"])))
elif "volumes" in options:
start, end = options["volumes"].split("-")
to_fetch = []
for v in xrange(int(start), int(end)+1):
to_fetch.extend(glob.glob(root_dir + "/*/STATUTE-" + str(v)))
elif "year" in options:
to_fetch = glob.glob(root_dir + "/" + str(int(options["year"])) + "/STATUTE-*")
elif "years" in options:
start, end = options["years"].split("-")
to_fetch = []
for y in xrange(int(start), int(end)+1):
to_fetch.extend(glob.glob(root_dir + "/" + str(y) + "/STATUTE-*"))
else:
to_fetch = sorted(glob.glob(root_dir + "/*/STATUTE-*"))
logging.warn("Going to process %i volumes" % len(to_fetch))
utils.process_set(to_fetch, proc_statute_volume, options)
def run(options):
bill_id = options.get('bill_id', None)
processor_func = process_bill
if options.get("reparse_actions"):
# Overrid default behavior.
processor_func = reparse_actions
if bill_id:
to_fetch = bill_id.split(",")
else:
to_fetch = get_bills_to_process(options)
if not to_fetch:
logging.warn("No bills changed.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
utils.process_set(to_fetch, processor_func, options)
def run(options):
amendment_id = options.get('amendment_id', None)
bill_id = options.get('bill_id', None)
search_state = { }
if amendment_id:
amendment_type, number, congress = utils.split_bill_id(amendment_id)
to_fetch = [amendment_id]
elif bill_id:
# first, crawl the bill
bill_type, number, congress = utils.split_bill_id(bill_id)
bill_status = fetch_bill(bill_id, options)
if bill_status['ok']:
bill = json.loads(utils.read(output_for_bill(bill_id, "json")))
to_fetch = [x["amendment_id"] for x in bill["amendments"]]
else:
logging.error("Couldn't download information for that bill.")
return None
else:
congress = options.get('congress', utils.current_congress())
to_fetch = bill_ids_for(congress, utils.merge(options, {'amendments': True}), bill_states=search_state)
if not to_fetch:
if options.get("fast", False):
logging.warn("No amendments changed.")
else:
logging.error("Error figuring out which amendments to download, aborting.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
if options.get('pages_only', False):
return None
logging.warn("Going to fetch %i amendments from congress #%s" % (len(to_fetch), congress))
saved_amendments = utils.process_set(to_fetch, fetch_amendment, options)
# keep record of the last state of all these amendments, for later fast-searching
save_bill_search_state(saved_amendments, search_state)
def run(options):
bill_id = options.get('bill_id', None)
bill_version_id = options.get('bill_version_id', None)
# using a specific bill or version overrides the congress flag/default
if bill_id:
bill_type, number, congress = utils.split_bill_id(bill_id)
elif bill_version_id:
bill_type, number, congress, version_code = utils.split_bill_version_id(bill_version_id)
else:
congress = options.get('congress', utils.current_congress())
if bill_version_id:
to_fetch = [bill_version_id]
else:
to_fetch = bill_version_ids_for(congress, options)
if not to_fetch:
logging.error("Error figuring out which bills to download, aborting.")
return None
saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
def run(options):
bill_id = options.get('bill_id', None)
bill_version_id = options.get('bill_version_id', None)
# using a specific bill or version overrides the congress flag/default
if bill_id:
bill_type, number, congress = utils.split_bill_id(bill_id)
elif bill_version_id:
bill_type, number, congress, version_code = utils.split_bill_version_id(
bill_version_id)
else:
congress = options.get('congress', utils.current_congress())
if bill_version_id:
to_fetch = [bill_version_id]
else:
to_fetch = bill_version_ids_for(congress, options)
if not to_fetch:
logging.error(
"Error figuring out which bills to download, aborting.")
return None
saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
def run(options):
nomination_id = options.get('nomination_id', None)
if nomination_id:
nomination_type, number, congress = utils.split_nomination_id(nomination_id)
to_fetch = [nomination_id]
else:
congress = options.get('congress', utils.current_congress())
to_fetch = nomination_ids_for(congress, options)
if not to_fetch:
if options.get("fast", False):
logging.warn("No nominations changed.")
else:
logging.error("Error figuring out which nominations to download, aborting.")
return None
limit = options.get('limit', None)
if limit:
to_fetch = to_fetch[:int(limit)]
logging.warn("Going to fetch %i nominations from congress #%s" % (len(to_fetch), congress))
saved_nominations = utils.process_set(to_fetch, nomination_info.fetch_nomination, options)
def run(options):
bill_version_id = options.get("bill_version_id", None)
if bill_version_id:
bill_type, bill_number, congress, version_code = utils.split_bill_version_id(bill_version_id)
bill_id = utils.build_bill_id(bill_type, bill_number, congress)
else:
version_code = None
bill_id = options.get("bill_id", None)
if bill_id:
bill_type, bill_number, congress = utils.split_bill_id(bill_id)
else:
bill_type = bill_number = None
congress = options.get("congress", utils.current_congress())
force = options.get("force", False)
to_fetch = bill_version_ids_for(congress, bill_type, bill_number, version_code, force)
if not to_fetch:
return None
saved_versions = utils.process_set(to_fetch, write_bill_catoxml, options)
