def process_bill(bill_id, options):
fdsys_xml_path = _path_to_billstatus_file(bill_id)
logging.info("[%s] Processing %s..." % (bill_id, fdsys_xml_path))
# Read FDSys bulk data file.
xml_as_dict = read_fdsys_bulk_bill_status_file(fdsys_xml_path, bill_id)
bill_data = form_bill_json_dict(xml_as_dict)
# Convert and write out data.json and data.xml.
utils.write(
unicode(json.dumps(bill_data, indent=2, sort_keys=True)),
os.path.dirname(fdsys_xml_path) + '/data.json')
from bill_info import create_govtrack_xml
with open(os.path.dirname(fdsys_xml_path) + '/data.xml', 'wb') as xml_file:
xml_file.write(create_govtrack_xml(bill_data, options))
if options.get("amendments", True):
process_amendments(bill_id, xml_as_dict, options)
# Mark this bulk data file as processed by saving its lastmod
# file under a new path.
utils.write(
utils.read(_path_to_billstatus_file(bill_id).replace(".xml", "-lastmod.txt")),
os.path.join(os.path.dirname(fdsys_xml_path), "data-fromfdsys-lastmod.txt"))
return {
"ok": True,
"saved": True,
}
def process_bill(bill_id, options):
fdsys_xml_path = _path_to_billstatus_file(bill_id)
logging.info("[%s] Processing %s..." % (bill_id, fdsys_xml_path))
# Read FDSys bulk data file.
xml_as_dict = read_fdsys_bulk_bill_status_file(fdsys_xml_path, bill_id)
bill_data = form_bill_json_dict(xml_as_dict)
# Convert and write out data.json and data.xml.
utils.write(
unicode(json.dumps(bill_data, indent=2, sort_keys=True)),
os.path.dirname(fdsys_xml_path) + '/data.json')
from bill_info import create_govtrack_xml
with open(os.path.dirname(fdsys_xml_path) + '/data.xml', 'wb') as xml_file:
xml_file.write(create_govtrack_xml(bill_data, options))
if options.get("amendments", True):
process_amendments(bill_id, xml_as_dict, options)
# Mark this bulk data file as processed by saving its lastmod
# file under a new path.
utils.write(
utils.read(_path_to_billstatus_file(bill_id).replace(".xml", "-lastmod.txt")),
os.path.join(os.path.dirname(fdsys_xml_path), "data-fromfdsys-lastmod.txt"))
return {
"ok": True,
"saved": True,
}
def reparse_actions(bill_id, options):
# Load an existing bill status JSON file.
data_json_fn = output_for_bill(bill_id, 'json')
source = utils.read(data_json_fn)
bill_data = json.loads(source)
# Munge data.
from bill_info import parse_bill_action
title = bill_info.current_title_for(bill_data['titles'], 'official')
old_status = None
for action in bill_data['actions']:
new_action, new_status = parse_bill_action(action, old_status, bill_id, title)
if new_status:
old_status = new_status
action['status'] = new_status
# clear out deleted keys
for key in ('vote_type', 'how', 'where', 'result', 'roll', 'suspension', 'calendar', 'under', 'number', 'committee', 'pocket', 'law', 'congress'):
if key in action and key not in new_action:
del action['key']
action.update(new_action)
status, status_date = bill_info.latest_status(bill_data['actions'], bill_data['introduced_at'])
bill_data['status'] = status
bill_data['status_at'] = status_date
# Show user a diff on the console to accept changes.
def show_diff_ask_ok(source, revised, fn):
if source == revised: return False # nothing to do
def split_lines(s): return [l+"\n" for l in s.split("\n")]
import sys
from difflib import unified_diff
sys.stdout.writelines(unified_diff(split_lines(source), split_lines(revised), fromfile=fn, tofile=fn))
return raw_input("Apply change? (y/n) ").strip() == "y"
wrote_any = False
# Write new data.json file.
revised = json.dumps(bill_data, indent=2, sort_keys=True)
if show_diff_ask_ok(source, revised, data_json_fn):
utils.write(revised, data_json_fn)
wrote_any = True
# Write new data.xml file.
from bill_info import create_govtrack_xml
data_xml_fn = data_json_fn.replace(".json", ".xml")
with open(data_xml_fn, 'r') as xml_file:
source = xml_file.read()
revised = create_govtrack_xml(bill_data, options)
if show_diff_ask_ok(source, revised.decode("utf8"), data_xml_fn):
with open(data_xml_fn, 'wb') as xml_file:
xml_file.write(revised)
wrote_any = True
return {
"ok": True,
"saved": wrote_any,
"reason": "no changes or changes skipped by user",
}
def process_bill(bill_id, options):
#pdb.set_trace()
fdsys_xml_path = _path_to_billstatus_file(bill_id)
logging.info("[%s] Processing %s..." % (bill_id, fdsys_xml_path))
# Read FDSys bulk data file.
xml_as_dict = read_fdsys_bulk_bill_status_file(fdsys_xml_path, bill_id)
bill_data = form_bill_json_dict(xml_as_dict)
# Convert and write out data.json and data.xml.
utils.write(unicode(json.dumps(bill_data, indent=2, sort_keys=True)),
os.path.dirname(fdsys_xml_path) + '/data.json')
try:
formatted = convert_congress_bill(bill_data)
formatted = json.loads(formatted)
insert_bills(formatted)
utils.write(unicode(json.dumps(formatted, indent=2, sort_keys=True)),
os.path.dirname(fdsys_xml_path) + '/formatted.json')
# maybe a logging.info call instead
print("Formatted and inserted %s" % fdsys_xml_path)
except:
print("FAILED FORMATTING %s" % fdsys_xml_path)
from bill_info import create_govtrack_xml
with open(os.path.dirname(fdsys_xml_path) + '/data.xml', 'wb') as xml_file:
xml_file.write(create_govtrack_xml(bill_data, options))
if options.get("amendments", True):
process_amendments(bill_id, xml_as_dict, options)
# Mark this bulk data file as processed by saving its lastmod
# file under a new path.
utils.write(
utils.read(
_path_to_billstatus_file(bill_id).replace(".xml", "-lastmod.txt")),
os.path.join(os.path.dirname(fdsys_xml_path),
"data-fromfdsys-lastmod.txt"))
return {
"ok": True,
"saved": True,
}