def form_bill_json_dict(xml_as_dict): """ Handles converting a government bulk XML file to legacy dictionary form. @param bill_id: id of the bill in format [type][number]-[congress] e.x. s934-113 @type bill_id: str @return: dictionary of bill attributes @rtype: dict """ bill_dict = xml_as_dict['billStatus']['bill'] bill_id = build_bill_id(bill_dict['billType'].lower(), bill_dict['billNumber'], bill_dict['congress']) titles = bill_info.titles_for(bill_dict['titles']['item']) actions = bill_info.actions_for(bill_dict['actions']['item'], bill_id, bill_info.current_title_for(titles, 'official')) status, status_date = bill_info.latest_status(actions, bill_dict.get('introducedDate', '')) bill_data = { 'bill_id': bill_id, 'bill_type': bill_dict.get('billType').lower(), 'number': bill_dict.get('billNumber'), 'congress': bill_dict.get('congress'), 'url': billstatus_url_for(bill_id), 'introduced_at': bill_dict.get('introducedDate', ''), 'by_request': bill_dict['sponsors']['item'][0]['byRequestType'] is not None, 'sponsor': bill_info.sponsor_for(bill_dict['sponsors']['item'][0]), 'cosponsors': bill_info.cosponsors_for(bill_dict['cosponsors']), 'actions': actions, 'history': bill_info.history_from_actions(actions), 'status': status, 'status_at': status_date, 'enacted_as': bill_info.slip_law_from(actions), 'titles': titles, 'official_title': bill_info.current_title_for(titles, 'official'), 'short_title': bill_info.current_title_for(titles, 'short'), 'popular_title': bill_info.current_title_for(titles, 'popular'), 'summary': bill_info.summary_for(bill_dict['summaries']['billSummaries']), # The top term's case has changed with the new bulk data. It's now in # Title Case. For backwards compatibility, the top term is run through # '.capitalize()' so it matches the old string. TODO: Remove one day? 'subjects_top_term': _fixup_top_term_case(bill_dict['policyArea']['name']) if bill_dict['policyArea'] else None, 'subjects': sorted( ([_fixup_top_term_case(bill_dict['policyArea']['name'])] if bill_dict['policyArea'] else []) + ([item['name'] for item in bill_dict['subjects']['billSubjects']['legislativeSubjects']['item']] if bill_dict['subjects']['billSubjects']['legislativeSubjects'] else []) ), 'related_bills': bill_info.related_bills_for(bill_dict['relatedBills']), 'committees': bill_info.committees_for(bill_dict['committees']['billCommittees']), 'amendments': bill_info.amendments_for(bill_dict['amendments']), 'updated_at': bill_dict.get('updateDate', ''), } return bill_data
def fetch_amendment(amdt_id, options): logging.info("\n[%s] Fetching..." % amdt_id) # fetch bill details body body = utils.download( amdt_url_for(amdt_id), amdt_cache_for(amdt_id, "information.html"), options) if not body: return {'saved': False, 'ok': False, 'reason': "failed to download"} if options.get("download_only", False): return {'saved': False, 'ok': True, 'reason': "requested download only"} amdt_type, number, congress = utils.split_bill_id(amdt_id) actions = actions_for(body, amdt_id, is_amendment=True) if actions is None: actions = [] parse_amendment_actions(actions) amdt = { 'amendment_id': amdt_id, 'amendment_type': amdt_type, 'chamber': amdt_type[0], 'number': number, 'congress': congress, 'amends': amends_for(body, grab_bill=False), 'amends_bill': amends_for(body, grab_bill=True), 'house_number': house_number_for(body), 'offered_at': offered_at_for(body, 'offered'), 'submitted_at': offered_at_for(body, 'submitted'), 'proposed_at': offered_at_for(body, 'proposed'), 'sponsor': sponsor_for(body), 'title': amendment_simple_text_for(body, "title"), 'description': amendment_simple_text_for(body, "description"), 'purpose': amendment_simple_text_for(body, "purpose"), 'actions': actions, 'updated_at': datetime.datetime.fromtimestamp(time.time()), } set_amendment_status(amdt) output_amendment(amdt, options) return {'ok': True, 'saved': True}
def form_bill_json_dict(xml_as_dict): """ Handles converting a government bulk XML file to legacy dictionary form. @param bill_id: id of the bill in format [type][number]-[congress] e.x. s934-113 @type bill_id: str @return: dictionary of bill attributes @rtype: dict """ bill_dict = xml_as_dict['billStatus']['bill'] bill_id = build_bill_id(bill_dict['billType'].lower(), bill_dict['billNumber'], bill_dict['congress']) titles = bill_info.titles_for(bill_dict['titles']['item']) actions = bill_info.actions_for(bill_dict['actions']['item'], bill_id, bill_info.current_title_for(titles, 'official')) status, status_date = bill_info.latest_status(actions, bill_dict.get('introducedDate', '')) bill_data = { 'bill_id': bill_id, 'bill_type': bill_dict.get('billType').lower(), 'number': bill_dict.get('billNumber'), 'congress': bill_dict.get('congress'), 'url': billstatus_url_for(bill_id), 'introduced_at': bill_dict.get('introducedDate', ''), 'by_request': bill_dict['sponsors']['item'][0]['byRequestType'] is not None, 'sponsor': bill_info.sponsor_for(bill_dict['sponsors']['item'][0]), 'cosponsors': bill_info.cosponsors_for(bill_dict['cosponsors']), 'actions': actions, 'history': bill_info.history_from_actions(actions), 'status': status, 'status_at': status_date, 'enacted_as': bill_info.slip_law_from(actions), 'titles': titles, 'official_title': bill_info.current_title_for(titles, 'official'), 'short_title': bill_info.current_title_for(titles, 'short'), 'popular_title': bill_info.current_title_for(titles, 'popular'), 'summary': bill_info.summary_for(bill_dict['summaries']['billSummaries']), # The top term's case has changed with the new bulk data. It's now in # Title Case. For backwards compatibility, the top term is run through # '.capitalize()' so it matches the old string. TODO: Remove one day? 'subjects_top_term': _fixup_top_term_case(bill_dict['policyArea']['name']) if bill_dict['policyArea'] else None, 'subjects': sorted( ([_fixup_top_term_case(bill_dict['policyArea']['name'])] if bill_dict['policyArea'] else []) + ([item['name'] for item in bill_dict['subjects']['billSubjects']['legislativeSubjects']['item']] if bill_dict['subjects']['billSubjects']['legislativeSubjects'] else []) ), 'related_bills': bill_info.related_bills_for(bill_dict['relatedBills']), 'committees': bill_info.committees_for(bill_dict['committees']['billCommittees']), 'amendments': bill_info.amendments_for(bill_dict['amendments']), 'committee_reports': bill_info.committee_reports_for(bill_dict['committeeReports']), 'updated_at': bill_dict.get('updateDate', ''), } return bill_data
def fetch_amendment(amendment_id, options): logging.info("\n[%s] Fetching..." % amendment_id) body = utils.download( amendment_url_for(amendment_id), amendment_cache_for(amendment_id, "information.html"), options) if not body: return {'saved': False, 'ok': False, 'reason': "failed to download"} if options.get("download_only", False): return { 'saved': False, 'ok': True, 'reason': "requested download only" } if "Amends:" not in body: return {'saved': False, 'ok': True, 'reason': "orphaned amendment"} amendment_type, number, congress = utils.split_bill_id(amendment_id) actions = actions_for(body, amendment_id, is_amendment=True) if actions is None: actions = [] parse_amendment_actions(actions) chamber = amendment_type[0] # good set of tests for each situation: # samdt712-113 - amendment to bill # samdt112-113 - amendment to amendment on bill # samdt4904-111 - amendment to treaty # samdt4922-111 - amendment to amendment to treaty amends_bill = amends_bill_for(body) # almost always present amends_treaty = amends_treaty_for(body) # present if bill is missing amends_amendment = amends_amendment_for(body) # sometimes present if not amends_bill and not amends_treaty: raise Exception( "Choked finding out what bill or treaty the amendment amends.") amdt = { 'amendment_id': amendment_id, 'amendment_type': amendment_type, 'chamber': chamber, 'number': int(number), 'congress': congress, 'amends_bill': amends_bill, 'amends_treaty': amends_treaty, 'amends_amendment': amends_amendment, 'sponsor': sponsor_for(body), 'description': amendment_simple_text_for(body, "description"), 'purpose': amendment_simple_text_for(body, "purpose"), 'actions': actions, 'updated_at': datetime.datetime.fromtimestamp(time.time()), } if chamber == 'h': amdt['introduced_at'] = offered_at_for(body, 'offered') elif chamber == 's': amdt['introduced_at'] = offered_at_for(body, 'submitted') amdt['proposed_at'] = offered_at_for(body, 'proposed') if not amdt.get('introduced_at', None): raise Exception( "Couldn't find a reliable introduction date for amendment.") # needs to come *after* the setting of introduced_at amdt['status'], amdt['status_at'] = amendment_status_for(amdt) # only set a house_number if it's a House bill - # this lets us choke if it's not found. if amdt['chamber'] == 'h': # numbers found in vote XML # summary = amdt['purpose'] if amdt['purpose'] else amdt['description'] # amdt['house_number'] = house_simple_number_for(amdt['amendment_id'], summary) if int(amdt['congress']) > 100: # A___-style numbers, present only starting with the 101st Congress amdt['house_number'] = house_number_for(body) output_amendment(amdt, options) return {'ok': True, 'saved': True}
def fetch_amendment(amendment_id, options): logging.info("\n[%s] Fetching..." % amendment_id) body = utils.download( amendment_url_for(amendment_id), amendment_cache_for(amendment_id, "information.html"), options) if not body: return {'saved': False, 'ok': False, 'reason': "failed to download"} if options.get("download_only", False): return {'saved': False, 'ok': True, 'reason': "requested download only"} if "Amends:" not in body: return {'saved': False, 'ok': True, 'reason': "orphaned amendment"} amendment_type, number, congress = utils.split_bill_id(amendment_id) actions = actions_for(body, amendment_id, is_amendment=True) if actions is None: actions = [] parse_amendment_actions(actions) chamber = amendment_type[0] # good set of tests for each situation: # samdt712-113 - amendment to bill # samdt112-113 - amendment to amendment on bill # samdt4904-111 - amendment to treaty # samdt4922-111 - amendment to amendment to treaty amends_bill = amends_bill_for(body) # almost always present amends_treaty = amends_treaty_for(body) # present if bill is missing amends_amendment = amends_amendment_for(body) # sometimes present if not amends_bill and not amends_treaty: raise Exception("Choked finding out what bill or treaty the amendment amends.") amdt = { 'amendment_id': amendment_id, 'amendment_type': amendment_type, 'chamber': chamber, 'number': int(number), 'congress': congress, 'amends_bill': amends_bill, 'amends_treaty': amends_treaty, 'amends_amendment': amends_amendment, 'sponsor': sponsor_for(body), 'description': amendment_simple_text_for(body, "description"), 'purpose': amendment_simple_text_for(body, "purpose"), 'actions': actions, 'updated_at': datetime.datetime.fromtimestamp(time.time()), } if chamber == 'h': amdt['introduced_at'] = offered_at_for(body, 'offered') elif chamber == 's': amdt['introduced_at'] = offered_at_for(body, 'submitted') amdt['proposed_at'] = offered_at_for(body, 'proposed') if not amdt.get('introduced_at', None): raise Exception("Couldn't find a reliable introduction date for amendment.") # needs to come *after* the setting of introduced_at amdt['status'], amdt['status_at'] = amendment_status_for(amdt) # only set a house_number if it's a House bill - # this lets us choke if it's not found. if amdt['chamber'] == 'h': # numbers found in vote XML # summary = amdt['purpose'] if amdt['purpose'] else amdt['description'] # amdt['house_number'] = house_simple_number_for(amdt['amendment_id'], summary) if int(amdt['congress']) > 100: # A___-style numbers, present only starting with the 101st Congress amdt['house_number'] = house_number_for(body) output_amendment(amdt, options) return {'ok': True, 'saved': True}