def fetch_notices_json(self): self.notices_json = fetch_notice_json(self.cfr_title, self.cfr_part, only_final=True) self.notice_doc_numbers = [ notice_json['document_number'] for notice_json in self.notices_json ]
def _fdsys_to_doc_number(xml, title, title_part): """Pull out a document number from an FDSYS document, i.e. an annual edition of a reg""" original_date_els = xml.xpath("//FDSYS/ORIGINALDATE") if len(original_date_els) > 0: date = original_date_els[0].text # Grab oldest document number from Federal register API notices = fetch_notice_json(title, title_part, only_final=True, max_effective_date=date) if notices: return notices[0]["document_number"]
def _fdsys_to_doc_number(xml, title, title_part): """Pull out a document number from an FDSYS document, i.e. an annual edition of a reg""" original_date_els = xml.xpath('//FDSYS/ORIGINALDATE') if len(original_date_els) > 0: date = original_date_els[0].text # Grab oldest document number from Federal register API notices = fetch_notice_json(title, title_part, only_final=True, max_effective_date=date) if notices: return notices[0]['document_number']
def fetch_version_ids(cfr_title, cfr_part, notice_dir): """Returns a list of version ids after looking them up between the federal register and the local filesystem""" version_ids = [] final_rules = fetch_notice_json(cfr_title, cfr_part, only_final=True) for document_number in (fr['document_number'] for fr in final_rules): # Document number followed by a date regex = re.compile(re.escape(document_number) + r"_\d{8}") version_ids.extend([name for name in notice_dir if regex.match(name)] or [document_number]) return version_ids
def _fdsys_to_doc_number(xml, title, title_part): """Pull out a document number from an FDSYS document, i.e. an annual edition of a reg""" original_date_els = xml.xpath('//FDSYS/ORIGINALDATE') if len(original_date_els) > 0: date = original_date_els[0].text # Grab closest notice to this effective date from the Federal Register notices = fetch_notice_json(title, title_part, only_final=True, max_effective_date=date) comparer = lambda n: (n['effective_on'], n['publication_date']) notices = sorted(notices, key=comparer, reverse=True) if notices: return notices[0]['document_number']
def fetch_version_ids(cfr_title, cfr_part, notice_dir): """Returns a list of version ids after looking them up between the federal register and the local filesystem""" version_ids = [] final_rules = fetch_notice_json(cfr_title, cfr_part, only_final=True) for document_number in (fr['document_number'] for fr in final_rules): # Document number followed by a date regex = re.compile(re.escape(document_number) + r"_\d{8}") version_ids.extend( [name for name in notice_dir if regex.match(name)] or [document_number]) return version_ids
def fetch_version_ids(cfr_title, cfr_part, notice_dir): """Returns a list of version ids after looking them up between the federal register and the local filesystem""" present_ids = [v.path[-1] for v in notice_dir.sub_entries()] final_rules = fetch_notice_json(cfr_title, cfr_part, only_final=True) version_ids = [] for fr_id in map(itemgetter('document_number'), final_rules): # Version_id concatenated with the date regex = re.compile(re.escape(fr_id) + r"_\d{8}") split_entries = [vid for vid in present_ids if regex.match(vid)] # Add either the split entries or the original version_id version_ids.extend(split_entries or [fr_id]) return version_ids
def fetch_version_ids(cfr_title, cfr_part, notice_dir): """Returns a list of version ids after looking them up between the federal register and the local filesystem""" present_ids = [v.path[-1] for v in notice_dir.sub_entries()] final_rules = fetch_notice_json(cfr_title, cfr_part, only_final=True) version_ids = [] for fr_id in map(itemgetter("document_number"), final_rules): # Version_id concatenated with the date regex = re.compile(re.escape(fr_id) + r"_\d{8}") split_entries = [vid for vid in present_ids if regex.match(vid)] # Add either the split entries or the original version_id version_ids.extend(split_entries or [fr_id]) return version_ids
def versions(title, part, from_fr=False, from_regml=True): """ List notices for regulation title/part """ if from_fr: # Get notices from the FR fr_notices = fetch_notice_json(title, part, only_final=True) print(colored("The Federal Register reports the following " "final notices:", attrs=['bold'])) for notice in fr_notices: print("\t", notice['document_number']) print("\t\tinitially effective on", notice['effective_on']) # Look for locally available notices regml_notice_dir = os.path.join(settings.XML_ROOT, 'notice', part, '*.xml') regml_notice_files = glob.glob(regml_notice_dir) # regml_notice_files = os.listdir(regml_notice_dir) print(colored("RegML Notices are available for:", attrs=['bold'])) regml_notices = [] for notice_file in regml_notice_files: with open(os.path.join(notice_file), 'r') as f: notice_xml = f.read() xml_tree = etree.fromstring(notice_xml) doc_number = xml_tree.find( './{eregs}preamble/{eregs}documentNumber').text effective_date = xml_tree.find( './{eregs}preamble/{eregs}effectiveDate').text applies_to = xml_tree.find( './{eregs}changeset').get('leftDocumentNumber') regml_notices.append((doc_number, effective_date, applies_to)) regml_notices.sort(key=lambda n: n[1]) for notice in regml_notices: print("\t", notice[0]) print("\t\teffective on", notice[1]) print("\t\tapplies to", notice[2]) # Verify that there's a logical sequence of applies_to index = regml_notices.index(notice) if index == 0: continue previous_notice = regml_notices[regml_notices.index(notice)-1] if previous_notice[0] != notice[2]: print(colored("\t\tWarning: {} does not apply to " "previous notice {}".format( notice[0], previous_notice[0]), 'yellow'))
def _fdsys_to_doc_number(xml, title, title_part): """Pull out a document number from an FDSYS document, i.e. an annual edition of a reg""" original_date_els = xml.xpath('//FDSYS/ORIGINALDATE') if len(original_date_els) > 0: date = original_date_els[0].text # Grab closest notice to this effective date from the Federal Register notices = fetch_notice_json(title, title_part, only_final=True, max_effective_date=date) def comparer(n): return (n['effective_on'], n['publication_date']) notices = sorted(notices, key=comparer, reverse=True) if notices: return notices[0]['document_number']
def fetch_version_ids(cfr_title, cfr_part, notice_dir): """Returns a list of version ids after looking them up between the federal register and the local filesystem""" present_ids = [v.path[-1] for v in notice_dir.sub_entries()] final_rules = fetch_notice_json(cfr_title, cfr_part, only_final=True) version_ids = [] pair_fn = itemgetter('document_number', 'full_text_xml_url') for fr_id, xml_url in map(pair_fn, final_rules): if xml_url: # Version_id concatenated with the date regex = re.compile(re.escape(fr_id) + r"_\d{8}") split_entries = [vid for vid in present_ids if regex.match(vid)] # Add either the split entries or the original version_id version_ids.extend(split_entries or [fr_id]) else: logger.warning("No XML for %s; skipping", fr_id) return version_ids
def first_notice_and_xml(title, part): """Find the first annual xml and its associated notice""" notices = [build_notice(title, part, n, do_process_xml=False) for n in fetch_notice_json(title, part, only_final=True) if n['full_text_xml_url'] and n['effective_on']] modify_effective_dates(notices) notices = sorted(notices, key=lambda n: (n['effective_on'], n['publication_date'])) years = {} for n in notices: year = annual_edition_for(title, n) years[year] = n for year, notice in sorted(years.iteritems()): volume = find_volume(year, title, part) if volume: part_xml = volume.find_part_xml(part) if part_xml is not None: return (notice, part_xml)
def first_notice_and_xml(title, part): """Find the first annual xml and its associated notice""" notices = [ build_notice(title, part, n, do_process_xml=False) for n in fetch_notice_json(title, part, only_final=True) if n['full_text_xml_url'] and n['effective_on'] ] modify_effective_dates(notices) notices = sorted(notices, key=lambda n: (n['effective_on'], n['publication_date'])) years = {} for n in notices: year = annual_edition_for(title, n) years[year] = n for year, notice in sorted(years.iteritems()): volume = find_volume(year, title, part) if volume: part_xml = volume.find_part_xml(part) if part_xml is not None: return (notice, part_xml)
def fetch_notices_json(self): self.notices_json = fetch_notice_json(self.cfr_title, self.cfr_part, only_final=True) self.notice_doc_numbers = [notice_json['document_number'] for notice_json in self.notices_json]