def fetch_notices_json(self):
self.notices_json = fetch_notice_json(self.cfr_title,
self.cfr_part,
only_final=True)
self.notice_doc_numbers = [
notice_json['document_number'] for notice_json in self.notices_json
]
def _fdsys_to_doc_number(xml, title, title_part):
"""Pull out a document number from an FDSYS document, i.e. an annual
edition of a reg"""
original_date_els = xml.xpath("//FDSYS/ORIGINALDATE")
if len(original_date_els) > 0:
date = original_date_els[0].text
# Grab oldest document number from Federal register API
notices = fetch_notice_json(title, title_part, only_final=True, max_effective_date=date)
if notices:
return notices[0]["document_number"]
def _fdsys_to_doc_number(xml, title, title_part):
"""Pull out a document number from an FDSYS document, i.e. an annual
edition of a reg"""
original_date_els = xml.xpath('//FDSYS/ORIGINALDATE')
if len(original_date_els) > 0:
date = original_date_els[0].text
# Grab oldest document number from Federal register API
notices = fetch_notice_json(title, title_part, only_final=True,
max_effective_date=date)
if notices:
return notices[0]['document_number']
def fetch_version_ids(cfr_title, cfr_part, notice_dir):
"""Returns a list of version ids after looking them up between the federal
register and the local filesystem"""
version_ids = []
final_rules = fetch_notice_json(cfr_title, cfr_part, only_final=True)
for document_number in (fr['document_number'] for fr in final_rules):
# Document number followed by a date
regex = re.compile(re.escape(document_number) + r"_\d{8}")
version_ids.extend([name for name in notice_dir if regex.match(name)]
or [document_number])
return version_ids
def _fdsys_to_doc_number(xml, title, title_part):
"""Pull out a document number from an FDSYS document, i.e. an annual
edition of a reg"""
original_date_els = xml.xpath('//FDSYS/ORIGINALDATE')
if len(original_date_els) > 0:
date = original_date_els[0].text
# Grab closest notice to this effective date from the Federal Register
notices = fetch_notice_json(title, title_part, only_final=True,
max_effective_date=date)
comparer = lambda n: (n['effective_on'], n['publication_date'])
notices = sorted(notices, key=comparer, reverse=True)
if notices:
return notices[0]['document_number']
def fetch_version_ids(cfr_title, cfr_part, notice_dir):
"""Returns a list of version ids after looking them up between the federal
register and the local filesystem"""
version_ids = []
final_rules = fetch_notice_json(cfr_title, cfr_part, only_final=True)
for document_number in (fr['document_number'] for fr in final_rules):
# Document number followed by a date
regex = re.compile(re.escape(document_number) + r"_\d{8}")
version_ids.extend(
[name for name in notice_dir if regex.match(name)]
or [document_number])
return version_ids
def fetch_version_ids(cfr_title, cfr_part, notice_dir):
"""Returns a list of version ids after looking them up between the federal
register and the local filesystem"""
present_ids = [v.path[-1] for v in notice_dir.sub_entries()]
final_rules = fetch_notice_json(cfr_title, cfr_part, only_final=True)
version_ids = []
for fr_id in map(itemgetter('document_number'), final_rules):
# Version_id concatenated with the date
regex = re.compile(re.escape(fr_id) + r"_\d{8}")
split_entries = [vid for vid in present_ids if regex.match(vid)]
# Add either the split entries or the original version_id
version_ids.extend(split_entries or [fr_id])
return version_ids
def fetch_version_ids(cfr_title, cfr_part, notice_dir):
"""Returns a list of version ids after looking them up between the federal
register and the local filesystem"""
present_ids = [v.path[-1] for v in notice_dir.sub_entries()]
final_rules = fetch_notice_json(cfr_title, cfr_part, only_final=True)
version_ids = []
for fr_id in map(itemgetter("document_number"), final_rules):
# Version_id concatenated with the date
regex = re.compile(re.escape(fr_id) + r"_\d{8}")
split_entries = [vid for vid in present_ids if regex.match(vid)]
# Add either the split entries or the original version_id
version_ids.extend(split_entries or [fr_id])
return version_ids
def versions(title, part, from_fr=False, from_regml=True):
""" List notices for regulation title/part """
if from_fr:
# Get notices from the FR
fr_notices = fetch_notice_json(title, part, only_final=True)
print(colored("The Federal Register reports the following "
"final notices:", attrs=['bold']))
for notice in fr_notices:
print("\t", notice['document_number'])
print("\t\tinitially effective on", notice['effective_on'])
# Look for locally available notices
regml_notice_dir = os.path.join(settings.XML_ROOT, 'notice', part, '*.xml')
regml_notice_files = glob.glob(regml_notice_dir)
# regml_notice_files = os.listdir(regml_notice_dir)
print(colored("RegML Notices are available for:", attrs=['bold']))
regml_notices = []
for notice_file in regml_notice_files:
with open(os.path.join(notice_file), 'r') as f:
notice_xml = f.read()
xml_tree = etree.fromstring(notice_xml)
doc_number = xml_tree.find(
'./{eregs}preamble/{eregs}documentNumber').text
effective_date = xml_tree.find(
'./{eregs}preamble/{eregs}effectiveDate').text
applies_to = xml_tree.find(
'./{eregs}changeset').get('leftDocumentNumber')
regml_notices.append((doc_number, effective_date, applies_to))
regml_notices.sort(key=lambda n: n[1])
for notice in regml_notices:
print("\t", notice[0])
print("\t\teffective on", notice[1])
print("\t\tapplies to", notice[2])
# Verify that there's a logical sequence of applies_to
index = regml_notices.index(notice)
if index == 0:
continue
previous_notice = regml_notices[regml_notices.index(notice)-1]
if previous_notice[0] != notice[2]:
print(colored("\t\tWarning: {} does not apply to "
"previous notice {}".format(
notice[0],
previous_notice[0]), 'yellow'))
def _fdsys_to_doc_number(xml, title, title_part):
"""Pull out a document number from an FDSYS document, i.e. an annual
edition of a reg"""
original_date_els = xml.xpath('//FDSYS/ORIGINALDATE')
if len(original_date_els) > 0:
date = original_date_els[0].text
# Grab closest notice to this effective date from the Federal Register
notices = fetch_notice_json(title,
title_part,
only_final=True,
max_effective_date=date)
def comparer(n):
return (n['effective_on'], n['publication_date'])
notices = sorted(notices, key=comparer, reverse=True)
if notices:
return notices[0]['document_number']
def fetch_version_ids(cfr_title, cfr_part, notice_dir):
"""Returns a list of version ids after looking them up between the federal
register and the local filesystem"""
present_ids = [v.path[-1] for v in notice_dir.sub_entries()]
final_rules = fetch_notice_json(cfr_title, cfr_part, only_final=True)
version_ids = []
pair_fn = itemgetter('document_number', 'full_text_xml_url')
for fr_id, xml_url in map(pair_fn, final_rules):
if xml_url:
# Version_id concatenated with the date
regex = re.compile(re.escape(fr_id) + r"_\d{8}")
split_entries = [vid for vid in present_ids if regex.match(vid)]
# Add either the split entries or the original version_id
version_ids.extend(split_entries or [fr_id])
else:
logger.warning("No XML for %s; skipping", fr_id)
return version_ids
def first_notice_and_xml(title, part):
"""Find the first annual xml and its associated notice"""
notices = [build_notice(title, part, n, do_process_xml=False)
for n in fetch_notice_json(title, part, only_final=True)
if n['full_text_xml_url'] and n['effective_on']]
modify_effective_dates(notices)
notices = sorted(notices,
key=lambda n: (n['effective_on'], n['publication_date']))
years = {}
for n in notices:
year = annual_edition_for(title, n)
years[year] = n
for year, notice in sorted(years.iteritems()):
volume = find_volume(year, title, part)
if volume:
part_xml = volume.find_part_xml(part)
if part_xml is not None:
return (notice, part_xml)
def first_notice_and_xml(title, part):
"""Find the first annual xml and its associated notice"""
notices = [
build_notice(title, part, n, do_process_xml=False)
for n in fetch_notice_json(title, part, only_final=True)
if n['full_text_xml_url'] and n['effective_on']
]
modify_effective_dates(notices)
notices = sorted(notices,
key=lambda n: (n['effective_on'], n['publication_date']))
years = {}
for n in notices:
year = annual_edition_for(title, n)
years[year] = n
for year, notice in sorted(years.iteritems()):
volume = find_volume(year, title, part)
if volume:
part_xml = volume.find_part_xml(part)
if part_xml is not None:
return (notice, part_xml)
def fetch_notices_json(self):
self.notices_json = fetch_notice_json(self.cfr_title, self.cfr_part,
only_final=True)
self.notice_doc_numbers = [notice_json['document_number']
for notice_json in self.notices_json]