def fetch_grouped_history(part):
client = api_reader.ApiReader()
versions = [
version for version in client.regversions(part)['versions']
if 'by_date' in version
]
for version in versions:
version['notices'] = []
versions = sorted(convert_to_python(versions), reverse=True,
key=lambda v: v['by_date'])
today = datetime.today()
seen_present = False
for version in versions:
if version['by_date'] > today:
version['timeline'] = Timeline.future
elif not seen_present:
seen_present = True
version['timeline'] = Timeline.present
else:
version['timeline'] = Timeline.past
for notice in client.notices(part)['results']:
notice = convert_to_python(notice)
for version in versions:
if version['by_date'] == notice.get('effective_on'):
version['notices'].append(notice)
for version in versions:
version['notices'] = sorted(version['notices'], reverse=True,
key=lambda n: n['publication_date'])
return versions
def fetch_grouped_history(part):
client = api_reader.ApiReader()
versions = filter(lambda v: 'by_date' in v,
client.regversions(part)['versions'])
for version in versions:
version['notices'] = []
versions = sorted(convert_to_python(versions),
reverse=True,
key=lambda v: v['by_date'])
today = datetime.today()
seen_current = False
for version in versions:
if version['by_date'] > today:
version['timeline'] = 'future'
elif not seen_current:
seen_current = True
version['timeline'] = 'current'
else:
version['timeline'] = 'past'
for notice in client.notices(part)['results']:
notice = convert_to_python(notice)
for v in (v for v in versions
if v['by_date'] == notice['effective_on']):
v['notices'].append(notice)
for version in versions:
version['notices'] = sorted(version['notices'],
reverse=True,
key=lambda n: n['publication_date'])
return versions
def fetch_toc(reg_part, version, flatten=False):
"""Fetch the toc, transform it into a list usable by navigation, etc."""
toc = api_reader.ApiReader().layer('toc', 'cfr', reg_part, version)
if toc is None:
logger.warning("404 when fetching TOC for %s@%s", reg_part, version)
toc = []
toc_list = []
if reg_part in toc:
for data in toc[reg_part]:
if 'Subpart' in data['index']:
toc_list.append(toc_subpart(data, toc_list, toc))
elif 'Subjgrp' in data['index']:
toc_list.append(toc_subjgrp(data, toc_list, toc))
elif 'Interp' in data['index']:
toc_list.append(toc_interp(data, toc_list, toc))
else:
toc_list.append(toc_sect_appendix(data, toc_list))
if flatten:
flattened = []
for el in toc_list:
if 'sub_toc' in el:
flattened.extend(el['sub_toc'])
else:
flattened.append(el)
return flattened
return toc_list
def __init__(self):
self.appliers = {
LayerBase.INLINE: InlineLayersApplier(),
LayerBase.PARAGRAPH: ParagraphLayersApplier(),
LayerBase.SEARCH_REPLACE: SearchReplaceLayersApplier()}
self.api = api_reader.ApiReader()
def check_regulation(reg_part):
""" If versions of the reg_part given don't exist, raise
a MissingContentException(). """
client = api_reader.ApiReader()
vr = client.regversions(reg_part)
if not vr:
raise MissingContentException()
def get_context_data(self, **kwargs):
# We don't want to run the content data of PartialView -- it assumes
# we will be applying layers
context = super(PartialView, self).get_context_data(**kwargs)
context['q'] = self.request.GET.get('q')
context['version'] = self.request.GET.get('version')
context['regulation'] = context['label_id'].split('-')[0]
try:
page = int(self.request.GET.get('page', '0'))
except ValueError:
page = 0
api_page = page // (API_PAGE_SIZE / PAGE_SIZE)
context['warnings'] = []
if not context['q']:
context['warnings'].append('Please provide a query.')
if not context['version']:
context['warnings'].append('Please provide a version.')
if context['warnings']:
results = {'results': [], 'total_hits': 0}
else:
results = api_reader.ApiReader().search(context['q'],
context['version'],
context['regulation'],
api_page)
self.reduce_results(results, page)
section_url = SectionUrl()
for result in results['results']:
result['header'] = node_types.label_to_text(result['label'])
if 'title' in result:
result['header'] += ' ' + title(result['title'])
result['section_id'] = section_url.view_label_id(
result['label'], context['version'])
result['url'] = section_url.fetch(result['label'],
context['version'],
sectional=True)
context['results'] = results
for version in fetch_grouped_history(context['regulation']):
for notice in version['notices']:
if notice['document_number'] == context['version']:
context['version_by_date'] = notice['effective_on']
self.add_prev_next(page, context)
self.final_context = context
return context
def get_context_data(self, **kwargs):
context = super(SideBarView, self).get_context_data(**kwargs)
label = context['label_id'].split('-')
context['human_label_id'] = label_to_text(label, include_marker=True)
client = api_reader.ApiReader()
node_trees = self._get_node_trees(client, label, context['version'])
self.add_sxs(client, node_trees, context)
return context
def get_context_data(self, **kwargs):
# We don't want to run the content data of PartialView -- it assumes
# we will be applying layers
context = super(PartialView, self).get_context_data(**kwargs)
context['regulation'] = context['label_id'].split('-')[0]
try:
page = int(self.request.GET.get('page', '0'))
except ValueError:
page = 0
# API page size is API_PAGE_SIZE, but we show only PAGE_SIZE
api_page = page // (API_PAGE_SIZE / PAGE_SIZE)
page_idx = (page % (API_PAGE_SIZE / PAGE_SIZE)) * PAGE_SIZE
results = api_reader.ApiReader().search(context['q'],
context['version'],
context['regulation'],
api_page)
# Ignore results found in the root (i.e. not a section), adjust
# the number of results accordingly.
original_count = len(results['results'])
results['results'] = [
r for r in results['results'] if len(r['label']) > 1
]
num_results_ignored = original_count - len(results['results'])
results['total_hits'] -= num_results_ignored
results['results'] = results['results'][page_idx:page_idx + PAGE_SIZE]
section_url = SectionUrl()
for result in results['results']:
result['header'] = label_to_text(result['label'])
if 'title' in result:
result['header'] += ' ' + title(result['title'])
result['section_id'] = section_url.view_label_id(
result['label'], context['version'])
result['url'] = section_url.fetch(result['label'],
context['version'],
sectional=True)
context['results'] = results
for version in fetch_grouped_history(context['regulation']):
for notice in version['notices']:
if notice['document_number'] == context['version']:
context['version_by_date'] = notice['effective_on']
self.add_prev_next(page, context)
self.final_context = context
return context
def layer_fn(layer_name):
api_layer_name = DATA_LAYERS[layer_name].data_source
reader = api_reader.ApiReader()
older_layer = reader.layer(api_layer_name, 'cfr', label_id,
versions.older)
newer_layer = reader.layer(api_layer_name, 'cfr', label_id,
versions.newer)
older_layer = older_layer or {}
newer_layer = newer_layer or {}
layer_json = dict(newer_layer) # copy
layer_json.update(older_layer) # older layer takes precedence
return layer_json
def check_version(label_id, version):
""" We check if the version of this regulation exists, and the user is only
referencing a section that does not exist. """
reg_part = label_id.split('-')[0]
client = api_reader.ApiReader()
vr = client.regversions(reg_part)
requested_version = [v for v in vr['versions'] if v['version'] == version]
if len(requested_version) > 0:
requested_version = convert_to_python(requested_version)
return requested_version[0]
def further_analyses(self, label_id, notice_id, fr_page, version):
"""Grab other analyses for this same paragraph (limiting to those
visible from this regulation version.) Make them in descending
order"""
sxs_layer_data = api_reader.ApiReader().layer(
'analyses', 'cfr', label_id, version)
if label_id not in sxs_layer_data:
return []
else:
return [convert_to_python(a)
for a in reversed(sxs_layer_data[label_id])
if (a['reference'] != [notice_id, label_id] or
a['fr_page'] != fr_page)]
def get_context_data(self, doc_type, **kwargs):
# We don't want to run the content data of PartialView -- it assumes
# we will be applying layers
context = super(PartialView, self).get_context_data(**kwargs)
context['q'] = self.request.GET.get('q')
context['version'] = self.request.GET.get('version')
context['doc_type'] = doc_type
context['regulation'] = context['label_id'].split('-')[0]
context['url_rule'] = url_rules[doc_type]
try:
page = int(self.request.GET.get('page', '0'))
except ValueError:
page = 0
context['warnings'] = []
if not context['q']:
context['warnings'].append('Please provide a query.')
if doc_type == 'cfr' and not context['version']:
context['warnings'].append('Please provide a version.')
if context['warnings']:
results = {'results': [], 'total_hits': 0}
else:
results = api_reader.ApiReader().search(
context['q'],
context['doc_type'],
version=context['version'],
regulation=context['regulation'],
page=page,
page_size=PAGE_SIZE,
is_root='false',
is_subpart='false',
)
if doc_type == 'cfr':
context['results'] = process_cfr_results(results,
context['version'])
for version in fetch_grouped_history(context['regulation']):
for notice in version['notices']:
if notice['document_number'] == context['version']:
context['version_by_date'] = notice['effective_on']
else:
context['results'] = process_preamble_results(results)
self.add_prev_next(page, context)
return context
def regulation_meta(cfr_part, version):
""" Return the contents of the meta layer, without using a tree. """
layer_data = api_reader.ApiReader().layer('meta', 'cfr', cfr_part, version)
if layer_data is None:
logger.warning("404 when fetching Meta for %s@%s", cfr_part, version)
layer_data = {}
if not layer_data.get(cfr_part):
logger.warning("Empty meta data for %s@%s. Misparsed?", cfr_part,
version)
meta = {}
for data in layer_data.get(cfr_part, []):
meta.update(data)
return convert_to_python(meta)
def fetch_regulations_and_future_versions():
""" Returns a dict for all the regulations in the API. The dict includes
lists of future versions for each regulation. """
client = api_reader.ApiReader()
all_versions = client.all_regulations_versions()
all_versions = convert_to_python(all_versions)
regulations_future = {}
# We're only interested in future endpoint versions
for v in all_versions['versions']:
if v['regulation'] not in regulations_future:
regulations_future[v['regulation']] = []
if 'by_date' in v:
regulations_future[v['regulation']].append(v)
return regulations_future
def get_regulation(regulation, version):
""" Get the regulation JSON tree. Manipulate the label a bit for easier
access in the templates."""
api = api_reader.ApiReader()
reg = api.regulation(regulation, version)
if reg:
title = reg['title']
# up till the paren
match = re.search('part \d+[^\w]*([^\(]*)', title, re.I)
if match:
reg['title_clean'] = match.group(1).strip()
match = re.search('\(regulation (\w+)\)', title, re.I)
if match:
reg['reg_letter'] = match.group(1)
return reg
def get_context_data(self, **kwargs):
# Skip ChromeView's implementation
context = super(ChromeView, self).get_context_data(**kwargs)
context['reg_part'] = context['label_id'].split('-')[0]
context['version'] = self.request.GET.get('from_version')
meta = api_reader.ApiReader().layer(
'meta', 'cfr', context['reg_part'], context['version'])
context['meta'] = meta[context['reg_part']][0]
context['formatted_id'] = label_to_text(context['label_id'].split('-'))
content = self.content(context)
if isinstance(content, HttpResponse): # error occurred
return content
context['partial_content'] = content
return context
def get_context_data(self, **kwargs):
context = super(SideBarView, self).get_context_data(**kwargs)
client = api_reader.ApiReader()
klasses = []
for class_or_class_path in self.components:
if isinstance(class_or_class_path, six.string_types):
module, class_name = class_or_class_path.rsplit('.', 1)
klasses.append(getattr(import_module(module), class_name))
else:
klasses.append(class_or_class_path)
sidebars = [
klass(context['label_id'], context['version']) for klass in klasses
]
context['sidebars'] = [
sidebar.full_context(client, self.request) for sidebar in sidebars
]
return context
def get_all_notices():
api = api_reader.ApiReader()
return notices.fetch_all(api)
def layer_fn(layer_name):
api_layer_name = DATA_LAYERS[layer_name].data_source
reader = api_reader.ApiReader()
return reader.layer(api_layer_name, doc_type, label_id, version)
def get_notice(document_number):
""" Get a the data from a particular notice, given the Federal Register
document number. """
api = api_reader.ApiReader()
return api.notice(document_number)
def regulation_exists(label_id):
client = api_reader.ApiReader()
vr = client.regversions(label_id)
return (vr and len(vr) > 0)
def get_tree_paragraph(paragraph_id, version):
"""Get a single level of the regulation tree."""
api = api_reader.ApiReader()
return api.regulation(paragraph_id, version)
def get_diff_json(regulation, older, newer):
api = api_reader.ApiReader()
return api.diff(regulation, older, newer)