def interp(self, citation, version):
"""Subterps throw a big monkey-wrench into things. Citations to
interpretations must be converted into their corresponding subterp,
which requires loading the toc."""
reg_part = citation[0]
key = (reg_part, version)
prefix = list(takewhile(lambda l: l != "Interp", citation))
prefix.append("Interp")
if key not in self.toc_cache:
self.toc_cache[key] = fetch_toc(reg_part, version)
prefix_section = prefix[:2]
for el in self.toc_cache[key]:
if el["index"] == prefix_section and el.get("is_section"):
# No subpart
return reg_part + "-Subpart-Interp"
for sub in el.get("sub_toc", []):
# In a subpart
if sub["index"] == prefix_section and el.get("is_subpart"):
return "-".join(el["index"][:3] + ["Interp"])
# In a subterp
if sub["index"] == prefix:
return "-".join(prefix)
# Full match: interpretation header
if sub["index"] == citation:
return sub["section_id"]
# Couldn't find it; most likely in the appendix
return reg_part + "-Appendices-Interp"
def add_cfr_part(self, doc_number, version_info, amendment):
"""While processing an amendment, if it refers to a CFR part which
hasn't been seen before, we need to perform some accounting, fetching
related meta data, etc."""
part = amendment['cfr_part']
if part not in version_info:
logger.warning("No version info for %s", part)
elif self.cfr_part is None or self.cfr_part != amendment['cfr_part']:
meta = utils.regulation_meta(part, version_info[part]['right'])
flat_toc = fetch_toc(part, version_info[part]['right'],
flatten=True)
self.section_titles = {
elt['index'][1]: elt['title']
for elt in flat_toc if len(elt['index']) == 2}
self.cfr_part = part
self.cfr_title = meta.get('cfr_title_number')
self.section = None
title = '{} CFR {}'.format(self.cfr_title, part)
markup_id = '{}-cfr-{}'.format(doc_number, part)
self.toc.append(NavItem(
url=reverse('cfr_changes', kwargs={
'doc_number': doc_number, 'section': part}),
title=Title('Authority', title, 'Authority'),
markup_id=markup_id,
category=title,
section_id='')) # disable AJAX
def first_section(reg_part, version):
""" Use the table of contents for a regulation, to get the label of the
first section of the regulation. In most regulations, this is -1, but in
some it's -101. """
toc = fetch_toc(reg_part, version, flatten=True)
return toc[0]['section_id']
def new_cfr_part(self, amendment):
"""While processing an amendment, if it refers to a CFR part which
hasn't been seen before, we need to perform some accounting, fetching
related meta data, etc."""
part = amendment['cfr_part']
meta = utils.regulation_meta(part, self.version_info[part]['right'])
flat_toc = fetch_toc(part,
self.version_info[part]['right'],
flatten=True)
self.section_titles = {
elt['index'][1]: elt['title']
for elt in flat_toc if len(elt['index']) == 2
}
self.current_part = ToCPart(title=meta.get('cfr_title_number'),
part=part,
name=meta.get('statutory_name'),
sections=[],
authority_url=reverse('cfr_changes',
kwargs={
'doc_number':
self.doc_number,
'section': part
}))
self.current_section = None
self.toc.append(self.current_part)
def interp(self, citation, version):
"""Subterps throw a big monkey-wrench into things. Citations to
interpretations must be converted into their corresponding subterp,
which requires loading the toc."""
reg_part = citation[0]
key = (reg_part, version)
prefix = list(takewhile(lambda l: l != 'Interp', citation))
prefix.append('Interp')
if key not in self.toc_cache:
self.toc_cache[key] = fetch_toc(reg_part, version)
prefix_section = prefix[:2]
for el in self.toc_cache[key]:
if el['index'] == prefix_section and el.get('is_section'):
# No subpart
return reg_part + '-Subpart-Interp'
for sub in el.get('sub_toc', []):
# In a subpart
if sub['index'] == prefix_section and el.get('is_subpart'):
return '-'.join(el['index'][:3] + ['Interp'])
# In a subterp
if sub['index'] == prefix:
return '-'.join(prefix)
# Other match: interpretation header
if sub['index'] == citation[:len(sub['index'])]:
return sub['section_id']
# Couldn't find it; most likely in the appendix
return reg_part + '-Appendices-Interp'
def first_section(reg_part, version):
""" Use the table of contents for a regulation, to get the label of the
first section of the regulation. In most regulations, this is -1, but in
some it's -101. """
toc = fetch_toc(reg_part, version, flatten=True)
return toc[0]['section_id']
def add_cfr_part(self, doc_number, version_info, amendment):
"""While processing an amendment, if it refers to a CFR part which
hasn't been seen before, we need to perform some accounting, fetching
related meta data, etc."""
part = amendment['cfr_part']
if part not in version_info:
logger.error("No version info for %s", part)
elif self.cfr_part is None or self.cfr_part != amendment['cfr_part']:
meta = utils.regulation_meta(part, version_info[part]['right'])
flat_toc = fetch_toc(part,
version_info[part]['right'],
flatten=True)
self.section_titles = {
elt['index'][1]: elt['title']
for elt in flat_toc if len(elt['index']) == 2
}
self.cfr_part = part
self.cfr_title = meta.get('cfr_title_number')
self.section = None
title = '{} CFR {}'.format(self.cfr_title, part)
markup_id = '{}-cfr-{}'.format(doc_number, part)
self.toc.append(
NavItem(url=reverse('cfr_changes',
kwargs={
'doc_number': doc_number,
'section': part
}),
title=Title('Authority', title, 'Authority'),
markup_id=markup_id,
category=title,
section_id='')) # disable AJAX
def new_cfr_part(self, amendment):
"""While processing an amendment, if it refers to a CFR part which
hasn't been seen before, we need to perform some accounting, fetching
related meta data, etc."""
part = amendment['cfr_part']
meta = utils.regulation_meta(part, self.version_info[part]['right'])
flat_toc = fetch_toc(part, self.version_info[part]['right'],
flatten=True)
self.section_titles = {elt['index'][1]: elt['title']
for elt in flat_toc if len(elt['index']) == 2}
self.current_part = ToCPart(
title=meta.get('cfr_title_number'), part=part,
name=meta.get('statutory_name'), sections=[],
authority_url=reverse('cfr_changes', kwargs={
'doc_number': self.doc_number, 'section': part}))
self.current_section = None
self.toc.append(self.current_part)
def set_chrome_context(self, context, reg_part, version):
context['reg_part'] = reg_part
context['history'] = fetch_grouped_history(reg_part)
toc = fetch_toc(reg_part, version)
for el in toc:
el['url'] = SectionUrl().of(
el['index'], version, self.partial_class.sectional_links)
for sub in el.get('sub_toc', []):
sub['url'] = SectionUrl().of(
sub['index'], version, self.partial_class.sectional_links)
context['TOC'] = toc
context['meta'] = utils.regulation_meta(reg_part, version)
context['version_switch_view'] = self.version_switch_view
context['diff_redirect_label'] = self.diff_redirect_label(
context['label_id'], toc)
def set_chrome_context(self, context, reg_part, version):
context['reg_part'] = reg_part
context['history'] = fetch_grouped_history(reg_part)
toc = fetch_toc(reg_part, version)
for el in toc:
el['url'] = SectionUrl().of(el['index'], version,
self.partial_class.sectional_links)
for sub in el.get('sub_toc', []):
sub['url'] = SectionUrl().of(
sub['index'], version, self.partial_class.sectional_links)
context['TOC'] = toc
context['meta'] = utils.regulation_meta(reg_part, version)
context['version_switch_view'] = self.version_switch_view
context['diff_redirect_label'] = self.diff_redirect_label(
context['label_id'], toc)
def nav_sections(current, version):
labels = get_labels(current)
reg_part = labels[0]
toc = fetch_toc(reg_part, version, flatten=True)
for idx, el in enumerate(toc):
if el['index'] == labels:
if idx == 0:
previous_section = None
else:
previous_section = _add_extra(toc[idx - 1], version)
if idx == len(toc) - 1:
next_section = None
else:
next_section = _add_extra(toc[idx + 1], version)
return (previous_section, next_section)
def filter_by_subterp(nodes, subterp_label, version):
"""Given an interp node (e.g. a node with label ['1005', 'Interp']),
remove all irrelevant sub-nodes. Note that stripping out nodes is more
efficient than compiling nodes when API calls are involved. We use
takewhile and dropwhile in case extra, non-node interpretations are in
the range"""
def is_section(n):
return n['label'][1].isdigit()
def not_section(n):
return not is_section(n)
if subterp_label[1:] == ['Subpart', 'Interp']: # Empty part
skip_intros = dropwhile(not_section, nodes)
sections = takewhile(is_section, skip_intros)
return list(sections)
elif subterp_label[1:] == ['Appendices', 'Interp']: # Appendices
skip_intros = dropwhile(not_section, nodes)
skip_sections = dropwhile(is_section, skip_intros)
return list(skip_sections)
else: # A Subpart. Most costly as we need to know the toc
subpart_label = subterp_label[:-1]
toc = fetch_toc(subterp_label[0], version)
toc = list(
dropwhile(
lambda el: el['index'] != subpart_label, # not subpart
toc))
if toc:
sections = set(
tuple(el['index'] + ['Interp']) for el in toc[0]['sub_toc'])
relevant_interps = dropwhile(
lambda el: tuple(el['label']) not in sections, # not relevant
nodes)
other_sections = set()
for el in toc[1:]:
other_sections.add(tuple(el['index'] + ['Interp']))
for sub in el.get('sub_toc', []):
other_sections.add(tuple(sub['index'] + ['Interp']))
relevant_interps = takewhile(
lambda n: tuple(n['label']) not in other_sections, # relevant
relevant_interps)
return list(relevant_interps)
def filter_by_subterp(nodes, subterp_label, version):
"""Given an interp node (e.g. a node with label ['1005', 'Interp']),
remove all irrelevant sub-nodes. Note that stripping out nodes is more
efficient than compiling nodes when API calls are involved. We use
takewhile and dropwhile in case extra, non-node interpretations are in
the range"""
def is_section(n):
return n['label'][1].isdigit()
def not_section(n):
return not is_section(n)
if subterp_label[1:] == ['Subpart', 'Interp']: # Empty part
skip_intros = dropwhile(not_section, nodes)
sections = takewhile(is_section, skip_intros)
return list(sections)
elif subterp_label[1:] == ['Appendices', 'Interp']: # Appendices
skip_intros = dropwhile(not_section, nodes)
skip_sections = dropwhile(is_section, skip_intros)
return list(skip_sections)
else: # A Subpart. Most costly as we need to know the toc
subpart_label = subterp_label[:-1]
toc = fetch_toc(subterp_label[0], version)
toc = list(dropwhile(
lambda el: el['index'] != subpart_label, # not subpart
toc))
if toc:
sections = set(tuple(el['index'] + ['Interp'])
for el in toc[0]['sub_toc'])
relevant_interps = dropwhile(
lambda el: tuple(el['label']) not in sections, # not relevant
nodes)
other_sections = set()
for el in toc[1:]:
other_sections.add(tuple(el['index'] + ['Interp']))
for sub in el.get('sub_toc', []):
other_sections.add(tuple(sub['index'] + ['Interp']))
relevant_interps = takewhile(
lambda n: tuple(n['label']) not in other_sections, # relevant
relevant_interps)
return list(relevant_interps)
def add_diff_content(self, context):
context['from_version'] = self.request.GET.get('from_version',
context['version'])
context['left_version'] = context['version']
context['right_version'] = \
context['main_content_context']['newer_version']
from_version = self.request.GET.get('from_version', context['version'])
context['TOC'] = context['main_content_context']['TOC']
# Add reference to the first subterp, so we know how to redirect
toc = fetch_toc(context['label_id'].split('-')[0], from_version)
for entry in toc:
if entry.get('is_supplement') and entry.get('sub_toc'):
el = entry['sub_toc'][0]
el['url'] = SectionUrl().of(el['index'], from_version,
self.partial_class.sectional_links)
context['first_subterp'] = el
return context
def get_context_data(self, **kwargs):
# We don't want to run the content data of PartialView -- it assumes
# we will be applying layers
context = super(PartialView, self).get_context_data(**kwargs)
label_id = context['label_id']
older = context['version']
newer = context['newer_version']
tree = generator.get_tree_paragraph(label_id, older)
if tree is None:
#TODO We need a more complicated check here to see if the diffs
#add the requested section. If not -> 404
tree = {}
appliers = get_appliers(label_id, older, newer)
builder = HTMLBuilder(*appliers)
builder.tree = tree
if not builder.tree:
return error_handling.handle_generic_404(self.request)
builder.generate_html()
child_of_root = builder.tree
if builder.tree['node_type'] == REGTEXT:
child_of_root = {
'node_type': EMPTYPART,
'children': [builder.tree]
}
context['tree'] = {'children': [child_of_root]}
context['markup_page_type'] = 'diff'
regpart = label_id.split('-')[0]
old_toc = fetch_toc(regpart, older)
diff = generator.get_diff_json(regpart, older, newer)
from_version = self.request.GET.get('from_version', older)
context['TOC'] = diff_toc(older, newer, old_toc, diff, from_version)
context['navigation'] = self.footer_nav(label_id, context['TOC'],
older, newer, from_version)
return context
def get_context_data(self, **kwargs):
# We don't want to run the content data of PartialView -- it assumes
# we will be applying layers
context = super(PartialView, self).get_context_data(**kwargs)
label_id = context['label_id']
older = context['version']
newer = context['newer_version']
tree = generator.get_tree_paragraph(label_id, older)
if tree is None:
#TODO We need a more complicated check here to see if the diffs
#add the requested section. If not -> 404
tree = {}
appliers = get_appliers(label_id, older, newer)
builder = HTMLBuilder(*appliers)
builder.tree = tree
if not builder.tree:
return error_handling.handle_generic_404(self.request)
builder.generate_html()
child_of_root = builder.tree
if builder.tree['node_type'] == REGTEXT:
child_of_root = {
'node_type': EMPTYPART,
'children': [builder.tree]}
context['tree'] = {'children': [child_of_root]}
context['markup_page_type'] = 'diff'
regpart = label_id.split('-')[0]
old_toc = fetch_toc(regpart, older)
diff = generator.get_diff_json(regpart, older, newer)
from_version = self.request.GET.get('from_version', older)
context['TOC'] = diff_toc(older, newer, old_toc, diff, from_version)
context['navigation'] = self.footer_nav(label_id, context['TOC'],
older, newer, from_version)
return context
def get_context_data(self, **kwargs):
# We don't want to run the content data of PartialView -- it assumes
# we will be applying layers
context = super(PartialView, self).get_context_data(**kwargs)
label_id = context['label_id']
versions = Versions(context['version'], context['newer_version'],
self.request.GET.get('from_version'))
tree = generator.get_tree_paragraph(label_id, versions.older)
if tree is None:
# TODO We need a more complicated check here to see if the diffs
# add the requested section. If not -> 404
tree = {}
diff_applier = generator.get_diff_applier(label_id, versions.older,
versions.newer)
if diff_applier is None:
raise error_handling.MissingContentException()
layers = list(generator.diff_layers(versions, label_id))
builder = CFRHTMLBuilder(layers, diff_applier)
builder.tree = tree
builder.generate_html()
child_of_root = builder.tree
if builder.tree['node_type'] == REGTEXT:
child_of_root = {
'node_type': EMPTYPART,
'children': [builder.tree]
}
context['tree'] = {'children': [child_of_root]}
context['markup_page_type'] = 'diff'
regpart = label_id.split('-')[0]
old_toc = fetch_toc(regpart, versions.older)
diff = generator.get_diff_json(regpart, versions.older, versions.newer)
context['TOC'] = diff_toc(versions, old_toc, diff)
context['navigation'] = self.footer_nav(label_id, context['TOC'],
versions)
return context
def add_diff_content(self, context):
context['from_version'] = self.request.GET.get(
'from_version', context['version'])
context['left_version'] = context['version']
context['right_version'] = \
context['main_content_context']['newer_version']
from_version = self.request.GET.get('from_version', context['version'])
context['TOC'] = context['main_content_context']['TOC']
# Add reference to the first subterp, so we know how to redirect
toc = fetch_toc(context['label_id'].split('-')[0], from_version)
for entry in toc:
if entry.get('is_supplement') and entry.get('sub_toc'):
el = entry['sub_toc'][0]
el['url'] = SectionUrl().of(
el['index'], from_version,
self.partial_class.sectional_links)
context['first_subterp'] = el
return context
def new_cfr_part(self, amendment):
"""While processing an amendment, if it refers to a CFR part which
hasn't been seen before, we need to perform some accounting, fetching
related meta data, etc."""
part = amendment["cfr_part"]
if part not in self.version_info:
logger.warning("No version info for %s", part)
else:
meta = utils.regulation_meta(part, self.version_info[part]["right"])
flat_toc = fetch_toc(part, self.version_info[part]["right"], flatten=True)
self.section_titles = {elt["index"][1]: elt["title"] for elt in flat_toc if len(elt["index"]) == 2}
self.current_part = ToCPart(
title=meta.get("cfr_title_number"),
part=part,
name=meta.get("statutory_name"),
sections=[],
authority_url=reverse("cfr_changes", kwargs={"doc_number": self.doc_number, "section": part}),
)
self.current_section = None
self.toc.append(self.current_part)
def get_context_data(self, **kwargs):
# We don't want to run the content data of PartialView -- it assumes
# we will be applying layers
context = super(PartialView, self).get_context_data(**kwargs)
label_id = context['label_id']
versions = Versions(context['version'], context['newer_version'],
self.request.GET.get('from_version'))
tree = generator.get_tree_paragraph(label_id, versions.older)
if tree is None:
# TODO We need a more complicated check here to see if the diffs
# add the requested section. If not -> 404
tree = {}
diff_applier = generator.get_diff_applier(
label_id, versions.older, versions.newer)
if diff_applier is None:
raise error_handling.MissingContentException()
layers = list(generator.diff_layers(versions, label_id))
builder = CFRHTMLBuilder(layers, diff_applier)
builder.tree = tree
builder.generate_html()
child_of_root = builder.tree
if builder.tree['node_type'] == REGTEXT:
child_of_root = {
'node_type': EMPTYPART,
'children': [builder.tree]}
context['tree'] = {'children': [child_of_root]}
context['markup_page_type'] = 'diff'
regpart = label_id.split('-')[0]
old_toc = fetch_toc(regpart, versions.older)
diff = generator.get_diff_json(regpart, versions.older, versions.newer)
context['TOC'] = diff_toc(versions, old_toc, diff)
context['navigation'] = self.footer_nav(
label_id, context['TOC'], versions)
return context
def set_chrome_context(self, context, reg_part, version):
context['reg_part'] = reg_part
context['history'] = fetch_grouped_history(reg_part)
toc = fetch_toc(reg_part, version)
for el in toc:
el['url'] = SectionUrl().of(el['index'], version,
self.partial_class.sectional_links)
for sub in el.get('sub_toc', []):
sub['url'] = SectionUrl().of(
sub['index'], version, self.partial_class.sectional_links)
context['TOC'] = toc
context['meta'] = utils.regulation_meta(reg_part, version)
# Throw 404 if regulation doesn't exist
if not context['meta']:
raise error_handling.MissingContentException()
context['version_span'] = version_span(
context['history'], context['meta']['effective_date'])
context['version_switch_view'] = self.version_switch_view
context['diff_redirect_label'] = self.diff_redirect_label(
context['label_id'], toc)
def set_chrome_context(self, context, reg_part, version):
context['reg_part'] = reg_part
context['history'] = fetch_grouped_history(reg_part)
toc = fetch_toc(reg_part, version)
for el in toc:
el['url'] = SectionUrl().of(
el['index'], version, self.partial_class.sectional_links)
for sub in el.get('sub_toc', []):
sub['url'] = SectionUrl().of(
sub['index'], version, self.partial_class.sectional_links)
context['TOC'] = toc
context['meta'] = utils.regulation_meta(reg_part, version)
# Throw 404 if regulation doesn't exist
if not context['meta']:
raise error_handling.MissingContentException()
context['version_span'] = version_span(
context['history'], context['meta']['effective_date'])
context['version_switch_view'] = self.version_switch_view
context['diff_redirect_label'] = self.diff_redirect_label(
context['label_id'], toc)
def test_fetch_toc_404(self, api_reader):
"""Should not crash if there is no TOC data"""
api_reader.ApiReader.return_value.layer.return_value = None
self.assertEqual([], toc.fetch_toc('111', 'vvv'))
self.assertEqual([], toc.fetch_toc('111', 'vvv', True))
def test_fetch_toc_404(self, api_reader):
"""Should not crash if there is no TOC data"""
api_reader.ApiReader.return_value.layer.return_value = None
self.assertEqual([], toc.fetch_toc('111', 'vvv'))
self.assertEqual([], toc.fetch_toc('111', 'vvv', True))
