def test_compile_designate(self):
root = self.tree_with_subparts()
change = {
'action': 'POST',
'subpart': ['205', 'Subpart', 'B'],
'node': {
'text': '2 text',
'label': ['205', '2'],
'node_type': 'regtext'}}
notice_changes = {'205-2': [change]}
reg = compiler.compile_regulation(root, notice_changes)
subpart_b = find(reg, '205-Subpart-B')
self.assertEqual(len(subpart_b.children), 1)
subpart_a = find(reg, '205-Subpart-A')
self.assertEqual(len(subpart_a.children), 0)
change = {
'action': 'DESIGNATE',
'destination': ['205', 'Subpart', 'A']}
notice_changes = {'205-2': [change]}
new_reg = compiler.compile_regulation(reg, notice_changes)
self.assertEqual(None, find(new_reg, '205-Subpart-B'))
subpart_a = find(new_reg, '205-Subpart-A')
self.assertEqual(len(subpart_a.children), 1)
def test_compile_designate(self):
root = self.tree_with_subparts()
change = {
"action": "POST",
"subpart": ["205", "Subpart", "B"],
"node": Node(text="2 text", label=["205", "2"], node_type=Node.REGTEXT),
}
notice_changes = {"205-2": [change]}
reg = compiler.compile_regulation(root, notice_changes)
subpart_b = find(reg, "205-Subpart-B")
self.assertEqual(len(subpart_b.children), 1)
subpart_a = find(reg, "205-Subpart-A")
self.assertEqual(len(subpart_a.children), 0)
change = {"action": "DESIGNATE", "destination": ["205", "Subpart", "A"]}
notice_changes = {"205-2": [change]}
new_reg = compiler.compile_regulation(reg, notice_changes)
self.assertEqual(None, find(new_reg, "205-Subpart-B"))
subpart_a = find(new_reg, "205-Subpart-A")
self.assertEqual(len(subpart_a.children), 1)
def test_compile_regulation_delete_move(self):
prev_tree = self.tree_with_paragraphs()
changes = {
"205-2-a": [
{"action": "MOVE", "destination": ["205", "2", "b"]},
{"action": "POST", "node": Node(text="aaa", label=["205", "2", "a"], node_type=Node.REGTEXT)},
],
"205-2-b": [{"action": "DELETE"}],
}
class SortedKeysDict(object):
def keys(self):
return list(sorted(changes.keys()))
def __getitem__(self, key):
return changes[key]
def __contains__(self, key):
if key in changes:
return True
else:
return False
new_tree = compiler.compile_regulation(prev_tree, SortedKeysDict())
s1, s2, s4 = new_tree.children
self.assertEqual(2, len(s2.children))
s2a, s2b = s2.children
self.assertEqual("aaa", s2a.text)
self.assertEqual("n2a", s2b.text)
def test_compile_regulation_delete_move(self):
prev_tree = self.tree_with_paragraphs()
print prev_tree.label[0]
changes = {
'205-2-a': [
{'action': 'MOVE', 'destination': ['205', '2', 'b']},
{'action': 'POST', 'node': {'text': 'aaa',
'label': ['205', '2', 'a'],
'node_type': Node.REGTEXT}}],
'205-2-b': [{'action': 'DELETE'}]}
class SortedKeysDict(object):
def keys(self):
return list(sorted(changes.keys()))
def __getitem__(self, key):
return changes[key]
def __contains__(self, key):
if key in changes:
return True
else:
return False
new_tree = compiler.compile_regulation(prev_tree, SortedKeysDict())
print new_tree
s1, s2, s4 = new_tree.children
self.assertEqual(2, len(s2.children))
s2a, s2b = s2.children
self.assertEqual("aaa", s2a.text)
self.assertEqual("n2a", s2b.text)
def test_compile_reg_put_replace_whole_tree(self):
root = self.tree_with_paragraphs()
change2a = {
'action': 'PUT',
'node': {
'text': 'new text',
'label': ['205', '2', 'a'],
'node_type': 'regtext'}}
change2a1 = {
'action': 'PUT',
'node': {
'text': '2a1 text',
'label': ['205', '2', 'a', '1'],
'node_type': 'regtext'}}
notice_changes = {
'205-2-a-1': [change2a1],
'205-2-a': [change2a]
}
reg = compiler.compile_regulation(root, notice_changes)
added_node = find(reg, '205-2-a')
self.assertEqual(added_node.text, 'new text')
deeper = find(reg, '205-2-a-1')
self.assertEqual(deeper.text, '2a1 text')
def ecfr_notice(title, cfr_part, notice, applies_to, act_title,
act_section, with_version=False, without_notice=False):
""" Generate RegML for a single notice from eCFR XML. """
# Get the notice the new one applies to
with open(find_file(os.path.join(cfr_part, applies_to)), 'r') as f:
reg_xml = f.read()
parser = etree.XMLParser(huge_tree=True)
xml_tree = etree.fromstring(reg_xml, parser)
doc_number = xml_tree.find('.//{eregs}documentNumber').text
# Validate the file relative to schema
validator = get_validator(xml_tree)
# Get the ecfr builder
builder = Builder(cfr_title=title,
cfr_part=cfr_part,
doc_number=doc_number,
checkpointer=None,
writer_type='XML')
# Fetch the notices from the FR API and find the notice we're
# looking for
builder.fetch_notices_json()
print([n['document_number'] for n in builder.notices_json])
notice_json = next((n for n in builder.notices_json
if n['document_number'] == notice))
# Build the notice
notice = builder.build_single_notice(notice_json)[0]
if 'changes' not in notice:
print('There are no changes in this notice to apply.')
return
# We've successfully fetched and parsed the new notice.
# Build a the reg tree and layers for the notice it applies to.
old_tree = build_reg_tree(xml_tree)
# Build the new reg tree from the old_tree + notice changes
last_version = doc_number
version = notice['document_number']
merged_changes = builder.merge_changes(version, notice['changes'])
reg_tree = compile_regulation(old_tree, merged_changes)
layer_cache = LayerCacheAggregator()
layers = builder.generate_layers(reg_tree,
[act_title, act_section],
layer_cache)
# Write the notice file
if not without_notice:
builder.write_notice(version,
old_tree=old_tree,
reg_tree=reg_tree,
layers=layers,
last_version=last_version)
# Write the regulation file for the new notice
if with_version:
builder.write_regulation(new_tree, layers=layers)
def process(tree_path, previous, version_id):
"""Build and write a tree by combining the preceding tree with changes
present in the associated rule"""
prev_tree = (tree_path / previous).read()
notice = entry.RuleChanges(version_id).read()
changes = merge_changes(version_id, notice.get('changes', {}))
new_tree = compile_regulation(prev_tree, changes)
(tree_path / version_id).write(new_tree)
def test_compile_reg_move_wrong_reg(self):
"""Changes applied to other regulations shouldn't affect the
regulation we care about, even if that has the same textual prefix"""
root = self.tree_with_paragraphs()
notice_changes = {"2055-2-a": [{"action": "MOVE", "destination": ["2055", "2", "b"]}]}
reg = compiler.compile_regulation(root, notice_changes)
self.assertEqual(find(reg, "205-2-a").text, "n2a")
self.assertEqual(find(reg, "205-2-b").text, "n2b")
self.assertEqual(find(reg, "2055-2-b"), None)
def test_compile_reg_move_wrong_reg(self):
"""Changes applied to other regulations shouldn't affect the
regulation we care about, even if that has the same textual prefix"""
root = self.tree_with_paragraphs()
notice_changes = {'2055-2-a': [{'action': 'MOVE',
'destination': ['2055', '2', 'b']}]}
reg = compiler.compile_regulation(root, notice_changes)
self.assertEqual(find(reg, '205-2-a').text, 'n2a')
self.assertEqual(find(reg, '205-2-b').text, 'n2b')
self.assertEqual(find(reg, '2055-2-b'), None)
def process(tree_path, previous, version_id):
"""Build and write a tree by combining the preceding tree with changes
present in the associated rule"""
prev_tree = (tree_path / previous).read()
notice = entry.Notice(version_id).read()
notice_changes = defaultdict(list)
for amendment in notice.amendments:
for label, change_list in amendment.get('changes', []):
notice_changes[label].extend(change_list)
new_tree = compile_regulation(prev_tree, notice_changes)
(tree_path / version_id).write(new_tree)
def test_compile_reg_post_no_subpart(self):
root = self.tree_with_paragraphs()
change2a1 = {
"action": "POST",
"node": Node(text="2a1 text", label=["205", "2", "a", "1"], node_type=Node.REGTEXT),
}
notice_changes = {"205-2-a-1": [change2a1]}
reg = compiler.compile_regulation(root, notice_changes)
added_node = find(reg, "205-2-a-1")
self.assertNotEqual(None, added_node)
self.assertEqual(added_node.text, "2a1 text")
def test_compile_reg_put_text_only(self):
root = self.tree_with_paragraphs()
change2a = {
"action": "PUT",
"field": "[text]",
"node": Node(text="new text", label=["205", "2", "a"], node_type=Node.REGTEXT),
}
notice_changes = {"205-2-a": [change2a]}
reg = compiler.compile_regulation(root, notice_changes)
changed_node = find(reg, "205-2-a")
self.assertEqual(changed_node.text, "new text")
def revision_generator(self, reg_tree):
relevant_notices = []
for date in sorted(self.eff_notices.keys()):
relevant_notices.extend(
n for n in self.eff_notices[date]
if 'changes' in n and n['document_number'] != self.doc_number)
for notice in relevant_notices:
version = notice['document_number']
old_tree = reg_tree
merged_changes = self.merge_changes(version, notice['changes'])
reg_tree = compile_regulation(old_tree, merged_changes)
notices = applicable_notices(self.notices, version)
yield notice, old_tree, reg_tree, notices
def test_compile_add_to_subpart(self):
root = self.tree_with_subparts()
change = {
"action": "POST",
"subpart": ["205", "Subpart", "B"],
"node": Node(text="2 text", label=["205", "2"], node_type=Node.REGTEXT),
}
notice_changes = {"205-2": [change]}
reg = compiler.compile_regulation(root, notice_changes)
added_node = find(reg, "205-2")
self.assertNotEqual(None, added_node)
self.assertEqual(added_node.text, "2 text")
def test_compile_reg_post_no_subpart(self):
root = self.tree_with_paragraphs()
change2a1 = {
'action': 'POST',
'node': {
'text': '2a1 text',
'label': ['205', '2', 'a', '1'],
'node_type': 'regtext'}}
notice_changes = {'205-2-a-1': [change2a1]}
reg = compiler.compile_regulation(root, notice_changes)
added_node = find(reg, '205-2-a-1')
self.assertNotEqual(None, added_node)
self.assertEqual(added_node.text, '2a1 text')
def test_compile_reg_post_no_subpart(self):
root = self.tree_with_paragraphs()
change2a1 = {
'action': 'POST',
'node': {
'text': '2a1 text',
'label': ['205', '2', 'a', '1'],
'node_type': 'regtext'}}
notice_changes = {'205-2-a-1': [change2a1]}
reg = compiler.compile_regulation(root, notice_changes)
added_node = find(reg, '205-2-a-1')
self.assertNotEqual(None, added_node)
self.assertEqual(added_node.text, '2a1 text')
def revision_generator(self, reg_tree):
"""Given an initial regulation tree, this will emit (and checkpoint)
new versions of the tree, along with the notice that caused the
change. This is a generator, so processing only occurs as needed"""
for version, merged_changes in self.changes_in_sequence():
old_tree = reg_tree
reg_tree = self.checkpointer.checkpoint(
"compiled-" + version,
lambda: compile_regulation(old_tree, merged_changes))
notices = applicable_notices(self.notices, version)
first_notice = None
for notice in notices:
if notice['document_number'] == version:
first_notice = notice
yield first_notice, old_tree, reg_tree, notices
def test_compile_reg_put_text_only(self):
root = self.tree_with_paragraphs()
change2a = {
'action': 'PUT',
'field': '[text]',
'node': {
'text': 'new text',
'label': ['205', '2', 'a'],
'node_type': 'regtext'}}
notice_changes = {'205-2-a': [change2a]}
reg = compiler.compile_regulation(root, notice_changes)
changed_node = find(reg, '205-2-a')
self.assertEqual(changed_node.text, 'new text')
def test_compile_reg_put_text_only(self):
root = self.tree_with_paragraphs()
change2a = {
'action': 'PUT',
'field': '[text]',
'node': {
'text': 'new text',
'label': ['205', '2', 'a'],
'node_type': 'regtext'}}
notice_changes = {'205-2-a': [change2a]}
reg = compiler.compile_regulation(root, notice_changes)
changed_node = find(reg, '205-2-a')
self.assertEqual(changed_node.text, 'new text')
def revision_generator(self, reg_tree):
"""Given an initial regulation tree, this will emit (and checkpoint)
new versions of the tree, along with the notice that caused the
change. This is a generator, so processing only occurs as needed"""
for version, merged_changes in self.changes_in_sequence():
old_tree = reg_tree
reg_tree = self.checkpointer.checkpoint(
"compiled-" + version,
lambda: compile_regulation(old_tree, merged_changes))
notices = applicable_notices(self.notices, version)
first_notice = None
for notice in notices:
if notice['document_number'] == version:
first_notice = notice
yield first_notice, old_tree, reg_tree, notices
def test_compile_reg_keep_child(self):
root = self.tree_with_paragraphs()
change2 = {"action": "PUT", "node": Node(text="n2n2", label=["205", "2"], node_type=Node.REGTEXT)}
change2a = {"action": "KEEP", "node": Node(text="(a) * * *", label=["205", "2", "a"], node_type=Node.REGTEXT)}
change2b = {"action": "PUT", "node": Node(text="(b) A Test", label=["205", "2", "b"], node_type=Node.REGTEXT)}
notice_changes = {"205-2": [change2], "205-2-a": [change2a], "205-2-b": [change2b]}
reg = compiler.compile_regulation(root, notice_changes)
changed = find(reg, "205-2")
self.assertEqual(changed.text, "n2n2")
self.assertEqual(2, len(changed.children))
changed2a, changed2b = changed.children
self.assertEqual("n2a", changed2a.text) # text didn't change
self.assertEqual("(b) A Test", changed2b.text)
def test_compile_add_to_subpart(self):
root = self.tree_with_subparts()
change = {
'action': 'POST',
'subpart': ['205', 'Subpart', 'B'],
'node': {
'text': '2 text',
'label': ['205', '2'],
'node_type': 'regtext'}}
notice_changes = {'205-2': [change]}
reg = compiler.compile_regulation(root, notice_changes)
added_node = find(reg, '205-2')
self.assertNotEqual(None, added_node)
self.assertEqual(added_node.text, '2 text')
def test_compile_add_to_subpart(self):
root = self.tree_with_subparts()
change = {
'action': 'POST',
'subpart': ['205', 'Subpart', 'B'],
'node': {
'text': '2 text',
'label': ['205', '2'],
'node_type': 'regtext'}}
notice_changes = {'205-2': [change]}
reg = compiler.compile_regulation(root, notice_changes)
added_node = find(reg, '205-2')
self.assertNotEqual(None, added_node)
self.assertEqual(added_node.text, '2 text')
def test_compile_reg_put_replace_whole_tree(self):
root = self.tree_with_paragraphs()
change2a = {"action": "PUT", "node": Node(text="new text", label=["205", "2", "a"], node_type=Node.REGTEXT)}
change2a1 = {
"action": "PUT",
"node": Node(text="2a1 text", label=["205", "2", "a", "1"], node_type=Node.REGTEXT),
}
notice_changes = {"205-2-a-1": [change2a1], "205-2-a": [change2a]}
reg = compiler.compile_regulation(root, notice_changes)
added_node = find(reg, "205-2-a")
self.assertEqual(added_node.text, "new text")
deeper = find(reg, "205-2-a-1")
self.assertEqual(deeper.text, "2a1 text")
def test_compile_regulation_delete_move(self):
prev_tree = self.tree_with_paragraphs()
print prev_tree.label[0]
changes = {
'205-2-a': [{
'action': 'MOVE',
'destination': ['205', '2', 'b']
}, {
'action': 'POST',
'node': {
'text': 'aaa',
'label': ['205', '2', 'a'],
'node_type': Node.REGTEXT
}
}],
'205-2-b': [{
'action': 'DELETE'
}]
}
class SortedKeysDict(object):
def keys(self):
return list(sorted(changes.keys()))
def __getitem__(self, key):
return changes[key]
def __contains__(self, key):
if key in changes:
return True
else:
return False
new_tree = compiler.compile_regulation(prev_tree, SortedKeysDict())
print new_tree
s1, s2, s4 = new_tree.children
self.assertEqual(2, len(s2.children))
s2a, s2b = s2.children
self.assertEqual("aaa", s2a.text)
self.assertEqual("n2a", s2b.text)
def test_compile_reg_put_children_only(self):
root = self.tree_with_paragraphs()
change2 = {'action': 'PUT',
'field': '[children]',
'node': {'text': '* * *', 'label': ['205', '2'],
'node_type': 'regtext'}}
change2a = {'action': 'PUT',
'node': {'text': '(a) A Test', 'label': ['205', '2', 'a'],
'node_type': 'regtext'}}
notice_changes = {'205-2': [change2], '205-2-a': [change2a]}
reg = compiler.compile_regulation(root, notice_changes)
changed = find(reg, '205-2')
self.assertEqual(changed.text, 'n2') # text didn't change
self.assertEqual(1, len(changed.children))
changed2a = changed.children[0]
self.assertEqual(['205', '2', 'a'], changed2a.label)
self.assertEqual('(a) A Test', changed2a.text)
self.assertEqual([], changed2a.children)
def test_compile_reg_keep_root(self):
root = self.tree_with_paragraphs()
change2 = {'action': 'KEEP',
'node': {'text': '* * *', 'label': ['205', '2'],
'node_type': 'regtext'}}
change2a = {'action': 'PUT',
'node': {'text': '(a) A Test', 'label': ['205', '2', 'a'],
'node_type': 'regtext'}}
notice_changes = {'205-2': [change2], '205-2-a': [change2a]}
reg = compiler.compile_regulation(root, notice_changes)
changed = find(reg, '205-2')
self.assertEqual(changed.text, 'n2') # text didn't change
self.assertEqual(2, len(changed.children))
changed2a, changed2b = changed.children
self.assertEqual(['205', '2', 'a'], changed2a.label)
self.assertEqual('(a) A Test', changed2a.text)
self.assertEqual([], changed2a.children)
self.assertEqual(['205', '2', 'b'], changed2b.label)
def test_compile_reg_keep_child(self):
root = self.tree_with_paragraphs()
change2 = {'action': 'PUT',
'node': {'text': 'n2n2', 'label': ['205', '2'],
'node_type': 'regtext'}}
change2a = {'action': 'KEEP',
'node': {'text': '(a) * * *', 'label': ['205', '2', 'a'],
'node_type': 'regtext'}}
change2b = {'action': 'PUT',
'node': {'text': '(b) A Test', 'label': ['205', '2', 'b'],
'node_type': 'regtext'}}
notice_changes = {'205-2': [change2], '205-2-a': [change2a],
'205-2-b': [change2b]}
reg = compiler.compile_regulation(root, notice_changes)
changed = find(reg, '205-2')
self.assertEqual(changed.text, 'n2n2')
self.assertEqual(2, len(changed.children))
changed2a, changed2b = changed.children
self.assertEqual('n2a', changed2a.text) # text didn't change
self.assertEqual('(b) A Test', changed2b.text)
def test_compile_reg_move_wrong_reg(self):
root = self.tree_with_paragraphs()
notice_changes = {'202-2-a': [{'action': 'MOVE',
'destination': ['202', '2', 'b']}]}
reg = compiler.compile_regulation(root, notice_changes)
self.assertEqual(find(reg, '205-2-a').text, 'n2a')
def test_compile_reg_move_wrong_reg(self):
root = self.tree_with_paragraphs()
notice_changes = {'202-2-a': [{'action': 'MOVE',
'destination': ['202', '2', 'b']}]}
reg = compiler.compile_regulation(root, notice_changes)
self.assertEqual(find(reg, '205-2-a').text, 'n2a')