def add_trees(self):
"""Add some versions to the index"""
entry.Tree('12', '1000', 'v2').write(Node('v2'))
entry.Tree('12', '1000', 'v3').write(Node('v3'))
entry.Tree('12', '1000', 'v4').write(Node('v4'))
entry.Tree('11', '1000', 'v5').write(Node('v5'))
entry.Tree('12', '1001', 'v6').write(Node('v6'))
def test_process(monkeypatch):
"""Verify that the correct changes are found"""
compile_regulation = Mock(return_value=Node())
monkeypatch.setattr(fill_with_rules, 'compile_regulation',
compile_regulation)
notice_mock = Mock()
# entry.Notice('new').read().amendments
notice_mock.return_value.read.return_value.amendments = [
{"instruction": "Something something",
"cfr_part": "1000",
"authority": "USC Numbers"},
{"instruction": "More things",
"cfr_part": "1000",
"changes": [["1000-2-b", ["2b changes"]],
["1000-2-c", ["2c changes"]]]},
{"instruction": "Yet more changes",
"cfr_part": "1000",
"changes": [["1000-4-a", ["4a changes"]]]}
]
monkeypatch.setattr(fill_with_rules.entry, 'Notice', notice_mock)
tree_dir = entry.Tree('12', '1000')
(tree_dir / 'old').write(Node())
entry.Entry('notice_xml', 'new').write(b'')
fill_with_rules.process(tree_dir, 'old', 'new')
changes = dict(compile_regulation.call_args[0][1])
assert changes == {"1000-2-b": ["2b changes"],
"1000-2-c": ["2c changes"],
"1000-4-a": ["4a changes"]}
def test_dependencies():
"""Expect nonexistent trees to depend on their predecessor, associated
rule changes and version files. Shouldn't add dependencies for the
first version, if missing"""
versions = [Version(str(i)*3, date(2001, i, i), date(2002, i, i))
for i in range(1, 7)]
parents = Version.parents_of(versions)
tree_dir = entry.Tree('12', '1000')
notice_dir = entry.Notice()
vers_dir = entry.Version('12', '1000')
# Existing trees
(tree_dir / '222').write(Node())
(tree_dir / '555').write(Node())
deps = fill_with_rules.dependencies(tree_dir, vers_dir,
list(zip(versions, parents)))
# First is skipped, as we can't build it from a rule
assert str(tree_dir / '111') not in deps
# Second can also be skipped as a tree already exists
assert deps.dependencies(str(tree_dir / '222')) == []
# Third relies on the associated versions and the second tree
expected = {str(tree_dir / '222'), str(notice_dir / '333'),
str(vers_dir / '333')}
assert set(deps.dependencies(str(tree_dir / '333'))) == expected
# Fourth relies on the third, even though it's not been built
expected = {str(tree_dir / '333'), str(notice_dir / '444'),
str(vers_dir / '444')}
assert set(deps.dependencies(str(tree_dir / '444'))) == expected
# Fifth can be skipped as the tree already exists
assert deps.dependencies(str(tree_dir / '555')) == []
# Six relies on the fifth
expected = {str(tree_dir / '555'), str(notice_dir / '666'),
str(vers_dir / '666')}
assert set(deps.dependencies(str(tree_dir / '666'))) == expected
def test_stale_layers(self):
"""We should have dependencies between all of the layers and their
associated trees. We should also tie the meta layer to the version"""
configured_layers = {'cfr': {'keyterms': None, 'other': None}}
with self.cli.isolated_filesystem(), patch.dict(
layers.LAYER_CLASSES, configured_layers):
version_entry = entry.Version(111, 22, 'aaa')
version_entry.write(Version('aaa', date.today(), date.today()))
tree_entry = entry.Tree(111, 22, 'aaa')
# Use list() to instantiate
self.assertRaises(dependency.Missing, list,
layers.stale_layers(tree_entry, 'cfr'))
entry.Entry('tree', 111, 22, 'bbb').write(b'') # wrong version
self.assertRaises(dependency.Missing, list,
layers.stale_layers(tree_entry, 'cfr'))
entry.Entry('tree', 111, 22, 'aaa').write(b'')
six.assertCountEqual(self, layers.stale_layers(tree_entry, 'cfr'),
['keyterms', 'other'])
self.assertIn(
str(version_entry),
dependency.Graph().dependencies(
str(entry.Layer.cfr(111, 22, 'aaa', 'meta'))))
def integration_setup(self):
self.cli = CliRunner()
with self.cli.isolated_filesystem():
self.tree_dir = entry.Tree('12', '1000')
self.diff_dir = entry.Diff('12', '1000')
(self.tree_dir / 'v1').write(Node(text='V1V1V1', label=['1000']))
(self.tree_dir / 'v2').write(Node(text='V2V2V2', label=['1000']))
yield
def process_cfr_layers(stale_names, cfr_title, version_entry):
"""Build all of the stale layers for this version, writing them into the
index. Assumes all dependencies have already been checked"""
tree = entry.Tree(*version_entry.path).read()
version = version_entry.read()
layer_dir = entry.Layer.cfr(*version_entry.path)
for layer_name in stale_names:
layer_json = LAYER_CLASSES['cfr'][layer_name](
tree, cfr_title=int(cfr_title), version=version).build()
(layer_dir / layer_name).write(layer_json)
def test_process_cfr_layers():
"""All layers for a single version should get written."""
version_entry = entry.Version(12, 1000, '1234')
version_entry.write(Version('1234', date.today(), Citation(1, 1)))
entry.Tree('12', '1000', '1234').write(Node())
layers.process_cfr_layers(['keyterms', 'meta'], 12, version_entry)
assert entry.Layer.cfr(12, 1000, '1234', 'keyterms').exists()
assert entry.Layer.cfr(12, 1000, '1234', 'meta').exists()
def test_process_cfr_layers(self):
"""All layers for a single version should get written."""
with self.cli.isolated_filesystem():
version_entry = entry.Version(12, 1000, '1234')
version_entry.write(Version('1234', date.today(), date.today()))
entry.Tree('12', '1000', '1234').write(Node())
layers.process_cfr_layers(['keyterms', 'meta'], 12, version_entry)
self.assertTrue(
entry.Layer.cfr(12, 1000, '1234', 'keyterms').exists())
self.assertTrue(entry.Layer.cfr(12, 1000, '1234', 'meta').exists())
def is_stale(cfr_title, cfr_part, version_id):
"""Modify and process dependency graph related to a single SxS layer"""
deps = dependency.Graph()
layer_entry = entry.Layer(cfr_title, cfr_part, version_id, 'analyses')
# Layers depend on their associated tree
deps.add(layer_entry, entry.Tree(cfr_title, cfr_part, version_id))
# And on all notices which came before
for sxs_entry in previous_sxs(cfr_title, cfr_part, version_id):
deps.add(layer_entry, sxs_entry)
deps.validate_for(layer_entry)
return deps.is_stale(layer_entry)
def sxs_layers(cfr_title, cfr_part):
"""Build SxS layers for all known versions."""
logger.info("Build SxS layers - %s CFR %s", cfr_title, cfr_part)
for tree_entry in entry.Tree(cfr_title, cfr_part).sub_entries():
version_id = tree_entry.path[-1]
if is_stale(cfr_title, cfr_part, version_id):
tree = tree_entry.read()
notices = [sxs.read() for sxs in previous_sxs(
cfr_title, cfr_part, version_id)]
layer_json = SectionBySection(tree, notices).build()
entry.Layer.cfr(cfr_title, cfr_part, version_id, 'analyses').write(
layer_json)
def test_is_derived():
"""Should filter version ids to only those with a dependency on
changes derived from a rule"""
tree_dir = entry.Tree('12', '1000')
deps = dependency.Graph()
deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001))
deps.add(tree_dir / 222, entry.Notice(222))
deps.add(tree_dir / 333, entry.Notice(333))
deps.add(tree_dir / 333, entry.Version(333))
assert not fill_with_rules.is_derived('111', deps, tree_dir)
assert fill_with_rules.is_derived('222', deps, tree_dir)
assert fill_with_rules.is_derived('333', deps, tree_dir)
assert not fill_with_rules.is_derived('444', deps, tree_dir)
def process_tree_if_needed(cfr_title, cfr_part, version_id):
"""Creates and writes a regulation tree if the appropriate notice
exists"""
notice_entry = entry.Notice(version_id)
tree_entry = entry.Tree(cfr_title, cfr_part, version_id)
deps = dependency.Graph()
deps.add(tree_entry, notice_entry)
deps.validate_for(tree_entry)
if deps.is_stale(tree_entry):
notice_xml = notice_entry.read()
tree = build_tree(regtext_for_part(notice_xml, cfr_title, cfr_part))
tree_entry.write(tree)
def test_derived_from_rules(self):
"""Should filter a set of version ids to only those with a dependency
on changes derived from a rule"""
with self.cli.isolated_filesystem():
tree_dir = entry.Tree('12', '1000')
deps = dependency.Graph()
deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001))
deps.add(tree_dir / 222, entry.RuleChanges(222))
deps.add(tree_dir / 333, entry.RuleChanges(333))
deps.add(tree_dir / 333, entry.Version(333))
derived = fill_with_rules.derived_from_rules(
['111', '222', '333', '444'], deps, tree_dir)
self.assertEqual(derived, ['222', '333'])
def test_process_tree_if_needed_success(monkeypatch):
"""If the requirements are present we should call tree-parsing function"""
mock_regtext = Mock(return_value=Node('root'))
monkeypatch.setattr(full_issuance, 'build_tree', mock_regtext)
with XMLBuilder() as ctx:
ctx.REGTEXT(TITLE=1, PART=2)
entry.Notice('vvv').write(NoticeXML(ctx.xml))
full_issuance.process_tree_if_needed('1', '2', 'vvv')
result = entry.Tree('1', '2', 'vvv').read()
assert result.text == 'root'
xml_given = mock_regtext.call_args[0][0]
assert etree.tostring(xml_given) == etree.tostring(ctx.xml[0])
def sxs_layers(cfr_title, cfr_part):
"""Build SxS layers for all known versions."""
logger.info("Build SxS layers - %s CFR %s", cfr_title, cfr_part)
tree_dir = entry.Tree(cfr_title, cfr_part)
for version_id in tree_dir:
if is_stale(cfr_title, cfr_part, version_id):
tree = (tree_dir / version_id).read()
notices = [
sxs.read()
for sxs in previous_sxs(cfr_title, cfr_part, version_id)
]
layer_json = SectionBySection(tree, notices).build()
entry.Layer(cfr_title, cfr_part, version_id,
'analyses').write(layer_json)
def fill_with_rules(cfr_title, cfr_part):
"""Fill in missing trees using data from rules. When a regulation tree
cannot be derived through annual editions, it must be built by parsing the
changes in final rules. This command builds those missing trees"""
logger.info("Fill with rules - %s CFR %s", cfr_title, cfr_part)
tree_path = entry.Tree(cfr_title, cfr_part)
version_ids = list(entry.Version(cfr_title, cfr_part))
deps = dependencies(tree_path, version_ids, cfr_title, cfr_part)
preceeded_by = dict(zip(version_ids[1:], version_ids))
derived = derived_from_rules(version_ids, deps, tree_path)
for version_id in derived:
deps.validate_for(tree_path / version_id)
if deps.is_stale(tree_path / version_id):
process(tree_path, preceeded_by[version_id], version_id)
def test_is_derived(self):
"""Should filter version ids to only those with a dependency on
changes derived from a rule"""
with self.cli.isolated_filesystem():
tree_dir = entry.Tree('12', '1000')
deps = dependency.Graph()
deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001))
deps.add(tree_dir / 222, entry.Notice(222))
deps.add(tree_dir / 333, entry.Notice(333))
deps.add(tree_dir / 333, entry.Version(333))
self.assertFalse(fill_with_rules.is_derived('111', deps, tree_dir))
self.assertTrue(fill_with_rules.is_derived('222', deps, tree_dir))
self.assertTrue(fill_with_rules.is_derived('333', deps, tree_dir))
self.assertFalse(fill_with_rules.is_derived('444', deps, tree_dir))
def layers(cfr_title, cfr_part):
"""Build all layers for all known versions."""
logger.info("Build layers - %s CFR %s", cfr_title, cfr_part)
for tree_entry in utils.relevant_paths(entry.Tree(), cfr_title, cfr_part):
tree_title, tree_part, version_id = tree_entry.path
version_entry = entry.Version(tree_title, tree_part, version_id)
stale = stale_layers(tree_entry, 'cfr')
if stale:
process_cfr_layers(stale, tree_title, version_entry)
if cfr_title is None and cfr_part is None:
for preamble_entry in entry.Preamble().sub_entries():
stale = stale_layers(preamble_entry, 'preamble')
if stale:
process_preamble_layers(stale, preamble_entry)
def process_if_needed(volume, cfr_part):
"""Review dependencies; if they're out of date, parse the annual edition
into a tree and store that"""
version_id = _version_id(volume.year, cfr_part)
annual_entry = entry.Annual(volume.title, cfr_part, volume.year)
tree_entry = entry.Tree(volume.title, cfr_part, version_id)
notice_entry = entry.Notice(version_id)
deps = dependency.Graph()
deps.add(tree_entry, annual_entry)
deps.validate_for(tree_entry)
if deps.is_stale(tree_entry):
tree = xml_parser.reg_text.build_tree(annual_entry.read().xml)
tree_entry.write(tree)
notice_entry.write(
build_fake_notice(version_id, volume.publication_date,
volume.title, cfr_part))
def fill_with_rules(cfr_title, cfr_part):
"""Fill in missing trees using data from rules. When a regulation tree
cannot be derived through annual editions, it must be built by parsing the
changes in final rules. This command builds those missing trees"""
logger.info("Fill with rules - %s CFR %s", cfr_title, cfr_part)
tree_dir = entry.Tree(cfr_title, cfr_part)
version_dir = entry.Version(cfr_title, cfr_part)
versions = [c.read() for c in version_dir.sub_entries()]
versions_with_parents = list(zip(versions, Version.parents_of(versions)))
deps = dependencies(tree_dir, version_dir, versions_with_parents)
derived = [(version.identifier, parent.identifier)
for version, parent in versions_with_parents
if is_derived(version.identifier, deps, tree_dir)]
for version_id, parent_id in derived:
deps.validate_for(tree_dir / version_id)
if deps.is_stale(tree_dir / version_id):
process(tree_dir, parent_id, version_id)
deps.rebuild()
def process_if_needed(cfr_title, cfr_part, last_version_list):
"""Calculate dependencies between input and output files for these annual
editions. If an output is missing or out of date, process it"""
annual_path = entry.Annual(cfr_title, cfr_part)
tree_path = entry.Tree(cfr_title, cfr_part)
version_path = entry.Version(cfr_title, cfr_part)
deps = dependency.Graph()
for last_version in last_version_list:
deps.add(tree_path / last_version.version_id,
version_path / last_version.version_id)
deps.add(tree_path / last_version.version_id,
annual_path / last_version.year)
for last_version in last_version_list:
tree_entry = tree_path / last_version.version_id
deps.validate_for(tree_entry)
if deps.is_stale(tree_entry):
input_entry = annual_path / last_version.year
tree = gpo_cfr.builder.build_tree(input_entry.read().xml)
tree_entry.write(tree)
def test_dependencies(self):
"""Expect nonexistent trees to depend on their predecessor, associated
rule changes and version files. Shouldn't add dependencies for the
first version, if missing"""
with self.cli.isolated_filesystem():
version_ids = ['111', '222', '333', '444', '555', '666']
tree_dir = entry.Tree('12', '1000')
rule_dir = entry.RuleChanges()
vers_dir = entry.Version('12', '1000')
# Existing trees
(tree_dir / '222').write(Node())
(tree_dir / '555').write(Node())
deps = fill_with_rules.dependencies(tree_dir, version_ids, '12',
'1000')
# First is skipped, as we can't build it from a rule
self.assertNotIn(str(tree_dir / '111'), deps)
# Second can also be skipped as a tree already exists
self.assertEqual(deps.dependencies(str(tree_dir / '222')), [])
# Third relies on the associated versions and the second tree
self.assertItemsEqual(deps.dependencies(str(tree_dir / '333')), [
str(tree_dir / '222'),
str(rule_dir / '333'),
str(vers_dir / '333')
])
# Fourth relies on the third, even though it's not been built
self.assertItemsEqual(deps.dependencies(str(tree_dir / '444')), [
str(tree_dir / '333'),
str(rule_dir / '444'),
str(vers_dir / '444')
])
# Fifth can be skipped as the tree already exists
self.assertEqual(deps.dependencies(str(tree_dir / '555')), [])
# Six relies on the fifth
self.assertItemsEqual(deps.dependencies(str(tree_dir / '666')), [
str(tree_dir / '555'),
str(rule_dir / '666'),
str(vers_dir / '666')
])
def test_process(self, Notice, compile_regulation):
"""Verify that the correct changes are found"""
compile_regulation.return_value = Node()
# entry.Notice('new').read().amendments
Notice.return_value.read.return_value.amendments = [{
"instruction":
"Something something",
"cfr_part":
"1000",
"authority":
"USC Numbers"
}, {
"instruction":
"More things",
"cfr_part":
"1000",
"changes": [["1000-2-b", ["2b changes"]],
["1000-2-c", ["2c changes"]]]
}, {
"instruction":
"Yet more changes",
"cfr_part":
"1000",
"changes": [["1000-4-a", ["4a changes"]]]
}]
with self.cli.isolated_filesystem():
tree_dir = entry.Tree('12', '1000')
(tree_dir / 'old').write(Node())
entry.Entry('notice_xml', 'new').write(b'')
fill_with_rules.process(tree_dir, 'old', 'new')
changes = dict(compile_regulation.call_args[0][1])
self.assertEqual(
changes, {
"1000-2-b": ["2b changes"],
"1000-2-c": ["2c changes"],
"1000-4-a": ["4a changes"]
})
def test_stale_layers(monkeypatch):
"""We should have dependencies between all of the layers and their
associated trees. We should also tie the meta layer to the version"""
monkeypatch.setattr(layers, 'LAYER_CLASSES',
{'cfr': {
'keyterms': None,
'other': None
}})
version_entry = entry.Version(111, 22, 'aaa')
version_entry.write(Version('aaa', date.today(), Citation(1, 1)))
tree_entry = entry.Tree(111, 22, 'aaa')
with pytest.raises(dependency.Missing):
layers.stale_layers(tree_entry, 'cfr')
entry.Entry('tree', 111, 22, 'bbb').write(b'') # wrong version
with pytest.raises(dependency.Missing):
layers.stale_layers(tree_entry, 'cfr')
entry.Entry('tree', 111, 22, 'aaa').write(b'')
assert set(layers.stale_layers(tree_entry, 'cfr')) == {'keyterms', 'other'}
assert str(version_entry) in dependency.Graph().dependencies(
str(entry.Layer.cfr(111, 22, 'aaa', 'meta')))
def write_trees(client, only_title, only_part):
for tree_entry in utils.relevant_paths(entry.Tree(), only_title,
only_part):
_, cfr_part, version_id = tree_entry.path
content = tree_entry.read()
client.regulation(cfr_part, version_id).write(content)
