def add_trees(self): """Add some versions to the index""" entry.Tree('12', '1000', 'v2').write(Node('v2')) entry.Tree('12', '1000', 'v3').write(Node('v3')) entry.Tree('12', '1000', 'v4').write(Node('v4')) entry.Tree('11', '1000', 'v5').write(Node('v5')) entry.Tree('12', '1001', 'v6').write(Node('v6'))
def test_process(monkeypatch): """Verify that the correct changes are found""" compile_regulation = Mock(return_value=Node()) monkeypatch.setattr(fill_with_rules, 'compile_regulation', compile_regulation) notice_mock = Mock() # entry.Notice('new').read().amendments notice_mock.return_value.read.return_value.amendments = [ {"instruction": "Something something", "cfr_part": "1000", "authority": "USC Numbers"}, {"instruction": "More things", "cfr_part": "1000", "changes": [["1000-2-b", ["2b changes"]], ["1000-2-c", ["2c changes"]]]}, {"instruction": "Yet more changes", "cfr_part": "1000", "changes": [["1000-4-a", ["4a changes"]]]} ] monkeypatch.setattr(fill_with_rules.entry, 'Notice', notice_mock) tree_dir = entry.Tree('12', '1000') (tree_dir / 'old').write(Node()) entry.Entry('notice_xml', 'new').write(b'') fill_with_rules.process(tree_dir, 'old', 'new') changes = dict(compile_regulation.call_args[0][1]) assert changes == {"1000-2-b": ["2b changes"], "1000-2-c": ["2c changes"], "1000-4-a": ["4a changes"]}
def test_dependencies(): """Expect nonexistent trees to depend on their predecessor, associated rule changes and version files. Shouldn't add dependencies for the first version, if missing""" versions = [Version(str(i)*3, date(2001, i, i), date(2002, i, i)) for i in range(1, 7)] parents = Version.parents_of(versions) tree_dir = entry.Tree('12', '1000') notice_dir = entry.Notice() vers_dir = entry.Version('12', '1000') # Existing trees (tree_dir / '222').write(Node()) (tree_dir / '555').write(Node()) deps = fill_with_rules.dependencies(tree_dir, vers_dir, list(zip(versions, parents))) # First is skipped, as we can't build it from a rule assert str(tree_dir / '111') not in deps # Second can also be skipped as a tree already exists assert deps.dependencies(str(tree_dir / '222')) == [] # Third relies on the associated versions and the second tree expected = {str(tree_dir / '222'), str(notice_dir / '333'), str(vers_dir / '333')} assert set(deps.dependencies(str(tree_dir / '333'))) == expected # Fourth relies on the third, even though it's not been built expected = {str(tree_dir / '333'), str(notice_dir / '444'), str(vers_dir / '444')} assert set(deps.dependencies(str(tree_dir / '444'))) == expected # Fifth can be skipped as the tree already exists assert deps.dependencies(str(tree_dir / '555')) == [] # Six relies on the fifth expected = {str(tree_dir / '555'), str(notice_dir / '666'), str(vers_dir / '666')} assert set(deps.dependencies(str(tree_dir / '666'))) == expected
def test_stale_layers(self): """We should have dependencies between all of the layers and their associated trees. We should also tie the meta layer to the version""" configured_layers = {'cfr': {'keyterms': None, 'other': None}} with self.cli.isolated_filesystem(), patch.dict( layers.LAYER_CLASSES, configured_layers): version_entry = entry.Version(111, 22, 'aaa') version_entry.write(Version('aaa', date.today(), date.today())) tree_entry = entry.Tree(111, 22, 'aaa') # Use list() to instantiate self.assertRaises(dependency.Missing, list, layers.stale_layers(tree_entry, 'cfr')) entry.Entry('tree', 111, 22, 'bbb').write(b'') # wrong version self.assertRaises(dependency.Missing, list, layers.stale_layers(tree_entry, 'cfr')) entry.Entry('tree', 111, 22, 'aaa').write(b'') six.assertCountEqual(self, layers.stale_layers(tree_entry, 'cfr'), ['keyterms', 'other']) self.assertIn( str(version_entry), dependency.Graph().dependencies( str(entry.Layer.cfr(111, 22, 'aaa', 'meta'))))
def integration_setup(self): self.cli = CliRunner() with self.cli.isolated_filesystem(): self.tree_dir = entry.Tree('12', '1000') self.diff_dir = entry.Diff('12', '1000') (self.tree_dir / 'v1').write(Node(text='V1V1V1', label=['1000'])) (self.tree_dir / 'v2').write(Node(text='V2V2V2', label=['1000'])) yield
def process_cfr_layers(stale_names, cfr_title, version_entry): """Build all of the stale layers for this version, writing them into the index. Assumes all dependencies have already been checked""" tree = entry.Tree(*version_entry.path).read() version = version_entry.read() layer_dir = entry.Layer.cfr(*version_entry.path) for layer_name in stale_names: layer_json = LAYER_CLASSES['cfr'][layer_name]( tree, cfr_title=int(cfr_title), version=version).build() (layer_dir / layer_name).write(layer_json)
def test_process_cfr_layers(): """All layers for a single version should get written.""" version_entry = entry.Version(12, 1000, '1234') version_entry.write(Version('1234', date.today(), Citation(1, 1))) entry.Tree('12', '1000', '1234').write(Node()) layers.process_cfr_layers(['keyterms', 'meta'], 12, version_entry) assert entry.Layer.cfr(12, 1000, '1234', 'keyterms').exists() assert entry.Layer.cfr(12, 1000, '1234', 'meta').exists()
def test_process_cfr_layers(self): """All layers for a single version should get written.""" with self.cli.isolated_filesystem(): version_entry = entry.Version(12, 1000, '1234') version_entry.write(Version('1234', date.today(), date.today())) entry.Tree('12', '1000', '1234').write(Node()) layers.process_cfr_layers(['keyterms', 'meta'], 12, version_entry) self.assertTrue( entry.Layer.cfr(12, 1000, '1234', 'keyterms').exists()) self.assertTrue(entry.Layer.cfr(12, 1000, '1234', 'meta').exists())
def is_stale(cfr_title, cfr_part, version_id): """Modify and process dependency graph related to a single SxS layer""" deps = dependency.Graph() layer_entry = entry.Layer(cfr_title, cfr_part, version_id, 'analyses') # Layers depend on their associated tree deps.add(layer_entry, entry.Tree(cfr_title, cfr_part, version_id)) # And on all notices which came before for sxs_entry in previous_sxs(cfr_title, cfr_part, version_id): deps.add(layer_entry, sxs_entry) deps.validate_for(layer_entry) return deps.is_stale(layer_entry)
def sxs_layers(cfr_title, cfr_part): """Build SxS layers for all known versions.""" logger.info("Build SxS layers - %s CFR %s", cfr_title, cfr_part) for tree_entry in entry.Tree(cfr_title, cfr_part).sub_entries(): version_id = tree_entry.path[-1] if is_stale(cfr_title, cfr_part, version_id): tree = tree_entry.read() notices = [sxs.read() for sxs in previous_sxs( cfr_title, cfr_part, version_id)] layer_json = SectionBySection(tree, notices).build() entry.Layer.cfr(cfr_title, cfr_part, version_id, 'analyses').write( layer_json)
def test_is_derived(): """Should filter version ids to only those with a dependency on changes derived from a rule""" tree_dir = entry.Tree('12', '1000') deps = dependency.Graph() deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001)) deps.add(tree_dir / 222, entry.Notice(222)) deps.add(tree_dir / 333, entry.Notice(333)) deps.add(tree_dir / 333, entry.Version(333)) assert not fill_with_rules.is_derived('111', deps, tree_dir) assert fill_with_rules.is_derived('222', deps, tree_dir) assert fill_with_rules.is_derived('333', deps, tree_dir) assert not fill_with_rules.is_derived('444', deps, tree_dir)
def process_tree_if_needed(cfr_title, cfr_part, version_id): """Creates and writes a regulation tree if the appropriate notice exists""" notice_entry = entry.Notice(version_id) tree_entry = entry.Tree(cfr_title, cfr_part, version_id) deps = dependency.Graph() deps.add(tree_entry, notice_entry) deps.validate_for(tree_entry) if deps.is_stale(tree_entry): notice_xml = notice_entry.read() tree = build_tree(regtext_for_part(notice_xml, cfr_title, cfr_part)) tree_entry.write(tree)
def test_derived_from_rules(self): """Should filter a set of version ids to only those with a dependency on changes derived from a rule""" with self.cli.isolated_filesystem(): tree_dir = entry.Tree('12', '1000') deps = dependency.Graph() deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001)) deps.add(tree_dir / 222, entry.RuleChanges(222)) deps.add(tree_dir / 333, entry.RuleChanges(333)) deps.add(tree_dir / 333, entry.Version(333)) derived = fill_with_rules.derived_from_rules( ['111', '222', '333', '444'], deps, tree_dir) self.assertEqual(derived, ['222', '333'])
def test_process_tree_if_needed_success(monkeypatch): """If the requirements are present we should call tree-parsing function""" mock_regtext = Mock(return_value=Node('root')) monkeypatch.setattr(full_issuance, 'build_tree', mock_regtext) with XMLBuilder() as ctx: ctx.REGTEXT(TITLE=1, PART=2) entry.Notice('vvv').write(NoticeXML(ctx.xml)) full_issuance.process_tree_if_needed('1', '2', 'vvv') result = entry.Tree('1', '2', 'vvv').read() assert result.text == 'root' xml_given = mock_regtext.call_args[0][0] assert etree.tostring(xml_given) == etree.tostring(ctx.xml[0])
def sxs_layers(cfr_title, cfr_part): """Build SxS layers for all known versions.""" logger.info("Build SxS layers - %s CFR %s", cfr_title, cfr_part) tree_dir = entry.Tree(cfr_title, cfr_part) for version_id in tree_dir: if is_stale(cfr_title, cfr_part, version_id): tree = (tree_dir / version_id).read() notices = [ sxs.read() for sxs in previous_sxs(cfr_title, cfr_part, version_id) ] layer_json = SectionBySection(tree, notices).build() entry.Layer(cfr_title, cfr_part, version_id, 'analyses').write(layer_json)
def fill_with_rules(cfr_title, cfr_part): """Fill in missing trees using data from rules. When a regulation tree cannot be derived through annual editions, it must be built by parsing the changes in final rules. This command builds those missing trees""" logger.info("Fill with rules - %s CFR %s", cfr_title, cfr_part) tree_path = entry.Tree(cfr_title, cfr_part) version_ids = list(entry.Version(cfr_title, cfr_part)) deps = dependencies(tree_path, version_ids, cfr_title, cfr_part) preceeded_by = dict(zip(version_ids[1:], version_ids)) derived = derived_from_rules(version_ids, deps, tree_path) for version_id in derived: deps.validate_for(tree_path / version_id) if deps.is_stale(tree_path / version_id): process(tree_path, preceeded_by[version_id], version_id)
def test_is_derived(self): """Should filter version ids to only those with a dependency on changes derived from a rule""" with self.cli.isolated_filesystem(): tree_dir = entry.Tree('12', '1000') deps = dependency.Graph() deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001)) deps.add(tree_dir / 222, entry.Notice(222)) deps.add(tree_dir / 333, entry.Notice(333)) deps.add(tree_dir / 333, entry.Version(333)) self.assertFalse(fill_with_rules.is_derived('111', deps, tree_dir)) self.assertTrue(fill_with_rules.is_derived('222', deps, tree_dir)) self.assertTrue(fill_with_rules.is_derived('333', deps, tree_dir)) self.assertFalse(fill_with_rules.is_derived('444', deps, tree_dir))
def layers(cfr_title, cfr_part): """Build all layers for all known versions.""" logger.info("Build layers - %s CFR %s", cfr_title, cfr_part) for tree_entry in utils.relevant_paths(entry.Tree(), cfr_title, cfr_part): tree_title, tree_part, version_id = tree_entry.path version_entry = entry.Version(tree_title, tree_part, version_id) stale = stale_layers(tree_entry, 'cfr') if stale: process_cfr_layers(stale, tree_title, version_entry) if cfr_title is None and cfr_part is None: for preamble_entry in entry.Preamble().sub_entries(): stale = stale_layers(preamble_entry, 'preamble') if stale: process_preamble_layers(stale, preamble_entry)
def process_if_needed(volume, cfr_part): """Review dependencies; if they're out of date, parse the annual edition into a tree and store that""" version_id = _version_id(volume.year, cfr_part) annual_entry = entry.Annual(volume.title, cfr_part, volume.year) tree_entry = entry.Tree(volume.title, cfr_part, version_id) notice_entry = entry.Notice(version_id) deps = dependency.Graph() deps.add(tree_entry, annual_entry) deps.validate_for(tree_entry) if deps.is_stale(tree_entry): tree = xml_parser.reg_text.build_tree(annual_entry.read().xml) tree_entry.write(tree) notice_entry.write( build_fake_notice(version_id, volume.publication_date, volume.title, cfr_part))
def fill_with_rules(cfr_title, cfr_part): """Fill in missing trees using data from rules. When a regulation tree cannot be derived through annual editions, it must be built by parsing the changes in final rules. This command builds those missing trees""" logger.info("Fill with rules - %s CFR %s", cfr_title, cfr_part) tree_dir = entry.Tree(cfr_title, cfr_part) version_dir = entry.Version(cfr_title, cfr_part) versions = [c.read() for c in version_dir.sub_entries()] versions_with_parents = list(zip(versions, Version.parents_of(versions))) deps = dependencies(tree_dir, version_dir, versions_with_parents) derived = [(version.identifier, parent.identifier) for version, parent in versions_with_parents if is_derived(version.identifier, deps, tree_dir)] for version_id, parent_id in derived: deps.validate_for(tree_dir / version_id) if deps.is_stale(tree_dir / version_id): process(tree_dir, parent_id, version_id) deps.rebuild()
def process_if_needed(cfr_title, cfr_part, last_version_list): """Calculate dependencies between input and output files for these annual editions. If an output is missing or out of date, process it""" annual_path = entry.Annual(cfr_title, cfr_part) tree_path = entry.Tree(cfr_title, cfr_part) version_path = entry.Version(cfr_title, cfr_part) deps = dependency.Graph() for last_version in last_version_list: deps.add(tree_path / last_version.version_id, version_path / last_version.version_id) deps.add(tree_path / last_version.version_id, annual_path / last_version.year) for last_version in last_version_list: tree_entry = tree_path / last_version.version_id deps.validate_for(tree_entry) if deps.is_stale(tree_entry): input_entry = annual_path / last_version.year tree = gpo_cfr.builder.build_tree(input_entry.read().xml) tree_entry.write(tree)
def test_dependencies(self): """Expect nonexistent trees to depend on their predecessor, associated rule changes and version files. Shouldn't add dependencies for the first version, if missing""" with self.cli.isolated_filesystem(): version_ids = ['111', '222', '333', '444', '555', '666'] tree_dir = entry.Tree('12', '1000') rule_dir = entry.RuleChanges() vers_dir = entry.Version('12', '1000') # Existing trees (tree_dir / '222').write(Node()) (tree_dir / '555').write(Node()) deps = fill_with_rules.dependencies(tree_dir, version_ids, '12', '1000') # First is skipped, as we can't build it from a rule self.assertNotIn(str(tree_dir / '111'), deps) # Second can also be skipped as a tree already exists self.assertEqual(deps.dependencies(str(tree_dir / '222')), []) # Third relies on the associated versions and the second tree self.assertItemsEqual(deps.dependencies(str(tree_dir / '333')), [ str(tree_dir / '222'), str(rule_dir / '333'), str(vers_dir / '333') ]) # Fourth relies on the third, even though it's not been built self.assertItemsEqual(deps.dependencies(str(tree_dir / '444')), [ str(tree_dir / '333'), str(rule_dir / '444'), str(vers_dir / '444') ]) # Fifth can be skipped as the tree already exists self.assertEqual(deps.dependencies(str(tree_dir / '555')), []) # Six relies on the fifth self.assertItemsEqual(deps.dependencies(str(tree_dir / '666')), [ str(tree_dir / '555'), str(rule_dir / '666'), str(vers_dir / '666') ])
def test_process(self, Notice, compile_regulation): """Verify that the correct changes are found""" compile_regulation.return_value = Node() # entry.Notice('new').read().amendments Notice.return_value.read.return_value.amendments = [{ "instruction": "Something something", "cfr_part": "1000", "authority": "USC Numbers" }, { "instruction": "More things", "cfr_part": "1000", "changes": [["1000-2-b", ["2b changes"]], ["1000-2-c", ["2c changes"]]] }, { "instruction": "Yet more changes", "cfr_part": "1000", "changes": [["1000-4-a", ["4a changes"]]] }] with self.cli.isolated_filesystem(): tree_dir = entry.Tree('12', '1000') (tree_dir / 'old').write(Node()) entry.Entry('notice_xml', 'new').write(b'') fill_with_rules.process(tree_dir, 'old', 'new') changes = dict(compile_regulation.call_args[0][1]) self.assertEqual( changes, { "1000-2-b": ["2b changes"], "1000-2-c": ["2c changes"], "1000-4-a": ["4a changes"] })
def test_stale_layers(monkeypatch): """We should have dependencies between all of the layers and their associated trees. We should also tie the meta layer to the version""" monkeypatch.setattr(layers, 'LAYER_CLASSES', {'cfr': { 'keyterms': None, 'other': None }}) version_entry = entry.Version(111, 22, 'aaa') version_entry.write(Version('aaa', date.today(), Citation(1, 1))) tree_entry = entry.Tree(111, 22, 'aaa') with pytest.raises(dependency.Missing): layers.stale_layers(tree_entry, 'cfr') entry.Entry('tree', 111, 22, 'bbb').write(b'') # wrong version with pytest.raises(dependency.Missing): layers.stale_layers(tree_entry, 'cfr') entry.Entry('tree', 111, 22, 'aaa').write(b'') assert set(layers.stale_layers(tree_entry, 'cfr')) == {'keyterms', 'other'} assert str(version_entry) in dependency.Graph().dependencies( str(entry.Layer.cfr(111, 22, 'aaa', 'meta')))
def write_trees(client, only_title, only_part): for tree_entry in utils.relevant_paths(entry.Tree(), only_title, only_part): _, cfr_part, version_id = tree_entry.path content = tree_entry.read() client.regulation(cfr_part, version_id).write(content)