def create_versions(self): entry.Version(11, 222, 'aaa').write( Version('aaa', date(2002, 2, 2), date(2002, 2, 2))) entry.Version(11, 222, 'bbb').write( Version('bbb', date(2001, 1, 1), date(2001, 1, 1))) entry.Version(11, 222, 'ccc').write( Version('ccc', date(2003, 3, 3), date(2003, 3, 3)))
def create_versions(): """Generate some dummy data""" entry.Version(11, 222, 'aaa').write(Version('aaa', date(2002, 2, 2), Citation(2, 2))) entry.Version(11, 222, 'bbb').write(Version('bbb', date(2001, 1, 1), Citation(1, 1))) entry.Version(11, 222, 'ccc').write(Version('ccc', date(2003, 3, 3), Citation(3, 3)))
def test_process_if_needed_missing_writes(self, xml_parser): """If output isn't already present, we should process. If it is present, we don't need to, unless a dependency has changed.""" with self.cli.isolated_filesystem(): build_tree = xml_parser.reg_text.build_tree build_tree.return_value = Node() last_versions = [annual_editions.LastVersionInYear('1111', 2000)] entry.Version('12', '1000', '1111').write( Version('1111', date(2000, 1, 1), date(2000, 1, 1))) entry.Entry('annual', '12', '1000', 2000).write( b'<ROOT></ROOT>') annual_editions.process_if_needed('12', '1000', last_versions) self.assertTrue(build_tree.called) build_tree.reset_mock() entry.Entry('tree', '12', '1000', '1111').write(b'tree-here') annual_editions.process_if_needed('12', '1000', last_versions) self.assertFalse(build_tree.called) # Simulate a change to an input file label_id = str(entry.Annual(12, 1000, 2000)) new_time = timezone.now() + timedelta(hours=1) DBEntry.objects.filter(label_id=label_id).update(modified=new_time) annual_editions.process_if_needed('12', '1000', last_versions) self.assertTrue(build_tree.called)
def test_dependencies(): """Expect nonexistent trees to depend on their predecessor, associated rule changes and version files. Shouldn't add dependencies for the first version, if missing""" versions = [Version(str(i)*3, date(2001, i, i), date(2002, i, i)) for i in range(1, 7)] parents = Version.parents_of(versions) tree_dir = entry.Tree('12', '1000') notice_dir = entry.Notice() vers_dir = entry.Version('12', '1000') # Existing trees (tree_dir / '222').write(Node()) (tree_dir / '555').write(Node()) deps = fill_with_rules.dependencies(tree_dir, vers_dir, list(zip(versions, parents))) # First is skipped, as we can't build it from a rule assert str(tree_dir / '111') not in deps # Second can also be skipped as a tree already exists assert deps.dependencies(str(tree_dir / '222')) == [] # Third relies on the associated versions and the second tree expected = {str(tree_dir / '222'), str(notice_dir / '333'), str(vers_dir / '333')} assert set(deps.dependencies(str(tree_dir / '333'))) == expected # Fourth relies on the third, even though it's not been built expected = {str(tree_dir / '333'), str(notice_dir / '444'), str(vers_dir / '444')} assert set(deps.dependencies(str(tree_dir / '444'))) == expected # Fifth can be skipped as the tree already exists assert deps.dependencies(str(tree_dir / '555')) == [] # Six relies on the fifth expected = {str(tree_dir / '555'), str(notice_dir / '666'), str(vers_dir / '666')} assert set(deps.dependencies(str(tree_dir / '666'))) == expected
def test_process_if_needed_missing_writes(monkeypatch): """If output isn't already present, we should process. If it is present, we don't need to, unless a dependency has changed.""" monkeypatch.setattr(annual_editions, 'gpo_cfr', Mock()) build_tree = annual_editions.gpo_cfr.builder.build_tree build_tree.return_value = Node() last_versions = [annual_editions.LastVersionInYear('1111', 2000)] entry.Version('12', '1000', '1111').write( Version('1111', date(2000, 1, 1), Citation(1, 1))) entry.Entry('annual', '12', '1000', 2000).write(b'<ROOT></ROOT>') annual_editions.process_if_needed('12', '1000', last_versions) assert build_tree.called build_tree.reset_mock() entry.Entry('tree', '12', '1000', '1111').write(b'tree-here') annual_editions.process_if_needed('12', '1000', last_versions) assert not build_tree.called # Simulate a change to an input file label_id = str(entry.Annual(12, 1000, 2000)) new_time = timezone.now() + timedelta(hours=1) DBEntry.objects.filter(label_id=label_id).update(modified=new_time) annual_editions.process_if_needed('12', '1000', last_versions) assert build_tree.called
def test_write_to_disk_no_effective(self): """If a version is somehow associated with a proposed rule (or a final rule has been misparsed), we should get an exception""" xml = Mock() xml.effective = None with self.assertRaises(versions.InvalidEffectiveDate): versions.write_to_disk(xml, entry.Version('12', '1000', '11'))
def test_stale_layers(self): """We should have dependencies between all of the layers and their associated trees. We should also tie the meta layer to the version""" configured_layers = {'cfr': {'keyterms': None, 'other': None}} with self.cli.isolated_filesystem(), patch.dict( layers.LAYER_CLASSES, configured_layers): version_entry = entry.Version(111, 22, 'aaa') version_entry.write(Version('aaa', date.today(), date.today())) tree_entry = entry.Tree(111, 22, 'aaa') # Use list() to instantiate self.assertRaises(dependency.Missing, list, layers.stale_layers(tree_entry, 'cfr')) entry.Entry('tree', 111, 22, 'bbb').write(b'') # wrong version self.assertRaises(dependency.Missing, list, layers.stale_layers(tree_entry, 'cfr')) entry.Entry('tree', 111, 22, 'aaa').write(b'') six.assertCountEqual(self, layers.stale_layers(tree_entry, 'cfr'), ['keyterms', 'other']) self.assertIn( str(version_entry), dependency.Graph().dependencies( str(entry.Layer.cfr(111, 22, 'aaa', 'meta'))))
def previous_sxs(cfr_title, cfr_part, stop_version): """The SxS layer relies on all notices that came before a particular version""" for previous_version in entry.Version(cfr_title, cfr_part): yield entry.SxS(previous_version) if previous_version == stop_version: break
def test_create_version(self): """Creates a version associated with the part and year""" with CliRunner().isolated_filesystem(): current_version.create_version_entry_if_needed( Volume(2010, 20, 1), 1001) version = entry.Version(20, 1001, '2010-annual-1001').read() self.assertEqual(version.effective, date(2010, 4, 1)) self.assertEqual(version.published, date(2010, 4, 1))
def test_transform_notice(self, process_sxs, add_footnotes): """We should add version information and the SxS functions should be called""" with CliRunner().isolated_filesystem(): entry.Version(11, 222, 'v1').write( Version('v1', date(2001, 1, 1), date(2002, 2, 2))) entry.Version(11, 222, 'v2').write( Version('v2', date(2002, 2, 2), date(2003, 3, 3))) entry.Version(11, 222, 'v3').write( Version('v3', date(2003, 3, 3), date(2004, 4, 4))) entry.Version(11, 223, 'v1').write( Version('v1', date(2001, 1, 1), date(2002, 2, 2))) entry.Version(11, 224, 'v1').write( Version('v1', date(2001, 1, 1), date(2002, 2, 2))) entry.Version(11, 222, 'proposal').write( Version('proposal', date(2003, 6, 6), None)) entry.Version(11, 223, 'proposal').write( Version('proposal', date(2003, 6, 6), None)) notice_xml = Mock() notice_xml.as_dict.return_value = {} notice_xml.version_id = 'proposal' notice_xml.cfr_ref_pairs = [(11, 222), (11, 223)] result = transform_notice(notice_xml) self.assertEqual(result['versions'], { 222: {'left': 'v3', 'right': 'proposal'}, 223: {'left': 'v1', 'right': 'proposal'}}) self.assertTrue(process_sxs.called) self.assertTrue(add_footnotes.called)
def test_create_version(): """Creates a version associated with the part and year""" vol_num = randint(1, 99) annual_version.create_version_entry_if_needed(Volume(2010, 20, vol_num), 1001) version = entry.Version(20, 1001, '2010-annual-1001').read() assert version.effective == date(2010, 4, 1) assert version.fr_citation.volume == vol_num assert version.fr_citation.page == 1
def test_write_to_disk_no_effective(monkeypatch): """If a version is somehow associated with a proposed rule (or a final rule has been misparsed), we should get a warning""" xml = Mock(effective=None, version_id='vv123') monkeypatch.setattr(versions, 'logger', Mock()) versions.write_to_disk(xml, entry.Version('12', '1000', '11')) assert versions.logger.warning.called assert 'vv123' in versions.logger.warning.call_args[0]
def test_process_cfr_layers(): """All layers for a single version should get written.""" version_entry = entry.Version(12, 1000, '1234') version_entry.write(Version('1234', date.today(), Citation(1, 1))) entry.Tree('12', '1000', '1234').write(Node()) layers.process_cfr_layers(['keyterms', 'meta'], 12, version_entry) assert entry.Layer.cfr(12, 1000, '1234', 'keyterms').exists() assert entry.Layer.cfr(12, 1000, '1234', 'meta').exists()
def test_process_cfr_layers(self): """All layers for a single version should get written.""" with self.cli.isolated_filesystem(): version_entry = entry.Version(12, 1000, '1234') version_entry.write(Version('1234', date.today(), date.today())) entry.Tree('12', '1000', '1234').write(Node()) layers.process_cfr_layers(['keyterms', 'meta'], 12, version_entry) self.assertTrue( entry.Layer.cfr(12, 1000, '1234', 'keyterms').exists()) self.assertTrue(entry.Layer.cfr(12, 1000, '1234', 'meta').exists())
def test_last_versions_not_printed(monkeypatch): """We should only find the annual editions which have been published already""" # 2001 exists; no other years do monkeypatch.setattr(annual_editions.annual, 'find_volume', Mock()) annual_editions.annual.find_volume = lambda year, title, part: year == 2001 path = entry.Version('12', '1000') (path / '1111').write(Version('1111', date(2000, 12, 1), Citation(1, 1))) (path / '2222').write(Version('2222', date(2001, 12, 1), Citation(1, 1))) results = list(annual_editions.last_versions(12, 1000)) assert results == [annual_editions.LastVersionInYear('1111', 2001)]
def write_if_needed(cfr_title, cfr_part, version_ids, xmls, delays): """All versions which are stale (either because they were never create or because their dependency has been updated) are written to disk. If any dependency is missing, an exception is raised""" version_dir = entry.Version(cfr_title, cfr_part) deps = generate_dependencies(version_dir, version_ids, delays) for version_id in version_ids: version_entry = version_dir / version_id deps.validate_for(version_entry) if deps.is_stale(version_entry): write_to_disk(xmls[version_id], version_entry, delays.get(version_id))
def proposal_versions(doc_number): """Generate version entries associated with a proposal.""" notice = entry.Notice(doc_number) if not notice.exists(): raise dependency.Missing(str(notice), str(notice)) notice = notice.read() version = Version(doc_number, notice.published, None) for cfr_title, cfr_part in notice.cfr_ref_pairs: version_entry = entry.Version(cfr_title, cfr_part, doc_number) if not version_entry.exists() or version_entry.read() != version: version_entry.write(version)
def test_process_if_needed_missing_dependency_error(): """If the annual XML or version isn't present, we should see a dependency error.""" last_versions = [annual_editions.LastVersionInYear('1111', 2000)] with pytest.raises(dependency.Missing): annual_editions.process_if_needed('12', '1000', last_versions) entry.Version('12', '1000', '1111').write( Version('1111', date(2000, 1, 1), Citation(1, 1))) with pytest.raises(dependency.Missing): annual_editions.process_if_needed('12', '1000', last_versions)
def test_process_version_if_needed_success(): """If the requirements are present we should write the version data""" notice_xml = NoticeXML(XMLBuilder().xml) notice_xml.effective = date(2001, 1, 1) notice_xml.fr_volume = 2 notice_xml.start_page = 3 entry.Notice('vvv').write(notice_xml) full_issuance.process_version_if_needed('title', 'part', 'vvv') result = entry.Version('title', 'part', 'vvv').read() assert result.identifier == 'vvv' assert result.effective == date(2001, 1, 1) assert result.fr_citation == Citation(2, 3)
def test_is_derived(): """Should filter version ids to only those with a dependency on changes derived from a rule""" tree_dir = entry.Tree('12', '1000') deps = dependency.Graph() deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001)) deps.add(tree_dir / 222, entry.Notice(222)) deps.add(tree_dir / 333, entry.Notice(333)) deps.add(tree_dir / 333, entry.Version(333)) assert not fill_with_rules.is_derived('111', deps, tree_dir) assert fill_with_rules.is_derived('222', deps, tree_dir) assert fill_with_rules.is_derived('333', deps, tree_dir) assert not fill_with_rules.is_derived('444', deps, tree_dir)
def test_derived_from_rules(self): """Should filter a set of version ids to only those with a dependency on changes derived from a rule""" with self.cli.isolated_filesystem(): tree_dir = entry.Tree('12', '1000') deps = dependency.Graph() deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001)) deps.add(tree_dir / 222, entry.RuleChanges(222)) deps.add(tree_dir / 333, entry.RuleChanges(333)) deps.add(tree_dir / 333, entry.Version(333)) derived = fill_with_rules.derived_from_rules( ['111', '222', '333', '444'], deps, tree_dir) self.assertEqual(derived, ['222', '333'])
def test_process_if_needed_missing_dependency_error(self): """If the annual XML or version isn't present, we should see a dependency error.""" with self.cli.isolated_filesystem(): last_versions = [annual_editions.LastVersionInYear('1111', 2000)] with self.assertRaises(dependency.Missing): annual_editions.process_if_needed('12', '1000', last_versions) entry.Version('12', '1000', '1111').write( Version('1111', date(2000, 1, 1), date(2000, 1, 1))) with self.assertRaises(dependency.Missing): annual_editions.process_if_needed('12', '1000', last_versions)
def process_version_if_needed(cfr_title, cfr_part, version_id): """Creates and writes a version struct after validating the Notice has been created""" notice_entry = entry.Notice(version_id) version_entry = entry.Version(cfr_title, cfr_part, version_id) deps = dependency.Graph() deps.add(version_entry, notice_entry) deps.validate_for(version_entry) if deps.is_stale(version_entry): notice_xml = notice_entry.read() version = Version(version_id, notice_xml.effective, notice_xml.fr_citation) version_entry.write(version)
def test_last_versions_multiple_versions(monkeypatch): """If multiple versions affect the same annual edition, we should only receive the last""" monkeypatch.setattr(annual_editions.annual, 'find_volume', Mock()) annual_editions.annual.find_volume.return_value = True path = entry.Version('12', '1000') (path / '1111').write(Version('1111', date(2000, 12, 1), Citation(1, 1))) (path / '2222').write(Version('2222', date(2000, 12, 2), Citation(1, 2))) (path / '3333').write(Version('3333', date(2001, 12, 1), Citation(1, 1))) results = list(annual_editions.last_versions(12, 1000)) assert results == [ annual_editions.LastVersionInYear('2222', 2001), annual_editions.LastVersionInYear('3333', 2002) ]
def test_write_to_disk(): """If a version has been delayed, its effective date should be part of the serialized json""" xml = Mock(effective=date(2002, 2, 2), version_id='111', fr_citation=Citation(1, 1)) path = entry.Version('12', '1000') versions.write_to_disk(xml, path / '111') xml.version_id = '222' versions.write_to_disk(xml, path / '222', versions.Delay(by='333', until=date(2004, 4, 4))) assert (path / '111').read().effective == date(2002, 2, 2) assert (path / '222').read().effective == date(2004, 4, 4)
def fill_with_rules(cfr_title, cfr_part): """Fill in missing trees using data from rules. When a regulation tree cannot be derived through annual editions, it must be built by parsing the changes in final rules. This command builds those missing trees""" logger.info("Fill with rules - %s CFR %s", cfr_title, cfr_part) tree_path = entry.Tree(cfr_title, cfr_part) version_ids = list(entry.Version(cfr_title, cfr_part)) deps = dependencies(tree_path, version_ids, cfr_title, cfr_part) preceeded_by = dict(zip(version_ids[1:], version_ids)) derived = derived_from_rules(version_ids, deps, tree_path) for version_id in derived: deps.validate_for(tree_path / version_id) if deps.is_stale(tree_path / version_id): process(tree_path, preceeded_by[version_id], version_id)
def test_is_derived(self): """Should filter version ids to only those with a dependency on changes derived from a rule""" with self.cli.isolated_filesystem(): tree_dir = entry.Tree('12', '1000') deps = dependency.Graph() deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001)) deps.add(tree_dir / 222, entry.Notice(222)) deps.add(tree_dir / 333, entry.Notice(333)) deps.add(tree_dir / 333, entry.Version(333)) self.assertFalse(fill_with_rules.is_derived('111', deps, tree_dir)) self.assertTrue(fill_with_rules.is_derived('222', deps, tree_dir)) self.assertTrue(fill_with_rules.is_derived('333', deps, tree_dir)) self.assertFalse(fill_with_rules.is_derived('444', deps, tree_dir))
def test_last_versions_not_printed(self, find_volume): """We should only find the annual editions which have been published already""" # 2001 exists; no other years do find_volume.side_effect = lambda year, title, part: year == 2001 with self.cli.isolated_filesystem(): path = entry.Version('12', '1000') (path / '1111').write( Version('1111', date(2000, 12, 1), date(2000, 12, 1))) (path / '2222').write( Version('2222', date(2001, 12, 1), date(2001, 12, 1))) results = list(annual_editions.last_versions(12, 1000)) self.assertEqual(results, [annual_editions.LastVersionInYear('1111', 2001)])
def dependencies(tree_path, version_ids, cfr_title, cfr_part): """Set up the dependency graph for this regulation. First calculates "gaps" -- versions for which there is no existing tree. In this calculation, we ignore the first version, as we won't be able to build anything for it. Add dependencies for any gaps, tying the output tree to the preceding tree, the version info and the parsed rule""" existing_ids = set(tree_path) gaps = [(prev, curr) for prev, curr in zip(version_ids, version_ids[1:]) if curr not in existing_ids] deps = dependency.Graph() for prev, curr in gaps: deps.add(tree_path / curr, tree_path / prev) deps.add(tree_path / curr, entry.RuleChanges(curr)) deps.add(tree_path / curr, entry.Version(cfr_title, cfr_part, curr)) return deps
def layers(cfr_title, cfr_part): """Build all layers for all known versions.""" logger.info("Build layers - %s CFR %s", cfr_title, cfr_part) for tree_entry in utils.relevant_paths(entry.Tree(), cfr_title, cfr_part): tree_title, tree_part, version_id = tree_entry.path version_entry = entry.Version(tree_title, tree_part, version_id) stale = stale_layers(tree_entry, 'cfr') if stale: process_cfr_layers(stale, tree_title, version_entry) if cfr_title is None and cfr_part is None: for preamble_entry in entry.Preamble().sub_entries(): stale = stale_layers(preamble_entry, 'preamble') if stale: process_preamble_layers(stale, preamble_entry)