def create_versions(self):
entry.Version(11, 222, 'aaa').write(
Version('aaa', date(2002, 2, 2), date(2002, 2, 2)))
entry.Version(11, 222, 'bbb').write(
Version('bbb', date(2001, 1, 1), date(2001, 1, 1)))
entry.Version(11, 222, 'ccc').write(
Version('ccc', date(2003, 3, 3), date(2003, 3, 3)))
def create_versions():
"""Generate some dummy data"""
entry.Version(11, 222,
'aaa').write(Version('aaa', date(2002, 2, 2), Citation(2,
2)))
entry.Version(11, 222,
'bbb').write(Version('bbb', date(2001, 1, 1), Citation(1,
1)))
entry.Version(11, 222,
'ccc').write(Version('ccc', date(2003, 3, 3), Citation(3,
3)))
def test_process_if_needed_missing_writes(self, xml_parser):
"""If output isn't already present, we should process. If it is
present, we don't need to, unless a dependency has changed."""
with self.cli.isolated_filesystem():
build_tree = xml_parser.reg_text.build_tree
build_tree.return_value = Node()
last_versions = [annual_editions.LastVersionInYear('1111', 2000)]
entry.Version('12', '1000', '1111').write(
Version('1111', date(2000, 1, 1), date(2000, 1, 1)))
entry.Entry('annual', '12', '1000', 2000).write(
b'<ROOT></ROOT>')
annual_editions.process_if_needed('12', '1000', last_versions)
self.assertTrue(build_tree.called)
build_tree.reset_mock()
entry.Entry('tree', '12', '1000', '1111').write(b'tree-here')
annual_editions.process_if_needed('12', '1000', last_versions)
self.assertFalse(build_tree.called)
# Simulate a change to an input file
label_id = str(entry.Annual(12, 1000, 2000))
new_time = timezone.now() + timedelta(hours=1)
DBEntry.objects.filter(label_id=label_id).update(modified=new_time)
annual_editions.process_if_needed('12', '1000', last_versions)
self.assertTrue(build_tree.called)
def test_dependencies():
"""Expect nonexistent trees to depend on their predecessor, associated
rule changes and version files. Shouldn't add dependencies for the
first version, if missing"""
versions = [Version(str(i)*3, date(2001, i, i), date(2002, i, i))
for i in range(1, 7)]
parents = Version.parents_of(versions)
tree_dir = entry.Tree('12', '1000')
notice_dir = entry.Notice()
vers_dir = entry.Version('12', '1000')
# Existing trees
(tree_dir / '222').write(Node())
(tree_dir / '555').write(Node())
deps = fill_with_rules.dependencies(tree_dir, vers_dir,
list(zip(versions, parents)))
# First is skipped, as we can't build it from a rule
assert str(tree_dir / '111') not in deps
# Second can also be skipped as a tree already exists
assert deps.dependencies(str(tree_dir / '222')) == []
# Third relies on the associated versions and the second tree
expected = {str(tree_dir / '222'), str(notice_dir / '333'),
str(vers_dir / '333')}
assert set(deps.dependencies(str(tree_dir / '333'))) == expected
# Fourth relies on the third, even though it's not been built
expected = {str(tree_dir / '333'), str(notice_dir / '444'),
str(vers_dir / '444')}
assert set(deps.dependencies(str(tree_dir / '444'))) == expected
# Fifth can be skipped as the tree already exists
assert deps.dependencies(str(tree_dir / '555')) == []
# Six relies on the fifth
expected = {str(tree_dir / '555'), str(notice_dir / '666'),
str(vers_dir / '666')}
assert set(deps.dependencies(str(tree_dir / '666'))) == expected
def test_process_if_needed_missing_writes(monkeypatch):
"""If output isn't already present, we should process. If it is present,
we don't need to, unless a dependency has changed."""
monkeypatch.setattr(annual_editions, 'gpo_cfr', Mock())
build_tree = annual_editions.gpo_cfr.builder.build_tree
build_tree.return_value = Node()
last_versions = [annual_editions.LastVersionInYear('1111', 2000)]
entry.Version('12', '1000', '1111').write(
Version('1111', date(2000, 1, 1), Citation(1, 1)))
entry.Entry('annual', '12', '1000', 2000).write(b'<ROOT></ROOT>')
annual_editions.process_if_needed('12', '1000', last_versions)
assert build_tree.called
build_tree.reset_mock()
entry.Entry('tree', '12', '1000', '1111').write(b'tree-here')
annual_editions.process_if_needed('12', '1000', last_versions)
assert not build_tree.called
# Simulate a change to an input file
label_id = str(entry.Annual(12, 1000, 2000))
new_time = timezone.now() + timedelta(hours=1)
DBEntry.objects.filter(label_id=label_id).update(modified=new_time)
annual_editions.process_if_needed('12', '1000', last_versions)
assert build_tree.called
def test_write_to_disk_no_effective(self):
"""If a version is somehow associated with a proposed rule (or a final
rule has been misparsed), we should get an exception"""
xml = Mock()
xml.effective = None
with self.assertRaises(versions.InvalidEffectiveDate):
versions.write_to_disk(xml, entry.Version('12', '1000', '11'))
def test_stale_layers(self):
"""We should have dependencies between all of the layers and their
associated trees. We should also tie the meta layer to the version"""
configured_layers = {'cfr': {'keyterms': None, 'other': None}}
with self.cli.isolated_filesystem(), patch.dict(
layers.LAYER_CLASSES, configured_layers):
version_entry = entry.Version(111, 22, 'aaa')
version_entry.write(Version('aaa', date.today(), date.today()))
tree_entry = entry.Tree(111, 22, 'aaa')
# Use list() to instantiate
self.assertRaises(dependency.Missing, list,
layers.stale_layers(tree_entry, 'cfr'))
entry.Entry('tree', 111, 22, 'bbb').write(b'') # wrong version
self.assertRaises(dependency.Missing, list,
layers.stale_layers(tree_entry, 'cfr'))
entry.Entry('tree', 111, 22, 'aaa').write(b'')
six.assertCountEqual(self, layers.stale_layers(tree_entry, 'cfr'),
['keyterms', 'other'])
self.assertIn(
str(version_entry),
dependency.Graph().dependencies(
str(entry.Layer.cfr(111, 22, 'aaa', 'meta'))))
def previous_sxs(cfr_title, cfr_part, stop_version):
"""The SxS layer relies on all notices that came before a particular
version"""
for previous_version in entry.Version(cfr_title, cfr_part):
yield entry.SxS(previous_version)
if previous_version == stop_version:
break
def test_create_version(self):
"""Creates a version associated with the part and year"""
with CliRunner().isolated_filesystem():
current_version.create_version_entry_if_needed(
Volume(2010, 20, 1), 1001)
version = entry.Version(20, 1001, '2010-annual-1001').read()
self.assertEqual(version.effective, date(2010, 4, 1))
self.assertEqual(version.published, date(2010, 4, 1))
def test_transform_notice(self, process_sxs, add_footnotes):
"""We should add version information and the SxS functions should be
called"""
with CliRunner().isolated_filesystem():
entry.Version(11, 222, 'v1').write(
Version('v1', date(2001, 1, 1), date(2002, 2, 2)))
entry.Version(11, 222, 'v2').write(
Version('v2', date(2002, 2, 2), date(2003, 3, 3)))
entry.Version(11, 222, 'v3').write(
Version('v3', date(2003, 3, 3), date(2004, 4, 4)))
entry.Version(11, 223, 'v1').write(
Version('v1', date(2001, 1, 1), date(2002, 2, 2)))
entry.Version(11, 224, 'v1').write(
Version('v1', date(2001, 1, 1), date(2002, 2, 2)))
entry.Version(11, 222, 'proposal').write(
Version('proposal', date(2003, 6, 6), None))
entry.Version(11, 223, 'proposal').write(
Version('proposal', date(2003, 6, 6), None))
notice_xml = Mock()
notice_xml.as_dict.return_value = {}
notice_xml.version_id = 'proposal'
notice_xml.cfr_ref_pairs = [(11, 222), (11, 223)]
result = transform_notice(notice_xml)
self.assertEqual(result['versions'], {
222: {'left': 'v3', 'right': 'proposal'},
223: {'left': 'v1', 'right': 'proposal'}})
self.assertTrue(process_sxs.called)
self.assertTrue(add_footnotes.called)
def test_create_version():
"""Creates a version associated with the part and year"""
vol_num = randint(1, 99)
annual_version.create_version_entry_if_needed(Volume(2010, 20, vol_num),
1001)
version = entry.Version(20, 1001, '2010-annual-1001').read()
assert version.effective == date(2010, 4, 1)
assert version.fr_citation.volume == vol_num
assert version.fr_citation.page == 1
def test_write_to_disk_no_effective(monkeypatch):
"""If a version is somehow associated with a proposed rule (or a final
rule has been misparsed), we should get a warning"""
xml = Mock(effective=None, version_id='vv123')
monkeypatch.setattr(versions, 'logger', Mock())
versions.write_to_disk(xml, entry.Version('12', '1000', '11'))
assert versions.logger.warning.called
assert 'vv123' in versions.logger.warning.call_args[0]
def test_process_cfr_layers():
"""All layers for a single version should get written."""
version_entry = entry.Version(12, 1000, '1234')
version_entry.write(Version('1234', date.today(), Citation(1, 1)))
entry.Tree('12', '1000', '1234').write(Node())
layers.process_cfr_layers(['keyterms', 'meta'], 12, version_entry)
assert entry.Layer.cfr(12, 1000, '1234', 'keyterms').exists()
assert entry.Layer.cfr(12, 1000, '1234', 'meta').exists()
def test_process_cfr_layers(self):
"""All layers for a single version should get written."""
with self.cli.isolated_filesystem():
version_entry = entry.Version(12, 1000, '1234')
version_entry.write(Version('1234', date.today(), date.today()))
entry.Tree('12', '1000', '1234').write(Node())
layers.process_cfr_layers(['keyterms', 'meta'], 12, version_entry)
self.assertTrue(
entry.Layer.cfr(12, 1000, '1234', 'keyterms').exists())
self.assertTrue(entry.Layer.cfr(12, 1000, '1234', 'meta').exists())
def test_last_versions_not_printed(monkeypatch):
"""We should only find the annual editions which have been published
already"""
# 2001 exists; no other years do
monkeypatch.setattr(annual_editions.annual, 'find_volume', Mock())
annual_editions.annual.find_volume = lambda year, title, part: year == 2001
path = entry.Version('12', '1000')
(path / '1111').write(Version('1111', date(2000, 12, 1), Citation(1, 1)))
(path / '2222').write(Version('2222', date(2001, 12, 1), Citation(1, 1)))
results = list(annual_editions.last_versions(12, 1000))
assert results == [annual_editions.LastVersionInYear('1111', 2001)]
def write_if_needed(cfr_title, cfr_part, version_ids, xmls, delays):
"""All versions which are stale (either because they were never create or
because their dependency has been updated) are written to disk. If any
dependency is missing, an exception is raised"""
version_dir = entry.Version(cfr_title, cfr_part)
deps = generate_dependencies(version_dir, version_ids, delays)
for version_id in version_ids:
version_entry = version_dir / version_id
deps.validate_for(version_entry)
if deps.is_stale(version_entry):
write_to_disk(xmls[version_id], version_entry,
delays.get(version_id))
def proposal_versions(doc_number):
"""Generate version entries associated with a proposal."""
notice = entry.Notice(doc_number)
if not notice.exists():
raise dependency.Missing(str(notice), str(notice))
notice = notice.read()
version = Version(doc_number, notice.published, None)
for cfr_title, cfr_part in notice.cfr_ref_pairs:
version_entry = entry.Version(cfr_title, cfr_part, doc_number)
if not version_entry.exists() or version_entry.read() != version:
version_entry.write(version)
def test_process_if_needed_missing_dependency_error():
"""If the annual XML or version isn't present, we should see a dependency
error."""
last_versions = [annual_editions.LastVersionInYear('1111', 2000)]
with pytest.raises(dependency.Missing):
annual_editions.process_if_needed('12', '1000', last_versions)
entry.Version('12', '1000', '1111').write(
Version('1111', date(2000, 1, 1), Citation(1, 1)))
with pytest.raises(dependency.Missing):
annual_editions.process_if_needed('12', '1000', last_versions)
def test_process_version_if_needed_success():
"""If the requirements are present we should write the version data"""
notice_xml = NoticeXML(XMLBuilder().xml)
notice_xml.effective = date(2001, 1, 1)
notice_xml.fr_volume = 2
notice_xml.start_page = 3
entry.Notice('vvv').write(notice_xml)
full_issuance.process_version_if_needed('title', 'part', 'vvv')
result = entry.Version('title', 'part', 'vvv').read()
assert result.identifier == 'vvv'
assert result.effective == date(2001, 1, 1)
assert result.fr_citation == Citation(2, 3)
def test_is_derived():
"""Should filter version ids to only those with a dependency on
changes derived from a rule"""
tree_dir = entry.Tree('12', '1000')
deps = dependency.Graph()
deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001))
deps.add(tree_dir / 222, entry.Notice(222))
deps.add(tree_dir / 333, entry.Notice(333))
deps.add(tree_dir / 333, entry.Version(333))
assert not fill_with_rules.is_derived('111', deps, tree_dir)
assert fill_with_rules.is_derived('222', deps, tree_dir)
assert fill_with_rules.is_derived('333', deps, tree_dir)
assert not fill_with_rules.is_derived('444', deps, tree_dir)
def test_derived_from_rules(self):
"""Should filter a set of version ids to only those with a dependency
on changes derived from a rule"""
with self.cli.isolated_filesystem():
tree_dir = entry.Tree('12', '1000')
deps = dependency.Graph()
deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001))
deps.add(tree_dir / 222, entry.RuleChanges(222))
deps.add(tree_dir / 333, entry.RuleChanges(333))
deps.add(tree_dir / 333, entry.Version(333))
derived = fill_with_rules.derived_from_rules(
['111', '222', '333', '444'], deps, tree_dir)
self.assertEqual(derived, ['222', '333'])
def test_process_if_needed_missing_dependency_error(self):
"""If the annual XML or version isn't present, we should see a
dependency error."""
with self.cli.isolated_filesystem():
last_versions = [annual_editions.LastVersionInYear('1111', 2000)]
with self.assertRaises(dependency.Missing):
annual_editions.process_if_needed('12', '1000', last_versions)
entry.Version('12', '1000', '1111').write(
Version('1111', date(2000, 1, 1), date(2000, 1, 1)))
with self.assertRaises(dependency.Missing):
annual_editions.process_if_needed('12', '1000', last_versions)
def process_version_if_needed(cfr_title, cfr_part, version_id):
"""Creates and writes a version struct after validating the Notice has
been created"""
notice_entry = entry.Notice(version_id)
version_entry = entry.Version(cfr_title, cfr_part, version_id)
deps = dependency.Graph()
deps.add(version_entry, notice_entry)
deps.validate_for(version_entry)
if deps.is_stale(version_entry):
notice_xml = notice_entry.read()
version = Version(version_id, notice_xml.effective,
notice_xml.fr_citation)
version_entry.write(version)
def test_last_versions_multiple_versions(monkeypatch):
"""If multiple versions affect the same annual edition, we should only
receive the last"""
monkeypatch.setattr(annual_editions.annual, 'find_volume', Mock())
annual_editions.annual.find_volume.return_value = True
path = entry.Version('12', '1000')
(path / '1111').write(Version('1111', date(2000, 12, 1), Citation(1, 1)))
(path / '2222').write(Version('2222', date(2000, 12, 2), Citation(1, 2)))
(path / '3333').write(Version('3333', date(2001, 12, 1), Citation(1, 1)))
results = list(annual_editions.last_versions(12, 1000))
assert results == [
annual_editions.LastVersionInYear('2222', 2001),
annual_editions.LastVersionInYear('3333', 2002)
]
def test_write_to_disk():
"""If a version has been delayed, its effective date should be part of the
serialized json"""
xml = Mock(effective=date(2002, 2, 2),
version_id='111',
fr_citation=Citation(1, 1))
path = entry.Version('12', '1000')
versions.write_to_disk(xml, path / '111')
xml.version_id = '222'
versions.write_to_disk(xml, path / '222',
versions.Delay(by='333', until=date(2004, 4, 4)))
assert (path / '111').read().effective == date(2002, 2, 2)
assert (path / '222').read().effective == date(2004, 4, 4)
def fill_with_rules(cfr_title, cfr_part):
"""Fill in missing trees using data from rules. When a regulation tree
cannot be derived through annual editions, it must be built by parsing the
changes in final rules. This command builds those missing trees"""
logger.info("Fill with rules - %s CFR %s", cfr_title, cfr_part)
tree_path = entry.Tree(cfr_title, cfr_part)
version_ids = list(entry.Version(cfr_title, cfr_part))
deps = dependencies(tree_path, version_ids, cfr_title, cfr_part)
preceeded_by = dict(zip(version_ids[1:], version_ids))
derived = derived_from_rules(version_ids, deps, tree_path)
for version_id in derived:
deps.validate_for(tree_path / version_id)
if deps.is_stale(tree_path / version_id):
process(tree_path, preceeded_by[version_id], version_id)
def test_is_derived(self):
"""Should filter version ids to only those with a dependency on
changes derived from a rule"""
with self.cli.isolated_filesystem():
tree_dir = entry.Tree('12', '1000')
deps = dependency.Graph()
deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001))
deps.add(tree_dir / 222, entry.Notice(222))
deps.add(tree_dir / 333, entry.Notice(333))
deps.add(tree_dir / 333, entry.Version(333))
self.assertFalse(fill_with_rules.is_derived('111', deps, tree_dir))
self.assertTrue(fill_with_rules.is_derived('222', deps, tree_dir))
self.assertTrue(fill_with_rules.is_derived('333', deps, tree_dir))
self.assertFalse(fill_with_rules.is_derived('444', deps, tree_dir))
def test_last_versions_not_printed(self, find_volume):
"""We should only find the annual editions which have been published
already"""
# 2001 exists; no other years do
find_volume.side_effect = lambda year, title, part: year == 2001
with self.cli.isolated_filesystem():
path = entry.Version('12', '1000')
(path / '1111').write(
Version('1111', date(2000, 12, 1), date(2000, 12, 1)))
(path / '2222').write(
Version('2222', date(2001, 12, 1), date(2001, 12, 1)))
results = list(annual_editions.last_versions(12, 1000))
self.assertEqual(results,
[annual_editions.LastVersionInYear('1111', 2001)])
def dependencies(tree_path, version_ids, cfr_title, cfr_part):
"""Set up the dependency graph for this regulation. First calculates
"gaps" -- versions for which there is no existing tree. In this
calculation, we ignore the first version, as we won't be able to build
anything for it. Add dependencies for any gaps, tying the output tree to
the preceding tree, the version info and the parsed rule"""
existing_ids = set(tree_path)
gaps = [(prev, curr) for prev, curr in zip(version_ids, version_ids[1:])
if curr not in existing_ids]
deps = dependency.Graph()
for prev, curr in gaps:
deps.add(tree_path / curr, tree_path / prev)
deps.add(tree_path / curr, entry.RuleChanges(curr))
deps.add(tree_path / curr, entry.Version(cfr_title, cfr_part, curr))
return deps
def layers(cfr_title, cfr_part):
"""Build all layers for all known versions."""
logger.info("Build layers - %s CFR %s", cfr_title, cfr_part)
for tree_entry in utils.relevant_paths(entry.Tree(), cfr_title, cfr_part):
tree_title, tree_part, version_id = tree_entry.path
version_entry = entry.Version(tree_title, tree_part, version_id)
stale = stale_layers(tree_entry, 'cfr')
if stale:
process_cfr_layers(stale, tree_title, version_entry)
if cfr_title is None and cfr_part is None:
for preamble_entry in entry.Preamble().sub_entries():
stale = stale_layers(preamble_entry, 'preamble')
if stale:
process_preamble_layers(stale, preamble_entry)
