def test_dependencies():
"""Expect nonexistent trees to depend on their predecessor, associated
rule changes and version files. Shouldn't add dependencies for the
first version, if missing"""
versions = [Version(str(i)*3, date(2001, i, i), date(2002, i, i))
for i in range(1, 7)]
parents = Version.parents_of(versions)
tree_dir = entry.Tree('12', '1000')
notice_dir = entry.Notice()
vers_dir = entry.Version('12', '1000')
# Existing trees
(tree_dir / '222').write(Node())
(tree_dir / '555').write(Node())
deps = fill_with_rules.dependencies(tree_dir, vers_dir,
list(zip(versions, parents)))
# First is skipped, as we can't build it from a rule
assert str(tree_dir / '111') not in deps
# Second can also be skipped as a tree already exists
assert deps.dependencies(str(tree_dir / '222')) == []
# Third relies on the associated versions and the second tree
expected = {str(tree_dir / '222'), str(notice_dir / '333'),
str(vers_dir / '333')}
assert set(deps.dependencies(str(tree_dir / '333'))) == expected
# Fourth relies on the third, even though it's not been built
expected = {str(tree_dir / '333'), str(notice_dir / '444'),
str(vers_dir / '444')}
assert set(deps.dependencies(str(tree_dir / '444'))) == expected
# Fifth can be skipped as the tree already exists
assert deps.dependencies(str(tree_dir / '555')) == []
# Six relies on the fifth
expected = {str(tree_dir / '555'), str(notice_dir / '666'),
str(vers_dir / '666')}
assert set(deps.dependencies(str(tree_dir / '666'))) == expected
def create_versions(self):
entry.Version(11, 222, 'aaa').write(
Version('aaa', date(2002, 2, 2), date(2002, 2, 2)))
entry.Version(11, 222, 'bbb').write(
Version('bbb', date(2001, 1, 1), date(2001, 1, 1)))
entry.Version(11, 222, 'ccc').write(
Version('ccc', date(2003, 3, 3), date(2003, 3, 3)))
def test_order():
v1 = Version('first', date(2001, 1, 1), Citation(1, 1))
v2 = Version('eff', date(2002, 2, 2), Citation(1, 1))
v3 = Version('cit', date(2002, 2, 2), Citation(3, 3))
v4 = Version('cit >id', date(2002, 2, 2), Citation(3, 3))
for permutation in permutations([v1, v2, v3, v4]):
assert list(sorted(permutation)) == [v1, v2, v3, v4]
def test_order(self):
v1 = Version('first', date(2001, 1, 1), date(2001, 1, 1))
v2 = Version('eff', date(2001, 1, 1), date(2002, 2, 2))
v3 = Version('pub', date(2003, 3, 3), date(2002, 2, 2))
v4 = Version('pub >id', date(2003, 3, 3), date(2002, 2, 2))
for permutation in permutations([v1, v2, v3, v4]):
self.assertEqual(sorted(permutation), [v1, v2, v3, v4])
def create_versions():
"""Generate some dummy data"""
entry.Version(11, 222,
'aaa').write(Version('aaa', date(2002, 2, 2), Citation(2,
2)))
entry.Version(11, 222,
'bbb').write(Version('bbb', date(2001, 1, 1), Citation(1,
1)))
entry.Version(11, 222,
'ccc').write(Version('ccc', date(2003, 3, 3), Citation(3,
3)))
def test_last_versions_not_printed(monkeypatch):
"""We should only find the annual editions which have been published
already"""
# 2001 exists; no other years do
monkeypatch.setattr(annual_editions.annual, 'find_volume', Mock())
annual_editions.annual.find_volume = lambda year, title, part: year == 2001
path = entry.Version('12', '1000')
(path / '1111').write(Version('1111', date(2000, 12, 1), Citation(1, 1)))
(path / '2222').write(Version('2222', date(2001, 12, 1), Citation(1, 1)))
results = list(annual_editions.last_versions(12, 1000))
assert results == [annual_editions.LastVersionInYear('1111', 2001)]
def test_last_versions_not_printed(self, find_volume):
"""We should only find the annual editions which have been published
already"""
# 2001 exists; no other years do
find_volume.side_effect = lambda year, title, part: year == 2001
with self.cli.isolated_filesystem():
path = entry.Version('12', '1000')
(path / '1111').write(
Version('1111', date(2000, 12, 1), date(2000, 12, 1)))
(path / '2222').write(
Version('2222', date(2001, 12, 1), date(2001, 12, 1)))
results = list(annual_editions.last_versions(12, 1000))
self.assertEqual(results,
[annual_editions.LastVersionInYear('1111', 2001)])
def test_last_versions_multiple_versions(monkeypatch):
"""If multiple versions affect the same annual edition, we should only
receive the last"""
monkeypatch.setattr(annual_editions.annual, 'find_volume', Mock())
annual_editions.annual.find_volume.return_value = True
path = entry.Version('12', '1000')
(path / '1111').write(Version('1111', date(2000, 12, 1), Citation(1, 1)))
(path / '2222').write(Version('2222', date(2000, 12, 2), Citation(1, 2)))
(path / '3333').write(Version('3333', date(2001, 12, 1), Citation(1, 1)))
results = list(annual_editions.last_versions(12, 1000))
assert results == [
annual_editions.LastVersionInYear('2222', 2001),
annual_editions.LastVersionInYear('3333', 2002)
]
def test_process_if_needed_missing_writes(self, xml_parser):
"""If output isn't already present, we should process. If it is
present, we don't need to, unless a dependency has changed."""
with self.cli.isolated_filesystem():
build_tree = xml_parser.reg_text.build_tree
build_tree.return_value = Node()
last_versions = [annual_editions.LastVersionInYear('1111', 2000)]
entry.Version('12', '1000', '1111').write(
Version('1111', date(2000, 1, 1), date(2000, 1, 1)))
entry.Entry('annual', '12', '1000', 2000).write(
b'<ROOT></ROOT>')
annual_editions.process_if_needed('12', '1000', last_versions)
self.assertTrue(build_tree.called)
build_tree.reset_mock()
entry.Entry('tree', '12', '1000', '1111').write(b'tree-here')
annual_editions.process_if_needed('12', '1000', last_versions)
self.assertFalse(build_tree.called)
# Simulate a change to an input file
label_id = str(entry.Annual(12, 1000, 2000))
new_time = timezone.now() + timedelta(hours=1)
DBEntry.objects.filter(label_id=label_id).update(modified=new_time)
annual_editions.process_if_needed('12', '1000', last_versions)
self.assertTrue(build_tree.called)
def test_stale_layers(self):
"""We should have dependencies between all of the layers and their
associated trees. We should also tie the meta layer to the version"""
configured_layers = {'cfr': {'keyterms': None, 'other': None}}
with self.cli.isolated_filesystem(), patch.dict(
layers.LAYER_CLASSES, configured_layers):
version_entry = entry.Version(111, 22, 'aaa')
version_entry.write(Version('aaa', date.today(), date.today()))
tree_entry = entry.Tree(111, 22, 'aaa')
# Use list() to instantiate
self.assertRaises(dependency.Missing, list,
layers.stale_layers(tree_entry, 'cfr'))
entry.Entry('tree', 111, 22, 'bbb').write(b'') # wrong version
self.assertRaises(dependency.Missing, list,
layers.stale_layers(tree_entry, 'cfr'))
entry.Entry('tree', 111, 22, 'aaa').write(b'')
six.assertCountEqual(self, layers.stale_layers(tree_entry, 'cfr'),
['keyterms', 'other'])
self.assertIn(
str(version_entry),
dependency.Graph().dependencies(
str(entry.Layer.cfr(111, 22, 'aaa', 'meta'))))
def test_process_effective_date(self):
"""The effective date is derived from a Version object"""
version = Version('v1', date(2004, 4, 4), date(2004, 4, 4))
m = Meta(None, cfr_title=8, version=version)
result = m.process(Node(label=['a']))
self.assertEqual(1, len(result))
self.assertEqual('2004-04-04', result[0].get('effective_date'))
def test_process_if_needed_missing_writes(monkeypatch):
"""If output isn't already present, we should process. If it is present,
we don't need to, unless a dependency has changed."""
monkeypatch.setattr(annual_editions, 'gpo_cfr', Mock())
build_tree = annual_editions.gpo_cfr.builder.build_tree
build_tree.return_value = Node()
last_versions = [annual_editions.LastVersionInYear('1111', 2000)]
entry.Version('12', '1000', '1111').write(
Version('1111', date(2000, 1, 1), Citation(1, 1)))
entry.Entry('annual', '12', '1000', 2000).write(b'<ROOT></ROOT>')
annual_editions.process_if_needed('12', '1000', last_versions)
assert build_tree.called
build_tree.reset_mock()
entry.Entry('tree', '12', '1000', '1111').write(b'tree-here')
annual_editions.process_if_needed('12', '1000', last_versions)
assert not build_tree.called
# Simulate a change to an input file
label_id = str(entry.Annual(12, 1000, 2000))
new_time = timezone.now() + timedelta(hours=1)
DBEntry.objects.filter(label_id=label_id).update(modified=new_time)
annual_editions.process_if_needed('12', '1000', last_versions)
assert build_tree.called
def test_last_versions_multiple_versions(self, find_volume):
"""If multiple versions affect the same annual edition, we should only
receive the last"""
find_volume.return_value = True
with self.cli.isolated_filesystem():
path = entry.Version('12', '1000')
(path / '1111').write(Version('1111', date(2000, 12, 1),
date(2000, 12, 1)))
(path / '2222').write(Version('2222', date(2000, 12, 2),
date(2000, 12, 2)))
(path / '3333').write(Version('3333', date(2001, 12, 1),
date(2001, 12, 1)))
results = list(annual_editions.last_versions(12, 1000))
self.assertEqual(results, [
annual_editions.LastVersionInYear('2222', 2001),
annual_editions.LastVersionInYear('3333', 2002)])
def write_to_disk(xml, version_entry, delay=None):
"""Serialize a Version instance to disk"""
effective = xml.effective if delay is None else delay.until
if not effective:
raise InvalidEffectiveDate(xml.version_id)
version = Version(identifier=xml.version_id, effective=effective,
published=xml.published)
version_entry.write(version)
def test_transform_notice(self, process_sxs, add_footnotes):
"""We should add version information and the SxS functions should be
called"""
with CliRunner().isolated_filesystem():
entry.Version(11, 222, 'v1').write(
Version('v1', date(2001, 1, 1), date(2002, 2, 2)))
entry.Version(11, 222, 'v2').write(
Version('v2', date(2002, 2, 2), date(2003, 3, 3)))
entry.Version(11, 222, 'v3').write(
Version('v3', date(2003, 3, 3), date(2004, 4, 4)))
entry.Version(11, 223, 'v1').write(
Version('v1', date(2001, 1, 1), date(2002, 2, 2)))
entry.Version(11, 224, 'v1').write(
Version('v1', date(2001, 1, 1), date(2002, 2, 2)))
entry.Version(11, 222, 'proposal').write(
Version('proposal', date(2003, 6, 6), None))
entry.Version(11, 223, 'proposal').write(
Version('proposal', date(2003, 6, 6), None))
notice_xml = Mock()
notice_xml.as_dict.return_value = {}
notice_xml.version_id = 'proposal'
notice_xml.cfr_ref_pairs = [(11, 222), (11, 223)]
result = transform_notice(notice_xml)
self.assertEqual(result['versions'], {
222: {'left': 'v3', 'right': 'proposal'},
223: {'left': 'v1', 'right': 'proposal'}})
self.assertTrue(process_sxs.called)
self.assertTrue(add_footnotes.called)
def test_iterator(self):
"""Versions should be correctly linearized"""
with CliRunner().isolated_filesystem():
path = entry.Version("12", "1000")
v1 = Version('1111',
effective=date(2004, 4, 4),
published=date(2004, 4, 4))
v2 = Version('2222',
effective=date(2002, 2, 2),
published=date(2004, 4, 4))
v3 = Version('3333',
effective=date(2004, 4, 4),
published=date(2003, 3, 3))
(path / '1111').write(v1)
(path / '2222').write(v2)
(path / '3333').write(v3)
self.assertEqual(['2222', '3333', '1111'], list(path))
def test_drop_initial_orphan_versions():
version_list = [Version(letter, None, None) for letter in 'abcdef']
version_pairs = list(zip(version_list, [None] + version_list[1:]))
existing = {'c', 'e'}
result = fill_with_rules.drop_initial_orphans(version_pairs, existing)
result = [pair[0].identifier for pair in result]
assert result == ['c', 'd', 'e', 'f']
def write_to_disk(xml, version_entry, delay=None):
"""Serialize a Version instance to disk"""
effective = xml.effective if delay is None else delay.until
if effective:
version = Version(xml.version_id, effective, xml.fr_citation)
version_entry.write(version)
else:
logger.warning("No effective date for this rule: %s. Skipping",
xml.version_id)
def test_iterator():
"""Versions should be correctly linearized"""
path = entry.Version("12", "1000")
v1 = Version('1111',
effective=date(2004, 4, 4),
published=date(2004, 4, 4))
v2 = Version('2222',
effective=date(2002, 2, 2),
published=date(2004, 4, 4))
v3 = Version('3333',
effective=date(2004, 4, 4),
published=date(2003, 3, 3))
(path / '1111').write(v1)
(path / '2222').write(v2)
(path / '3333').write(v3)
actual = [child.path[-1] for child in path.sub_entries()]
assert ['2222', '3333', '1111'] == actual
def test_process_cfr_layers():
"""All layers for a single version should get written."""
version_entry = entry.Version(12, 1000, '1234')
version_entry.write(Version('1234', date.today(), Citation(1, 1)))
entry.Tree('12', '1000', '1234').write(Node())
layers.process_cfr_layers(['keyterms', 'meta'], 12, version_entry)
assert entry.Layer.cfr(12, 1000, '1234', 'keyterms').exists()
assert entry.Layer.cfr(12, 1000, '1234', 'meta').exists()
def create_version_entry_if_needed(volume, cfr_part):
"""Only write the version entry if it doesn't already exist. If we
overwrote one, we'd be invalidating all related trees, etc."""
version_id = _version_id(volume.year, cfr_part)
version_entries = entry.FinalVersion(volume.title, cfr_part)
if version_id not in version_entries:
(version_entries / version_id).write(
Version(identifier=version_id,
effective=volume.publication_date,
published=volume.publication_date))
def create_version_entry_if_needed(volume, cfr_part):
"""Only write the version entry if it doesn't already exist. If we
overwrote one, we'd be invalidating all related trees, etc."""
version_id = _version_id(volume.year, cfr_part)
version_dir = entry.FinalVersion(volume.title, cfr_part)
if version_id not in [c.path[-1] for c in version_dir.sub_entries()]:
(version_dir / version_id).write(
Version(version_id,
effective=volume.publication_date,
fr_citation=Citation(volume.vol_num, 1)))
def test_process_cfr_layers(self):
"""All layers for a single version should get written."""
with self.cli.isolated_filesystem():
version_entry = entry.Version(12, 1000, '1234')
version_entry.write(Version('1234', date.today(), date.today()))
entry.Tree('12', '1000', '1234').write(Node())
layers.process_cfr_layers(['keyterms', 'meta'], 12, version_entry)
self.assertTrue(
entry.Layer.cfr(12, 1000, '1234', 'keyterms').exists())
self.assertTrue(entry.Layer.cfr(12, 1000, '1234', 'meta').exists())
def proposal_versions(doc_number):
"""Generate version entries associated with a proposal."""
notice = entry.Notice(doc_number)
if not notice.exists():
raise dependency.Missing(str(notice), str(notice))
notice = notice.read()
version = Version(doc_number, notice.published, None)
for cfr_title, cfr_part in notice.cfr_ref_pairs:
version_entry = entry.Version(cfr_title, cfr_part, doc_number)
if not version_entry.exists() or version_entry.read() != version:
version_entry.write(version)
def test_process_if_needed_missing_dependency_error():
"""If the annual XML or version isn't present, we should see a dependency
error."""
last_versions = [annual_editions.LastVersionInYear('1111', 2000)]
with pytest.raises(dependency.Missing):
annual_editions.process_if_needed('12', '1000', last_versions)
entry.Version('12', '1000', '1111').write(
Version('1111', date(2000, 1, 1), Citation(1, 1)))
with pytest.raises(dependency.Missing):
annual_editions.process_if_needed('12', '1000', last_versions)
def test_creates_version(self, entry):
notice = entry.Notice.return_value.read.return_value
notice.published = date.today()
notice.cfr_ref_pairs = [(11, 111), (11, 222), (22, 222), (22, 333)]
with self.cli.isolated_filesystem():
result = self.cli.invoke(proposal_versions, ['dddd'])
self.assertIsNone(result.exception)
self.assertEqual('dddd', entry.Notice.call_args[0][0])
self.assertEqual([lst[0] for lst in entry.Version.call_args_list],
[(11, 111, 'dddd'), (11, 222, 'dddd'),
(22, 222, 'dddd'), (22, 333, 'dddd')])
self.assertEqual(entry.Version.return_value.write.call_args[0][0],
Version('dddd', date.today(), None))
def test_process_if_needed_missing_dependency_error(self):
"""If the annual XML or version isn't present, we should see a
dependency error."""
with self.cli.isolated_filesystem():
last_versions = [annual_editions.LastVersionInYear('1111', 2000)]
with self.assertRaises(dependency.Missing):
annual_editions.process_if_needed('12', '1000', last_versions)
entry.Version('12', '1000', '1111').write(
Version('1111', date(2000, 1, 1), date(2000, 1, 1)))
with self.assertRaises(dependency.Missing):
annual_editions.process_if_needed('12', '1000', last_versions)
def process_version_if_needed(cfr_title, cfr_part, version_id):
"""Creates and writes a version struct after validating the Notice has
been created"""
notice_entry = entry.Notice(version_id)
version_entry = entry.Version(cfr_title, cfr_part, version_id)
deps = dependency.Graph()
deps.add(version_entry, notice_entry)
deps.validate_for(version_entry)
if deps.is_stale(version_entry):
notice_xml = notice_entry.read()
version = Version(version_id, notice_xml.effective,
notice_xml.fr_citation)
version_entry.write(version)
def test_parents_of():
final1 = Version(str(randrange(1000)), date(2002, 2, 2), Citation(1, 1))
prop1 = Version(str(randrange(1000)), None, Citation(2, 2))
final2 = Version(str(randrange(1000)), date(2004, 4, 4), Citation(3, 3))
prop2 = Version('222', None, Citation(4, 4))
prop3 = Version('333', None, Citation(6, 6))
prop4 = Version('444', None, Citation(6, 6))
correct_order = [final1, prop1, final2, prop2, prop3, prop4]
for permutation in permutations(correct_order):
assert list(sorted(permutation)) == correct_order
paired = list(zip(correct_order, Version.parents_of(correct_order)))
assert paired == [(final1, None), (prop1, final1), (final2, final1),
(prop2, final2), (prop3, final2), (prop4, final2)]
def test_creates_version(monkeypatch):
monkeypatch.setattr(proposal_versions, 'entry', Mock())
notice = proposal_versions.entry.Notice.return_value.read.return_value
notice.fr_citation = Citation(1, 2)
notice.cfr_ref_pairs = [(11, 111), (11, 222), (22, 222), (22, 333)]
result = CliRunner().invoke(proposal_versions.proposal_versions, ['dddd'])
assert result.exception is None
assert proposal_versions.entry.Notice.call_args == call('dddd')
assert proposal_versions.entry.Version.call_args_list == [
call(11, 111, 'dddd'),
call(11, 222, 'dddd'),
call(22, 222, 'dddd'),
call(22, 333, 'dddd')
]
write_args = proposal_versions.entry.Version.return_value.write.call_args
assert write_args == call(Version('dddd', None, Citation(1, 2)))
def test_parents_of(self):
final1 = Version(str(randrange(1000)),
date(2001, 1, 1), date(2002, 2, 2))
prop1 = Version(str(randrange(1000)), date(2001, 6, 6), None)
final2 = Version(str(randrange(1000)),
date(2003, 3, 3), date(2004, 4, 4))
prop2 = Version('222', date(2003, 4, 4), None)
prop3 = Version('333', date(2006, 6, 6), None)
prop4 = Version('444', date(2006, 6, 6), None)
correct_order = [final1, prop1, final2, prop2, prop3, prop4]
for permutation in permutations(correct_order):
self.assertEqual(sorted(permutation), correct_order)
paired = list(zip(correct_order, Version.parents_of(correct_order)))
self.assertEqual(
paired,
[(final1, None), (prop1, final1), (final2, final1),
(prop2, final2), (prop3, final2), (prop4, final2)])
def transform_notice(notice_xml):
"""The API has a different format for notices than the local XML. We'll
need to convert and add appropriate fields"""
as_dict = notice_xml.as_dict()
as_dict['versions'] = {}
for cfr_title, cfr_part in notice_xml.cfr_ref_pairs:
version_dir = entry.Version(cfr_title, cfr_part)
versions = [(version_dir / id).read() for id in version_dir]
with_parents = zip(versions, Version.parents_of(versions))
for version, parent in with_parents:
if version.identifier == notice_xml.version_id and parent:
as_dict['versions'][cfr_part] = {"left": parent.identifier,
"right": version.identifier}
# @todo - SxS and footnotes aren't used outside of CFPB
add_footnotes(as_dict, notice_xml.xml)
if notice_xml.cfr_ref_pairs:
process_sxs(as_dict, notice_xml.xml)
return as_dict
def fill_with_rules(cfr_title, cfr_part):
"""Fill in missing trees using data from rules. When a regulation tree
cannot be derived through annual editions, it must be built by parsing the
changes in final rules. This command builds those missing trees"""
logger.info("Fill with rules - %s CFR %s", cfr_title, cfr_part)
tree_dir = entry.Tree(cfr_title, cfr_part)
version_dir = entry.Version(cfr_title, cfr_part)
versions = [(version_dir / v).read() for v in version_dir]
versions_with_parents = list(zip(versions, Version.parents_of(versions)))
deps = dependencies(tree_dir, version_dir, versions_with_parents)
derived = [(version.identifier, parent.identifier)
for version, parent in versions_with_parents
if is_derived(version.identifier, deps, tree_dir)]
for version_id, parent_id in derived:
deps.validate_for(tree_dir / version_id)
if deps.is_stale(tree_dir / version_id):
process(tree_dir, parent_id, version_id)
deps.rebuild()
def deserialize(self, content):
return VersionStruct.from_json(content)
def deserialize(self, content):
return VersionStruct.from_json(content.decode('utf-8'))
