def test_dependencies():
"""Expect nonexistent trees to depend on their predecessor, associated
rule changes and version files. Shouldn't add dependencies for the
first version, if missing"""
versions = [Version(str(i)*3, date(2001, i, i), date(2002, i, i))
for i in range(1, 7)]
parents = Version.parents_of(versions)
tree_dir = entry.Tree('12', '1000')
notice_dir = entry.Notice()
vers_dir = entry.Version('12', '1000')
# Existing trees
(tree_dir / '222').write(Node())
(tree_dir / '555').write(Node())
deps = fill_with_rules.dependencies(tree_dir, vers_dir,
list(zip(versions, parents)))
# First is skipped, as we can't build it from a rule
assert str(tree_dir / '111') not in deps
# Second can also be skipped as a tree already exists
assert deps.dependencies(str(tree_dir / '222')) == []
# Third relies on the associated versions and the second tree
expected = {str(tree_dir / '222'), str(notice_dir / '333'),
str(vers_dir / '333')}
assert set(deps.dependencies(str(tree_dir / '333'))) == expected
# Fourth relies on the third, even though it's not been built
expected = {str(tree_dir / '333'), str(notice_dir / '444'),
str(vers_dir / '444')}
assert set(deps.dependencies(str(tree_dir / '444'))) == expected
# Fifth can be skipped as the tree already exists
assert deps.dependencies(str(tree_dir / '555')) == []
# Six relies on the fifth
expected = {str(tree_dir / '555'), str(notice_dir / '666'),
str(vers_dir / '666')}
assert set(deps.dependencies(str(tree_dir / '666'))) == expected
def test_parents_of():
final1 = Version(str(randrange(1000)), date(2002, 2, 2), Citation(1, 1))
prop1 = Version(str(randrange(1000)), None, Citation(2, 2))
final2 = Version(str(randrange(1000)), date(2004, 4, 4), Citation(3, 3))
prop2 = Version('222', None, Citation(4, 4))
prop3 = Version('333', None, Citation(6, 6))
prop4 = Version('444', None, Citation(6, 6))
correct_order = [final1, prop1, final2, prop2, prop3, prop4]
for permutation in permutations(correct_order):
assert list(sorted(permutation)) == correct_order
paired = list(zip(correct_order, Version.parents_of(correct_order)))
assert paired == [(final1, None), (prop1, final1), (final2, final1),
(prop2, final2), (prop3, final2), (prop4, final2)]
def test_parents_of(self):
final1 = Version(str(randrange(1000)), date(2001, 1, 1),
date(2002, 2, 2))
prop1 = Version(str(randrange(1000)), date(2001, 6, 6), None)
final2 = Version(str(randrange(1000)), date(2003, 3, 3),
date(2004, 4, 4))
prop2 = Version('222', date(2003, 4, 4), None)
prop3 = Version('333', date(2006, 6, 6), None)
prop4 = Version('444', date(2006, 6, 6), None)
correct_order = [final1, prop1, final2, prop2, prop3, prop4]
for permutation in permutations(correct_order):
self.assertEqual(sorted(permutation), correct_order)
paired = list(zip(correct_order, Version.parents_of(correct_order)))
self.assertEqual(paired, [(final1, None), (prop1, final1),
(final2, final1), (prop2, final2),
(prop3, final2), (prop4, final2)])
def test_parents_of(self):
final1 = Version(str(randrange(1000)),
date(2001, 1, 1), date(2002, 2, 2))
prop1 = Version(str(randrange(1000)), date(2001, 6, 6), None)
final2 = Version(str(randrange(1000)),
date(2003, 3, 3), date(2004, 4, 4))
prop2 = Version('222', date(2003, 4, 4), None)
prop3 = Version('333', date(2006, 6, 6), None)
prop4 = Version('444', date(2006, 6, 6), None)
correct_order = [final1, prop1, final2, prop2, prop3, prop4]
for permutation in permutations(correct_order):
self.assertEqual(sorted(permutation), correct_order)
paired = list(zip(correct_order, Version.parents_of(correct_order)))
self.assertEqual(
paired,
[(final1, None), (prop1, final1), (final2, final1),
(prop2, final2), (prop3, final2), (prop4, final2)])
def transform_notice(notice_xml):
"""The API has a different format for notices than the local XML. We'll
need to convert and add appropriate fields"""
as_dict = notice_xml.as_dict()
as_dict['versions'] = {}
for cfr_title, cfr_part in notice_xml.cfr_ref_pairs:
version_dir = entry.Version(cfr_title, cfr_part)
versions = [(version_dir / id).read() for id in version_dir]
with_parents = zip(versions, Version.parents_of(versions))
for version, parent in with_parents:
if version.identifier == notice_xml.version_id and parent:
as_dict['versions'][cfr_part] = {"left": parent.identifier,
"right": version.identifier}
# @todo - SxS and footnotes aren't used outside of CFPB
add_footnotes(as_dict, notice_xml.xml)
if notice_xml.cfr_ref_pairs:
process_sxs(as_dict, notice_xml.xml)
return as_dict
def transform_notice(notice_xml):
"""The API has a different format for notices than the local XML. We'll
need to convert and add appropriate fields"""
as_dict = notice_xml.as_dict()
as_dict['versions'] = {}
for cfr_title, cfr_part in notice_xml.cfr_ref_pairs:
version_dir = entry.Version(cfr_title, cfr_part)
versions = [v.read() for v in version_dir.sub_entries()]
with_parents = zip(versions, Version.parents_of(versions))
for version, parent in with_parents:
if version.identifier == notice_xml.version_id and parent:
as_dict['versions'][cfr_part] = {"left": parent.identifier,
"right": version.identifier}
# @todo - SxS and footnotes aren't used outside of CFPB
add_footnotes(as_dict, notice_xml.xml)
if notice_xml.cfr_ref_pairs:
process_sxs(as_dict, notice_xml.xml)
return as_dict
def fill_with_rules(cfr_title, cfr_part):
"""Fill in missing trees using data from rules. When a regulation tree
cannot be derived through annual editions, it must be built by parsing the
changes in final rules. This command builds those missing trees"""
logger.info("Fill with rules - %s CFR %s", cfr_title, cfr_part)
tree_dir = entry.Tree(cfr_title, cfr_part)
version_dir = entry.Version(cfr_title, cfr_part)
versions = [c.read() for c in version_dir.sub_entries()]
versions_with_parents = list(zip(versions, Version.parents_of(versions)))
deps = dependencies(tree_dir, version_dir, versions_with_parents)
derived = [(version.identifier, parent.identifier)
for version, parent in versions_with_parents
if is_derived(version.identifier, deps, tree_dir)]
for version_id, parent_id in derived:
deps.validate_for(tree_dir / version_id)
if deps.is_stale(tree_dir / version_id):
process(tree_dir, parent_id, version_id)
deps.rebuild()
def fill_with_rules(cfr_title, cfr_part):
"""Fill in missing trees using data from rules. When a regulation tree
cannot be derived through annual editions, it must be built by parsing the
changes in final rules. This command builds those missing trees"""
logger.info("Fill with rules - %s CFR %s", cfr_title, cfr_part)
tree_dir = entry.Tree(cfr_title, cfr_part)
version_dir = entry.Version(cfr_title, cfr_part)
versions = [(version_dir / v).read() for v in version_dir]
versions_with_parents = list(zip(versions, Version.parents_of(versions)))
deps = dependencies(tree_dir, version_dir, versions_with_parents)
derived = [(version.identifier, parent.identifier)
for version, parent in versions_with_parents
if is_derived(version.identifier, deps, tree_dir)]
for version_id, parent_id in derived:
deps.validate_for(tree_dir / version_id)
if deps.is_stale(tree_dir / version_id):
process(tree_dir, parent_id, version_id)
deps.rebuild()
