def test_dependencies(): """Expect nonexistent trees to depend on their predecessor, associated rule changes and version files. Shouldn't add dependencies for the first version, if missing""" versions = [Version(str(i)*3, date(2001, i, i), date(2002, i, i)) for i in range(1, 7)] parents = Version.parents_of(versions) tree_dir = entry.Tree('12', '1000') notice_dir = entry.Notice() vers_dir = entry.Version('12', '1000') # Existing trees (tree_dir / '222').write(Node()) (tree_dir / '555').write(Node()) deps = fill_with_rules.dependencies(tree_dir, vers_dir, list(zip(versions, parents))) # First is skipped, as we can't build it from a rule assert str(tree_dir / '111') not in deps # Second can also be skipped as a tree already exists assert deps.dependencies(str(tree_dir / '222')) == [] # Third relies on the associated versions and the second tree expected = {str(tree_dir / '222'), str(notice_dir / '333'), str(vers_dir / '333')} assert set(deps.dependencies(str(tree_dir / '333'))) == expected # Fourth relies on the third, even though it's not been built expected = {str(tree_dir / '333'), str(notice_dir / '444'), str(vers_dir / '444')} assert set(deps.dependencies(str(tree_dir / '444'))) == expected # Fifth can be skipped as the tree already exists assert deps.dependencies(str(tree_dir / '555')) == [] # Six relies on the fifth expected = {str(tree_dir / '555'), str(notice_dir / '666'), str(vers_dir / '666')} assert set(deps.dependencies(str(tree_dir / '666'))) == expected
def test_parents_of(): final1 = Version(str(randrange(1000)), date(2002, 2, 2), Citation(1, 1)) prop1 = Version(str(randrange(1000)), None, Citation(2, 2)) final2 = Version(str(randrange(1000)), date(2004, 4, 4), Citation(3, 3)) prop2 = Version('222', None, Citation(4, 4)) prop3 = Version('333', None, Citation(6, 6)) prop4 = Version('444', None, Citation(6, 6)) correct_order = [final1, prop1, final2, prop2, prop3, prop4] for permutation in permutations(correct_order): assert list(sorted(permutation)) == correct_order paired = list(zip(correct_order, Version.parents_of(correct_order))) assert paired == [(final1, None), (prop1, final1), (final2, final1), (prop2, final2), (prop3, final2), (prop4, final2)]
def test_parents_of(self): final1 = Version(str(randrange(1000)), date(2001, 1, 1), date(2002, 2, 2)) prop1 = Version(str(randrange(1000)), date(2001, 6, 6), None) final2 = Version(str(randrange(1000)), date(2003, 3, 3), date(2004, 4, 4)) prop2 = Version('222', date(2003, 4, 4), None) prop3 = Version('333', date(2006, 6, 6), None) prop4 = Version('444', date(2006, 6, 6), None) correct_order = [final1, prop1, final2, prop2, prop3, prop4] for permutation in permutations(correct_order): self.assertEqual(sorted(permutation), correct_order) paired = list(zip(correct_order, Version.parents_of(correct_order))) self.assertEqual(paired, [(final1, None), (prop1, final1), (final2, final1), (prop2, final2), (prop3, final2), (prop4, final2)])
def test_parents_of(self): final1 = Version(str(randrange(1000)), date(2001, 1, 1), date(2002, 2, 2)) prop1 = Version(str(randrange(1000)), date(2001, 6, 6), None) final2 = Version(str(randrange(1000)), date(2003, 3, 3), date(2004, 4, 4)) prop2 = Version('222', date(2003, 4, 4), None) prop3 = Version('333', date(2006, 6, 6), None) prop4 = Version('444', date(2006, 6, 6), None) correct_order = [final1, prop1, final2, prop2, prop3, prop4] for permutation in permutations(correct_order): self.assertEqual(sorted(permutation), correct_order) paired = list(zip(correct_order, Version.parents_of(correct_order))) self.assertEqual( paired, [(final1, None), (prop1, final1), (final2, final1), (prop2, final2), (prop3, final2), (prop4, final2)])
def transform_notice(notice_xml): """The API has a different format for notices than the local XML. We'll need to convert and add appropriate fields""" as_dict = notice_xml.as_dict() as_dict['versions'] = {} for cfr_title, cfr_part in notice_xml.cfr_ref_pairs: version_dir = entry.Version(cfr_title, cfr_part) versions = [(version_dir / id).read() for id in version_dir] with_parents = zip(versions, Version.parents_of(versions)) for version, parent in with_parents: if version.identifier == notice_xml.version_id and parent: as_dict['versions'][cfr_part] = {"left": parent.identifier, "right": version.identifier} # @todo - SxS and footnotes aren't used outside of CFPB add_footnotes(as_dict, notice_xml.xml) if notice_xml.cfr_ref_pairs: process_sxs(as_dict, notice_xml.xml) return as_dict
def transform_notice(notice_xml): """The API has a different format for notices than the local XML. We'll need to convert and add appropriate fields""" as_dict = notice_xml.as_dict() as_dict['versions'] = {} for cfr_title, cfr_part in notice_xml.cfr_ref_pairs: version_dir = entry.Version(cfr_title, cfr_part) versions = [v.read() for v in version_dir.sub_entries()] with_parents = zip(versions, Version.parents_of(versions)) for version, parent in with_parents: if version.identifier == notice_xml.version_id and parent: as_dict['versions'][cfr_part] = {"left": parent.identifier, "right": version.identifier} # @todo - SxS and footnotes aren't used outside of CFPB add_footnotes(as_dict, notice_xml.xml) if notice_xml.cfr_ref_pairs: process_sxs(as_dict, notice_xml.xml) return as_dict
def fill_with_rules(cfr_title, cfr_part): """Fill in missing trees using data from rules. When a regulation tree cannot be derived through annual editions, it must be built by parsing the changes in final rules. This command builds those missing trees""" logger.info("Fill with rules - %s CFR %s", cfr_title, cfr_part) tree_dir = entry.Tree(cfr_title, cfr_part) version_dir = entry.Version(cfr_title, cfr_part) versions = [c.read() for c in version_dir.sub_entries()] versions_with_parents = list(zip(versions, Version.parents_of(versions))) deps = dependencies(tree_dir, version_dir, versions_with_parents) derived = [(version.identifier, parent.identifier) for version, parent in versions_with_parents if is_derived(version.identifier, deps, tree_dir)] for version_id, parent_id in derived: deps.validate_for(tree_dir / version_id) if deps.is_stale(tree_dir / version_id): process(tree_dir, parent_id, version_id) deps.rebuild()
def fill_with_rules(cfr_title, cfr_part): """Fill in missing trees using data from rules. When a regulation tree cannot be derived through annual editions, it must be built by parsing the changes in final rules. This command builds those missing trees""" logger.info("Fill with rules - %s CFR %s", cfr_title, cfr_part) tree_dir = entry.Tree(cfr_title, cfr_part) version_dir = entry.Version(cfr_title, cfr_part) versions = [(version_dir / v).read() for v in version_dir] versions_with_parents = list(zip(versions, Version.parents_of(versions))) deps = dependencies(tree_dir, version_dir, versions_with_parents) derived = [(version.identifier, parent.identifier) for version, parent in versions_with_parents if is_derived(version.identifier, deps, tree_dir)] for version_id, parent_id in derived: deps.validate_for(tree_dir / version_id) if deps.is_stale(tree_dir / version_id): process(tree_dir, parent_id, version_id) deps.rebuild()