コード例 #1
0
    def test_compile_designate(self):
        root = self.tree_with_subparts()
        change = {
            'action': 'POST',
            'subpart': ['205', 'Subpart', 'B'],
            'node': {
                'text': '2 text',
                'label': ['205', '2'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2': [change]}
        reg = compiler.compile_regulation(root, notice_changes)

        subpart_b = find(reg, '205-Subpart-B')
        self.assertEqual(len(subpart_b.children), 1)

        subpart_a = find(reg, '205-Subpart-A')
        self.assertEqual(len(subpart_a.children), 0)

        change = {
            'action': 'DESIGNATE',
            'destination': ['205', 'Subpart', 'A']}

        notice_changes = {'205-2': [change]}

        new_reg = compiler.compile_regulation(reg, notice_changes)

        self.assertEqual(None, find(new_reg, '205-Subpart-B'))

        subpart_a = find(new_reg, '205-Subpart-A')
        self.assertEqual(len(subpart_a.children), 1)
コード例 #2
0
    def test_compile_designate(self):
        root = self.tree_with_subparts()
        change = {
            "action": "POST",
            "subpart": ["205", "Subpart", "B"],
            "node": Node(text="2 text", label=["205", "2"], node_type=Node.REGTEXT),
        }

        notice_changes = {"205-2": [change]}
        reg = compiler.compile_regulation(root, notice_changes)

        subpart_b = find(reg, "205-Subpart-B")
        self.assertEqual(len(subpart_b.children), 1)

        subpart_a = find(reg, "205-Subpart-A")
        self.assertEqual(len(subpart_a.children), 0)

        change = {"action": "DESIGNATE", "destination": ["205", "Subpart", "A"]}

        notice_changes = {"205-2": [change]}

        new_reg = compiler.compile_regulation(reg, notice_changes)

        self.assertEqual(None, find(new_reg, "205-Subpart-B"))

        subpart_a = find(new_reg, "205-Subpart-A")
        self.assertEqual(len(subpart_a.children), 1)
コード例 #3
0
    def test_compile_regulation_delete_move(self):
        prev_tree = self.tree_with_paragraphs()
        changes = {
            "205-2-a": [
                {"action": "MOVE", "destination": ["205", "2", "b"]},
                {"action": "POST", "node": Node(text="aaa", label=["205", "2", "a"], node_type=Node.REGTEXT)},
            ],
            "205-2-b": [{"action": "DELETE"}],
        }

        class SortedKeysDict(object):
            def keys(self):
                return list(sorted(changes.keys()))

            def __getitem__(self, key):
                return changes[key]

            def __contains__(self, key):
                if key in changes:
                    return True
                else:
                    return False

        new_tree = compiler.compile_regulation(prev_tree, SortedKeysDict())

        s1, s2, s4 = new_tree.children
        self.assertEqual(2, len(s2.children))
        s2a, s2b = s2.children
        self.assertEqual("aaa", s2a.text)
        self.assertEqual("n2a", s2b.text)
コード例 #4
0
    def test_compile_regulation_delete_move(self):
        prev_tree = self.tree_with_paragraphs()
        print prev_tree.label[0]
        changes = {
            '205-2-a': [
                {'action': 'MOVE', 'destination': ['205', '2', 'b']},
                {'action': 'POST', 'node': {'text': 'aaa',
                                            'label': ['205', '2', 'a'],
                                            'node_type': Node.REGTEXT}}],
            '205-2-b': [{'action': 'DELETE'}]}

        class SortedKeysDict(object):
            def keys(self):
                return list(sorted(changes.keys()))

            def __getitem__(self, key):
                return changes[key]

            def __contains__(self, key):
                if key in changes:
                    return True
                else:
                    return False

        new_tree = compiler.compile_regulation(prev_tree, SortedKeysDict())

        print new_tree

        s1, s2, s4 = new_tree.children
        self.assertEqual(2, len(s2.children))
        s2a, s2b = s2.children
        self.assertEqual("aaa", s2a.text)
        self.assertEqual("n2a", s2b.text)
コード例 #5
0
    def test_compile_reg_put_replace_whole_tree(self):
        root = self.tree_with_paragraphs()

        change2a = {
            'action': 'PUT',
            'node': {
                'text': 'new text',
                'label': ['205', '2', 'a'],
                'node_type': 'regtext'}}

        change2a1 = {
            'action': 'PUT',
            'node': {
                'text': '2a1 text',
                'label': ['205', '2', 'a', '1'],
                'node_type': 'regtext'}}

        notice_changes = {
            '205-2-a-1': [change2a1],
            '205-2-a': [change2a]
        }

        reg = compiler.compile_regulation(root, notice_changes)

        added_node = find(reg, '205-2-a')
        self.assertEqual(added_node.text, 'new text')

        deeper = find(reg, '205-2-a-1')
        self.assertEqual(deeper.text, '2a1 text')
コード例 #6
0
ファイル: regml.py プロジェクト: cfpb/regulations-xml-parser
def ecfr_notice(title, cfr_part, notice, applies_to, act_title,
        act_section, with_version=False, without_notice=False):
    """ Generate RegML for a single notice from eCFR XML. """

    # Get the notice the new one applies to
    with open(find_file(os.path.join(cfr_part, applies_to)), 'r') as f:
        reg_xml = f.read()
    parser = etree.XMLParser(huge_tree=True)
    xml_tree = etree.fromstring(reg_xml, parser)
    doc_number = xml_tree.find('.//{eregs}documentNumber').text

    # Validate the file relative to schema
    validator = get_validator(xml_tree)

    # Get the ecfr builder
    builder = Builder(cfr_title=title,
                      cfr_part=cfr_part,
                      doc_number=doc_number,
                      checkpointer=None,
                      writer_type='XML')

    # Fetch the notices from the FR API and find the notice we're
    # looking for
    builder.fetch_notices_json()
    print([n['document_number'] for n in builder.notices_json])
    notice_json = next((n for n in builder.notices_json
                        if n['document_number'] == notice))

    # Build the notice
    notice = builder.build_single_notice(notice_json)[0]

    if 'changes' not in notice:
        print('There are no changes in this notice to apply.')
        return

    # We've successfully fetched and parsed the new notice.
    # Build a the reg tree and layers for the notice it applies to.
    old_tree = build_reg_tree(xml_tree)

    # Build the new reg tree from the old_tree + notice changes
    last_version = doc_number
    version = notice['document_number']
    merged_changes = builder.merge_changes(version, notice['changes'])
    reg_tree = compile_regulation(old_tree, merged_changes)
    layer_cache = LayerCacheAggregator()
    layers = builder.generate_layers(reg_tree,
                                     [act_title, act_section],
                                     layer_cache)

    # Write the notice file
    if not without_notice:
        builder.write_notice(version,
                             old_tree=old_tree,
                             reg_tree=reg_tree,
                             layers=layers,
                             last_version=last_version)

    # Write the regulation file for the new notice
    if with_version:
        builder.write_regulation(new_tree, layers=layers)
コード例 #7
0
def process(tree_path, previous, version_id):
    """Build and write a tree by combining the preceding tree with changes
    present in the associated rule"""
    prev_tree = (tree_path / previous).read()
    notice = entry.RuleChanges(version_id).read()
    changes = merge_changes(version_id, notice.get('changes', {}))
    new_tree = compile_regulation(prev_tree, changes)
    (tree_path / version_id).write(new_tree)
コード例 #8
0
 def test_compile_reg_move_wrong_reg(self):
     """Changes applied to other regulations shouldn't affect the
     regulation we care about, even if that has the same textual prefix"""
     root = self.tree_with_paragraphs()
     notice_changes = {"2055-2-a": [{"action": "MOVE", "destination": ["2055", "2", "b"]}]}
     reg = compiler.compile_regulation(root, notice_changes)
     self.assertEqual(find(reg, "205-2-a").text, "n2a")
     self.assertEqual(find(reg, "205-2-b").text, "n2b")
     self.assertEqual(find(reg, "2055-2-b"), None)
コード例 #9
0
 def test_compile_reg_move_wrong_reg(self):
     """Changes applied to other regulations shouldn't affect the
     regulation we care about, even if that has the same textual prefix"""
     root = self.tree_with_paragraphs()
     notice_changes = {'2055-2-a': [{'action': 'MOVE',
                                    'destination': ['2055', '2', 'b']}]}
     reg = compiler.compile_regulation(root, notice_changes)
     self.assertEqual(find(reg, '205-2-a').text, 'n2a')
     self.assertEqual(find(reg, '205-2-b').text, 'n2b')
     self.assertEqual(find(reg, '2055-2-b'), None)
コード例 #10
0
def process(tree_path, previous, version_id):
    """Build and write a tree by combining the preceding tree with changes
    present in the associated rule"""
    prev_tree = (tree_path / previous).read()
    notice = entry.Notice(version_id).read()
    notice_changes = defaultdict(list)
    for amendment in notice.amendments:
        for label, change_list in amendment.get('changes', []):
            notice_changes[label].extend(change_list)
    new_tree = compile_regulation(prev_tree, notice_changes)
    (tree_path / version_id).write(new_tree)
コード例 #11
0
    def test_compile_reg_post_no_subpart(self):
        root = self.tree_with_paragraphs()
        change2a1 = {
            "action": "POST",
            "node": Node(text="2a1 text", label=["205", "2", "a", "1"], node_type=Node.REGTEXT),
        }

        notice_changes = {"205-2-a-1": [change2a1]}
        reg = compiler.compile_regulation(root, notice_changes)
        added_node = find(reg, "205-2-a-1")
        self.assertNotEqual(None, added_node)
        self.assertEqual(added_node.text, "2a1 text")
コード例 #12
0
    def test_compile_reg_put_text_only(self):
        root = self.tree_with_paragraphs()
        change2a = {
            "action": "PUT",
            "field": "[text]",
            "node": Node(text="new text", label=["205", "2", "a"], node_type=Node.REGTEXT),
        }

        notice_changes = {"205-2-a": [change2a]}
        reg = compiler.compile_regulation(root, notice_changes)

        changed_node = find(reg, "205-2-a")
        self.assertEqual(changed_node.text, "new text")
コード例 #13
0
 def revision_generator(self, reg_tree):
     relevant_notices = []
     for date in sorted(self.eff_notices.keys()):
         relevant_notices.extend(
             n for n in self.eff_notices[date]
             if 'changes' in n and n['document_number'] != self.doc_number)
     for notice in relevant_notices:
         version = notice['document_number']
         old_tree = reg_tree
         merged_changes = self.merge_changes(version, notice['changes'])
         reg_tree = compile_regulation(old_tree, merged_changes)
         notices = applicable_notices(self.notices, version)
         yield notice, old_tree, reg_tree, notices
コード例 #14
0
    def test_compile_add_to_subpart(self):
        root = self.tree_with_subparts()

        change = {
            "action": "POST",
            "subpart": ["205", "Subpart", "B"],
            "node": Node(text="2 text", label=["205", "2"], node_type=Node.REGTEXT),
        }

        notice_changes = {"205-2": [change]}
        reg = compiler.compile_regulation(root, notice_changes)
        added_node = find(reg, "205-2")
        self.assertNotEqual(None, added_node)
        self.assertEqual(added_node.text, "2 text")
コード例 #15
0
    def test_compile_reg_post_no_subpart(self):
        root = self.tree_with_paragraphs()
        change2a1 = {
            'action': 'POST',
            'node': {
                'text': '2a1 text',
                'label': ['205', '2', 'a', '1'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2-a-1': [change2a1]}
        reg = compiler.compile_regulation(root, notice_changes)
        added_node = find(reg, '205-2-a-1')
        self.assertNotEqual(None, added_node)
        self.assertEqual(added_node.text, '2a1 text')
コード例 #16
0
    def test_compile_reg_post_no_subpart(self):
        root = self.tree_with_paragraphs()
        change2a1 = {
            'action': 'POST',
            'node': {
                'text': '2a1 text',
                'label': ['205', '2', 'a', '1'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2-a-1': [change2a1]}
        reg = compiler.compile_regulation(root, notice_changes)
        added_node = find(reg, '205-2-a-1')
        self.assertNotEqual(None, added_node)
        self.assertEqual(added_node.text, '2a1 text')
コード例 #17
0
 def revision_generator(self, reg_tree):
     """Given an initial regulation tree, this will emit (and checkpoint)
     new versions of the tree, along with the notice that caused the
     change. This is a generator, so processing only occurs as needed"""
     for version, merged_changes in self.changes_in_sequence():
         old_tree = reg_tree
         reg_tree = self.checkpointer.checkpoint(
             "compiled-" + version,
             lambda: compile_regulation(old_tree, merged_changes))
         notices = applicable_notices(self.notices, version)
         first_notice = None
         for notice in notices:
             if notice['document_number'] == version:
                 first_notice = notice
         yield first_notice, old_tree, reg_tree, notices
コード例 #18
0
    def test_compile_reg_put_text_only(self):
        root = self.tree_with_paragraphs()
        change2a = {
            'action': 'PUT',
            'field': '[text]',
            'node': {
                'text': 'new text',
                'label': ['205', '2', 'a'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2-a': [change2a]}
        reg = compiler.compile_regulation(root, notice_changes)

        changed_node = find(reg, '205-2-a')
        self.assertEqual(changed_node.text, 'new text')
コード例 #19
0
    def test_compile_reg_put_text_only(self):
        root = self.tree_with_paragraphs()
        change2a = {
            'action': 'PUT',
            'field': '[text]',
            'node': {
                'text': 'new text',
                'label': ['205', '2', 'a'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2-a': [change2a]}
        reg = compiler.compile_regulation(root, notice_changes)

        changed_node = find(reg, '205-2-a')
        self.assertEqual(changed_node.text, 'new text')
コード例 #20
0
 def revision_generator(self, reg_tree):
     """Given an initial regulation tree, this will emit (and checkpoint)
     new versions of the tree, along with the notice that caused the
     change. This is a generator, so processing only occurs as needed"""
     for version, merged_changes in self.changes_in_sequence():
         old_tree = reg_tree
         reg_tree = self.checkpointer.checkpoint(
             "compiled-" + version,
             lambda: compile_regulation(old_tree, merged_changes))
         notices = applicable_notices(self.notices, version)
         first_notice = None
         for notice in notices:
             if notice['document_number'] == version:
                 first_notice = notice
         yield first_notice, old_tree, reg_tree, notices
コード例 #21
0
    def test_compile_reg_keep_child(self):
        root = self.tree_with_paragraphs()
        change2 = {"action": "PUT", "node": Node(text="n2n2", label=["205", "2"], node_type=Node.REGTEXT)}
        change2a = {"action": "KEEP", "node": Node(text="(a) * * *", label=["205", "2", "a"], node_type=Node.REGTEXT)}
        change2b = {"action": "PUT", "node": Node(text="(b) A Test", label=["205", "2", "b"], node_type=Node.REGTEXT)}

        notice_changes = {"205-2": [change2], "205-2-a": [change2a], "205-2-b": [change2b]}
        reg = compiler.compile_regulation(root, notice_changes)

        changed = find(reg, "205-2")
        self.assertEqual(changed.text, "n2n2")
        self.assertEqual(2, len(changed.children))
        changed2a, changed2b = changed.children
        self.assertEqual("n2a", changed2a.text)  # text didn't change
        self.assertEqual("(b) A Test", changed2b.text)
コード例 #22
0
    def test_compile_add_to_subpart(self):
        root = self.tree_with_subparts()

        change = {
            'action': 'POST',
            'subpart': ['205', 'Subpart', 'B'],
            'node': {
                'text': '2 text',
                'label': ['205', '2'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2': [change]}
        reg = compiler.compile_regulation(root, notice_changes)
        added_node = find(reg, '205-2')
        self.assertNotEqual(None, added_node)
        self.assertEqual(added_node.text, '2 text')
コード例 #23
0
    def test_compile_add_to_subpart(self):
        root = self.tree_with_subparts()

        change = {
            'action': 'POST',
            'subpart': ['205', 'Subpart', 'B'],
            'node': {
                'text': '2 text',
                'label': ['205', '2'],
                'node_type': 'regtext'}}

        notice_changes = {'205-2': [change]}
        reg = compiler.compile_regulation(root, notice_changes)
        added_node = find(reg, '205-2')
        self.assertNotEqual(None, added_node)
        self.assertEqual(added_node.text, '2 text')
コード例 #24
0
    def test_compile_reg_put_replace_whole_tree(self):
        root = self.tree_with_paragraphs()

        change2a = {"action": "PUT", "node": Node(text="new text", label=["205", "2", "a"], node_type=Node.REGTEXT)}

        change2a1 = {
            "action": "PUT",
            "node": Node(text="2a1 text", label=["205", "2", "a", "1"], node_type=Node.REGTEXT),
        }

        notice_changes = {"205-2-a-1": [change2a1], "205-2-a": [change2a]}

        reg = compiler.compile_regulation(root, notice_changes)

        added_node = find(reg, "205-2-a")
        self.assertEqual(added_node.text, "new text")

        deeper = find(reg, "205-2-a-1")
        self.assertEqual(deeper.text, "2a1 text")
コード例 #25
0
    def test_compile_regulation_delete_move(self):
        prev_tree = self.tree_with_paragraphs()
        print prev_tree.label[0]
        changes = {
            '205-2-a': [{
                'action': 'MOVE',
                'destination': ['205', '2', 'b']
            }, {
                'action': 'POST',
                'node': {
                    'text': 'aaa',
                    'label': ['205', '2', 'a'],
                    'node_type': Node.REGTEXT
                }
            }],
            '205-2-b': [{
                'action': 'DELETE'
            }]
        }

        class SortedKeysDict(object):
            def keys(self):
                return list(sorted(changes.keys()))

            def __getitem__(self, key):
                return changes[key]

            def __contains__(self, key):
                if key in changes:
                    return True
                else:
                    return False

        new_tree = compiler.compile_regulation(prev_tree, SortedKeysDict())

        print new_tree

        s1, s2, s4 = new_tree.children
        self.assertEqual(2, len(s2.children))
        s2a, s2b = s2.children
        self.assertEqual("aaa", s2a.text)
        self.assertEqual("n2a", s2b.text)
コード例 #26
0
    def test_compile_reg_put_children_only(self):
        root = self.tree_with_paragraphs()
        change2 = {'action': 'PUT',
                   'field': '[children]',
                   'node': {'text': '* * *', 'label': ['205', '2'],
                            'node_type': 'regtext'}}
        change2a = {'action': 'PUT',
                    'node': {'text': '(a) A Test', 'label': ['205', '2', 'a'],
                             'node_type': 'regtext'}}

        notice_changes = {'205-2': [change2], '205-2-a': [change2a]}
        reg = compiler.compile_regulation(root, notice_changes)

        changed = find(reg, '205-2')
        self.assertEqual(changed.text, 'n2')    # text didn't change
        self.assertEqual(1, len(changed.children))
        changed2a = changed.children[0]
        self.assertEqual(['205', '2', 'a'], changed2a.label)
        self.assertEqual('(a) A Test', changed2a.text)
        self.assertEqual([], changed2a.children)
コード例 #27
0
    def test_compile_reg_keep_root(self):
        root = self.tree_with_paragraphs()
        change2 = {'action': 'KEEP',
                   'node': {'text': '* * *', 'label': ['205', '2'],
                            'node_type': 'regtext'}}
        change2a = {'action': 'PUT',
                    'node': {'text': '(a) A Test', 'label': ['205', '2', 'a'],
                             'node_type': 'regtext'}}

        notice_changes = {'205-2': [change2], '205-2-a': [change2a]}
        reg = compiler.compile_regulation(root, notice_changes)

        changed = find(reg, '205-2')
        self.assertEqual(changed.text, 'n2')    # text didn't change
        self.assertEqual(2, len(changed.children))
        changed2a, changed2b = changed.children
        self.assertEqual(['205', '2', 'a'], changed2a.label)
        self.assertEqual('(a) A Test', changed2a.text)
        self.assertEqual([], changed2a.children)

        self.assertEqual(['205', '2', 'b'], changed2b.label)
コード例 #28
0
    def test_compile_reg_keep_child(self):
        root = self.tree_with_paragraphs()
        change2 = {'action': 'PUT',
                   'node': {'text': 'n2n2', 'label': ['205', '2'],
                            'node_type': 'regtext'}}
        change2a = {'action': 'KEEP',
                    'node': {'text': '(a) * * *', 'label': ['205', '2', 'a'],
                             'node_type': 'regtext'}}
        change2b = {'action': 'PUT',
                    'node': {'text': '(b) A Test', 'label': ['205', '2', 'b'],
                             'node_type': 'regtext'}}

        notice_changes = {'205-2': [change2], '205-2-a': [change2a],
                          '205-2-b': [change2b]}
        reg = compiler.compile_regulation(root, notice_changes)

        changed = find(reg, '205-2')
        self.assertEqual(changed.text, 'n2n2')
        self.assertEqual(2, len(changed.children))
        changed2a, changed2b = changed.children
        self.assertEqual('n2a', changed2a.text)     # text didn't change
        self.assertEqual('(b) A Test', changed2b.text)
コード例 #29
0
 def test_compile_reg_move_wrong_reg(self):
     root = self.tree_with_paragraphs()
     notice_changes = {'202-2-a': [{'action': 'MOVE',
                                    'destination': ['202', '2', 'b']}]}
     reg = compiler.compile_regulation(root, notice_changes)
     self.assertEqual(find(reg, '205-2-a').text, 'n2a')
コード例 #30
0
 def test_compile_reg_move_wrong_reg(self):
     root = self.tree_with_paragraphs()
     notice_changes = {'202-2-a': [{'action': 'MOVE',
                                    'destination': ['202', '2', 'b']}]}
     reg = compiler.compile_regulation(root, notice_changes)
     self.assertEqual(find(reg, '205-2-a').text, 'n2a')