def test_process_changes_added_end(self):
     """ If we can't guess a valid sibling, ensure the added element
         is at the end of the parent. """
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="added" label="1234-5">
               <paragraph label="1234-5">An added paragraph</paragraph>
             </change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <paragraph label="1234-1">An existing paragraph</paragraph>
               <paragraph label="1234-3">Another existing paragraph</paragraph>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     new_para = new_xml.find('.//{eregs}paragraph[@label="1234-5"]')
     self.assertEqual(new_para.getparent().index(new_para), 2)
 def test_process_changes_added_interp(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="added" label="1234-Interp">
               <interpretations label="1234-Interp">
                 <title>Supplement I to Part 1234</title>
               </interpretations>
             </change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <subpart><content/></subpart>
               <appendix label="1234-A"></appendix>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     new_interp = new_xml.find('.//{eregs}interpretations[@label="1234-Interp"]')
     self.assertNotEqual(new_interp, None)
     self.assertEqual(new_interp.getparent().index(new_interp), 2)
Example #3
0
 def test_process_changes_added_interp(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="added" label="1234-Interp">
               <interpretations label="1234-Interp">
                 <title>Supplement I to Part 1234</title>
               </interpretations>
             </change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <subpart><content/></subpart>
               <appendix label="1234-A"></appendix>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     new_interp = new_xml.find(
         './/{eregs}interpretations[@label="1234-Interp"]')
     self.assertNotEqual(new_interp, None)
     self.assertEqual(new_interp.getparent().index(new_interp), 2)
Example #4
0
def apply_notices(cfr_part, version, notices):
    regulation_file = find_file(os.path.join(cfr_part, version))
    with open(regulation_file, 'r') as f:
        left_reg_xml = f.read()
    parser = etree.XMLParser(huge_tree=True)
    left_xml_tree = etree.fromstring(left_reg_xml, parser)

    prev_notice = version
    prev_tree = left_xml_tree
    for notice in notices:
        print('Applying notice {} to version {}'.format(notice, prev_notice))
        notice_file = find_file(os.path.join(cfr_part, notice), is_notice=True)
        with open(notice_file, 'r') as f:
            notice_string = f.read()
        parser = etree.XMLParser(huge_tree=True)
        notice_xml = etree.fromstring(notice_string, parser)

        # Process the notice changeset
        new_xml_tree = process_changes(prev_tree, notice_xml)

        # Write the new xml tree
        new_xml_string = etree.tostring(new_xml_tree,
                                        pretty_print=True,
                                        xml_declaration=True,
                                        encoding='UTF-8')
        new_path = os.path.join(
            os.path.dirname(regulation_file),
            os.path.basename(notice_file))
        with open(new_path, 'w') as f:
            print("Writing regulation to {}".format(new_path))
            f.write(new_xml_string)

        prev_notice = notice
        prev_tree = new_xml_tree
Example #5
0
 def test_process_changes_moved_xpath(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="moved" label="1234-Subpart-A" subpath='title' parent="1234-Subpart-B"></change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <subpart label="1234-Subpart-A">
                 <title>Test Title</title>
                 <content>
                   <paragraph label="1234-1">An existing paragraph</paragraph>
                 </content>
               </subpart>
               <subpart label="1234-Subpart-B">
                 <content>
                   <paragraph label="1234-2">Another existing paragraph</paragraph>
                 </content>
               </subpart>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     moved_title = new_xml.find('.//{eregs}title')
     self.assertEqual(moved_title.getparent().get('label'),
                      '1234-Subpart-B')
     old_title = new_xml.find(
         './/{eregs}subpart[@label="1234-Subpart-A"]/{eregs}title')
     self.assertEqual(old_title, None)
Example #6
0
 def test_process_changes_moved_after(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="moved" label="1234-1" parent="1234-Subpart-B" after="1234-2"></change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <subpart label="1234-Subpart-A">
                 <content>
                   <paragraph label="1234-1">An existing paragraph</paragraph>
                 </content>
               </subpart>
               <subpart label="1234-Subpart-B">
                 <content>
                   <paragraph label="1234-2">Another existing paragraph</paragraph>
                   <paragraph label="1234-3">One more existing paragraph</paragraph>
                 </content>
               </subpart>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     moved_para = new_xml.find('.//{eregs}paragraph[@label="1234-1"]')
     self.assertEqual(moved_para.getparent().getparent().get('label'),
                      '1234-Subpart-B')
     self.assertEqual(moved_para.getparent().index(moved_para), 1)
Example #7
0
 def test_process_changes_change_target_text(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="changeTarget" oldTarget="1234-1" newTarget="1234-3">reference to 1234-1</change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <subpart label="1234-Subpart-A">
                 <content>
                   <paragraph label="1234-1">An existing paragraph</paragraph>
                 </content>
               </subpart>
               <subpart label="1234-Subpart-B">
                 <content>
                   <paragraph label="1234-2">Another existing paragraph with a
                   <ref target="1234-1" reftype="internal">reference to 1234-1</ref></paragraph>
                   <paragraph label="1234-3">One more existing paragraph with <ref target="1234-1" reftype="internal">another reference to 1234-1</ref></paragraph>
                 </content>
               </subpart>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     old_refs = new_xml.findall('.//{eregs}ref[@target="1234-1"]')
     new_refs = new_xml.findall('.//{eregs}ref[@target="1234-3"]')
     self.assertEqual(len(old_refs), 1)
     self.assertEqual(len(new_refs), 1)
Example #8
0
 def test_process_changes_added_end(self):
     """ If we can't guess a valid sibling, ensure the added element
         is at the end of the parent. """
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="added" label="1234-5">
               <paragraph label="1234-5">An added paragraph</paragraph>
             </change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <paragraph label="1234-1">An existing paragraph</paragraph>
               <paragraph label="1234-3">Another existing paragraph</paragraph>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     new_para = new_xml.find('.//{eregs}paragraph[@label="1234-5"]')
     self.assertEqual(new_para.getparent().index(new_para), 2)
 def test_process_changes_moved_after(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="moved" label="1234-1" parent="1234-Subpart-B" after="1234-2"></change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <subpart label="1234-Subpart-A">
                 <content>
                   <paragraph label="1234-1">An existing paragraph</paragraph>
                 </content>
               </subpart>
               <subpart label="1234-Subpart-B">
                 <content>
                   <paragraph label="1234-2">Another existing paragraph</paragraph>
                   <paragraph label="1234-3">One more existing paragraph</paragraph>
                 </content>
               </subpart>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     moved_para = new_xml.find('.//{eregs}paragraph[@label="1234-1"]')
     self.assertEqual(moved_para.getparent().getparent().get('label'),
                      '1234-Subpart-B')
     self.assertEqual(moved_para.getparent().index(moved_para), 1)
 def test_process_changes_moved_xpath(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="moved" label="1234-Subpart-A" subpath='title' parent="1234-Subpart-B"></change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <subpart label="1234-Subpart-A">
                 <title>Test Title</title>
                 <content>
                   <paragraph label="1234-1">An existing paragraph</paragraph>
                 </content>
               </subpart>
               <subpart label="1234-Subpart-B">
                 <content>
                   <paragraph label="1234-2">Another existing paragraph</paragraph>
                 </content>
               </subpart>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     moved_title = new_xml.find('.//{eregs}title')
     self.assertEqual(moved_title.getparent().get('label'),
                      '1234-Subpart-B')
     old_title = new_xml.find('.//{eregs}subpart[@label="1234-Subpart-A"]/{eregs}title')
     self.assertEqual(old_title, None)
 def test_process_changes_added(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="added" label="1234-2">
               <paragraph label="1234-2">An added paragraph</paragraph>
             </change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <paragraph label="1234-1">An existing paragraph</paragraph>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     new_para = new_xml.find('.//{eregs}paragraph[@label="1234-2"]')
     self.assertNotEqual(new_para, None)
     self.assertEqual("An added paragraph", new_para.text)
     self.assertEqual(new_para.getparent().index(new_para), 1)
 def test_process_changes_modified_xpath(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="modified" label="1234" subpath='title'>
               <title>Modified Title</title>
             </change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <title>Test Title</title>
             <content>
               <paragraph label="1234-1">An existing paragraph</paragraph>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     mod_title = new_xml.findall('.//{eregs}title')
     self.assertEqual(len(mod_title), 1)
     self.assertNotEqual(mod_title[0], None)
     self.assertEqual("Modified Title", mod_title[0].text)
Example #13
0
 def test_process_changes_added(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="added" label="1234-2">
               <paragraph label="1234-2">An added paragraph</paragraph>
             </change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <paragraph label="1234-1">An existing paragraph</paragraph>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     new_para = new_xml.find('.//{eregs}paragraph[@label="1234-2"]')
     self.assertNotEqual(new_para, None)
     self.assertEqual("An added paragraph", new_para.text)
     self.assertEqual(new_para.getparent().index(new_para), 1)
 def test_process_changes_change_target_text(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="changeTarget" oldTarget="1234-1" newTarget="1234-3">reference to 1234-1</change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <subpart label="1234-Subpart-A">
                 <content>
                   <paragraph label="1234-1">An existing paragraph</paragraph>
                 </content>
               </subpart>
               <subpart label="1234-Subpart-B">
                 <content>
                   <paragraph label="1234-2">Another existing paragraph with a
                   <ref target="1234-1" reftype="internal">reference to 1234-1</ref></paragraph>
                   <paragraph label="1234-3">One more existing paragraph with <ref target="1234-1" reftype="internal">another reference to 1234-1</ref></paragraph>
                 </content>
               </subpart>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     old_refs = new_xml.findall('.//{eregs}ref[@target="1234-1"]')
     new_refs = new_xml.findall('.//{eregs}ref[@target="1234-3"]')
     self.assertEqual(len(old_refs), 1)
     self.assertEqual(len(new_refs), 1)
Example #15
0
 def test_process_changes_modified_xpath(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="modified" label="1234" subpath='title'>
               <title>Modified Title</title>
             </change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <title>Test Title</title>
             <content>
               <paragraph label="1234-1">An existing paragraph</paragraph>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     mod_title = new_xml.findall('.//{eregs}title')
     self.assertEqual(len(mod_title), 1)
     self.assertNotEqual(mod_title[0], None)
     self.assertEqual("Modified Title", mod_title[0].text)
Example #16
0
def apply_notice(regulation_file, notice_file):
    """ Apply notice changes """
    # Read the RegML starting point
    regulation_file = find_file(regulation_file)

    with open(regulation_file, 'r') as f:
        left_reg_xml = f.read()
    left_xml_tree = etree.fromstring(left_reg_xml)

    # Read the notice file
    notice_file = find_file(notice_file, is_notice=True)
    with open(notice_file, 'r') as f:
        notice_string = f.read()
    notice_xml = etree.fromstring(notice_string)

    # Process the notice changeset
    new_xml_tree = process_changes(left_xml_tree, notice_xml)

    # Write the new xml tree
    new_xml_string = etree.tostring(new_xml_tree,
                                    pretty_print=True,
                                    xml_declaration=True,
                                    encoding='UTF-8')
    new_path = os.path.join(
        os.path.dirname(regulation_file),
        os.path.basename(notice_file))
    with open(new_path, 'w') as f:
        print("Writing regulation to {}".format(new_path))
        f.write(new_xml_string)
 def test_process_changes_added_existing(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="added" label="1234-2">
               <paragraph label="1234-2">An added paragraph</paragraph>
             </change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <paragraph label="1234-1">An existing paragraph</paragraph>
               <paragraph label="1234-2">Another existing paragraph</paragraph>
             </content>
           </part>
         </regulation>""")
     with self.assertRaises(KeyError):
         process_changes(original_xml, notice_xml)
Example #18
0
 def test_process_changes_added_existing(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="added" label="1234-2">
               <paragraph label="1234-2">An added paragraph</paragraph>
             </change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <paragraph label="1234-1">An existing paragraph</paragraph>
               <paragraph label="1234-2">Another existing paragraph</paragraph>
             </content>
           </part>
         </regulation>""")
     with self.assertRaises(KeyError):
         process_changes(original_xml, notice_xml)
Example #19
0
 def test_process_changes_deleted(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="deleted" label="1234-1"></change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <paragraph label="1234-1">An existing paragraph</paragraph>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     del_paras = new_xml.findall('.//{eregs}paragraph[@label="1234-1"]')
     self.assertEqual(len(del_paras), 0)
 def test_process_changes_deleted(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys><preamble></preamble>
           <changeset>
             <change operation="deleted" label="1234-1"></change>
           </changeset>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys></fdsys>
           <preamble></preamble>
           <part label="1234">
             <content>
               <paragraph label="1234-1">An existing paragraph</paragraph>
             </content>
           </part>
         </regulation>""")
     new_xml = process_changes(original_xml, notice_xml)
     del_paras = new_xml.findall('.//{eregs}paragraph[@label="1234-1"]')
     self.assertEqual(len(del_paras), 0)
Example #21
0
def check_keyterms(file, with_notice=None):
    """ Check for keyterm fragments in a RegML file

        If --with-notice is used, only *new* keyterm fragments
        introduced in the notice will be given. """

    file = find_file(file)
    with open(file, 'r') as f:
        reg_string = f.read()
    parser = etree.XMLParser(huge_tree=True)
    reg_tree = etree.fromstring(reg_string, parser)

    if reg_tree.tag == '{eregs}notice':
        print("Cannot check terms in notice files directly.")
        print("Use a regulation file and --with-notice to specify the notice that applies.")
        sys.exit(1)

    # If we're given a notice, apply it to the given regulation file,
    # then check terms in the result and write it out to the notice file
    # as changes.
    notice_tree = None
    if with_notice is not None:
        # file is changed here so the term checker will write the notice
        # instead of the regulation
        file = find_file(with_notice, is_notice=True)
        with open(file, 'r') as f:
            notice_xml = f.read()
        notice_tree = etree.fromstring(notice_xml)

        # Process the notice changeset
        print(colored('Applying notice...', attrs=['bold']))
        reg_tree = process_changes(reg_tree, notice_tree)

    # Validate the file relative to schema
    validator = get_validator(reg_tree)
    validator.validate_keyterms(reg_tree, notice_tree=notice_tree)

    for event in validator.events:
        print(str(event))
Example #22
0
def check_terms(file, label=None, term=None, with_notice=None):
    """ Check the terms in a RegML file """

    file = find_file(file)
    with open(file, 'r') as f:
        reg_string = f.read()
    parser = etree.XMLParser(huge_tree=True)
    reg_tree = etree.fromstring(reg_string, parser)

    if reg_tree.tag == '{eregs}notice':
        print("Cannot check terms in notice files directly.")
        print("Use a regulation file and --with-notice to specify the notice that applies.")
        sys.exit(1)

    # If we're given a notice, apply it to the given regulation file,
    # then check terms in the result and write it out to the notice file
    # as changes.
    notice_tree = None
    if with_notice is not None:
        # file is changed here so the term checker will write the notice
        # instead of the regulation
        file = find_file(with_notice, is_notice=True)
        with open(file, 'r') as f:
            notice_xml = f.read()
        notice_tree = etree.fromstring(notice_xml)

        # Process the notice changeset
        print(colored('Applying notice...', attrs=['bold']))
        reg_tree = process_changes(reg_tree, notice_tree)

    # Validate the file relative to schema
    validator = get_validator(reg_tree)

    terms = build_terms_layer(reg_tree)
    validator.validate_terms(reg_tree, terms)
    validator.validate_term_references(reg_tree, terms, file,
            label=label, term=term, notice=notice_tree)
 def test_process_changes_meta(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys>
             This is an fdsys 
           </fdsys>
           <preamble>
             This is the preamble
           </preamble>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys>
             Old fdsys
           </fdsys>
           <preamble>
             Old preamble
           </preamble>
         </regulation>""") # noqa
     new_xml = process_changes(original_xml, notice_xml)
     fdsys = new_xml.find('./{eregs}fdsys')
     preamble = new_xml.find('./{eregs}preamble')
     self.assertTrue("This is an fdsys" in fdsys.text)
     self.assertTrue("This is the preamble" in preamble.text)
Example #24
0
 def test_process_changes_meta(self):
     notice_xml = etree.fromstring("""
         <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys>
             This is an fdsys 
           </fdsys>
           <preamble>
             This is the preamble
           </preamble>
         </notice>""")
     original_xml = etree.fromstring("""
         <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd">
           <fdsys>
             Old fdsys
           </fdsys>
           <preamble>
             Old preamble
           </preamble>
         </regulation>""")  # noqa
     new_xml = process_changes(original_xml, notice_xml)
     fdsys = new_xml.find('./{eregs}fdsys')
     preamble = new_xml.find('./{eregs}preamble')
     self.assertTrue("This is an fdsys" in fdsys.text)
     self.assertTrue("This is the preamble" in preamble.text)
Example #25
0
def apply_through(cfr_title, cfr_part, start=None, through=None,
                  fix_notices=False, skip_fix_notices=[],
                  skip_fix_notices_through=None):
    # Get list of notices that apply to this reg
    # Look for locally available notices
    regml_notice_files = find_all(cfr_part, is_notice=True)

    regml_notices = []
    for notice_file in regml_notice_files:
        file_name = os.path.join(notice_file)
        with open(file_name, 'r') as f:
            notice_xml = f.read()
        parser = etree.XMLParser(huge_tree=True)

        try:
            xml_tree = etree.fromstring(notice_xml, parser)
        except etree.XMLSyntaxError as e:
            print(colored('Syntax error in {}'.format(notice_file), 'red'))
            print(e)
            return

        doc_number = xml_tree.find(
            './{eregs}preamble/{eregs}documentNumber').text
        effective_date = xml_tree.find(
            './{eregs}preamble/{eregs}effectiveDate').text
        applies_to = xml_tree.find(
            './{eregs}changeset').get('leftDocumentNumber')
        if applies_to is None:
            # Major problem here
            print(colored("Error locating"),
                  colored("leftDocumentNumber", attrs=['bold']),
                  colored("attribute in"),
                  colored("{}".format(doc_number), 'red',
                          attrs=['bold']))
            return

        regml_notices.append((doc_number, effective_date, applies_to, file_name))

    if cfr_part in settings.CUSTOM_NOTICE_ORDER:
        order = settings.CUSTOM_NOTICE_ORDER[cfr_part]
        regml_notices.sort(key=lambda n: order.index(n[0]))

    else:
        regml_notices.sort(key=lambda n: n[1])
    
    regs = [nn[2] for nn in regml_notices]
    regs.sort()

    # If no notices found, issue error message
    if not regml_notices:
        print(colored("\nNo available notices for reg {} in part {}".format(cfr_part, cfr_title)))
        return

    # If initial version is not findable, issue error message
    if regs[0] is None:
        print(colored("\nError reading initial version and apply order for reg {} in part {}. No changes have been made.".format(cfr_part, cfr_title),
                      attrs=['bold']))
        return

    # Generate prompt for user
    print(colored("\nAvailable notices for reg {}:".format(cfr_part),
          attrs=['bold']))
    print("{:>3}. {:<22}(Initial version)".format(0, regs[0]))
    # Process notices found
    for kk in range(len(regml_notices)):
        print("{0:>3}. {1[0]:<22}(Effective: {1[1]})".format(kk+1,
                                               regml_notices[kk]))
    print()

    # Possible answers are blank (all), the numbers, or the notice names
    possible_indices = [str(kk) for kk in range(len(regml_notices) + 1)]
    possible_notices = [nn[0] for nn in regml_notices]

    # If notice number is supplied, use that one
    if through is not None:
        print("Command-line option selected notice '{}'".format(through))
        answer = through
    else:
        # Get user input to specify end version
        answer = None
        while answer not in [""] + possible_indices + possible_notices:
            answer = raw_input('Press enter to apply all or enter notice number: [all] ')

    if len(answer) == 0:
        # Apply notices
        last_ver_idx = len(regml_notices) - 1
    elif answer is "0":
        # Cancel - this is just the initial version
        print(colored("CANCELED: Version", attrs=['bold']),
              colored("{}".format(regs[0]), 'yellow', attrs=['bold']),
              colored("is the initial version - no changes have been made.", attrs=['bold']))
        return
    elif answer in possible_indices:
        # Apply notices through answer-1 to adjust for the initial ver offset
        last_ver_idx = int(answer) - 1
    elif answer in possible_notices:
        # Find index to stop at in notice list
        last_ver_idx = possible_notices.index(answer)
    else:
        print(colored("ERROR: Notice", attrs=['bold']),
              colored("{}".format(answer), 'red', attrs=['bold']),
              colored("does not exist - no changes have been made.", attrs=['bold']))
        return

    print(colored("\nApplying notices through {0[0]}\n".format(regml_notices[last_ver_idx]),
          attrs=['bold']))

    # Perform the notice application process
    reg_path = os.path.abspath(os.path.join(settings.XML_ROOT,
                                            'regulation',
                                            cfr_part,
                                            '{}.xml'.format(regs[0])))
    print("Opening initial version {}".format(reg_path))
    regulation_file = find_file(reg_path)
    with open(regulation_file, 'r') as f:
        left_reg_xml = f.read()
    parser = etree.XMLParser(huge_tree=True)
    left_xml_tree = etree.fromstring(left_reg_xml, parser)

    kk = 1
    prev_tree = left_xml_tree
    for notice in regml_notices[:last_ver_idx+1]:
        doc_number, effective_date, prev_notice, file_name = notice

        print("[{}] Applying notice {} from {} to version {}".format(kk,
                                                                     doc_number,
                                                                     file_name,
                                                                     prev_notice))

        # Open the notice file
        notice_file = find_file(file_name, is_notice=True)
        with open(notice_file, 'r') as f:
            notice_string = f.read()
        parser = etree.XMLParser(huge_tree=True)

        notice_xml = etree.fromstring(notice_string, parser)

        # TODO: Validate labels for json-compliance?
        # Example: JSON fails on upload only for interpParagraphs without "Interp" in them

        # Validate the files
        regulation_validator = get_validator(prev_tree)
        terms_layer = build_terms_layer(prev_tree)

        try:
            notice_validator = get_validator(notice_xml, raise_instead_of_exiting=True)
        except Exception as e:
            print("[{}]".format(kk),
                  colored("Exception occurred in notice", 'red'),
                  colored(doc_number, attrs=['bold']),
                  colored("; details are below. ", 'red'),
                  "To retry this single notice, use:\n\n",
                  colored("> ./regml.py apply-notice {0}/{1} {0}/{2}\n".format(cfr_part,
                                                                               prev_notice,
                                                                               doc_number),
                          attrs=['bold']))
            sys.exit(0)

        # validate the notice XML with the layers derived from the
        # tree of the previous version
        reload_notice = False
        skip_notices = list(skip_fix_notices)

        if skip_fix_notices_through is not None:
            if skip_fix_notices_through in possible_notices:
                last_fix_idx = possible_notices.index(skip_fix_notices_through)
                skip_notices.extend(possible_notices[:last_fix_idx + 1])

        if fix_notices and doc_number not in skip_notices:
            print('Fixing notice number {}:'.format(doc_number))
            notice_validator.validate_terms(notice_xml, terms_layer)
            notice_validator.validate_term_references(notice_xml, terms_layer, notice_file)
            notice_terms_layer = build_terms_layer(notice_xml)
            notice_validator.validate_term_references(notice_xml, notice_terms_layer, notice_file)
            notice_validator.fix_omitted_cites(notice_xml, notice_file)
            reload_notice = True

        # at this point the file has possibly changed, so we should really reload it
        if reload_notice:
            with open(notice_file, 'r') as f:
                notice_string = f.read()
            parser = etree.XMLParser(huge_tree=True)

            notice_xml = etree.fromstring(notice_string, parser)

        # Process the notice changeset
        try:
            new_xml_tree = process_changes(prev_tree, notice_xml)
        except Exception as e:
            print("[{}]".format(kk),
                  colored("Exception occurred; details are below. ".format(kk), 'red'),
                  "To retry this single notice, use:\n\n",
                  colored("> ./regml.py apply-notice {0}/{1} {0}/{2}\n".format(cfr_part,
                                                                               prev_notice,
                                                                               doc_number),
                          attrs=['bold']))
            raise e

        # Add in any new analysis
        new_xml_tree = process_analysis(new_xml_tree, notice_xml)

        # Write the new xml tree
        new_xml_string = etree.tostring(new_xml_tree,
                                        pretty_print=True,
                                        xml_declaration=True,
                                        encoding='UTF-8')
        new_path = os.path.join(
            os.path.dirname(regulation_file),
            os.path.basename(notice_file))
        with open(new_path, 'w') as f:
            print("[{}] Writing regulation to {}".format(kk, new_path))
            f.write(new_xml_string)

        prev_tree = new_xml_tree
        kk += 1