def test_process_analysis_no_existing(self): notice_xml = etree.fromstring(""" <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd"> <fdsys> <date>2015-11-17</date> </fdsys> <preamble> <documentNumber>2015-12345</documentNumber> </preamble> <changeset></changeset> <analysis label="1234-Analysis"> <analysisSection target="1234-2" notice="2015-12345" date="2015-11-17">An existing analysis</analysisSection> <analysisSection target="1234-3" notice="2015-12345" date="2015-11-17">An unchanging analysis</analysisSection> </analysis> </notice>""") regulation_xml = etree.fromstring(""" <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd"> <part label="1234"></part> </regulation>""") result = process_analysis(regulation_xml, notice_xml) analysis = result.find('.//{eregs}analysis') self.assertTrue(analysis is not None) sections = analysis.findall('{eregs}analysisSection') self.assertEquals(len(sections), 2)
def test_process_analysis(self): notice_xml = etree.fromstring(""" <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd"> <fdsys> <date>2014-11-17</date> </fdsys> <preamble> <documentNumber>2015-12345</documentNumber> </preamble> <changeset></changeset> <analysis label="1234-Analysis"> <analysisSection target="1234-1" notice="2015-12345" date="2015-11-17">An added analysis</analysisSection> <analysisSection target="1234-2" notice="2015-12345" date="2015-11-17">An updated analysis</analysisSection> </analysis> </notice>""") regulation_xml = etree.fromstring(""" <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd"> <part label="1234"></part> <analysis label="1234-Analysis"> <analysisSection target="1234-2" notice="2014-12345" date="2014-11-17">An existing analysis</analysisSection> <analysisSection target="1234-3" notice="2014-12345" date="2014-11-17">An unchanging analysis</analysisSection> </analysis> </regulation>""") result = process_analysis(regulation_xml, notice_xml) sections = result.findall('.//{eregs}analysisSection') self.assertEquals(len(sections), 4) first_analysis = result.find( './/{eregs}analysisSection[@target="1234-1"]') third_analysis = result.find( './/{eregs}analysisSection[@target="1234-3"]') self.assertEquals(first_analysis.get('notice'), '2015-12345') self.assertEquals(third_analysis.get('notice'), '2014-12345') self.assertEquals(first_analysis.get('date'), '2015-11-17') self.assertEquals(third_analysis.get('date'), '2014-11-17') second_analysis = result.findall( './/{eregs}analysisSection[@target="1234-2"]') self.assertEquals(len(second_analysis), 2) self.assertEquals(second_analysis[0].get('date'), '2014-11-17') self.assertEquals(second_analysis[0].get('notice'), '2014-12345') self.assertEquals(second_analysis[1].get('date'), '2015-11-17') self.assertEquals(second_analysis[1].get('notice'), '2015-12345')
def test_process_analysis(self): notice_xml = etree.fromstring(""" <notice xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd"> <fdsys> <date>2014-11-17</date> </fdsys> <preamble> <documentNumber>2015-12345</documentNumber> </preamble> <changeset></changeset> <analysis label="1234-Analysis"> <analysisSection target="1234-1" notice="2015-12345" date="2015-11-17">An added analysis</analysisSection> <analysisSection target="1234-2" notice="2015-12345" date="2015-11-17">An updated analysis</analysisSection> </analysis> </notice>""") regulation_xml = etree.fromstring(""" <regulation xmlns="eregs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="eregs ../../eregs.xsd"> <part label="1234"></part> <analysis label="1234-Analysis"> <analysisSection target="1234-2" notice="2014-12345" date="2014-11-17">An existing analysis</analysisSection> <analysisSection target="1234-3" notice="2014-12345" date="2014-11-17">An unchanging analysis</analysisSection> </analysis> </regulation>""") result = process_analysis(regulation_xml, notice_xml) sections = result.findall('.//{eregs}analysisSection') self.assertEquals(len(sections), 4) first_analysis = result.find('.//{eregs}analysisSection[@target="1234-1"]') third_analysis = result.find('.//{eregs}analysisSection[@target="1234-3"]') self.assertEquals(first_analysis.get('notice'), '2015-12345') self.assertEquals(third_analysis.get('notice'), '2014-12345') self.assertEquals(first_analysis.get('date'), '2015-11-17') self.assertEquals(third_analysis.get('date'), '2014-11-17') second_analysis = result.findall('.//{eregs}analysisSection[@target="1234-2"]') self.assertEquals(len(second_analysis), 2) self.assertEquals(second_analysis[0].get('date'), '2014-11-17') self.assertEquals(second_analysis[0].get('notice'), '2014-12345') self.assertEquals(second_analysis[1].get('date'), '2015-11-17') self.assertEquals(second_analysis[1].get('notice'), '2015-12345')
def apply_notice(regulation_file, notice_file): """ Apply notice changes """ # Read the RegML starting point regulation_file = find_file(regulation_file) with open(regulation_file, 'r') as f: left_reg_xml = f.read() parser = etree.XMLParser(huge_tree=True) left_xml_tree = etree.fromstring(left_reg_xml, parser) # Read the notice file notice_file = find_file(notice_file, is_notice=True) with open(notice_file, 'r') as f: notice_string = f.read() parser = etree.XMLParser(huge_tree=True) notice_xml = etree.fromstring(notice_string, parser) # Validate the files regulation_validator = get_validator(left_xml_tree) notice_validator = get_validator(notice_xml) # Process the notice changeset new_xml_tree = process_changes(left_xml_tree, notice_xml) # Add in any new analysis new_xml_tree = process_analysis(new_xml_tree, notice_xml) # Write the new xml tree new_xml_string = etree.tostring(new_xml_tree, pretty_print=True, xml_declaration=True, encoding='UTF-8') new_path = os.path.join( os.path.dirname(regulation_file), os.path.basename(notice_file)) with open(new_path, 'w') as f: print("Writing regulation to {}".format(new_path)) f.write(new_xml_string)
def apply_through(cfr_title, cfr_part, start=None, through=None, fix_notices=False, skip_fix_notices=[], skip_fix_notices_through=None): # Get list of notices that apply to this reg # Look for locally available notices regml_notice_files = find_all(cfr_part, is_notice=True) regml_notices = [] for notice_file in regml_notice_files: file_name = os.path.join(notice_file) with open(file_name, 'r') as f: notice_xml = f.read() parser = etree.XMLParser(huge_tree=True) try: xml_tree = etree.fromstring(notice_xml, parser) except etree.XMLSyntaxError as e: print(colored('Syntax error in {}'.format(notice_file), 'red')) print(e) return doc_number = xml_tree.find( './{eregs}preamble/{eregs}documentNumber').text effective_date = xml_tree.find( './{eregs}preamble/{eregs}effectiveDate').text applies_to = xml_tree.find( './{eregs}changeset').get('leftDocumentNumber') if applies_to is None: # Major problem here print(colored("Error locating"), colored("leftDocumentNumber", attrs=['bold']), colored("attribute in"), colored("{}".format(doc_number), 'red', attrs=['bold'])) return regml_notices.append((doc_number, effective_date, applies_to, file_name)) if cfr_part in settings.CUSTOM_NOTICE_ORDER: order = settings.CUSTOM_NOTICE_ORDER[cfr_part] regml_notices.sort(key=lambda n: order.index(n[0])) else: regml_notices.sort(key=lambda n: n[1]) regs = [nn[2] for nn in regml_notices] regs.sort() # If no notices found, issue error message if not regml_notices: print(colored("\nNo available notices for reg {} in part {}".format(cfr_part, cfr_title))) return # If initial version is not findable, issue error message if regs[0] is None: print(colored("\nError reading initial version and apply order for reg {} in part {}. No changes have been made.".format(cfr_part, cfr_title), attrs=['bold'])) return # Generate prompt for user print(colored("\nAvailable notices for reg {}:".format(cfr_part), attrs=['bold'])) print("{:>3}. {:<22}(Initial version)".format(0, regs[0])) # Process notices found for kk in range(len(regml_notices)): print("{0:>3}. {1[0]:<22}(Effective: {1[1]})".format(kk+1, regml_notices[kk])) print() # Possible answers are blank (all), the numbers, or the notice names possible_indices = [str(kk) for kk in range(len(regml_notices) + 1)] possible_notices = [nn[0] for nn in regml_notices] # If notice number is supplied, use that one if through is not None: print("Command-line option selected notice '{}'".format(through)) answer = through else: # Get user input to specify end version answer = None while answer not in [""] + possible_indices + possible_notices: answer = raw_input('Press enter to apply all or enter notice number: [all] ') if len(answer) == 0: # Apply notices last_ver_idx = len(regml_notices) - 1 elif answer is "0": # Cancel - this is just the initial version print(colored("CANCELED: Version", attrs=['bold']), colored("{}".format(regs[0]), 'yellow', attrs=['bold']), colored("is the initial version - no changes have been made.", attrs=['bold'])) return elif answer in possible_indices: # Apply notices through answer-1 to adjust for the initial ver offset last_ver_idx = int(answer) - 1 elif answer in possible_notices: # Find index to stop at in notice list last_ver_idx = possible_notices.index(answer) else: print(colored("ERROR: Notice", attrs=['bold']), colored("{}".format(answer), 'red', attrs=['bold']), colored("does not exist - no changes have been made.", attrs=['bold'])) return print(colored("\nApplying notices through {0[0]}\n".format(regml_notices[last_ver_idx]), attrs=['bold'])) # Perform the notice application process reg_path = os.path.abspath(os.path.join(settings.XML_ROOT, 'regulation', cfr_part, '{}.xml'.format(regs[0]))) print("Opening initial version {}".format(reg_path)) regulation_file = find_file(reg_path) with open(regulation_file, 'r') as f: left_reg_xml = f.read() parser = etree.XMLParser(huge_tree=True) left_xml_tree = etree.fromstring(left_reg_xml, parser) kk = 1 prev_tree = left_xml_tree for notice in regml_notices[:last_ver_idx+1]: doc_number, effective_date, prev_notice, file_name = notice print("[{}] Applying notice {} from {} to version {}".format(kk, doc_number, file_name, prev_notice)) # Open the notice file notice_file = find_file(file_name, is_notice=True) with open(notice_file, 'r') as f: notice_string = f.read() parser = etree.XMLParser(huge_tree=True) notice_xml = etree.fromstring(notice_string, parser) # TODO: Validate labels for json-compliance? # Example: JSON fails on upload only for interpParagraphs without "Interp" in them # Validate the files regulation_validator = get_validator(prev_tree) terms_layer = build_terms_layer(prev_tree) try: notice_validator = get_validator(notice_xml, raise_instead_of_exiting=True) except Exception as e: print("[{}]".format(kk), colored("Exception occurred in notice", 'red'), colored(doc_number, attrs=['bold']), colored("; details are below. ", 'red'), "To retry this single notice, use:\n\n", colored("> ./regml.py apply-notice {0}/{1} {0}/{2}\n".format(cfr_part, prev_notice, doc_number), attrs=['bold'])) sys.exit(0) # validate the notice XML with the layers derived from the # tree of the previous version reload_notice = False skip_notices = list(skip_fix_notices) if skip_fix_notices_through is not None: if skip_fix_notices_through in possible_notices: last_fix_idx = possible_notices.index(skip_fix_notices_through) skip_notices.extend(possible_notices[:last_fix_idx + 1]) if fix_notices and doc_number not in skip_notices: print('Fixing notice number {}:'.format(doc_number)) notice_validator.validate_terms(notice_xml, terms_layer) notice_validator.validate_term_references(notice_xml, terms_layer, notice_file) notice_terms_layer = build_terms_layer(notice_xml) notice_validator.validate_term_references(notice_xml, notice_terms_layer, notice_file) notice_validator.fix_omitted_cites(notice_xml, notice_file) reload_notice = True # at this point the file has possibly changed, so we should really reload it if reload_notice: with open(notice_file, 'r') as f: notice_string = f.read() parser = etree.XMLParser(huge_tree=True) notice_xml = etree.fromstring(notice_string, parser) # Process the notice changeset try: new_xml_tree = process_changes(prev_tree, notice_xml) except Exception as e: print("[{}]".format(kk), colored("Exception occurred; details are below. ".format(kk), 'red'), "To retry this single notice, use:\n\n", colored("> ./regml.py apply-notice {0}/{1} {0}/{2}\n".format(cfr_part, prev_notice, doc_number), attrs=['bold'])) raise e # Add in any new analysis new_xml_tree = process_analysis(new_xml_tree, notice_xml) # Write the new xml tree new_xml_string = etree.tostring(new_xml_tree, pretty_print=True, xml_declaration=True, encoding='UTF-8') new_path = os.path.join( os.path.dirname(regulation_file), os.path.basename(notice_file)) with open(new_path, 'w') as f: print("[{}] Writing regulation to {}".format(kk, new_path)) f.write(new_xml_string) prev_tree = new_xml_tree kk += 1