def test_build_formatting_layer_callout(self): tree = etree.fromstring(""" <section xmlns="eregs"> <paragraph label="foo"> <content> <callout type="note"> <line>Note:</line> <line>Some notes</line> </callout> </content> </paragraph> </section> """, parser=etree.XMLParser(remove_blank_text=True)) expected_result = { 'foo': [{ 'fence_data': { 'lines': [ 'Note:', 'Some notes' ], 'type': 'note' }, 'locations': [ 0 ], 'text': 'Note:Some notes', }] } result = build_formatting_layer(tree) self.assertEqual(expected_result, result)
def test_build_formatting_layer_variable(self): tree = etree.fromstring(""" <section xmlns="eregs"> <paragraph label="foo"> <content> <variable>Val<subscript>n</subscript></variable> </content> </paragraph> </section> """) expected_result = { 'foo': [{ 'locations': [0], 'subscript_data': { 'subscript': 'n', 'variable': 'Val' }, 'text': 'Val_{n}' }] } result = build_formatting_layer(tree) self.assertEqual(expected_result, result)
def test_build_formatting_layer_dash(self): tree = etree.fromstring(""" <section xmlns="eregs"> <paragraph label="foo"> <content> <dash>Date</dash> </content> </paragraph> </section> """, parser=etree.XMLParser(remove_blank_text=True)) expected_result = { 'foo': [{ 'dash_data': { 'text': 'Date' }, 'locations': [ 0 ], 'text': 'Date_____', }] } result = build_formatting_layer(tree) self.assertEqual(expected_result, result)
def generate_json(regulation_file, check_terms=False): with open(find_file(regulation_file), 'r') as f: reg_xml = f.read() parser = etree.XMLParser(huge_tree=True) xml_tree = etree.fromstring(reg_xml, parser) # Validate the file relative to schema validator = get_validator(xml_tree) reg_tree = build_reg_tree(xml_tree) reg_number = reg_tree.label[0] paragraph_markers = build_paragraph_marker_layer(xml_tree) internal_citations = build_internal_citations_layer(xml_tree) external_citations = build_external_citations_layer(xml_tree) terms = build_terms_layer(xml_tree) meta = build_meta_layer(xml_tree) toc = build_toc_layer(xml_tree) keyterms = build_keyterm_layer(xml_tree) graphics = build_graphics_layer(xml_tree) formatting = build_formatting_layer(xml_tree) interps = build_interp_layer(xml_tree) analysis = build_analysis(xml_tree) notice_dict = build_notice(xml_tree) # if the validator had problems then we should report them and bail out validator.validate_terms(xml_tree, terms) validator.validate_internal_cites(xml_tree, internal_citations) if check_terms: validator.validate_term_references(xml_tree, terms, regulation_file) for event in validator.events: print(str(event)) reg_tree.include_children = True reg_json = reg_tree.to_json() notice = xml_tree.find('.//{eregs}documentNumber').text version = os.path.split(regulation_file)[-1].replace('.xml', '') if notice != version: print('Notice ({}) different from version ({}), ' 'using version'.format(notice, version)) notice = version write_layer(reg_json, reg_number, notice, 'regulation') write_layer(meta, reg_number, notice, 'layer/meta') write_layer(paragraph_markers, reg_number, notice, 'layer/paragraph-markers') write_layer(internal_citations, reg_number, notice, 'layer/internal-citations') write_layer(external_citations, reg_number, notice, 'layer/external-citations') write_layer(terms, reg_number, notice, 'layer/terms') write_layer(toc, reg_number, notice, 'layer/toc') write_layer(keyterms, reg_number, notice, 'layer/keyterms') write_layer(graphics, reg_number, notice, 'layer/graphics') write_layer(formatting, reg_number, notice, 'layer/formatting') write_layer(interps, reg_number, notice, 'layer/interpretations') write_layer(analysis, reg_number, notice, 'layer/analyses') write_layer(notice_dict, reg_number, notice, 'notice') return reg_number, notice, xml_tree
def parser_driver(regulation_file, check_terms=False, correct_interps=False, headerize_interps=False, fix_missed_cites=False): with open(regulation_file, 'r') as f: reg_xml = f.read() xml_tree = etree.fromstring(reg_xml) # validate relative to schema validator = EregsValidator(settings.XSD_FILE) validator.validate_reg(xml_tree) if not validator.is_valid: for event in validator.events: print(str(event)) sys.exit(0) reg_tree = build_reg_tree(xml_tree) reg_number = reg_tree.label[0] # we can correct interps right away if necessary if correct_interps: validator.insert_interp_markers(xml_tree, regulation_file) if headerize_interps: validator.headerize_interps(xml_tree, regulation_file) if fix_missed_cites: validator.fix_omitted_cites(xml_tree, regulation_file) paragraph_markers = build_paragraph_marker_layer(xml_tree) internal_citations = build_internal_citations_layer(xml_tree) external_citations = build_external_citations_layer(xml_tree) terms = build_terms_layer(xml_tree) meta = build_meta_layer(xml_tree) toc = build_toc_layer(xml_tree) keyterms = build_keyterm_layer(xml_tree) graphics = build_graphics_layer(xml_tree) formatting = build_formatting_layer(xml_tree) interps = build_interp_layer(xml_tree) analysis = build_analysis(xml_tree) notice_dict = build_notice(xml_tree) # if the validator had problems then we should report them and bail out validator.validate_terms(xml_tree, terms) validator.validate_internal_cites(xml_tree, internal_citations) if check_terms: validator.validate_term_references(xml_tree, terms, regulation_file) for event in validator.events: print(str(event)) reg_tree.include_children = True reg_json = reg_tree.to_json() notice = xml_tree.find('.//{eregs}documentNumber').text version = os.path.split(regulation_file)[-1].replace('.xml', '') if notice != version: print('Notice ({}) different from version ({}), using version'.format(notice, version)) notice = version write_layer(reg_json, reg_number, notice, 'regulation') write_layer(meta, reg_number, notice, 'layer/meta') write_layer(paragraph_markers, reg_number, notice, 'layer/paragraph-markers') write_layer(internal_citations, reg_number, notice, 'layer/internal-citations') write_layer(external_citations, reg_number, notice, 'layer/external-citations') write_layer(terms, reg_number, notice, 'layer/terms') write_layer(toc, reg_number, notice, 'layer/toc') write_layer(keyterms, reg_number, notice, 'layer/keyterms') write_layer(graphics, reg_number, notice, 'layer/graphics') write_layer(formatting, reg_number, notice, 'layer/formatting') write_layer(interps, reg_number, notice, 'layer/interpretations') write_layer(analysis, reg_number, notice, 'layer/analyses') write_layer(notice_dict, reg_number, notice, 'notice')