def test_create_xml_changes_child_stars(self): labels_amended = [Amendment('PUT', '200-2-a')] xml = etree.fromstring("<ROOT><P>(a) Content</P><STARS /></ROOT>") n2a = Node('(a) Content', label=['200', '2', 'a'], source_xml=xml.xpath('//P')[0]) n2b = Node('(b) Content', label=['200', '2', 'b']) n2 = Node('n2', label=['200', '2'], children=[n2a, n2b]) root = Node('root', label=['200'], children=[n2]) notice_changes = changes.NoticeChanges() build.create_xml_changes(labels_amended, root, notice_changes) self.assertTrue('200-2-a' in notice_changes.changes) self.assertTrue(1, len(notice_changes.changes['200-2-a'])) change = notice_changes.changes['200-2-a'][0] self.assertEqual('PUT', change['action']) self.assertFalse('field' in change) n2a.text = n2a.text + ":" n2a.source_xml.text = n2a.source_xml.text + ":" notice_changes = changes.NoticeChanges() build.create_xml_changes(labels_amended, root, notice_changes) self.assertTrue('200-2-a' in notice_changes.changes) self.assertTrue(1, len(notice_changes.changes['200-2-a'])) change = notice_changes.changes['200-2-a'][0] self.assertEqual('PUT', change['action']) self.assertEqual('[text]', change.get('field'))
def test_create_xml_changes_child_stars(): labels_amended = [Amendment('PUT', '200-?-2-a')] with XMLBuilder("ROOT") as ctx: ctx.P("(a) Content") ctx.STARS() n2a = Node('(a) Content', label=['200', '2', 'a'], source_xml=ctx.xml.xpath('//P')[0]) n2b = Node('(b) Content', label=['200', '2', 'b']) n2 = Node('n2', label=['200', '2'], children=[n2a, n2b]) root = Node('root', label=['200'], children=[n2]) notice_changes = changes.NoticeChanges() fetch.create_xml_changes(labels_amended, root, notice_changes) data = notice_changes[None] assert '200-2-a' in data assert len(data['200-2-a']) == 1 change = data['200-2-a'][0] assert change['action'] == 'PUT' assert 'field' not in change n2a.text = n2a.text + ":" n2a.source_xml.text = n2a.source_xml.text + ":" notice_changes = changes.NoticeChanges() fetch.create_xml_changes(labels_amended, root, notice_changes) data = notice_changes[None] assert '200-2-a' in data assert len(data['200-2-a']) == 1 change = data['200-2-a'][0] assert change['action'] == 'PUT' assert change.get('field') == '[text]'
def test_create_xml_changes_child_stars(self): labels_amended = [Amendment('PUT', '200-?-2-a')] with XMLBuilder("ROOT") as ctx: ctx.P("(a) Content") ctx.STARS() n2a = Node('(a) Content', label=['200', '2', 'a'], source_xml=ctx.xml.xpath('//P')[0]) n2b = Node('(b) Content', label=['200', '2', 'b']) n2 = Node('n2', label=['200', '2'], children=[n2a, n2b]) root = Node('root', label=['200'], children=[n2]) notice_changes = changes.NoticeChanges() amendments.create_xml_changes(labels_amended, root, notice_changes) data = notice_changes.changes_by_xml[None] self.assertIn('200-2-a', data) self.assertTrue(1, len(data['200-2-a'])) change = data['200-2-a'][0] self.assertEqual('PUT', change['action']) self.assertNotIn('field', change) n2a.text = n2a.text + ":" n2a.source_xml.text = n2a.source_xml.text + ":" notice_changes = changes.NoticeChanges() amendments.create_xml_changes(labels_amended, root, notice_changes) data = notice_changes.changes_by_xml[None] self.assertIn('200-2-a', data) self.assertTrue(1, len(data['200-2-a'])) change = data['200-2-a'][0] self.assertEqual('PUT', change['action']) self.assertEqual('[text]', change.get('field'))
def test_update_duplicates(self): nc = changes.NoticeChanges() nc.add_changes(None, { '123-12': { 'action': 'DELETE' }, '123-22': { 'action': 'OTHER' } }) nc.add_changes(None, {'123-12': {'action': 'DELETE'}}) nc.add_changes(None, {'123-12': {'action': 'OTHER'}}) nc.add_changes(None, { '123-22': { 'action': 'OTHER' }, '123-32': { 'action': 'LAST' } }) data = nc.changes_by_xml[None] self.assertTrue('123-12' in data) self.assertTrue('123-22' in data) self.assertTrue('123-32' in data) self.assertEqual(data['123-12'], [{ 'action': 'DELETE' }, { 'action': 'OTHER' }]) self.assertEqual(data['123-22'], [{'action': 'OTHER'}]) self.assertEqual(data['123-32'], [{'action': 'LAST'}])
def test_update_duplicates(self): nc = changes.NoticeChanges() nc.update({ '123-12': { 'action': 'DELETE' }, '123-22': { 'action': 'OTHER' } }) nc.update({'123-12': {'action': 'DELETE'}}) nc.update({'123-12': {'action': 'OTHER'}}) nc.update({ '123-22': { 'action': 'OTHER' }, '123-32': { 'action': 'LAST' } }) self.assertTrue('123-12' in nc.changes) self.assertTrue('123-22' in nc.changes) self.assertTrue('123-32' in nc.changes) self.assertEqual(nc.changes['123-12'], [{ 'action': 'DELETE' }, { 'action': 'OTHER' }]) self.assertEqual(nc.changes['123-22'], [{'action': 'OTHER'}]) self.assertEqual(nc.changes['123-32'], [{'action': 'LAST'}])
def process_amendments(notice, notice_xml): """Process changes to the regulation that are expressed in the notice.""" all_amends = [] # will be added to the notice cfr_part = notice['cfr_parts'][0] notice_changes = changes.NoticeChanges() # process amendments in batches, based on their parent XML for amdparent in notice_xml.xpath('//AMDPAR/..'): context = [amdparent.get('PART') or cfr_part] amendments_by_section = defaultdict(list) normal_amends = [] # amendments not moving or adding a subpart for amdpar in amdparent.xpath('.//AMDPAR'): instructions = amdpar.xpath('./EREGS_INSTRUCTIONS') if not instructions: logger.warning('No <EREGS_INSTRUCTIONS>. Was this notice ' 'preprocessed?') continue instructions = instructions[0] amendments = [amendment_from_xml(el) for el in instructions] context = [None if l is '?' else l for l in instructions.get('final_context').split('-')] section_xml = find_section(amdpar) for amendment in amendments: all_amends.append(amendment) if isinstance(amendment, DesignateAmendment): subpart_changes = process_designate_subpart(amendment) if subpart_changes: notice_changes.update(subpart_changes) elif new_subpart_added(amendment): notice_changes.update(process_new_subpart( notice, amendment, amdpar)) elif section_xml is None: normal_amends.append(amendment) else: normal_amends.append(amendment) amendments_by_section[section_xml].append(amendment) cfr_part = context[0] # carry the part through to the next amdparent create_xmlless_changes(normal_amends, notice_changes) # Process amendments relating to a specific section in batches, too for section_xml, related_amends in amendments_by_section.items(): for section in reg_text.build_from_section(cfr_part, section_xml): create_xml_changes(related_amends, section, notice_changes) for appendix in parse_appendix_changes(normal_amends, cfr_part, amdparent): create_xml_changes(normal_amends, appendix, notice_changes) interp = parse_interp_changes(normal_amends, cfr_part, amdparent) if interp: create_xml_changes(normal_amends, interp, notice_changes) if all_amends: notice['amendments'] = all_amends notice['changes'] = notice_changes.changes return notice
def test_create_xmlless_changes(self): labels_amended = [Amendment('DELETE', '200-2-a'), Amendment('MOVE', '200-2-b', '200-2-c')] notice_changes = changes.NoticeChanges() build.create_xmlless_changes(labels_amended, notice_changes) delete = notice_changes.changes['200-2-a'][0] move = notice_changes.changes['200-2-b'][0] self.assertEqual({'action': 'DELETE'}, delete) self.assertEqual({'action': 'MOVE', 'destination': ['200', '2', 'c']}, move)
def test_create_xmlless_changes(): labels_amended = [Amendment('DELETE', '200-?-2-a'), Amendment('MOVE', '200-?-2-b', '200-?-2-c')] notice_changes = changes.NoticeChanges() for amendment in labels_amended: fetch.create_xmlless_change(amendment, notice_changes) delete = notice_changes[None]['200-2-a'][0] move = notice_changes[None]['200-2-b'][0] assert delete == {'action': 'DELETE'} assert move == {'action': 'MOVE', 'destination': ['200', '2', 'c']}
def test_create_xml_changes_reserve(self): labels_amended = [Amendment('RESERVE', '200-2-a')] n2a = Node('[Reserved]', label=['200', '2', 'a']) n2 = Node('n2', label=['200', '2'], children=[n2a]) root = Node('root', label=['200'], children=[n2]) notice_changes = changes.NoticeChanges() build.create_xml_changes(labels_amended, root, notice_changes) reserve = notice_changes.changes['200-2-a'][0] self.assertEqual(reserve['action'], 'RESERVE') self.assertEqual(reserve['node']['text'], u'[Reserved]')
def test_create_xml_changes_reserve(): labels_amended = [Amendment('RESERVE', '200-?-2-a')] n2a = Node('[Reserved]', label=['200', '2', 'a']) n2 = Node('n2', label=['200', '2'], children=[n2a]) root = Node('root', label=['200'], children=[n2]) notice_changes = changes.NoticeChanges() fetch.create_xml_changes(labels_amended, root, notice_changes) reserve = notice_changes[None]['200-2-a'][0] assert reserve['action'] == 'RESERVE' assert reserve['node']['text'] == '[Reserved]'
def fetch_amendments(notice_xml): """Process changes to the regulation that are expressed in the notice.""" notice_changes = changes.NoticeChanges() if notice_xml.xpath('.//AMDPAR[not(EREGS_INSTRUCTIONS)]'): logger.warning( 'No <EREGS_INSTRUCTIONS>. Was this notice preprocessed?') cache = ContentCache() authority_by_xml = {} for instruction_xml in notice_xml.xpath('.//EREGS_INSTRUCTIONS/*'): amendment = amendment_from_xml(instruction_xml) content = cache.content_of_change(instruction_xml) if instruction_xml.tag == 'MOVE_INTO_SUBPART': subpart_changes = process_designate_subpart(amendment) if subpart_changes: notice_changes.add_changes(amendment.amdpar_xml, subpart_changes) elif instruction_xml.tag == 'AUTHORITY': authority_by_xml[amendment.amdpar_xml] = instruction_xml.text elif changes.new_subpart_added(amendment): subpart_changes = {} for change in changes.create_subpart_amendment(content.struct): subpart_changes.update(change) notice_changes.add_changes(amendment.amdpar_xml, subpart_changes) elif content: content.amends.append(amendment) else: create_xmlless_change(amendment, notice_changes) for content in cache.by_xml.values(): create_xml_changes(content.amends, content.struct, notice_changes) amendments = [] for amdpar_xml in notice_xml.xpath('.//AMDPAR'): amendment = {"instruction": amdpar_xml.text} # There'll be at most one for inst_xml in amdpar_xml.xpath('./EREGS_INSTRUCTIONS'): context = inst_xml.get('final_context', '') amendment['cfr_part'] = context.split('-')[0] relevant_changes = notice_changes.changes_by_xml[amdpar_xml] if relevant_changes: amendment['changes'] = list(relevant_changes.items()) if amdpar_xml in authority_by_xml: amendment['authority'] = authority_by_xml[amdpar_xml] amendments.append(amendment) return amendments
def test_notice_changes_update_duplicates(): nc = changes.NoticeChanges() nc.add_change(None, changes.Change('123-12', {'action': 'DELETE'})) nc.add_change(None, changes.Change('123-22', {'action': 'OTHER'})) nc.add_change(None, changes.Change('123-12', {'action': 'DELETE'})) nc.add_change(None, changes.Change('123-12', {'action': 'OTHER'})) nc.add_change(None, changes.Change('123-22', {'action': 'OTHER'})) nc.add_change(None, changes.Change('123-32', {'action': 'LAST'})) data = nc[None] assert '123-12' in data assert '123-22' in data assert '123-32' in data assert data['123-12'] == [{'action': 'DELETE'}, {'action': 'OTHER'}] assert data['123-22'] == [{'action': 'OTHER'}] assert data['123-32'] == [{'action': 'LAST'}]
def test_create_xml_changes_stars_hole(self): labels_amended = [Amendment('PUT', '200-2-a')] n2a1 = Node('(1) * * *', label=['200', '2', 'a', '1']) n2a2 = Node('(2) a2a2a2', label=['200', '2', 'a', '2']) n2a = Node('(a) aaa', label=['200', '2', 'a'], children=[n2a1, n2a2]) n2 = Node('n2', label=['200', '2'], children=[n2a]) root = Node('root', label=['200'], children=[n2]) notice_changes = changes.NoticeChanges() build.create_xml_changes(labels_amended, root, notice_changes) for label in ('200-2-a', '200-2-a-2'): self.assertTrue(label in notice_changes.changes) self.assertEqual(1, len(notice_changes.changes[label])) change = notice_changes.changes[label][0] self.assertEqual('PUT', change['action']) self.assertFalse('field' in change) self.assertTrue('200-2-a-1' in notice_changes.changes) self.assertEqual(1, len(notice_changes.changes['200-2-a-1'])) change = notice_changes.changes['200-2-a-1'][0] self.assertEqual('KEEP', change['action']) self.assertFalse('field' in change)
def test_create_xml_changes_stars_hole(): labels_amended = [Amendment('PUT', '200-?-2-a')] n2a1 = Node('(1) * * *', label=['200', '2', 'a', '1']) n2a2 = Node('(2) a2a2a2', label=['200', '2', 'a', '2']) n2a = Node('(a) aaa', label=['200', '2', 'a'], children=[n2a1, n2a2]) n2 = Node('n2', label=['200', '2'], children=[n2a]) root = Node('root', label=['200'], children=[n2]) notice_changes = changes.NoticeChanges() fetch.create_xml_changes(labels_amended, root, notice_changes) data = notice_changes[None] for label in ('200-2-a', '200-2-a-2'): assert label in data assert len(data[label]) == 1 change = data[label][0] assert change['action'] == 'PUT' assert 'field' not in change assert '200-2-a-1' in data assert len(data['200-2-a-1']) == 1 change = data['200-2-a-1'][0] assert change['action'] == 'KEEP' assert 'field' not in change
def test_create_xml_changes_stars(self): labels_amended = [Amendment('PUT', '200-?-2-a')] n2a1 = Node('(1) Content', label=['200', '2', 'a', '1']) n2a2 = Node('(2) Content', label=['200', '2', 'a', '2']) n2a = Node('(a) * * *', label=['200', '2', 'a'], children=[n2a1, n2a2]) n2 = Node('n2', label=['200', '2'], children=[n2a]) root = Node('root', label=['200'], children=[n2]) notice_changes = changes.NoticeChanges() amendments.create_xml_changes(labels_amended, root, notice_changes) data = notice_changes.changes_by_xml[None] for label in ('200-2-a-1', '200-2-a-2'): self.assertIn(label, data) self.assertEqual(1, len(data[label])) change = data[label][0] self.assertEqual('PUT', change['action']) self.assertNotIn('field', change) self.assertTrue('200-2-a' in data) self.assertEqual(1, len(data['200-2-a'])) change = data['200-2-a'][0] self.assertEqual('KEEP', change['action']) self.assertNotIn('field', change)
def process_amendments(notice, notice_xml): """ Process the changes to the regulation that are expressed in the notice. """ amends = [] notice_changes = changes.NoticeChanges() amdpars_by_parent = [] for par in notice_xml.xpath('//AMDPAR'): parent = par.getparent() exists = filter(lambda aXp: aXp.parent == parent, amdpars_by_parent) if exists: exists[0].append(par) else: amdpars_by_parent.append(AmdparByParent(parent, par)) default_cfr_part = notice['cfr_part'] for aXp in amdpars_by_parent: amended_labels = [] designate_labels, other_labels = [], [] context = [default_cfr_part] for par in aXp.amdpars: als, context = parse_amdpar(par, context) amended_labels.extend(als) labels_by_part = defaultdict(list) for al in amended_labels: if isinstance(al, DesignateAmendment): subpart_changes = process_designate_subpart(al) if subpart_changes: notice_changes.update(subpart_changes) designate_labels.append(al) elif new_subpart_added(al): notice_changes.update(process_new_subpart(notice, al, par)) designate_labels.append(al) else: other_labels.append(al) labels_by_part[al.label[0]].append(al) create_xmlless_changes(other_labels, notice_changes) # for cfr_part, rel_labels in labels_by_part.iteritems(): labels_for_part = { part: labels for part, labels in labels_by_part.iteritems() if part == default_cfr_part } print(labels_for_part) for cfr_part, rel_labels in labels_for_part.iteritems(): section_xml = find_section(par) if section_xml is not None: subparts = aXp.parent.xpath('.//SUBPART/HD') if subparts: subpart_label = [ cfr_part, 'Subpart', subparts[0].text[8:9] ] else: subpart_label = None for section in reg_text.build_from_section( cfr_part, section_xml): create_xml_changes(rel_labels, section, notice_changes, subpart_label) for appendix in parse_appendix_changes(rel_labels, cfr_part, aXp.parent): create_xml_changes(rel_labels, appendix, notice_changes) interp = parse_interp_changes(rel_labels, cfr_part, aXp.parent) if interp: create_xml_changes(rel_labels, interp, notice_changes) amends.extend(designate_labels) amends.extend(other_labels) # if other_labels: # Carry cfr_part through amendments # default_cfr_part = other_labels[-1].label[0] if amends: notice['amendments'] = amends notice['changes'] = notice_changes.changes elif notice['document_number'] in settings.REISSUANCES: notice['changes'] = { default_cfr_part: [{ 'action': 'PUT', 'node': reg_text.build_tree(notice_xml) }] }
def process_amendments(notice, notice_xml): """ Process the changes to the regulation that are expressed in the notice. """ amends = [] notice_changes = changes.NoticeChanges() amdpars_by_parent = [] for par in notice_xml.xpath('//AMDPAR'): parent = par.getparent() exists = filter(lambda aXp: aXp.parent == parent, amdpars_by_parent) if exists: exists[0].append(par) else: amdpars_by_parent.append(AmdparByParent(parent, par)) default_cfr_part = notice['cfr_parts'][0] for aXp in amdpars_by_parent: amended_labels = [] designate_labels, other_labels = [], [] context = [aXp.parent.get('PART') or default_cfr_part] for par in aXp.amdpars: als, context = parse_amdpar(par, context) amended_labels.extend(als) labels_by_part = defaultdict(list) for al in amended_labels: if isinstance(al, DesignateAmendment): subpart_changes = process_designate_subpart(al) if subpart_changes: notice_changes.update(subpart_changes) designate_labels.append(al) elif new_subpart_added(al): notice_changes.update(process_new_subpart(notice, al, par)) designate_labels.append(al) else: other_labels.append(al) labels_by_part[al.label[0]].append(al) create_xmlless_changes(other_labels, notice_changes) for cfr_part, rel_labels in labels_by_part.iteritems(): section_xml = find_section(par) if section_xml is not None: for section in reg_text.build_from_section( cfr_part, section_xml): create_xml_changes(rel_labels, section, notice_changes) for appendix in parse_appendix_changes(rel_labels, cfr_part, aXp.parent): create_xml_changes(rel_labels, appendix, notice_changes) interp = parse_interp_changes(rel_labels, cfr_part, aXp.parent) if interp: create_xml_changes(rel_labels, interp, notice_changes) amends.extend(designate_labels) amends.extend(other_labels) if other_labels: # Carry cfr_part through amendments default_cfr_part = other_labels[-1].label[0] if amends: notice['amendments'] = amends notice['changes'] = notice_changes.changes