def test_create_xml_changes_child_stars(self):
labels_amended = [Amendment('PUT', '200-2-a')]
xml = etree.fromstring("<ROOT><P>(a) Content</P><STARS /></ROOT>")
n2a = Node('(a) Content', label=['200', '2', 'a'],
source_xml=xml.xpath('//P')[0])
n2b = Node('(b) Content', label=['200', '2', 'b'])
n2 = Node('n2', label=['200', '2'], children=[n2a, n2b])
root = Node('root', label=['200'], children=[n2])
notice_changes = changes.NoticeChanges()
build.create_xml_changes(labels_amended, root, notice_changes)
self.assertTrue('200-2-a' in notice_changes.changes)
self.assertTrue(1, len(notice_changes.changes['200-2-a']))
change = notice_changes.changes['200-2-a'][0]
self.assertEqual('PUT', change['action'])
self.assertFalse('field' in change)
n2a.text = n2a.text + ":"
n2a.source_xml.text = n2a.source_xml.text + ":"
notice_changes = changes.NoticeChanges()
build.create_xml_changes(labels_amended, root, notice_changes)
self.assertTrue('200-2-a' in notice_changes.changes)
self.assertTrue(1, len(notice_changes.changes['200-2-a']))
change = notice_changes.changes['200-2-a'][0]
self.assertEqual('PUT', change['action'])
self.assertEqual('[text]', change.get('field'))
def test_create_xml_changes_child_stars():
labels_amended = [Amendment('PUT', '200-?-2-a')]
with XMLBuilder("ROOT") as ctx:
ctx.P("(a) Content")
ctx.STARS()
n2a = Node('(a) Content', label=['200', '2', 'a'],
source_xml=ctx.xml.xpath('//P')[0])
n2b = Node('(b) Content', label=['200', '2', 'b'])
n2 = Node('n2', label=['200', '2'], children=[n2a, n2b])
root = Node('root', label=['200'], children=[n2])
notice_changes = changes.NoticeChanges()
fetch.create_xml_changes(labels_amended, root, notice_changes)
data = notice_changes[None]
assert '200-2-a' in data
assert len(data['200-2-a']) == 1
change = data['200-2-a'][0]
assert change['action'] == 'PUT'
assert 'field' not in change
n2a.text = n2a.text + ":"
n2a.source_xml.text = n2a.source_xml.text + ":"
notice_changes = changes.NoticeChanges()
fetch.create_xml_changes(labels_amended, root, notice_changes)
data = notice_changes[None]
assert '200-2-a' in data
assert len(data['200-2-a']) == 1
change = data['200-2-a'][0]
assert change['action'] == 'PUT'
assert change.get('field') == '[text]'
def test_create_xml_changes_child_stars(self):
labels_amended = [Amendment('PUT', '200-?-2-a')]
with XMLBuilder("ROOT") as ctx:
ctx.P("(a) Content")
ctx.STARS()
n2a = Node('(a) Content', label=['200', '2', 'a'],
source_xml=ctx.xml.xpath('//P')[0])
n2b = Node('(b) Content', label=['200', '2', 'b'])
n2 = Node('n2', label=['200', '2'], children=[n2a, n2b])
root = Node('root', label=['200'], children=[n2])
notice_changes = changes.NoticeChanges()
amendments.create_xml_changes(labels_amended, root, notice_changes)
data = notice_changes.changes_by_xml[None]
self.assertIn('200-2-a', data)
self.assertTrue(1, len(data['200-2-a']))
change = data['200-2-a'][0]
self.assertEqual('PUT', change['action'])
self.assertNotIn('field', change)
n2a.text = n2a.text + ":"
n2a.source_xml.text = n2a.source_xml.text + ":"
notice_changes = changes.NoticeChanges()
amendments.create_xml_changes(labels_amended, root, notice_changes)
data = notice_changes.changes_by_xml[None]
self.assertIn('200-2-a', data)
self.assertTrue(1, len(data['200-2-a']))
change = data['200-2-a'][0]
self.assertEqual('PUT', change['action'])
self.assertEqual('[text]', change.get('field'))
def test_update_duplicates(self):
nc = changes.NoticeChanges()
nc.add_changes(None, {
'123-12': {
'action': 'DELETE'
},
'123-22': {
'action': 'OTHER'
}
})
nc.add_changes(None, {'123-12': {'action': 'DELETE'}})
nc.add_changes(None, {'123-12': {'action': 'OTHER'}})
nc.add_changes(None, {
'123-22': {
'action': 'OTHER'
},
'123-32': {
'action': 'LAST'
}
})
data = nc.changes_by_xml[None]
self.assertTrue('123-12' in data)
self.assertTrue('123-22' in data)
self.assertTrue('123-32' in data)
self.assertEqual(data['123-12'], [{
'action': 'DELETE'
}, {
'action': 'OTHER'
}])
self.assertEqual(data['123-22'], [{'action': 'OTHER'}])
self.assertEqual(data['123-32'], [{'action': 'LAST'}])
def test_update_duplicates(self):
nc = changes.NoticeChanges()
nc.update({
'123-12': {
'action': 'DELETE'
},
'123-22': {
'action': 'OTHER'
}
})
nc.update({'123-12': {'action': 'DELETE'}})
nc.update({'123-12': {'action': 'OTHER'}})
nc.update({
'123-22': {
'action': 'OTHER'
},
'123-32': {
'action': 'LAST'
}
})
self.assertTrue('123-12' in nc.changes)
self.assertTrue('123-22' in nc.changes)
self.assertTrue('123-32' in nc.changes)
self.assertEqual(nc.changes['123-12'], [{
'action': 'DELETE'
}, {
'action': 'OTHER'
}])
self.assertEqual(nc.changes['123-22'], [{'action': 'OTHER'}])
self.assertEqual(nc.changes['123-32'], [{'action': 'LAST'}])
def process_amendments(notice, notice_xml):
"""Process changes to the regulation that are expressed in the notice."""
all_amends = [] # will be added to the notice
cfr_part = notice['cfr_parts'][0]
notice_changes = changes.NoticeChanges()
# process amendments in batches, based on their parent XML
for amdparent in notice_xml.xpath('//AMDPAR/..'):
context = [amdparent.get('PART') or cfr_part]
amendments_by_section = defaultdict(list)
normal_amends = [] # amendments not moving or adding a subpart
for amdpar in amdparent.xpath('.//AMDPAR'):
instructions = amdpar.xpath('./EREGS_INSTRUCTIONS')
if not instructions:
logger.warning('No <EREGS_INSTRUCTIONS>. Was this notice '
'preprocessed?')
continue
instructions = instructions[0]
amendments = [amendment_from_xml(el) for el in instructions]
context = [None if l is '?' else l
for l in instructions.get('final_context').split('-')]
section_xml = find_section(amdpar)
for amendment in amendments:
all_amends.append(amendment)
if isinstance(amendment, DesignateAmendment):
subpart_changes = process_designate_subpart(amendment)
if subpart_changes:
notice_changes.update(subpart_changes)
elif new_subpart_added(amendment):
notice_changes.update(process_new_subpart(
notice, amendment, amdpar))
elif section_xml is None:
normal_amends.append(amendment)
else:
normal_amends.append(amendment)
amendments_by_section[section_xml].append(amendment)
cfr_part = context[0] # carry the part through to the next amdparent
create_xmlless_changes(normal_amends, notice_changes)
# Process amendments relating to a specific section in batches, too
for section_xml, related_amends in amendments_by_section.items():
for section in reg_text.build_from_section(cfr_part, section_xml):
create_xml_changes(related_amends, section, notice_changes)
for appendix in parse_appendix_changes(normal_amends, cfr_part,
amdparent):
create_xml_changes(normal_amends, appendix, notice_changes)
interp = parse_interp_changes(normal_amends, cfr_part, amdparent)
if interp:
create_xml_changes(normal_amends, interp, notice_changes)
if all_amends:
notice['amendments'] = all_amends
notice['changes'] = notice_changes.changes
return notice
def test_create_xmlless_changes(self):
labels_amended = [Amendment('DELETE', '200-2-a'),
Amendment('MOVE', '200-2-b', '200-2-c')]
notice_changes = changes.NoticeChanges()
build.create_xmlless_changes(labels_amended, notice_changes)
delete = notice_changes.changes['200-2-a'][0]
move = notice_changes.changes['200-2-b'][0]
self.assertEqual({'action': 'DELETE'}, delete)
self.assertEqual({'action': 'MOVE', 'destination': ['200', '2', 'c']},
move)
def test_create_xmlless_changes():
labels_amended = [Amendment('DELETE', '200-?-2-a'),
Amendment('MOVE', '200-?-2-b', '200-?-2-c')]
notice_changes = changes.NoticeChanges()
for amendment in labels_amended:
fetch.create_xmlless_change(amendment, notice_changes)
delete = notice_changes[None]['200-2-a'][0]
move = notice_changes[None]['200-2-b'][0]
assert delete == {'action': 'DELETE'}
assert move == {'action': 'MOVE', 'destination': ['200', '2', 'c']}
def test_create_xml_changes_reserve(self):
labels_amended = [Amendment('RESERVE', '200-2-a')]
n2a = Node('[Reserved]', label=['200', '2', 'a'])
n2 = Node('n2', label=['200', '2'], children=[n2a])
root = Node('root', label=['200'], children=[n2])
notice_changes = changes.NoticeChanges()
build.create_xml_changes(labels_amended, root, notice_changes)
reserve = notice_changes.changes['200-2-a'][0]
self.assertEqual(reserve['action'], 'RESERVE')
self.assertEqual(reserve['node']['text'], u'[Reserved]')
def test_create_xml_changes_reserve():
labels_amended = [Amendment('RESERVE', '200-?-2-a')]
n2a = Node('[Reserved]', label=['200', '2', 'a'])
n2 = Node('n2', label=['200', '2'], children=[n2a])
root = Node('root', label=['200'], children=[n2])
notice_changes = changes.NoticeChanges()
fetch.create_xml_changes(labels_amended, root, notice_changes)
reserve = notice_changes[None]['200-2-a'][0]
assert reserve['action'] == 'RESERVE'
assert reserve['node']['text'] == '[Reserved]'
def fetch_amendments(notice_xml):
"""Process changes to the regulation that are expressed in the notice."""
notice_changes = changes.NoticeChanges()
if notice_xml.xpath('.//AMDPAR[not(EREGS_INSTRUCTIONS)]'):
logger.warning(
'No <EREGS_INSTRUCTIONS>. Was this notice preprocessed?')
cache = ContentCache()
authority_by_xml = {}
for instruction_xml in notice_xml.xpath('.//EREGS_INSTRUCTIONS/*'):
amendment = amendment_from_xml(instruction_xml)
content = cache.content_of_change(instruction_xml)
if instruction_xml.tag == 'MOVE_INTO_SUBPART':
subpart_changes = process_designate_subpart(amendment)
if subpart_changes:
notice_changes.add_changes(amendment.amdpar_xml,
subpart_changes)
elif instruction_xml.tag == 'AUTHORITY':
authority_by_xml[amendment.amdpar_xml] = instruction_xml.text
elif changes.new_subpart_added(amendment):
subpart_changes = {}
for change in changes.create_subpart_amendment(content.struct):
subpart_changes.update(change)
notice_changes.add_changes(amendment.amdpar_xml, subpart_changes)
elif content:
content.amends.append(amendment)
else:
create_xmlless_change(amendment, notice_changes)
for content in cache.by_xml.values():
create_xml_changes(content.amends, content.struct, notice_changes)
amendments = []
for amdpar_xml in notice_xml.xpath('.//AMDPAR'):
amendment = {"instruction": amdpar_xml.text}
# There'll be at most one
for inst_xml in amdpar_xml.xpath('./EREGS_INSTRUCTIONS'):
context = inst_xml.get('final_context', '')
amendment['cfr_part'] = context.split('-')[0]
relevant_changes = notice_changes.changes_by_xml[amdpar_xml]
if relevant_changes:
amendment['changes'] = list(relevant_changes.items())
if amdpar_xml in authority_by_xml:
amendment['authority'] = authority_by_xml[amdpar_xml]
amendments.append(amendment)
return amendments
def test_notice_changes_update_duplicates():
nc = changes.NoticeChanges()
nc.add_change(None, changes.Change('123-12', {'action': 'DELETE'}))
nc.add_change(None, changes.Change('123-22', {'action': 'OTHER'}))
nc.add_change(None, changes.Change('123-12', {'action': 'DELETE'}))
nc.add_change(None, changes.Change('123-12', {'action': 'OTHER'}))
nc.add_change(None, changes.Change('123-22', {'action': 'OTHER'}))
nc.add_change(None, changes.Change('123-32', {'action': 'LAST'}))
data = nc[None]
assert '123-12' in data
assert '123-22' in data
assert '123-32' in data
assert data['123-12'] == [{'action': 'DELETE'}, {'action': 'OTHER'}]
assert data['123-22'] == [{'action': 'OTHER'}]
assert data['123-32'] == [{'action': 'LAST'}]
def test_create_xml_changes_stars_hole(self):
labels_amended = [Amendment('PUT', '200-2-a')]
n2a1 = Node('(1) * * *', label=['200', '2', 'a', '1'])
n2a2 = Node('(2) a2a2a2', label=['200', '2', 'a', '2'])
n2a = Node('(a) aaa', label=['200', '2', 'a'], children=[n2a1, n2a2])
n2 = Node('n2', label=['200', '2'], children=[n2a])
root = Node('root', label=['200'], children=[n2])
notice_changes = changes.NoticeChanges()
build.create_xml_changes(labels_amended, root, notice_changes)
for label in ('200-2-a', '200-2-a-2'):
self.assertTrue(label in notice_changes.changes)
self.assertEqual(1, len(notice_changes.changes[label]))
change = notice_changes.changes[label][0]
self.assertEqual('PUT', change['action'])
self.assertFalse('field' in change)
self.assertTrue('200-2-a-1' in notice_changes.changes)
self.assertEqual(1, len(notice_changes.changes['200-2-a-1']))
change = notice_changes.changes['200-2-a-1'][0]
self.assertEqual('KEEP', change['action'])
self.assertFalse('field' in change)
def test_create_xml_changes_stars_hole():
labels_amended = [Amendment('PUT', '200-?-2-a')]
n2a1 = Node('(1) * * *', label=['200', '2', 'a', '1'])
n2a2 = Node('(2) a2a2a2', label=['200', '2', 'a', '2'])
n2a = Node('(a) aaa', label=['200', '2', 'a'], children=[n2a1, n2a2])
n2 = Node('n2', label=['200', '2'], children=[n2a])
root = Node('root', label=['200'], children=[n2])
notice_changes = changes.NoticeChanges()
fetch.create_xml_changes(labels_amended, root, notice_changes)
data = notice_changes[None]
for label in ('200-2-a', '200-2-a-2'):
assert label in data
assert len(data[label]) == 1
change = data[label][0]
assert change['action'] == 'PUT'
assert 'field' not in change
assert '200-2-a-1' in data
assert len(data['200-2-a-1']) == 1
change = data['200-2-a-1'][0]
assert change['action'] == 'KEEP'
assert 'field' not in change
def test_create_xml_changes_stars(self):
labels_amended = [Amendment('PUT', '200-?-2-a')]
n2a1 = Node('(1) Content', label=['200', '2', 'a', '1'])
n2a2 = Node('(2) Content', label=['200', '2', 'a', '2'])
n2a = Node('(a) * * *', label=['200', '2', 'a'], children=[n2a1, n2a2])
n2 = Node('n2', label=['200', '2'], children=[n2a])
root = Node('root', label=['200'], children=[n2])
notice_changes = changes.NoticeChanges()
amendments.create_xml_changes(labels_amended, root, notice_changes)
data = notice_changes.changes_by_xml[None]
for label in ('200-2-a-1', '200-2-a-2'):
self.assertIn(label, data)
self.assertEqual(1, len(data[label]))
change = data[label][0]
self.assertEqual('PUT', change['action'])
self.assertNotIn('field', change)
self.assertTrue('200-2-a' in data)
self.assertEqual(1, len(data['200-2-a']))
change = data['200-2-a'][0]
self.assertEqual('KEEP', change['action'])
self.assertNotIn('field', change)
def process_amendments(notice, notice_xml):
""" Process the changes to the regulation that are expressed in the notice.
"""
amends = []
notice_changes = changes.NoticeChanges()
amdpars_by_parent = []
for par in notice_xml.xpath('//AMDPAR'):
parent = par.getparent()
exists = filter(lambda aXp: aXp.parent == parent, amdpars_by_parent)
if exists:
exists[0].append(par)
else:
amdpars_by_parent.append(AmdparByParent(parent, par))
default_cfr_part = notice['cfr_part']
for aXp in amdpars_by_parent:
amended_labels = []
designate_labels, other_labels = [], []
context = [default_cfr_part]
for par in aXp.amdpars:
als, context = parse_amdpar(par, context)
amended_labels.extend(als)
labels_by_part = defaultdict(list)
for al in amended_labels:
if isinstance(al, DesignateAmendment):
subpart_changes = process_designate_subpart(al)
if subpart_changes:
notice_changes.update(subpart_changes)
designate_labels.append(al)
elif new_subpart_added(al):
notice_changes.update(process_new_subpart(notice, al, par))
designate_labels.append(al)
else:
other_labels.append(al)
labels_by_part[al.label[0]].append(al)
create_xmlless_changes(other_labels, notice_changes)
# for cfr_part, rel_labels in labels_by_part.iteritems():
labels_for_part = {
part: labels
for part, labels in labels_by_part.iteritems()
if part == default_cfr_part
}
print(labels_for_part)
for cfr_part, rel_labels in labels_for_part.iteritems():
section_xml = find_section(par)
if section_xml is not None:
subparts = aXp.parent.xpath('.//SUBPART/HD')
if subparts:
subpart_label = [
cfr_part, 'Subpart', subparts[0].text[8:9]
]
else:
subpart_label = None
for section in reg_text.build_from_section(
cfr_part, section_xml):
create_xml_changes(rel_labels, section, notice_changes,
subpart_label)
for appendix in parse_appendix_changes(rel_labels, cfr_part,
aXp.parent):
create_xml_changes(rel_labels, appendix, notice_changes)
interp = parse_interp_changes(rel_labels, cfr_part, aXp.parent)
if interp:
create_xml_changes(rel_labels, interp, notice_changes)
amends.extend(designate_labels)
amends.extend(other_labels)
# if other_labels: # Carry cfr_part through amendments
# default_cfr_part = other_labels[-1].label[0]
if amends:
notice['amendments'] = amends
notice['changes'] = notice_changes.changes
elif notice['document_number'] in settings.REISSUANCES:
notice['changes'] = {
default_cfr_part: [{
'action': 'PUT',
'node': reg_text.build_tree(notice_xml)
}]
}
def process_amendments(notice, notice_xml):
""" Process the changes to the regulation that are expressed in the notice.
"""
amends = []
notice_changes = changes.NoticeChanges()
amdpars_by_parent = []
for par in notice_xml.xpath('//AMDPAR'):
parent = par.getparent()
exists = filter(lambda aXp: aXp.parent == parent, amdpars_by_parent)
if exists:
exists[0].append(par)
else:
amdpars_by_parent.append(AmdparByParent(parent, par))
default_cfr_part = notice['cfr_parts'][0]
for aXp in amdpars_by_parent:
amended_labels = []
designate_labels, other_labels = [], []
context = [aXp.parent.get('PART') or default_cfr_part]
for par in aXp.amdpars:
als, context = parse_amdpar(par, context)
amended_labels.extend(als)
labels_by_part = defaultdict(list)
for al in amended_labels:
if isinstance(al, DesignateAmendment):
subpart_changes = process_designate_subpart(al)
if subpart_changes:
notice_changes.update(subpart_changes)
designate_labels.append(al)
elif new_subpart_added(al):
notice_changes.update(process_new_subpart(notice, al, par))
designate_labels.append(al)
else:
other_labels.append(al)
labels_by_part[al.label[0]].append(al)
create_xmlless_changes(other_labels, notice_changes)
for cfr_part, rel_labels in labels_by_part.iteritems():
section_xml = find_section(par)
if section_xml is not None:
for section in reg_text.build_from_section(
cfr_part, section_xml):
create_xml_changes(rel_labels, section, notice_changes)
for appendix in parse_appendix_changes(rel_labels, cfr_part,
aXp.parent):
create_xml_changes(rel_labels, appendix, notice_changes)
interp = parse_interp_changes(rel_labels, cfr_part, aXp.parent)
if interp:
create_xml_changes(rel_labels, interp, notice_changes)
amends.extend(designate_labels)
amends.extend(other_labels)
if other_labels: # Carry cfr_part through amendments
default_cfr_part = other_labels[-1].label[0]
if amends:
notice['amendments'] = amends
notice['changes'] = notice_changes.changes
