def test_process_if_needed_missing_writes(self, xml_parser):
"""If output isn't already present, we should process. If it is
present, we don't need to, unless a dependency has changed."""
with self.cli.isolated_filesystem():
build_tree = xml_parser.reg_text.build_tree
build_tree.return_value = Node()
last_versions = [annual_editions.LastVersionInYear('1111', 2000)]
entry.Version('12', '1000', '1111').write(
Version('1111', date(2000, 1, 1), date(2000, 1, 1)))
entry.Entry('annual', '12', '1000', 2000).write(
b'<ROOT></ROOT>')
annual_editions.process_if_needed('12', '1000', last_versions)
self.assertTrue(build_tree.called)
build_tree.reset_mock()
entry.Entry('tree', '12', '1000', '1111').write(b'tree-here')
annual_editions.process_if_needed('12', '1000', last_versions)
self.assertFalse(build_tree.called)
# Simulate a change to an input file
label_id = str(entry.Annual(12, 1000, 2000))
new_time = timezone.now() + timedelta(hours=1)
DBEntry.objects.filter(label_id=label_id).update(modified=new_time)
annual_editions.process_if_needed('12', '1000', last_versions)
self.assertTrue(build_tree.called)
def test_process_if_needed_missing_writes(monkeypatch):
"""If output isn't already present, we should process. If it is present,
we don't need to, unless a dependency has changed."""
monkeypatch.setattr(annual_editions, 'gpo_cfr', Mock())
build_tree = annual_editions.gpo_cfr.builder.build_tree
build_tree.return_value = Node()
last_versions = [annual_editions.LastVersionInYear('1111', 2000)]
entry.Version('12', '1000', '1111').write(
Version('1111', date(2000, 1, 1), Citation(1, 1)))
entry.Entry('annual', '12', '1000', 2000).write(b'<ROOT></ROOT>')
annual_editions.process_if_needed('12', '1000', last_versions)
assert build_tree.called
build_tree.reset_mock()
entry.Entry('tree', '12', '1000', '1111').write(b'tree-here')
annual_editions.process_if_needed('12', '1000', last_versions)
assert not build_tree.called
# Simulate a change to an input file
label_id = str(entry.Annual(12, 1000, 2000))
new_time = timezone.now() + timedelta(hours=1)
DBEntry.objects.filter(label_id=label_id).update(modified=new_time)
annual_editions.process_if_needed('12', '1000', last_versions)
assert build_tree.called
def fetch_annual_edition(cfr_title, cfr_part, year):
"""Download an annual edition of a regulation"""
volume = annual.find_volume(year, cfr_title, cfr_part)
xml = volume.find_part_xml(cfr_part).preprocess()
annual_entry = entry.Annual(cfr_title, cfr_part, year)
annual_entry.write(xml)
if xml.source_is_local:
dependency.Graph().add(str(annual_entry), xml.source)
def test_is_derived():
"""Should filter version ids to only those with a dependency on
changes derived from a rule"""
tree_dir = entry.Tree('12', '1000')
deps = dependency.Graph()
deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001))
deps.add(tree_dir / 222, entry.Notice(222))
deps.add(tree_dir / 333, entry.Notice(333))
deps.add(tree_dir / 333, entry.Version(333))
assert not fill_with_rules.is_derived('111', deps, tree_dir)
assert fill_with_rules.is_derived('222', deps, tree_dir)
assert fill_with_rules.is_derived('333', deps, tree_dir)
assert not fill_with_rules.is_derived('444', deps, tree_dir)
def test_derived_from_rules(self):
"""Should filter a set of version ids to only those with a dependency
on changes derived from a rule"""
with self.cli.isolated_filesystem():
tree_dir = entry.Tree('12', '1000')
deps = dependency.Graph()
deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001))
deps.add(tree_dir / 222, entry.RuleChanges(222))
deps.add(tree_dir / 333, entry.RuleChanges(333))
deps.add(tree_dir / 333, entry.Version(333))
derived = fill_with_rules.derived_from_rules(
['111', '222', '333', '444'], deps, tree_dir)
self.assertEqual(derived, ['222', '333'])
def test_is_derived(self):
"""Should filter version ids to only those with a dependency on
changes derived from a rule"""
with self.cli.isolated_filesystem():
tree_dir = entry.Tree('12', '1000')
deps = dependency.Graph()
deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001))
deps.add(tree_dir / 222, entry.Notice(222))
deps.add(tree_dir / 333, entry.Notice(333))
deps.add(tree_dir / 333, entry.Version(333))
self.assertFalse(fill_with_rules.is_derived('111', deps, tree_dir))
self.assertTrue(fill_with_rules.is_derived('222', deps, tree_dir))
self.assertTrue(fill_with_rules.is_derived('333', deps, tree_dir))
self.assertFalse(fill_with_rules.is_derived('444', deps, tree_dir))
def process_if_needed(volume, cfr_part):
"""Review dependencies; if they're out of date, parse the annual edition
into a tree and store that"""
version_id = _version_id(volume.year, cfr_part)
annual_entry = entry.Annual(volume.title, cfr_part, volume.year)
tree_entry = entry.Tree(volume.title, cfr_part, version_id)
notice_entry = entry.Notice(version_id)
deps = dependency.Graph()
deps.add(tree_entry, annual_entry)
deps.validate_for(tree_entry)
if deps.is_stale(tree_entry):
tree = xml_parser.reg_text.build_tree(annual_entry.read().xml)
tree_entry.write(tree)
notice_entry.write(
build_fake_notice(version_id, volume.publication_date,
volume.title, cfr_part))
def process_if_needed(cfr_title, cfr_part, last_version_list):
"""Calculate dependencies between input and output files for these annual
editions. If an output is missing or out of date, process it"""
annual_path = entry.Annual(cfr_title, cfr_part)
tree_path = entry.Tree(cfr_title, cfr_part)
version_path = entry.Version(cfr_title, cfr_part)
deps = dependency.Graph()
for last_version in last_version_list:
deps.add(tree_path / last_version.version_id,
version_path / last_version.version_id)
deps.add(tree_path / last_version.version_id,
annual_path / last_version.year)
for last_version in last_version_list:
tree_entry = tree_path / last_version.version_id
deps.validate_for(tree_entry)
if deps.is_stale(tree_entry):
input_entry = annual_path / last_version.year
tree = gpo_cfr.builder.build_tree(input_entry.read().xml)
tree_entry.write(tree)
def test_process_if_needed_missing_writes(self, xml_parser):
"""If output isn't already present, we should process. If it is
present, we don't need to, unless a dependency has changed."""
with self.cli.isolated_filesystem():
build_tree = xml_parser.reg_text.build_tree
build_tree.return_value = Node()
last_versions = [annual_editions.LastVersionInYear('1111', 2000)]
entry.Version('12', '1000', '1111').write(
Version('1111', date(2000, 1, 1), date(2000, 1, 1)))
entry.Entry('annual', '12', '1000', 2000).write(b'<ROOT></ROOT>')
annual_editions.process_if_needed('12', '1000', last_versions)
self.assertTrue(build_tree.called)
build_tree.reset_mock()
entry.Entry('tree', '12', '1000', '1111').write(b'tree-here')
annual_editions.process_if_needed('12', '1000', last_versions)
self.assertFalse(build_tree.called)
# Simulate a change to an input file
os.utime(str(entry.Annual('12', '1000', '2000')),
(time() + 1000, time() + 1000))
annual_editions.process_if_needed('12', '1000', last_versions)
self.assertTrue(build_tree.called)
def fetch_annual_edition(cfr_title, cfr_part, year):
"""Download an annual edition of a regulation"""
volume = annual.find_volume(year, cfr_title, cfr_part)
xml = volume.find_part_xml(cfr_part).preprocess()
annual_entry = entry.Annual(cfr_title, cfr_part, year)
annual_entry.write(xml)
