def test_process_if_needed_missing_writes(self, xml_parser): """If output isn't already present, we should process. If it is present, we don't need to, unless a dependency has changed.""" with self.cli.isolated_filesystem(): build_tree = xml_parser.reg_text.build_tree build_tree.return_value = Node() last_versions = [annual_editions.LastVersionInYear('1111', 2000)] entry.Version('12', '1000', '1111').write( Version('1111', date(2000, 1, 1), date(2000, 1, 1))) entry.Entry('annual', '12', '1000', 2000).write( b'<ROOT></ROOT>') annual_editions.process_if_needed('12', '1000', last_versions) self.assertTrue(build_tree.called) build_tree.reset_mock() entry.Entry('tree', '12', '1000', '1111').write(b'tree-here') annual_editions.process_if_needed('12', '1000', last_versions) self.assertFalse(build_tree.called) # Simulate a change to an input file label_id = str(entry.Annual(12, 1000, 2000)) new_time = timezone.now() + timedelta(hours=1) DBEntry.objects.filter(label_id=label_id).update(modified=new_time) annual_editions.process_if_needed('12', '1000', last_versions) self.assertTrue(build_tree.called)
def test_process_if_needed_missing_writes(monkeypatch): """If output isn't already present, we should process. If it is present, we don't need to, unless a dependency has changed.""" monkeypatch.setattr(annual_editions, 'gpo_cfr', Mock()) build_tree = annual_editions.gpo_cfr.builder.build_tree build_tree.return_value = Node() last_versions = [annual_editions.LastVersionInYear('1111', 2000)] entry.Version('12', '1000', '1111').write( Version('1111', date(2000, 1, 1), Citation(1, 1))) entry.Entry('annual', '12', '1000', 2000).write(b'<ROOT></ROOT>') annual_editions.process_if_needed('12', '1000', last_versions) assert build_tree.called build_tree.reset_mock() entry.Entry('tree', '12', '1000', '1111').write(b'tree-here') annual_editions.process_if_needed('12', '1000', last_versions) assert not build_tree.called # Simulate a change to an input file label_id = str(entry.Annual(12, 1000, 2000)) new_time = timezone.now() + timedelta(hours=1) DBEntry.objects.filter(label_id=label_id).update(modified=new_time) annual_editions.process_if_needed('12', '1000', last_versions) assert build_tree.called
def fetch_annual_edition(cfr_title, cfr_part, year): """Download an annual edition of a regulation""" volume = annual.find_volume(year, cfr_title, cfr_part) xml = volume.find_part_xml(cfr_part).preprocess() annual_entry = entry.Annual(cfr_title, cfr_part, year) annual_entry.write(xml) if xml.source_is_local: dependency.Graph().add(str(annual_entry), xml.source)
def test_is_derived(): """Should filter version ids to only those with a dependency on changes derived from a rule""" tree_dir = entry.Tree('12', '1000') deps = dependency.Graph() deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001)) deps.add(tree_dir / 222, entry.Notice(222)) deps.add(tree_dir / 333, entry.Notice(333)) deps.add(tree_dir / 333, entry.Version(333)) assert not fill_with_rules.is_derived('111', deps, tree_dir) assert fill_with_rules.is_derived('222', deps, tree_dir) assert fill_with_rules.is_derived('333', deps, tree_dir) assert not fill_with_rules.is_derived('444', deps, tree_dir)
def test_derived_from_rules(self): """Should filter a set of version ids to only those with a dependency on changes derived from a rule""" with self.cli.isolated_filesystem(): tree_dir = entry.Tree('12', '1000') deps = dependency.Graph() deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001)) deps.add(tree_dir / 222, entry.RuleChanges(222)) deps.add(tree_dir / 333, entry.RuleChanges(333)) deps.add(tree_dir / 333, entry.Version(333)) derived = fill_with_rules.derived_from_rules( ['111', '222', '333', '444'], deps, tree_dir) self.assertEqual(derived, ['222', '333'])
def test_is_derived(self): """Should filter version ids to only those with a dependency on changes derived from a rule""" with self.cli.isolated_filesystem(): tree_dir = entry.Tree('12', '1000') deps = dependency.Graph() deps.add(tree_dir / 111, entry.Annual(12, 1000, 2001)) deps.add(tree_dir / 222, entry.Notice(222)) deps.add(tree_dir / 333, entry.Notice(333)) deps.add(tree_dir / 333, entry.Version(333)) self.assertFalse(fill_with_rules.is_derived('111', deps, tree_dir)) self.assertTrue(fill_with_rules.is_derived('222', deps, tree_dir)) self.assertTrue(fill_with_rules.is_derived('333', deps, tree_dir)) self.assertFalse(fill_with_rules.is_derived('444', deps, tree_dir))
def process_if_needed(volume, cfr_part): """Review dependencies; if they're out of date, parse the annual edition into a tree and store that""" version_id = _version_id(volume.year, cfr_part) annual_entry = entry.Annual(volume.title, cfr_part, volume.year) tree_entry = entry.Tree(volume.title, cfr_part, version_id) notice_entry = entry.Notice(version_id) deps = dependency.Graph() deps.add(tree_entry, annual_entry) deps.validate_for(tree_entry) if deps.is_stale(tree_entry): tree = xml_parser.reg_text.build_tree(annual_entry.read().xml) tree_entry.write(tree) notice_entry.write( build_fake_notice(version_id, volume.publication_date, volume.title, cfr_part))
def process_if_needed(cfr_title, cfr_part, last_version_list): """Calculate dependencies between input and output files for these annual editions. If an output is missing or out of date, process it""" annual_path = entry.Annual(cfr_title, cfr_part) tree_path = entry.Tree(cfr_title, cfr_part) version_path = entry.Version(cfr_title, cfr_part) deps = dependency.Graph() for last_version in last_version_list: deps.add(tree_path / last_version.version_id, version_path / last_version.version_id) deps.add(tree_path / last_version.version_id, annual_path / last_version.year) for last_version in last_version_list: tree_entry = tree_path / last_version.version_id deps.validate_for(tree_entry) if deps.is_stale(tree_entry): input_entry = annual_path / last_version.year tree = gpo_cfr.builder.build_tree(input_entry.read().xml) tree_entry.write(tree)
def test_process_if_needed_missing_writes(self, xml_parser): """If output isn't already present, we should process. If it is present, we don't need to, unless a dependency has changed.""" with self.cli.isolated_filesystem(): build_tree = xml_parser.reg_text.build_tree build_tree.return_value = Node() last_versions = [annual_editions.LastVersionInYear('1111', 2000)] entry.Version('12', '1000', '1111').write( Version('1111', date(2000, 1, 1), date(2000, 1, 1))) entry.Entry('annual', '12', '1000', 2000).write(b'<ROOT></ROOT>') annual_editions.process_if_needed('12', '1000', last_versions) self.assertTrue(build_tree.called) build_tree.reset_mock() entry.Entry('tree', '12', '1000', '1111').write(b'tree-here') annual_editions.process_if_needed('12', '1000', last_versions) self.assertFalse(build_tree.called) # Simulate a change to an input file os.utime(str(entry.Annual('12', '1000', '2000')), (time() + 1000, time() + 1000)) annual_editions.process_if_needed('12', '1000', last_versions) self.assertTrue(build_tree.called)
def fetch_annual_edition(cfr_title, cfr_part, year): """Download an annual edition of a regulation""" volume = annual.find_volume(year, cfr_title, cfr_part) xml = volume.find_part_xml(cfr_part).preprocess() annual_entry = entry.Annual(cfr_title, cfr_part, year) annual_entry.write(xml)