def _GeneratorFromPath(path, event_file_active_filter=None): """Create an event generator for file or directory at given path string.""" if not path: raise ValueError("path must be a valid string") if io_wrapper.IsTensorFlowEventsFile(path): return event_file_loader.EventFileLoader(path) elif event_file_active_filter: return directory_loader.DirectoryLoader( path, event_file_loader.TimestampedEventFileLoader, path_filter=io_wrapper.IsTensorFlowEventsFile, active_filter=event_file_active_filter, ) else: return directory_watcher.DirectoryWatcher( path, event_file_loader.EventFileLoader, io_wrapper.IsTensorFlowEventsFile)
def testDoesntCrashWhenCurrentFileIsDeleted(self): # Use actual file loader so it emits the real error. self._loader = directory_loader.DirectoryLoader( self._directory, event_file_loader.TimestampedEventFileLoader) with test_util.FileWriter(self._directory, filename_suffix=".a") as writer_a: writer_a.add_test_summary("a") events = list(self._loader.Load()) events.pop(0) # Ignore the file_version event. self.assertEqual(1, len(events)) self.assertEqual("a", events[0].summary.value[0].tag) os.remove(glob.glob(os.path.join(self._directory, "*.a"))[0]) with test_util.FileWriter(self._directory, filename_suffix=".b") as writer_b: writer_b.add_test_summary("b") events = list(self._loader.Load()) events.pop(0) # Ignore the file_version event. self.assertEqual(1, len(events)) self.assertEqual("b", events[0].summary.value[0].tag)
def testDoesntCrashWhenUpcomingFileIsDeleted(self): # Use actual file loader so it emits the real error. self._loader = directory_loader.DirectoryLoader( self._directory, event_file_loader.TimestampedEventFileLoader ) with test_util.FileWriter( self._directory, filename_suffix=".a" ) as writer_a: writer_a.add_test_summary("a") with test_util.FileWriter( self._directory, filename_suffix=".b" ) as writer_b: writer_b.add_test_summary("b") generator = self._loader.Load() next(generator) # Ignore the file_version event. event = next(generator) self.assertEqual("a", event.summary.value[0].tag) os.remove(glob.glob(os.path.join(self._directory, "*.b"))[0]) self.assertEmpty(list(generator))
def testActiveFilter_dynamicFilterBehavior(self): """Tests behavior of a dynamic active_filter.""" loader_registry = [] loader_factory = functools.partial( _TimestampedByteLoader, registry=loader_registry ) threshold = 0 active_filter = lambda timestamp: timestamp >= threshold self._loader = directory_loader.DirectoryLoader( self._directory, loader_factory, active_filter=active_filter ) def assertLoadersForPaths(paths): paths = [os.path.join(self._directory, path) for path in paths] self.assertEqual(loader_registry, paths) self._WriteToFile("a", ["A1", "A2"], [1, 2]) self._WriteToFile("b", ["B1", "B2", "B3"], [1, 2, 3]) self._WriteToFile("c", ["C1"], [1]) threshold = 2 # First load pass should leave file C marked inactive. self.assertLoaderYields(["A1", "A2", "B1", "B2", "B3", "C1"]) assertLoadersForPaths(["a", "b"]) self._WriteToFile("a", ["A4"], [4]) self._WriteToFile("b", ["B4"], [4]) self._WriteToFile("c", ["C4"], [4]) threshold = 3 # Second load pass should mark file A as inactive (due to newly # increased threshold) and thus skip reading data from it. self.assertLoaderYields(["B4"]) assertLoadersForPaths(["b"]) self._WriteToFile("b", ["B5", "B6"], [5, 6]) # Simulate a third pass in which the threshold increases while # we're processing a file, so it's still active at the start of the # load but should be marked inactive at the end. load_generator = self._loader.Load() self.assertEqual("B5", next(load_generator)) threshold = 7 self.assertEqual(["B6"], list(load_generator)) assertLoadersForPaths([]) # Confirm that all loaders are now inactive. self._WriteToFile("b", ["B7"], [7]) self.assertLoaderYields([])
def testActiveFilter_staticFilterBehavior(self): """Tests behavior of a static active_filter.""" loader_registry = [] loader_factory = functools.partial( _TimestampedByteLoader, registry=loader_registry ) active_filter = lambda timestamp: timestamp >= 2 self._loader = directory_loader.DirectoryLoader( self._directory, loader_factory, active_filter=active_filter ) def assertLoadersForPaths(paths): paths = [os.path.join(self._directory, path) for path in paths] self.assertEqual(loader_registry, paths) # a: normal-looking file. # b: file without sufficiently active data (should be marked inactive). # c: file with timestamps in reverse order (max computed correctly). # d: empty file (should be considered active in absence of timestamps). self._WriteToFile("a", ["A1", "A2"], [1, 2]) self._WriteToFile("b", ["B1"], [1]) self._WriteToFile("c", ["C2", "C1", "C0"], [2, 1, 0]) self._WriteToFile("d", [], []) self.assertLoaderYields(["A1", "A2", "B1", "C2", "C1", "C0"]) assertLoadersForPaths(["a", "c", "d"]) self._WriteToFile("a", ["A3"], [3]) self._WriteToFile("b", ["B3"], [3]) self._WriteToFile("c", ["C0"], [0]) self._WriteToFile("d", ["D3"], [3]) self.assertLoaderYields(["A3", "C0", "D3"]) assertLoadersForPaths(["a", "c", "d"]) # Check that a 0 timestamp in file C on the most recent load doesn't # override the max timestamp of 2 seen in the earlier load. self._WriteToFile("c", ["C4"], [4]) self.assertLoaderYields(["C4"]) assertLoadersForPaths(["a", "c", "d"])
def testRaisesWithBadArguments(self): with self.assertRaises(ValueError): directory_loader.DirectoryLoader(None, lambda x: None) with self.assertRaises(ValueError): directory_loader.DirectoryLoader("dir", None)
def setUp(self): # Put everything in a directory so it's easier to delete w/in tests. self._directory = os.path.join(self.get_temp_dir(), "testdir") os.mkdir(self._directory) self._loader = directory_loader.DirectoryLoader( self._directory, _TimestampedByteLoader)
def directory_loader_factory(path): return directory_loader.DirectoryLoader( path, event_file_loader.TimestampedEventFileLoader, path_filter=io_wrapper.IsTensorFlowEventsFile, )