def _GeneratorFromPath(path, event_file_active_filter=None):
"""Create an event generator for file or directory at given path string."""
if not path:
raise ValueError("path must be a valid string")
if io_wrapper.IsTensorFlowEventsFile(path):
return event_file_loader.EventFileLoader(path)
elif event_file_active_filter:
return directory_loader.DirectoryLoader(
path,
event_file_loader.TimestampedEventFileLoader,
path_filter=io_wrapper.IsTensorFlowEventsFile,
active_filter=event_file_active_filter,
)
else:
return directory_watcher.DirectoryWatcher(
path, event_file_loader.EventFileLoader,
io_wrapper.IsTensorFlowEventsFile)
def testDoesntCrashWhenCurrentFileIsDeleted(self):
# Use actual file loader so it emits the real error.
self._loader = directory_loader.DirectoryLoader(
self._directory, event_file_loader.TimestampedEventFileLoader)
with test_util.FileWriter(self._directory,
filename_suffix=".a") as writer_a:
writer_a.add_test_summary("a")
events = list(self._loader.Load())
events.pop(0) # Ignore the file_version event.
self.assertEqual(1, len(events))
self.assertEqual("a", events[0].summary.value[0].tag)
os.remove(glob.glob(os.path.join(self._directory, "*.a"))[0])
with test_util.FileWriter(self._directory,
filename_suffix=".b") as writer_b:
writer_b.add_test_summary("b")
events = list(self._loader.Load())
events.pop(0) # Ignore the file_version event.
self.assertEqual(1, len(events))
self.assertEqual("b", events[0].summary.value[0].tag)
def testDoesntCrashWhenUpcomingFileIsDeleted(self):
# Use actual file loader so it emits the real error.
self._loader = directory_loader.DirectoryLoader(
self._directory, event_file_loader.TimestampedEventFileLoader
)
with test_util.FileWriter(
self._directory, filename_suffix=".a"
) as writer_a:
writer_a.add_test_summary("a")
with test_util.FileWriter(
self._directory, filename_suffix=".b"
) as writer_b:
writer_b.add_test_summary("b")
generator = self._loader.Load()
next(generator) # Ignore the file_version event.
event = next(generator)
self.assertEqual("a", event.summary.value[0].tag)
os.remove(glob.glob(os.path.join(self._directory, "*.b"))[0])
self.assertEmpty(list(generator))
def testActiveFilter_dynamicFilterBehavior(self):
"""Tests behavior of a dynamic active_filter."""
loader_registry = []
loader_factory = functools.partial(
_TimestampedByteLoader, registry=loader_registry
)
threshold = 0
active_filter = lambda timestamp: timestamp >= threshold
self._loader = directory_loader.DirectoryLoader(
self._directory, loader_factory, active_filter=active_filter
)
def assertLoadersForPaths(paths):
paths = [os.path.join(self._directory, path) for path in paths]
self.assertEqual(loader_registry, paths)
self._WriteToFile("a", ["A1", "A2"], [1, 2])
self._WriteToFile("b", ["B1", "B2", "B3"], [1, 2, 3])
self._WriteToFile("c", ["C1"], [1])
threshold = 2
# First load pass should leave file C marked inactive.
self.assertLoaderYields(["A1", "A2", "B1", "B2", "B3", "C1"])
assertLoadersForPaths(["a", "b"])
self._WriteToFile("a", ["A4"], [4])
self._WriteToFile("b", ["B4"], [4])
self._WriteToFile("c", ["C4"], [4])
threshold = 3
# Second load pass should mark file A as inactive (due to newly
# increased threshold) and thus skip reading data from it.
self.assertLoaderYields(["B4"])
assertLoadersForPaths(["b"])
self._WriteToFile("b", ["B5", "B6"], [5, 6])
# Simulate a third pass in which the threshold increases while
# we're processing a file, so it's still active at the start of the
# load but should be marked inactive at the end.
load_generator = self._loader.Load()
self.assertEqual("B5", next(load_generator))
threshold = 7
self.assertEqual(["B6"], list(load_generator))
assertLoadersForPaths([])
# Confirm that all loaders are now inactive.
self._WriteToFile("b", ["B7"], [7])
self.assertLoaderYields([])
def testActiveFilter_staticFilterBehavior(self):
"""Tests behavior of a static active_filter."""
loader_registry = []
loader_factory = functools.partial(
_TimestampedByteLoader, registry=loader_registry
)
active_filter = lambda timestamp: timestamp >= 2
self._loader = directory_loader.DirectoryLoader(
self._directory, loader_factory, active_filter=active_filter
)
def assertLoadersForPaths(paths):
paths = [os.path.join(self._directory, path) for path in paths]
self.assertEqual(loader_registry, paths)
# a: normal-looking file.
# b: file without sufficiently active data (should be marked inactive).
# c: file with timestamps in reverse order (max computed correctly).
# d: empty file (should be considered active in absence of timestamps).
self._WriteToFile("a", ["A1", "A2"], [1, 2])
self._WriteToFile("b", ["B1"], [1])
self._WriteToFile("c", ["C2", "C1", "C0"], [2, 1, 0])
self._WriteToFile("d", [], [])
self.assertLoaderYields(["A1", "A2", "B1", "C2", "C1", "C0"])
assertLoadersForPaths(["a", "c", "d"])
self._WriteToFile("a", ["A3"], [3])
self._WriteToFile("b", ["B3"], [3])
self._WriteToFile("c", ["C0"], [0])
self._WriteToFile("d", ["D3"], [3])
self.assertLoaderYields(["A3", "C0", "D3"])
assertLoadersForPaths(["a", "c", "d"])
# Check that a 0 timestamp in file C on the most recent load doesn't
# override the max timestamp of 2 seen in the earlier load.
self._WriteToFile("c", ["C4"], [4])
self.assertLoaderYields(["C4"])
assertLoadersForPaths(["a", "c", "d"])
def testRaisesWithBadArguments(self):
with self.assertRaises(ValueError):
directory_loader.DirectoryLoader(None, lambda x: None)
with self.assertRaises(ValueError):
directory_loader.DirectoryLoader("dir", None)
def setUp(self):
# Put everything in a directory so it's easier to delete w/in tests.
self._directory = os.path.join(self.get_temp_dir(), "testdir")
os.mkdir(self._directory)
self._loader = directory_loader.DirectoryLoader(
self._directory, _TimestampedByteLoader)
def directory_loader_factory(path):
return directory_loader.DirectoryLoader(
path,
event_file_loader.TimestampedEventFileLoader,
path_filter=io_wrapper.IsTensorFlowEventsFile,
)
