def _GeneratorFromPath(path,
event_file_active_filter=None,
detect_file_replacement=None):
"""Create an event generator for file or directory at given path string."""
if not path:
raise ValueError("path must be a valid string")
if io_wrapper.IsSummaryEventsFile(path):
return event_file_loader.EventFileLoader(path, detect_file_replacement)
elif event_file_active_filter:
loader_factory = (
lambda path: event_file_loader.TimestampedEventFileLoader(
path, detect_file_replacement))
return directory_loader.DirectoryLoader(
path,
loader_factory,
path_filter=io_wrapper.IsSummaryEventsFile,
active_filter=event_file_active_filter,
)
else:
loader_factory = lambda path: event_file_loader.EventFileLoader(
path, detect_file_replacement)
return directory_watcher.DirectoryWatcher(
path,
loader_factory,
io_wrapper.IsSummaryEventsFile,
)
def setUp(self):
# Put everything in a directory so it's easier to delete.
self._directory = os.path.join(self.get_temp_dir(), 'monitor_dir')
os.mkdir(self._directory)
self._watcher = directory_watcher.DirectoryWatcher(
self._directory, _ByteLoader)
self.stubs = tf.compat.v1.test.StubOutForTesting()
def _GeneratorFromPath(path):
"""Create an event generator for file or directory at given path string."""
if not path:
raise ValueError('path must be a valid string')
if IsTensorFlowEventsFile(path):
return event_file_loader.EventFileLoader(path)
else:
return directory_watcher.DirectoryWatcher(
path, event_file_loader.EventFileLoader, IsTensorFlowEventsFile)
def testPathFilter(self):
self._watcher = directory_watcher.DirectoryWatcher(
self._directory, _ByteLoader,
lambda path: 'do_not_watch_me' not in path)
self._WriteToFile('a', 'a,1,1\n')
self._WriteToFile('do_not_watch_me', 'b,2,2\n')
self._WriteToFile('c', 'c,3,3\n')
self.assertWatcherYields(['a', 'c'])
self.assertFalse(self._watcher.OutOfOrderWritesDetected())
def _GeneratorFromPath(path):
"""Create an event generator for file or directory at given path string."""
if not path:
raise ValueError("path must be a valid string")
if io_wrapper.IsSummaryEventsFile(path):
return event_file_loader.LegacyEventFileLoader(path)
else:
return directory_watcher.DirectoryWatcher(
path,
event_file_loader.LegacyEventFileLoader,
io_wrapper.IsSummaryEventsFile,
)
def testPathFilter(self):
self._watcher = directory_watcher.DirectoryWatcher(
self._directory,
_ByteLoader,
lambda path: "do_not_watch_me" not in path,
)
self._WriteToFile("a", "a")
self._WriteToFile("do_not_watch_me", "b")
self._WriteToFile("c", "c")
self.assertWatcherYields(["a", "c"])
self.assertFalse(self._watcher.OutOfOrderWritesDetected())
def __init__(self, logdir, queue, namespace=None, save=True):
"""Uses tensorboard to watch a directory for tfevents files
and put them on a queue"""
self.namespace = namespace
self.queue = queue
self.logdir = logdir
# TODO: prepend the namespace here?
self._generator = directory_watcher.DirectoryWatcher(
logdir, loader(save, namespace), IsNewTensorFlowEventsFile)
self._first_event_timestamp = None
self._shutdown = False
self._thread = threading.Thread(target=self._thread_body)
self._thread.daemon = True
def __init__(self, subdir, experiment_name, run_name):
"""Constructs a `_RunLoader`.
Args:
subdir: string, filesystem path of the run directory
experiment_name: string, name of the run's experiment
run_name: string, name of the run
"""
self._subdir = subdir
self._experiment_name = experiment_name
self._run_name = run_name
self._directory_watcher = directory_watcher.DirectoryWatcher(
subdir, event_file_loader.RawEventFileLoader,
io_wrapper.IsTensorFlowEventsFile)
def parse_and_return_summary_value(self, path):
"""Parse the event file in the given path and return the
only summary value."""
event_value_list = []
event_file_generator = directory_watcher.DirectoryWatcher(
path, event_file_loader.EventFileLoader).Load()
event_values = itertools.chain.from_iterable(
map(_process_event, event_file_generator))
for value_dict in event_values:
event_value_list.append(value_dict)
self.assertLen(event_value_list, 1)
self.assertEqual(event_value_list[0]['step'], 1)
self.assertGreater(event_value_list[0]['wall_time'], 0.0)
return event_value_list[0]['value']
def _GeneratorFromPath(path, event_file_active_filter=None):
"""Create an event generator for file or directory at given path string."""
if not path:
raise ValueError('path must be a valid string')
if io_wrapper.IsTensorFlowEventsFile(path):
return event_file_loader.EventFileLoader(path)
elif event_file_active_filter:
return directory_loader.DirectoryLoader(
path,
event_file_loader.TimestampedEventFileLoader,
path_filter=io_wrapper.IsTensorFlowEventsFile,
active_filter=event_file_active_filter)
else:
return directory_watcher.DirectoryWatcher(
path, event_file_loader.EventFileLoader,
io_wrapper.IsTensorFlowEventsFile)
def __init__(self, db_path, subdir, experiment_name, run_name):
"""Constructs a `_RunImporter` and initializes a DB run.
Args:
db_path: string, filesystem path of the DB file to open
subdir: string, filesystem path of the run directory
experiment_name: string, name of the run's experiment
run_name: string, name of the run
"""
self._directory_generator = directory_watcher.DirectoryWatcher(
subdir, event_file_loader.RawEventFileLoader,
io_wrapper.IsTensorFlowEventsFile)
with tf.Graph().as_default():
self._placeholder = tf.placeholder(shape=[], dtype=tf.string)
writer = tf.contrib.summary.create_db_writer(
db_path, experiment_name=experiment_name, run_name=run_name)
with writer.as_default():
# TODO(nickfelt): running import_event() one record at a time is very
# slow; we should add an op that accepts a vector of records.
self._import_op = tf.contrib.summary.import_event(
self._placeholder)
self._session = tf.Session()
self._session.run(writer.init())
def get_tensorboard_log_watcher_from_path(path: Text):
"""Create an event generator for file or directory at given path string.
This method creates an event generator using tensorboard directory_watcher.
The generator.load() method will return event logs as they become available.
The generator does not repeat events.
Args:
path: Text representing a directory, file, or Google Cloud Storage
(GCS) for tensorboard logs.
Returns:
A tensorboard directory_watcher event generator.
Raises:
ValueError: if path is not defined.
"""
if not path:
raise ValueError("path must be a valid string")
if io_wrapper.IsSummaryEventsFile(path):
return event_file_loader.EventFileLoader(path)
return directory_watcher.DirectoryWatcher(
path,
event_file_loader.EventFileLoader,
io_wrapper.IsSummaryEventsFile,
)
def _make_events_generator(path):
"""Makes a generator yielding TensorBoard events from files in `path`."""
return directory_watcher.DirectoryWatcher(
path, event_file_loader.EventFileLoader,
io_wrapper.IsSummaryEventsFile).Load()
def testRaisesWithBadArguments(self):
with self.assertRaises(ValueError):
directory_watcher.DirectoryWatcher(None, lambda x: None)
with self.assertRaises(ValueError):
directory_watcher.DirectoryWatcher('dir', None)
