def _GeneratorFromPath(path, event_file_active_filter=None, detect_file_replacement=None): """Create an event generator for file or directory at given path string.""" if not path: raise ValueError("path must be a valid string") if io_wrapper.IsSummaryEventsFile(path): return event_file_loader.EventFileLoader(path, detect_file_replacement) elif event_file_active_filter: loader_factory = ( lambda path: event_file_loader.TimestampedEventFileLoader( path, detect_file_replacement)) return directory_loader.DirectoryLoader( path, loader_factory, path_filter=io_wrapper.IsSummaryEventsFile, active_filter=event_file_active_filter, ) else: loader_factory = lambda path: event_file_loader.EventFileLoader( path, detect_file_replacement) return directory_watcher.DirectoryWatcher( path, loader_factory, io_wrapper.IsSummaryEventsFile, )
def setUp(self): # Put everything in a directory so it's easier to delete. self._directory = os.path.join(self.get_temp_dir(), 'monitor_dir') os.mkdir(self._directory) self._watcher = directory_watcher.DirectoryWatcher( self._directory, _ByteLoader) self.stubs = tf.compat.v1.test.StubOutForTesting()
def _GeneratorFromPath(path): """Create an event generator for file or directory at given path string.""" if not path: raise ValueError('path must be a valid string') if IsTensorFlowEventsFile(path): return event_file_loader.EventFileLoader(path) else: return directory_watcher.DirectoryWatcher( path, event_file_loader.EventFileLoader, IsTensorFlowEventsFile)
def testPathFilter(self): self._watcher = directory_watcher.DirectoryWatcher( self._directory, _ByteLoader, lambda path: 'do_not_watch_me' not in path) self._WriteToFile('a', 'a,1,1\n') self._WriteToFile('do_not_watch_me', 'b,2,2\n') self._WriteToFile('c', 'c,3,3\n') self.assertWatcherYields(['a', 'c']) self.assertFalse(self._watcher.OutOfOrderWritesDetected())
def _GeneratorFromPath(path): """Create an event generator for file or directory at given path string.""" if not path: raise ValueError("path must be a valid string") if io_wrapper.IsSummaryEventsFile(path): return event_file_loader.LegacyEventFileLoader(path) else: return directory_watcher.DirectoryWatcher( path, event_file_loader.LegacyEventFileLoader, io_wrapper.IsSummaryEventsFile, )
def testPathFilter(self): self._watcher = directory_watcher.DirectoryWatcher( self._directory, _ByteLoader, lambda path: "do_not_watch_me" not in path, ) self._WriteToFile("a", "a") self._WriteToFile("do_not_watch_me", "b") self._WriteToFile("c", "c") self.assertWatcherYields(["a", "c"]) self.assertFalse(self._watcher.OutOfOrderWritesDetected())
def __init__(self, logdir, queue, namespace=None, save=True): """Uses tensorboard to watch a directory for tfevents files and put them on a queue""" self.namespace = namespace self.queue = queue self.logdir = logdir # TODO: prepend the namespace here? self._generator = directory_watcher.DirectoryWatcher( logdir, loader(save, namespace), IsNewTensorFlowEventsFile) self._first_event_timestamp = None self._shutdown = False self._thread = threading.Thread(target=self._thread_body) self._thread.daemon = True
def __init__(self, subdir, experiment_name, run_name): """Constructs a `_RunLoader`. Args: subdir: string, filesystem path of the run directory experiment_name: string, name of the run's experiment run_name: string, name of the run """ self._subdir = subdir self._experiment_name = experiment_name self._run_name = run_name self._directory_watcher = directory_watcher.DirectoryWatcher( subdir, event_file_loader.RawEventFileLoader, io_wrapper.IsTensorFlowEventsFile)
def parse_and_return_summary_value(self, path): """Parse the event file in the given path and return the only summary value.""" event_value_list = [] event_file_generator = directory_watcher.DirectoryWatcher( path, event_file_loader.EventFileLoader).Load() event_values = itertools.chain.from_iterable( map(_process_event, event_file_generator)) for value_dict in event_values: event_value_list.append(value_dict) self.assertLen(event_value_list, 1) self.assertEqual(event_value_list[0]['step'], 1) self.assertGreater(event_value_list[0]['wall_time'], 0.0) return event_value_list[0]['value']
def _GeneratorFromPath(path, event_file_active_filter=None): """Create an event generator for file or directory at given path string.""" if not path: raise ValueError('path must be a valid string') if io_wrapper.IsTensorFlowEventsFile(path): return event_file_loader.EventFileLoader(path) elif event_file_active_filter: return directory_loader.DirectoryLoader( path, event_file_loader.TimestampedEventFileLoader, path_filter=io_wrapper.IsTensorFlowEventsFile, active_filter=event_file_active_filter) else: return directory_watcher.DirectoryWatcher( path, event_file_loader.EventFileLoader, io_wrapper.IsTensorFlowEventsFile)
def __init__(self, db_path, subdir, experiment_name, run_name): """Constructs a `_RunImporter` and initializes a DB run. Args: db_path: string, filesystem path of the DB file to open subdir: string, filesystem path of the run directory experiment_name: string, name of the run's experiment run_name: string, name of the run """ self._directory_generator = directory_watcher.DirectoryWatcher( subdir, event_file_loader.RawEventFileLoader, io_wrapper.IsTensorFlowEventsFile) with tf.Graph().as_default(): self._placeholder = tf.placeholder(shape=[], dtype=tf.string) writer = tf.contrib.summary.create_db_writer( db_path, experiment_name=experiment_name, run_name=run_name) with writer.as_default(): # TODO(nickfelt): running import_event() one record at a time is very # slow; we should add an op that accepts a vector of records. self._import_op = tf.contrib.summary.import_event( self._placeholder) self._session = tf.Session() self._session.run(writer.init())
def get_tensorboard_log_watcher_from_path(path: Text): """Create an event generator for file or directory at given path string. This method creates an event generator using tensorboard directory_watcher. The generator.load() method will return event logs as they become available. The generator does not repeat events. Args: path: Text representing a directory, file, or Google Cloud Storage (GCS) for tensorboard logs. Returns: A tensorboard directory_watcher event generator. Raises: ValueError: if path is not defined. """ if not path: raise ValueError("path must be a valid string") if io_wrapper.IsSummaryEventsFile(path): return event_file_loader.EventFileLoader(path) return directory_watcher.DirectoryWatcher( path, event_file_loader.EventFileLoader, io_wrapper.IsSummaryEventsFile, )
def _make_events_generator(path): """Makes a generator yielding TensorBoard events from files in `path`.""" return directory_watcher.DirectoryWatcher( path, event_file_loader.EventFileLoader, io_wrapper.IsSummaryEventsFile).Load()
def testRaisesWithBadArguments(self): with self.assertRaises(ValueError): directory_watcher.DirectoryWatcher(None, lambda x: None) with self.assertRaises(ValueError): directory_watcher.DirectoryWatcher('dir', None)