def testInitialization(self): """Tests EventMultiplexer is created properly with its params.""" x = event_multiplexer.EventMultiplexer() self.assertEqual(x.Runs(), {}) x = event_multiplexer.EventMultiplexer({'run1': 'path1', 'run2': 'path2'}) self.assertItemsEqual(x.Runs(), ['run1', 'run2']) self.assertEqual(x.GetAccumulator('run1')._path, 'path1') self.assertEqual(x.GetAccumulator('run2')._path, 'path2')
def testInitialization(self): """Tests EventMultiplexer is created properly with its params.""" x = event_multiplexer.EventMultiplexer() self.assertEqual(x.Runs(), {}) x = event_multiplexer.EventMultiplexer({ "run1": "path1", "run2": "path2" }) self.assertItemsEqual(x.Runs(), ["run1", "run2"]) self.assertEqual(x.GetAccumulator("run1")._path, "path1") self.assertEqual(x.GetAccumulator("run2")._path, "path2")
def testGetOpsWithHealthPillsWhenHealthPillsAreNotAvailable(self): # The event accumulator lacks health pills for the run. x = event_multiplexer.EventMultiplexer({ 'run1': 'path1', 'run2': 'path2' }) self.assertItemsEqual([], x.GetOpsWithHealthPills('run1'))
def read_metrics_from_events_dir(events_dir, tags_to_ignore=None, use_run_name_prefix=False): """Collect the TensorBoard summary values for each metric. Args: events_dir (string): Path to location of TensorBoard summaries. tags_to_ignore (set[string]): Set of TensorBoard tag names to skip. use_run_name_prefix (bool): If True, prefix tag names with the name of the run that contains them (e.g. `train/` or `eval/`) Returns: raw_metrics (dict): Keys are TensorBoard tags and value is a list of MetricPoint for every data point for that Tensorboard tag. """ tags_to_ignore = tags_to_ignore or set() em = event_multiplexer.EventMultiplexer() em.AddRunsFromDirectory(events_dir) em.Reload() raw_metrics = collections.defaultdict(list) for run, tags in em.Runs().items(): # 'Old-style' runs have a simple format and store values directly. for tag in tags['scalars']: if use_run_name_prefix and run != '.': tag_final = '/'.join((run, tag)) else: tag_final = tag if tag_final in tags_to_ignore: continue raw_metrics[tag_final].extend([ MetricPoint(metric_value=x.value, wall_time=x.wall_time) for x in em.Scalars(run, tag) ]) # 'New-style' runs stores values inside of Tensor protos. for tag in tags['tensors']: if use_run_name_prefix and run != '.': tag_final = '/'.join((run, tag)) else: tag_final = tag if tag_final in tags_to_ignore: continue for t in em.Tensors(run, tag): tensor_dtype = tf.dtypes.as_dtype(t.tensor_proto.dtype) try: val = np.frombuffer(t.tensor_proto.tensor_content, tensor_dtype.as_numpy_dtype).tolist() assert len(val) == 1 # There should be 1 value per tensor. raw_metrics[tag_final].append( MetricPoint(metric_value=val[0], wall_time=t.wall_time)) except ValueError as e: logging.warning( 'Unable to parse tag: `{}` from tensor_content: {}. ' 'Error: {}. Consider adding this tag to tags_to_ignore ' 'in config.'.format(tag, t.tensor_proto.tensor_content, e)) return raw_metrics
def testAddRunMaintainsLoading(self): x = event_multiplexer.EventMultiplexer() x.Reload() x.AddRun('run1') x.AddRun('run2') self.assertTrue(x.GetAccumulator('run1').reload_called) self.assertTrue(x.GetAccumulator('run2').reload_called)
def _test(self, route, should_be_okay): temp_dir = tempfile.mkdtemp(prefix=self.get_temp_dir()) self.addCleanup(shutil.rmtree, temp_dir) multiplexer = event_multiplexer.EventMultiplexer( size_guidance=application.DEFAULT_SIZE_GUIDANCE, purge_orphaned_data=True) plugins = [ FakePlugin(None, plugin_name='foo', is_active_value=True, routes_mapping={ route: lambda environ, start_response: None }), ] if should_be_okay: application.TensorBoardWSGIApp(temp_dir, plugins, multiplexer, reload_interval=0) else: with six.assertRaisesRegex(self, ValueError, r'invalid route'): application.TensorBoardWSGIApp(temp_dir, plugins, multiplexer, reload_interval=0)
def standard_tensorboard_wsgi( logdir, purge_orphaned_data, reload_interval, plugins): """Construct a TensorBoardWSGIApp with standard plugins and multiplexer. Args: logdir: The path to the directory containing events files. purge_orphaned_data: Whether to purge orphaned data. reload_interval: The interval at which the backend reloads more data in seconds. plugins: A list of constructor functions for TBPlugin subclasses. Returns: The new TensorBoard WSGI application. """ multiplexer = event_multiplexer.EventMultiplexer( size_guidance=DEFAULT_SIZE_GUIDANCE, purge_orphaned_data=purge_orphaned_data) context = base_plugin.TBContext( assets_zip_provider=get_default_assets_zip_provider(), logdir=logdir, multiplexer=multiplexer) plugins = [constructor(context) for constructor in plugins] return TensorBoardWSGIApp(logdir, plugins, multiplexer, reload_interval)
def setUp(self): self.logdir = self.get_temp_dir() self._GenerateTestData(run_name='run1') self._multiplexer = event_multiplexer.EventMultiplexer( size_guidance=application.DEFAULT_SIZE_GUIDANCE, purge_orphaned_data=True) plugins = [ FakePlugin(None, plugin_name='foo', is_active_value=True, routes_mapping={}), FakePlugin(None, plugin_name='bar', is_active_value=False, routes_mapping={}), ] app = application.TensorBoardWSGIApp(self.logdir, plugins, self._multiplexer, reload_interval=0) try: self._server = serving.BaseWSGIServer('localhost', 0, app) # 0 to pick an unused port. except IOError: # BaseWSGIServer has a preference for IPv4. If that didn't work, try again # with an explicit IPv6 address. self._server = serving.BaseWSGIServer('::1', 0, app) self._server_thread = threading.Thread( target=self._server.serve_forever) self._server_thread.daemon = True self._server_thread.start() self._connection = http_client.HTTPConnection( 'localhost', self._server.server_address[1])
def standard_tensorboard_wsgi(logdir, purge_orphaned_data, reload_interval, plugins, db_uri="", assets_zip_provider=None): """Construct a TensorBoardWSGIApp with standard plugins and multiplexer. Args: logdir: The path to the directory containing events files. purge_orphaned_data: Whether to purge orphaned data. reload_interval: The interval at which the backend reloads more data in seconds. plugins: A list of constructor functions for TBPlugin subclasses. db_uri: A String containing the URI of the SQL database for persisting data, or empty for memory-only mode. assets_zip_provider: Delegates to TBContext or uses default if None. Returns: The new TensorBoard WSGI application. """ multiplexer = event_multiplexer.EventMultiplexer( size_guidance=DEFAULT_SIZE_GUIDANCE, purge_orphaned_data=purge_orphaned_data) db_module, db_connection_provider = get_database_info(db_uri) context = base_plugin.TBContext( db_module=db_module, db_connection_provider=db_connection_provider, logdir=logdir, multiplexer=multiplexer, assets_zip_provider=(assets_zip_provider or get_default_assets_zip_provider())) plugins = [constructor(context) for constructor in plugins] return TensorBoardWSGIApp(logdir, plugins, multiplexer, reload_interval)
def testAddRunsFromDirectoryThrowsException(self): x = event_multiplexer.EventMultiplexer() tmpdir = self.get_temp_dir() filepath = _AddEvents(tmpdir) with self.assertRaises(ValueError): x.AddRunsFromDirectory(filepath)
def testExceptions(self): logdir = '/fake/foo' multiplexer = event_multiplexer.EventMultiplexer() # Fails if there is an unnamed plugin with self.assertRaises(ValueError): # This plugin lacks a name. plugins = [ FakePlugin(None, plugin_name=None, is_active_value=True, routes_mapping={}), ] application.TensorBoardWSGIApp(logdir, plugins, multiplexer, 0) # Fails if there are two plugins with same name with self.assertRaises(ValueError): plugins = [ FakePlugin(None, plugin_name='foo', is_active_value=True, routes_mapping={}), FakePlugin(None, plugin_name='foo', is_active_value=True, routes_mapping={}), ] application.TensorBoardWSGIApp(logdir, plugins, multiplexer, 0)
def _init_multiplexer(self): dirs = [ os.path.join(self._events_dir, subdir) for subdir in self._subdirs ] run_path_map = dict([(self._RUN_NAME % i, d) for i, d in enumerate(dirs)]) return event_multiplexer.EventMultiplexer(run_path_map)
def setUp(self): self.log_dir = tempfile.mkdtemp() # We use numpy.random to generate images. We seed to avoid non-determinism # in this test. numpy.random.seed(42) # Create old-style image summaries for run "foo". tf.reset_default_graph() sess = tf.Session() placeholder = tf.placeholder(tf.uint8) tf.summary.image(name="baz", tensor=placeholder) merged_summary_op = tf.summary.merge_all() foo_directory = os.path.join(self.log_dir, "foo") writer = tf.summary.FileWriter(foo_directory) writer.add_graph(sess.graph) for step in xrange(2): writer.add_summary(sess.run(merged_summary_op, feed_dict={ placeholder: (numpy.random.rand(1, 16, 42, 3) * 255).astype(numpy.uint8) }), global_step=step) writer.close() # Create new-style image summaries for run bar. tf.reset_default_graph() sess = tf.Session() placeholder = tf.placeholder(tf.uint8) summary.op(name="quux", images=placeholder, description="how do you pronounce that, anyway?") merged_summary_op = tf.summary.merge_all() bar_directory = os.path.join(self.log_dir, "bar") writer = tf.summary.FileWriter(bar_directory) writer.add_graph(sess.graph) for step in xrange(2): writer.add_summary(sess.run(merged_summary_op, feed_dict={ placeholder: (numpy.random.rand(1, 6, 8, 3) * 255).astype(numpy.uint8) }), global_step=step) writer.close() # Start a server with the plugin. multiplexer = event_multiplexer.EventMultiplexer({ "foo": foo_directory, "bar": bar_directory, }) context = base_plugin.TBContext(logdir=self.log_dir, multiplexer=multiplexer) plugin = images_plugin.ImagesPlugin(context) wsgi_app = application.TensorBoardWSGIApp(self.log_dir, [plugin], multiplexer, reload_interval=0) self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse) self.routes = plugin.get_plugin_apps()
def testPluginIsActiveWhenTextRuns(self): """The plugin should be active when there are runs with text.""" multiplexer = event_multiplexer.EventMultiplexer() context = base_plugin.TBContext(logdir=None, multiplexer=multiplexer) plugin = text_plugin.TextPlugin(context) multiplexer.AddRunsFromDirectory(self.logdir) multiplexer.Reload() self.assertTrue(plugin.is_active())
def testReload(self): """EventAccumulators should Reload after EventMultiplexer call it.""" x = event_multiplexer.EventMultiplexer({'run1': 'path1', 'run2': 'path2'}) self.assertFalse(x.GetAccumulator('run1').reload_called) self.assertFalse(x.GetAccumulator('run2').reload_called) x.Reload() self.assertTrue(x.GetAccumulator('run1').reload_called) self.assertTrue(x.GetAccumulator('run2').reload_called)
def testScalars(self): """Tests Scalars function returns suitable values.""" x = event_multiplexer.EventMultiplexer({'run1': 'path1', 'run2': 'path2'}) run1_actual = x.Scalars('run1', 'sv1') run1_expected = ['path1/sv1'] self.assertEqual(run1_expected, run1_actual)
def setUp(self): self.logdir = self.get_temp_dir() self.generate_testdata() multiplexer = event_multiplexer.EventMultiplexer() multiplexer.AddRunsFromDirectory(self.logdir) multiplexer.Reload() context = base_plugin.TBContext(logdir=self.logdir, multiplexer=multiplexer) self.plugin = text_plugin.TextPlugin(context)
def testExceptions(self): """KeyError should be raised when accessing non-existing keys.""" x = event_multiplexer.EventMultiplexer({ 'run1': 'path1', 'run2': 'path2' }) with self.assertRaises(KeyError): x.Scalars('sv1', 'xxx')
def testExceptions(self): """KeyError should be raised when accessing non-existing keys.""" x = event_multiplexer.EventMultiplexer({ "run1": "path1", "run2": "path2" }) with self.assertRaises(KeyError): x.Scalars("sv1", "xxx")
def testRunNamesRespected(self): """Tests two EventAccumulators inserted/accessed in EventMultiplexer.""" x = event_multiplexer.EventMultiplexer({ 'run1': 'path1', 'run2': 'path2' }) self.assertItemsEqual(sorted(x.Runs().keys()), ['run1', 'run2']) self.assertEqual(x.GetAccumulator('run1')._path, 'path1') self.assertEqual(x.GetAccumulator('run2')._path, 'path2')
def set_up_with_runs(self, run_names): self.logdir = self.get_temp_dir() for run_name in run_names: self.generate_run(run_name) multiplexer = event_multiplexer.EventMultiplexer() multiplexer.AddRunsFromDirectory(self.logdir) multiplexer.Reload() context = base_plugin.TBContext(logdir=self.logdir, multiplexer=multiplexer) self.plugin = scalars_plugin.ScalarsPlugin(context)
def testRunNamesRespected(self): """Tests two EventAccumulators inserted/accessed in EventMultiplexer.""" x = event_multiplexer.EventMultiplexer({ "run1": "path1", "run2": "path2" }) self.assertItemsEqual(sorted(x.Runs().keys()), ["run1", "run2"]) self.assertEqual(x.GetAccumulator("run1")._path, "path1") self.assertEqual(x.GetAccumulator("run2")._path, "path2")
def testPluginIsActiveWhenRunsButNoText(self): """The plugin should be inactive when there are runs but none has text.""" multiplexer = event_multiplexer.EventMultiplexer() context = base_plugin.TBContext(logdir=None, multiplexer=multiplexer) plugin = text_plugin.TextPlugin(context) logdir = os.path.join(self.get_temp_dir(), 'runs_with_no_text') self.generate_testdata(include_text=False, logdir=logdir) multiplexer.AddRunsFromDirectory(logdir) multiplexer.Reload() self.assertFalse(plugin.is_active())
def testReload(self): """EventAccumulators should Reload after EventMultiplexer call it.""" x = event_multiplexer.EventMultiplexer({ "run1": "path1", "run2": "path2" }) self.assertFalse(x.GetAccumulator("run1").reload_called) self.assertFalse(x.GetAccumulator("run2").reload_called) x.Reload() self.assertTrue(x.GetAccumulator("run1").reload_called) self.assertTrue(x.GetAccumulator("run2").reload_called)
def testGetOpsWithHealthPillsWhenHealthPillsAreAvailable(self): # The event accumulator has health pills for the run. self.stubs.Set( event_accumulator, 'EventAccumulator', functools.partial(_GetFakeAccumulator, health_pill_mapping={'Add': ['hp1', 'hp2']})) x = event_multiplexer.EventMultiplexer({ 'run1': 'path1', 'run2': 'path2' }) self.assertItemsEqual(['Add'], x.GetOpsWithHealthPills('run1'))
def testScalars(self): """Tests Scalars function returns suitable values.""" x = event_multiplexer.EventMultiplexer({ "run1": "path1", "run2": "path2" }) run1_actual = x.Scalars("run1", "sv1") run1_expected = ["path1/sv1"] self.assertEqual(run1_expected, run1_actual)
def _SetupWSGIApp(self): multiplexer = event_multiplexer.EventMultiplexer( size_guidance=application.DEFAULT_SIZE_GUIDANCE, purge_orphaned_data=True) context = base_plugin.TBContext(logdir=self.log_dir, multiplexer=multiplexer) self.plugin = projector_plugin.ProjectorPlugin(context) wsgi_app = application.TensorBoardWSGIApp(self.log_dir, [self.plugin], multiplexer, reload_interval=0) self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse)
def setUp(self): self.log_dir = tempfile.mkdtemp() # We use numpy.random to generate audio. We seed to avoid non-determinism # in this test. numpy.random.seed(42) # Create audio summaries for run foo. tf.reset_default_graph() sess = tf.Session() placeholder = tf.placeholder(tf.float32) tf.summary.audio(name="baz", tensor=placeholder, sample_rate=44100) merged_summary_op = tf.summary.merge_all() foo_directory = os.path.join(self.log_dir, "foo") writer = tf.summary.FileWriter(foo_directory) writer.add_graph(sess.graph) for step in xrange(2): # The floats (sample data) range from -1 to 1. writer.add_summary(sess.run( merged_summary_op, feed_dict={placeholder: numpy.random.rand(42, 22050) * 2 - 1}), global_step=step) writer.close() # Create audio summaries for run bar. tf.reset_default_graph() sess = tf.Session() placeholder = tf.placeholder(tf.float32) tf.summary.audio(name="quux", tensor=placeholder, sample_rate=44100) merged_summary_op = tf.summary.merge_all() bar_directory = os.path.join(self.log_dir, "bar") writer = tf.summary.FileWriter(bar_directory) writer.add_graph(sess.graph) for step in xrange(2): # The floats (sample data) range from -1 to 1. writer.add_summary(sess.run( merged_summary_op, feed_dict={placeholder: numpy.random.rand(42, 11025) * 2 - 1}), global_step=step) writer.close() # Start a server with the plugin. multiplexer = event_multiplexer.EventMultiplexer({ "foo": foo_directory, "bar": bar_directory, }) context = base_plugin.TBContext(logdir=self.log_dir, multiplexer=multiplexer) self.plugin = audio_plugin.AudioPlugin(context) wsgi_app = application.TensorBoardWSGIApp(self.log_dir, [self.plugin], multiplexer, reload_interval=0) self.server = werkzeug_test.Client(wsgi_app, wrappers.BaseResponse)
def __init__(self, logdir): """Loads the events of all runs under the given directory. The directory should contain runs and event files in the same format used by TensorBoard. Args: logdir: The string directory name containing event files. """ self.logdir = logdir self.event_mux = emux.EventMultiplexer().AddRunsFromDirectory(self.logdir) self.event_mux.Reload()
def set_up_with_runs(self, with_graph=True, without_graph=True): self.logdir = self.get_temp_dir() if with_graph: self.generate_run(self._RUN_WITH_GRAPH, include_graph=True) if without_graph: self.generate_run(self._RUN_WITHOUT_GRAPH, include_graph=False) multiplexer = event_multiplexer.EventMultiplexer() multiplexer.AddRunsFromDirectory(self.logdir) multiplexer.Reload() context = base_plugin.TBContext(logdir=self.logdir, multiplexer=multiplexer) self.plugin = graphs_plugin.GraphsPlugin(context)