def annotations_impl(self, ctx, experiment): mapping = self._data_provider.list_tensors( ctx, experiment_id=experiment, plugin_name=self.plugin_name, run_tag_filter=provider.RunTagFilter( tags=[metadata.ANNOTATIONS_TAG]), ) result = {run: {} for run in mapping} for (run, _) in six.iteritems(mapping): all_annotations = self._data_provider.read_tensors( ctx, experiment_id=experiment, plugin_name=self.plugin_name, run_tag_filter=provider.RunTagFilter( runs=[run], tags=[metadata.ANNOTATIONS_TAG]), downsample=self._downsample_to, ) annotations = all_annotations.get(run, {}).get(metadata.ANNOTATIONS_TAG, {}) event_data = [ annotation.decode("utf-8") for annotation in annotations[0].numpy ] result[run] = event_data return result
def _build_session_groups(self): """Returns a list of SessionGroups protobuffers from the summary data.""" # Algorithm: We keep a dict 'groups_by_name' mapping a SessionGroup name # (str) to a SessionGroup protobuffer. We traverse the runs associated with # the plugin--each representing a single session. We form a Session # protobuffer from each run and add it to the relevant SessionGroup object # in the 'groups_by_name' dict. We create the SessionGroup object, if this # is the first session of that group we encounter. groups_by_name = {} run_to_tag_to_content = self._context.hparams_metadata( self._experiment_id, run_tag_filter=provider.RunTagFilter(tags=[ metadata.SESSION_START_INFO_TAG, metadata.SESSION_END_INFO_TAG, ]), ) # The TensorBoard runs with session start info are the # "sessions", which are not necessarily the runs that actually # contain metrics (may be in subdirectories). session_names = [ run for (run, tags) in run_to_tag_to_content.items() if metadata.SESSION_START_INFO_TAG in tags ] metric_runs = set() metric_tags = set() for session_name in session_names: for metric in self._experiment.metric_infos: metric_name = metric.name (run, tag) = metrics.run_tag_from_session_and_metric( session_name, metric_name) metric_runs.add(run) metric_tags.add(tag) all_metric_evals = self._context.read_last_scalars( self._experiment_id, run_tag_filter=provider.RunTagFilter(runs=metric_runs, tags=metric_tags), ) for (session_name, tag_to_content) in run_to_tag_to_content.items(): if metadata.SESSION_START_INFO_TAG not in tag_to_content: continue start_info = metadata.parse_session_start_info_plugin_data( tag_to_content[metadata.SESSION_START_INFO_TAG]) end_info = None if metadata.SESSION_END_INFO_TAG in tag_to_content: end_info = metadata.parse_session_end_info_plugin_data( tag_to_content[metadata.SESSION_END_INFO_TAG]) session = self._build_session(session_name, start_info, end_info, all_metric_evals) if session.status in self._request.allowed_statuses: self._add_session(session, start_info, groups_by_name) # Compute the session group's aggregated metrics for each group. groups = groups_by_name.values() for group in groups: # We sort the sessions in a group so that the order is deterministic. group.sessions.sort(key=operator.attrgetter("name")) self._aggregate_metrics(group) return groups
def embeddings_impl(self, ctx, experiment): mapping = self._data_provider.list_tensors( ctx, experiment_id=experiment, plugin_name=self.plugin_name, run_tag_filter=provider.RunTagFilter( tags=[metadata.EMBEDDINGS_TAG] ), ) result = {run: {} for run in mapping} for (run, _) in mapping.items(): all_embeddings = self._data_provider.read_tensors( ctx, experiment_id=experiment, plugin_name=self.plugin_name, run_tag_filter=provider.RunTagFilter( runs=[run], tags=[metadata.EMBEDDINGS_TAG] ), downsample=self._downsample_to, ) embeddings = all_embeddings.get(run, {}).get( metadata.EMBEDDINGS_TAG, {} ) event_data = embeddings[0].numpy.tolist() result[run] = event_data return result
def test_scalars(self): listing = self.with_unpfx.list_scalars(_ctx(), experiment_id="foo:123", plugin_name="scalars") self.assertEqual( listing, self.foo_provider.list_scalars(_ctx(), experiment_id="123", plugin_name="scalars"), ) reading = self.with_unpfx.read_scalars( _ctx(), experiment_id="foo:123", plugin_name="scalars", downsample=1000, run_tag_filter=provider.RunTagFilter(["123/train"], ["loss.scalars"]), ) expected_reading = self.foo_provider.read_scalars( _ctx(), experiment_id="123", plugin_name="scalars", downsample=1000, run_tag_filter=provider.RunTagFilter(["123/train"], ["loss.scalars"]), ) self.assertNotEmpty(expected_reading) self.assertEqual(reading, expected_reading)
def test_list_scalars_filters(self): provider = self.create_provider() result = provider.list_scalars( experiment_id="unused", plugin_name=scalar_metadata.PLUGIN_NAME, run_tag_filter=base_provider.RunTagFilter(["waves"], ["square"]), ) self.assertItemsEqual(result.keys(), ["waves"]) self.assertItemsEqual(result["waves"].keys(), ["square"]) result = provider.list_scalars( experiment_id="unused", plugin_name=scalar_metadata.PLUGIN_NAME, run_tag_filter=base_provider.RunTagFilter( tags=["square", "quartic"]), ) self.assertItemsEqual(result.keys(), ["polynomials", "waves"]) self.assertItemsEqual(result["polynomials"].keys(), ["square"]) self.assertItemsEqual(result["waves"].keys(), ["square"]) result = provider.list_scalars( experiment_id="unused", plugin_name=scalar_metadata.PLUGIN_NAME, run_tag_filter=base_provider.RunTagFilter(runs=["waves", "hugs"]), ) self.assertItemsEqual(result.keys(), ["waves"]) self.assertItemsEqual(result["waves"].keys(), ["sine", "square"]) result = provider.list_scalars( experiment_id="unused", plugin_name=scalar_metadata.PLUGIN_NAME, run_tag_filter=base_provider.RunTagFilter(["un"], ["likely"]), ) self.assertEqual(result, {})
def test_validates_runs_tags(self): # Accidentally passed scalar strings with six.assertRaisesRegex(self, TypeError, "runs:.*got.*str.*myrun"): provider.RunTagFilter(runs="myrun") with six.assertRaisesRegex(self, TypeError, "tags:.*got.*str.*mytag"): provider.RunTagFilter(tags="mytag") # Passed collections with non-string elements with six.assertRaisesRegex(self, TypeError, "runs:.*got item of type.*NoneType.*None"): provider.RunTagFilter(runs=[None]) with six.assertRaisesRegex(self, TypeError, "tags:.*got item of type.*int.*3"): provider.RunTagFilter(tags=["one", "two", 3])
def histograms_impl(self, ctx, tag, run, experiment, downsample_to=None): """Result of the form `(body, mime_type)`. At most `downsample_to` events will be returned. If this value is `None`, then default downsampling will be performed. Raises: tensorboard.errors.PublicError: On invalid request. """ sample_count = ( downsample_to if downsample_to is not None else self._downsample_to ) all_histograms = self._data_provider.read_tensors( ctx, experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME, downsample=sample_count, run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), ) histograms = all_histograms.get(run, {}).get(tag, None) if histograms is None: raise errors.NotFoundError( "No histogram tag %r for run %r" % (tag, run) ) events = [(e.wall_time, e.step, e.numpy.tolist()) for e in histograms] return (events, "application/json")
def scalars_impl(self, ctx, experiment, tag, run): """Returns scalar data for the specified tag and run. For details on how to use tags and runs, see https://github.com/tensorflow/tensorboard#tags-giving-names-to-data Args: tag: string run: string Returns: A list of ScalarEvents - tuples containing 3 numbers describing entries in the data series. Raises: NotFoundError if there are no scalars data for provided `run` and `tag`. """ all_scalars = self._data_provider.read_scalars( ctx, experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME, downsample=5000, run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), ) scalars = all_scalars.get(run, {}).get(tag, None) if scalars is None: raise errors.NotFoundError("No scalar data for run=%r, tag=%r" % (run, tag)) return [(x.wall_time, x.step, x.value) for x in scalars]
def test_list_tensors(self): res = data_provider_pb2.ListTensorsResponse() run1 = res.runs.add(run_name="val") tag11 = run1.tags.add(tag_name="weights") tag11.metadata.max_step = 7 tag11.metadata.max_wall_time = 7.77 tag11.metadata.summary_metadata.plugin_data.content = b"magic" tag11.metadata.summary_metadata.summary_description = "hey" tag12 = run1.tags.add(tag_name="other") tag12.metadata.max_step = 8 tag12.metadata.max_wall_time = 8.88 run2 = res.runs.add(run_name="test") tag21 = run2.tags.add(tag_name="weights") tag21.metadata.max_step = 9 tag21.metadata.max_wall_time = 9.99 self.stub.ListTensors.return_value = res actual = self.provider.list_tensors( self.ctx, experiment_id="123", plugin_name="histograms", run_tag_filter=provider.RunTagFilter(tags=["weights", "other"]), ) expected = { "val": { "weights": provider.TensorTimeSeries( max_step=7, max_wall_time=7.77, plugin_content=b"magic", description="hey", display_name="", ), "other": provider.TensorTimeSeries( max_step=8, max_wall_time=8.88, plugin_content=b"", description="", display_name="", ), }, "test": { "weights": provider.TensorTimeSeries( max_step=9, max_wall_time=9.99, plugin_content=b"", description="", display_name="", ), }, } self.assertEqual(actual, expected) req = data_provider_pb2.ListTensorsRequest() req.experiment_id = "123" req.plugin_filter.plugin_name = "histograms" req.run_tag_filter.tags.names.extend(["other", "weights"]) # sorted self.stub.ListTensors.assert_called_once_with(req)
def test_read_tensors(self): multiplexer = self.create_multiplexer() provider = data_provider.MultiplexerDataProvider( multiplexer, self.logdir) run_tag_filter = base_provider.RunTagFilter( runs=["lebesgue"], tags=["uniform", "bimodal"], ) result = provider.read_tensors( experiment_id="unused", plugin_name=histogram_metadata.PLUGIN_NAME, run_tag_filter=run_tag_filter, downsample=None, # not yet implemented ) self.assertItemsEqual(result.keys(), ["lebesgue"]) self.assertItemsEqual(result["lebesgue"].keys(), ["uniform", "bimodal"]) for run in result: for tag in result[run]: tensor_events = multiplexer.Tensors(run, tag) self.assertLen(result[run][tag], len(tensor_events)) for (datum, event) in zip(result[run][tag], tensor_events): self.assertEqual(datum.step, event.step) self.assertEqual(datum.wall_time, event.wall_time) np.testing.assert_equal( datum.numpy, tensor_util.make_ndarray(event.tensor_proto), )
def pr_curves_impl(self, ctx, experiment, runs, tag): """Creates the JSON object for the PR curves response for a run-tag combo. Arguments: runs: A list of runs to fetch the curves for. tag: The tag to fetch the curves for. Raises: ValueError: If no PR curves could be fetched for a run and tag. Returns: The JSON object for the PR curves route response. """ response_mapping = {} rtf = provider.RunTagFilter(runs, [tag]) read_result = self._data_provider.read_tensors( ctx, experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME, run_tag_filter=rtf, downsample=self._downsample_to, ) for run in runs: data = read_result.get(run, {}).get(tag) if data is None: raise ValueError( "No PR curves could be found for run %r and tag %r" % (run, tag)) response_mapping[run] = [self._process_datum(d) for d in data] return response_mapping
def histograms_impl(self, ctx, tag, run, experiment, downsample_to=None): """Result of the form `(body, mime_type)`. At most `downsample_to` events will be returned. If this value is `None`, then default downsampling will be performed. Raises: tensorboard.errors.PublicError: On invalid request. """ sample_count = ( downsample_to if downsample_to is not None else self._downsample_to ) all_histograms = self._data_provider.read_tensors( ctx, experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME, downsample=sample_count, run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), ) histograms = all_histograms.get(run, {}).get(tag, None) if histograms is None: raise errors.NotFoundError( "No histogram tag %r for run %r" % (tag, run) ) # Downsample again, even though the data provider is supposed to, # because the multiplexer provider currently doesn't. (For # well-behaved data providers, this is a no-op.) if downsample_to is not None: rng = random.Random(0) histograms = _downsample(rng, histograms, downsample_to) events = [(e.wall_time, e.step, e.numpy.tolist()) for e in histograms] return (events, "application/json")
def graph_impl( self, run, tag, is_conceptual, experiment=None, limit_attr_size=None, large_attrs_key=None, ): """Result of the form `(body, mime_type)`, or `None` if no graph exists.""" if self._data_provider: graph_blob_sequences = self._data_provider.read_blob_sequences( experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME, run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), ) blob_datum_list = graph_blob_sequences.get(run, {}).get(tag, ()) try: blob_ref = blob_datum_list[0].values[0] except IndexError: return None # Always use the blob_key approach for now, even if there is a direct url. graph_raw = self._data_provider.read_blob(blob_ref.blob_key) # This method ultimately returns pbtxt, but we have to deserialize and # later reserialize this anyway, because a) this way we accept binary # protobufs too, and b) below we run `prepare_graph_for_ui` on the graph. graph = graph_pb2.GraphDef.FromString(graph_raw) elif is_conceptual: tensor_events = self._multiplexer.Tensors(run, tag) # Take the first event if there are multiple events written from different # steps. keras_model_config = json.loads( tensor_events[0].tensor_proto.string_val[0] ) graph = keras_util.keras_model_to_graph_def(keras_model_config) elif tag: tensor_events = self._multiplexer.Tensors(run, tag) # Take the first event if there are multiple events written from different # steps. run_metadata = config_pb2.RunMetadata.FromString( tensor_events[0].tensor_proto.string_val[0] ) graph = graph_pb2.GraphDef() for func_graph in run_metadata.function_graphs: graph_util.combine_graph_defs( graph, func_graph.pre_optimization_graph ) else: graph = self._multiplexer.Graph(run) # This next line might raise a ValueError if the limit parameters # are invalid (size is negative, size present but key absent, etc.). process_graph.prepare_graph_for_ui( graph, limit_attr_size, large_attrs_key ) return (str(graph), "text/x-protobuf") # pbtxt
def _image_response_for_run(self, ctx, experiment, run, tag, sample): """Builds a JSON-serializable object with information about images. Args: run: The name of the run. tag: The name of the tag the images all belong to. sample: The zero-indexed sample of the image for which to retrieve information. For instance, setting `sample` to `2` will fetch information about only the third image of each batch. Steps with fewer than three images will be omitted from the results. Returns: A list of dictionaries containing the wall time, step, and URL for each image. Raises: KeyError, NotFoundError: If no image data exists for the given parameters. """ all_images = self._data_provider.read_blob_sequences( ctx, experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME, downsample=self._downsample_to, run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), ) images = all_images.get(run, {}).get(tag, None) if images is None: raise errors.NotFoundError("No image data for run=%r, tag=%r" % (run, tag)) return [{ "wall_time": datum.wall_time, "step": datum.step, "query": self._data_provider_query(datum.values[sample + 2]), } for datum in images if len(datum.values) - 2 > sample]
def read_blob_sequences(self, ctx, *, experiment_id, plugin_name, downsample=None, run_tag_filter=None): self._validate_eid(experiment_id) if run_tag_filter is None: run_tag_filter = provider.RunTagFilter() rtf = run_tag_filter expected_run = "%s/test" % experiment_id expected_tag = "input.%s" % plugin_name if rtf.runs is not None and expected_run not in rtf.runs: return {} if rtf.tags is not None and expected_tag not in rtf.tags: return {} return { expected_run: { expected_tag: [ provider.BlobSequenceDatum( step=0, wall_time=0.0, values=[ self._make_blob_reference("experiment: %s" % experiment_id), self._make_blob_reference("name: %s" % self._name), ], ), ] } }
def list_blob_sequences(self, experiment_id, plugin_name, run_tag_filter=None): self._validate_experiment_id(experiment_id) if run_tag_filter is None: run_tag_filter = provider.RunTagFilter(runs=None, tags=None) # TODO(davidsoergel, wchargin): consider images, etc. # Note this plugin_name can really just be 'graphs' for now; the # v2 cases are not handled yet. if plugin_name != graphs_metadata.PLUGIN_NAME: logger.warn("Directory has no blob data for plugin %r", plugin_name) return {} result = collections.defaultdict(lambda: {}) for (run, run_info) in six.iteritems(self._multiplexer.Runs()): tag = None if not self._test_run_tag(run_tag_filter, run, tag): continue if not run_info[plugin_event_accumulator.GRAPH]: continue result[run][tag] = provider.BlobSequenceTimeSeries( max_step=0, max_wall_time=0, latest_max_index=0, # Graphs are always one blob at a time plugin_content=None, description=None, display_name=None, ) return result
def list_scalars(self, experiment_id, plugin_name, run_tag_filter=None): del experiment_id # ignored for now run_tag_content = self._multiplexer.PluginRunToTagToContent( plugin_name) result = {} if run_tag_filter is None: run_tag_filter = provider.RunTagFilter(runs=None, tags=None) for (run, tag_to_content) in six.iteritems(run_tag_content): result_for_run = {} for tag in tag_to_content: if not self._test_run_tag(run_tag_filter, run, tag): continue result[run] = result_for_run max_step = None max_wall_time = None for event in self._multiplexer.Tensors(run, tag): if max_step is None or max_step < event.step: max_step = event.step if max_wall_time is None or max_wall_time < event.wall_time: max_wall_time = event.wall_time summary_metadata = self._multiplexer.SummaryMetadata(run, tag) result_for_run[tag] = provider.ScalarTimeSeries( max_step=max_step, max_wall_time=max_wall_time, plugin_content=summary_metadata.plugin_data.content, description=summary_metadata.summary_description, display_name=summary_metadata.display_name, ) return result
def read_scalars(self, ctx, *, experiment_id, plugin_name, downsample=None, run_tag_filter=None): self._validate_eid(experiment_id) if run_tag_filter is None: run_tag_filter = provider.RunTagFilter() rtf = run_tag_filter expected_run = "%s/train" % experiment_id expected_tag = "loss.%s" % plugin_name if rtf.runs is not None and expected_run not in rtf.runs: return {} if rtf.tags is not None and expected_tag not in rtf.tags: return {} return { expected_run: { expected_tag: [ provider.ScalarDatum(step=0, wall_time=0.0, value=float(len(plugin_name))), provider.ScalarDatum(step=1, wall_time=0.5, value=float(len(experiment_id))), ] } }
def _get_run_to_histogram_series(self, ctx, experiment, tag, runs): """Builds a run-to-histogram-series dict for client consumption. Args: ctx: A `tensorboard.context.RequestContext` value. experiment: a string experiment id. tag: string of the requested tag. runs: optional list of run names as strings. Returns: A map from string run names to `HistogramStepDatum` (see http_api.md). """ mapping = self._data_provider.read_tensors( ctx, experiment_id=experiment, plugin_name=histogram_metadata.PLUGIN_NAME, downsample=self._plugin_downsampling["histograms"], run_tag_filter=provider.RunTagFilter(runs=runs, tags=[tag]), ) run_to_series = {} for (result_run, tag_data) in mapping.items(): if tag not in tag_data: continue values = [{ "wallTime": datum.wall_time, "step": datum.step, "bins": self._format_histogram_datum_bins(datum), } for datum in tag_data[tag]] run_to_series[result_run] = values return run_to_series
def _collect_tensor_events(self, request, step=None): """Collects list of tensor events based on request.""" ctx = plugin_util.context(request.environ) experiment = plugin_util.experiment_id(request.environ) run = request.args.get("run") tag = request.args.get("tag") tensor_events = [] # List of tuples (meta, tensor) that contain tag. for instance_tag in self._instance_tags(ctx, experiment, run, tag): tensors = self._data_provider.read_tensors( ctx, experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME, run_tag_filter=provider.RunTagFilter( runs=[run], tags=[instance_tag] ), downsample=self._downsample_to, )[run][instance_tag] meta = self._instance_tag_metadata( ctx, experiment, run, instance_tag ) tensor_events += [(meta, tensor) for tensor in tensors] if step is not None: tensor_events = [ event for event in tensor_events if event[1].step == step ] else: # Make sure tensors sorted by step in ascending order. tensor_events = sorted( tensor_events, key=lambda tensor_data: tensor_data[1].step ) return tensor_events
def scalars_impl(self, tag, run, experiment, output_format): """Result of the form `(body, mime_type)`.""" if self._data_provider: # Downsample reads to 1000 scalars per time series, which is the # default size guidance for scalars under the multiplexer loading # logic. SAMPLE_COUNT = 1000 all_scalars = self._data_provider.read_scalars( experiment_id=None, # experiment support not yet implemented plugin_name=metadata.PLUGIN_NAME, downsample=SAMPLE_COUNT, run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), ) scalars = all_scalars.get(run, {}).get(tag, None) if scalars is None: raise ValueError('No scalar data for run=%r, tag=%r' % (run, tag)) values = [(x.wall_time, x.step, x.value) for x in scalars] elif self._db_connection_provider: db = self._db_connection_provider() # We select for steps greater than -1 because the writer inserts # placeholder rows en masse. The check for step filters out those rows. cursor = db.execute(''' SELECT Tensors.step, Tensors.computed_time, Tensors.data, Tensors.dtype FROM Tensors JOIN Tags ON Tensors.series = Tags.tag_id JOIN Runs ON Tags.run_id = Runs.run_id WHERE /* For backwards compatibility, ignore the experiment id for matching purposes if it is empty. */ (:exp == '' OR Runs.experiment_id == CAST(:exp AS INT)) AND Runs.run_name = :run AND Tags.tag_name = :tag AND Tags.plugin_name = :plugin AND Tensors.shape = '' AND Tensors.step > -1 ORDER BY Tensors.step ''', dict(exp=experiment, run=run, tag=tag, plugin=metadata.PLUGIN_NAME)) values = [(wall_time, step, self._get_value(data, dtype_enum)) for (step, wall_time, data, dtype_enum) in cursor] else: tensor_events = self._multiplexer.Tensors(run, tag) values = [(tensor_event.wall_time, tensor_event.step, tensor_util.make_ndarray(tensor_event.tensor_proto).item()) for tensor_event in tensor_events] if output_format == OutputFormat.CSV: string_io = StringIO() writer = csv.writer(string_io) writer.writerow(['Wall time', 'Step', 'Value']) writer.writerows(values) return (string_io.getvalue(), 'text/csv') else: return (values, 'application/json')
def _get_run_to_image_series(self, ctx, experiment, tag, sample, runs): """Builds a run-to-image-series dict for client consumption. Args: ctx: A `tensorboard.context.RequestContext` value. experiment: a string experiment id. tag: string of the requested tag. sample: zero-indexed integer for the requested sample. runs: optional list of run names as strings. Returns: A `RunToSeries` dict (see http_api.md). """ mapping = self._data_provider.read_blob_sequences( ctx, experiment_id=experiment, plugin_name=image_metadata.PLUGIN_NAME, downsample=self._plugin_downsampling["images"], run_tag_filter=provider.RunTagFilter(runs, tags=[tag]), ) run_to_series = {} for (result_run, tag_data) in mapping.items(): if tag not in tag_data: continue blob_sequence_datum_list = tag_data[tag] series = _format_image_blob_sequence_datum( blob_sequence_datum_list, sample) if series: run_to_series[result_run] = series return run_to_series
def test_list_blob_sequences(self): provider = self.create_provider() with self.subTest("finds all time series for a plugin"): result = provider.list_blob_sequences( self.ctx, experiment_id="unused", plugin_name=image_metadata.PLUGIN_NAME, ) self.assertItemsEqual(result.keys(), ["mondrian"]) self.assertItemsEqual(result["mondrian"].keys(), ["red", "blue", "yellow"]) sample = result["mondrian"]["blue"] self.assertIsInstance(sample, base_provider.BlobSequenceTimeSeries) self.assertEqual(sample.max_step, 10) # nothing to test for wall time, as it can't be mocked out self.assertEqual(sample.plugin_content, b"") self.assertEqual(sample.max_length, 6 + 2) self.assertEqual(sample.description, "bottom-left") self.assertEqual(sample.display_name, "") with self.subTest("filters by run/tag"): result = provider.list_blob_sequences( self.ctx, experiment_id="unused", plugin_name=image_metadata.PLUGIN_NAME, run_tag_filter=base_provider.RunTagFilter( runs=["mondrian", "picasso"], tags=["yellow", "green't"]), ) self.assertItemsEqual(result.keys(), ["mondrian"]) self.assertItemsEqual(result["mondrian"].keys(), ["yellow"]) self.assertIsInstance( result["mondrian"]["yellow"], base_provider.BlobSequenceTimeSeries, )
def graph_execution_digest_run_tag_filter(run, begin, end, trace_id=None): """Create a RunTagFilter for GraphExecutionTraceDigests. This differs from `graph_execution_data_run_tag_filter()` in that it is for the small-size digest objects for intra-graph execution debug events, instead of the full-size data objects. Args: run: tfdbg2 run name. begin: Beginning index of GraphExecutionTraceDigests. end: Ending index of GraphExecutionTraceDigests. Returns: `RunTagFilter` for the run and range of GraphExecutionTraceDigests. """ # TODO(cais): Implement support for trace_id once joining of eager # execution and intra-graph execution is supported by DebugDataReader. if trace_id is not None: raise NotImplementedError( "trace_id support for graph_execution_digest_run_tag_filter() is " "not implemented yet.") return provider.RunTagFilter( runs=[run], tags=[ "%s_%d_%d" % (GRAPH_EXECUTION_DIGESTS_BLOB_TAG_PREFIX, begin, end) ], )
def test_read_scalars(self): res = data_provider_pb2.ReadScalarsResponse() run = res.runs.add(run_name="test") tag = run.tags.add(tag_name="accuracy") tag.data.step.extend([0, 1, 2, 4]) tag.data.wall_time.extend([1234.0, 1235.0, 1236.0, 1237.0]) tag.data.value.extend([0.25, 0.50, 0.75, 1.00]) self.stub.ReadScalars.return_value = res actual = self.provider.read_scalars( self.ctx, experiment_id="123", plugin_name="scalars", run_tag_filter=provider.RunTagFilter(runs=["test", "nope"]), downsample=4, ) expected = { "test": { "accuracy": [ provider.ScalarDatum(step=0, wall_time=1234.0, value=0.25), provider.ScalarDatum(step=1, wall_time=1235.0, value=0.50), provider.ScalarDatum(step=2, wall_time=1236.0, value=0.75), provider.ScalarDatum(step=4, wall_time=1237.0, value=1.00), ], }, } self.assertEqual(actual, expected) req = data_provider_pb2.ReadScalarsRequest() req.experiment_id = "123" req.plugin_filter.plugin_name = "scalars" req.run_tag_filter.runs.names.extend(["nope", "test"]) # sorted req.downsample.num_points = 4 self.stub.ReadScalars.assert_called_once_with(req)
def test_read_scalars(self): multiplexer = self.create_multiplexer() provider = data_provider.MultiplexerDataProvider( multiplexer, self.logdir) run_tag_filter = base_provider.RunTagFilter( runs=["waves", "polynomials", "unicorns"], tags=["sine", "square", "cube", "iridescence"], ) result = provider.read_scalars( experiment_id="unused", plugin_name=scalar_metadata.PLUGIN_NAME, run_tag_filter=run_tag_filter, downsample=None, # not yet implemented ) self.assertItemsEqual(result.keys(), ["polynomials", "waves"]) self.assertItemsEqual(result["polynomials"].keys(), ["square", "cube"]) self.assertItemsEqual(result["waves"].keys(), ["square", "sine"]) for run in result: for tag in result[run]: tensor_events = multiplexer.Tensors(run, tag) self.assertLen(result[run][tag], len(tensor_events)) for (datum, event) in zip(result[run][tag], tensor_events): self.assertEqual(datum.step, event.step) self.assertEqual(datum.wall_time, event.wall_time) self.assertEqual( datum.value, tensor_util.make_ndarray(event.tensor_proto).item(), )
def _serve_greetings(self, request): """Serves greeting data for the specified tag and run. For details on how to use tags and runs, see https://github.com/tensorflow/tensorboard#tags-giving-names-to-data """ run = request.args.get("run") tag = request.args.get("tag") ctx = plugin_util.context(request.environ) experiment = plugin_util.experiment_id(request.environ) if run is None or tag is None: raise werkzeug.exceptions.BadRequest("Must specify run and tag") read_result = self.data_provider.read_tensors( ctx, downsample=1000, plugin_name=metadata.PLUGIN_NAME, experiment_id=experiment, run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), ) data = read_result.get(run, {}).get(tag, []) if not data: raise werkzeug.exceptions.BadRequest("Invalid run or tag") event_data = [datum.numpy.item().decode("utf-8") for datum in data] contents = json.dumps(event_data, sort_keys=True) return werkzeug.Response(contents, content_type="application/json")
def test_defensive_copy(self): runs = ["r1"] tags = ["t1"] f = provider.RunTagFilter(runs, tags) runs.append("r2") tags.pop() self.assertEqual(frozenset(f.runs), frozenset(["r1"])) self.assertEqual(frozenset(f.tags), frozenset(["t1"]))
def source_file_list_run_tag_filter(run): """Create a RunTagFilter for listing source files. Args: run: tfdbg2 run name. Returns: `RunTagFilter` for listing the source files in the tfdbg2 run. """ return provider.RunTagFilter(runs=[run], tags=[SOURCE_FILE_LIST_BLOB_TAG])
def test_read_blob_sequences_and_read_blob(self): provider = self.create_provider() with self.subTest("reads all time series for a plugin"): result = provider.read_blob_sequences( self.ctx, experiment_id="unused", plugin_name=image_metadata.PLUGIN_NAME, downsample=4, ) self.assertItemsEqual(result.keys(), ["mondrian"]) self.assertItemsEqual( result["mondrian"].keys(), ["red", "blue", "yellow"] ) sample = result["mondrian"]["blue"] self.assertLen(sample, 4) # downsampled from 10 last = sample[-1] self.assertIsInstance(last, base_provider.BlobSequenceDatum) self.assertEqual(last.step, 10) self.assertLen(last.values, 2 + 2) blobs = [ provider.read_blob(self.ctx, blob_key=v.blob_key) for v in last.values ] self.assertEqual(blobs[0], b"10") self.assertEqual(blobs[1], b"10") self.assertStartsWith(blobs[2], b"\x89PNG") self.assertStartsWith(blobs[3], b"\x89PNG") blue1 = blobs[2] blue2 = blobs[3] red1 = provider.read_blob( self.ctx, blob_key=result["mondrian"]["red"][-1].values[2].blob_key, ) self.assertEqual(blue1, blue2) self.assertNotEqual(blue1, red1) with self.subTest("filters by run/tag"): result = provider.read_blob_sequences( self.ctx, experiment_id="unused", plugin_name=image_metadata.PLUGIN_NAME, run_tag_filter=base_provider.RunTagFilter( runs=["mondrian", "picasso"], tags=["yellow", "green't"] ), downsample=1, ) self.assertItemsEqual(result.keys(), ["mondrian"]) self.assertItemsEqual(result["mondrian"].keys(), ["yellow"]) self.assertIsInstance( result["mondrian"]["yellow"][0], base_provider.BlobSequenceDatum, )