def annotations_impl(self, ctx, experiment):
mapping = self._data_provider.list_tensors(
ctx,
experiment_id=experiment,
plugin_name=self.plugin_name,
run_tag_filter=provider.RunTagFilter(
tags=[metadata.ANNOTATIONS_TAG]),
)
result = {run: {} for run in mapping}
for (run, _) in six.iteritems(mapping):
all_annotations = self._data_provider.read_tensors(
ctx,
experiment_id=experiment,
plugin_name=self.plugin_name,
run_tag_filter=provider.RunTagFilter(
runs=[run], tags=[metadata.ANNOTATIONS_TAG]),
downsample=self._downsample_to,
)
annotations = all_annotations.get(run,
{}).get(metadata.ANNOTATIONS_TAG,
{})
event_data = [
annotation.decode("utf-8")
for annotation in annotations[0].numpy
]
result[run] = event_data
return result
def _build_session_groups(self):
"""Returns a list of SessionGroups protobuffers from the summary
data."""
# Algorithm: We keep a dict 'groups_by_name' mapping a SessionGroup name
# (str) to a SessionGroup protobuffer. We traverse the runs associated with
# the plugin--each representing a single session. We form a Session
# protobuffer from each run and add it to the relevant SessionGroup object
# in the 'groups_by_name' dict. We create the SessionGroup object, if this
# is the first session of that group we encounter.
groups_by_name = {}
run_to_tag_to_content = self._context.hparams_metadata(
self._experiment_id,
run_tag_filter=provider.RunTagFilter(tags=[
metadata.SESSION_START_INFO_TAG,
metadata.SESSION_END_INFO_TAG,
]),
)
# The TensorBoard runs with session start info are the
# "sessions", which are not necessarily the runs that actually
# contain metrics (may be in subdirectories).
session_names = [
run for (run, tags) in run_to_tag_to_content.items()
if metadata.SESSION_START_INFO_TAG in tags
]
metric_runs = set()
metric_tags = set()
for session_name in session_names:
for metric in self._experiment.metric_infos:
metric_name = metric.name
(run, tag) = metrics.run_tag_from_session_and_metric(
session_name, metric_name)
metric_runs.add(run)
metric_tags.add(tag)
all_metric_evals = self._context.read_last_scalars(
self._experiment_id,
run_tag_filter=provider.RunTagFilter(runs=metric_runs,
tags=metric_tags),
)
for (session_name, tag_to_content) in run_to_tag_to_content.items():
if metadata.SESSION_START_INFO_TAG not in tag_to_content:
continue
start_info = metadata.parse_session_start_info_plugin_data(
tag_to_content[metadata.SESSION_START_INFO_TAG])
end_info = None
if metadata.SESSION_END_INFO_TAG in tag_to_content:
end_info = metadata.parse_session_end_info_plugin_data(
tag_to_content[metadata.SESSION_END_INFO_TAG])
session = self._build_session(session_name, start_info, end_info,
all_metric_evals)
if session.status in self._request.allowed_statuses:
self._add_session(session, start_info, groups_by_name)
# Compute the session group's aggregated metrics for each group.
groups = groups_by_name.values()
for group in groups:
# We sort the sessions in a group so that the order is deterministic.
group.sessions.sort(key=operator.attrgetter("name"))
self._aggregate_metrics(group)
return groups
def embeddings_impl(self, ctx, experiment):
mapping = self._data_provider.list_tensors(
ctx,
experiment_id=experiment,
plugin_name=self.plugin_name,
run_tag_filter=provider.RunTagFilter(
tags=[metadata.EMBEDDINGS_TAG]
),
)
result = {run: {} for run in mapping}
for (run, _) in mapping.items():
all_embeddings = self._data_provider.read_tensors(
ctx,
experiment_id=experiment,
plugin_name=self.plugin_name,
run_tag_filter=provider.RunTagFilter(
runs=[run], tags=[metadata.EMBEDDINGS_TAG]
),
downsample=self._downsample_to,
)
embeddings = all_embeddings.get(run, {}).get(
metadata.EMBEDDINGS_TAG, {}
)
event_data = embeddings[0].numpy.tolist()
result[run] = event_data
return result
def test_scalars(self):
listing = self.with_unpfx.list_scalars(_ctx(),
experiment_id="foo:123",
plugin_name="scalars")
self.assertEqual(
listing,
self.foo_provider.list_scalars(_ctx(),
experiment_id="123",
plugin_name="scalars"),
)
reading = self.with_unpfx.read_scalars(
_ctx(),
experiment_id="foo:123",
plugin_name="scalars",
downsample=1000,
run_tag_filter=provider.RunTagFilter(["123/train"],
["loss.scalars"]),
)
expected_reading = self.foo_provider.read_scalars(
_ctx(),
experiment_id="123",
plugin_name="scalars",
downsample=1000,
run_tag_filter=provider.RunTagFilter(["123/train"],
["loss.scalars"]),
)
self.assertNotEmpty(expected_reading)
self.assertEqual(reading, expected_reading)
def test_list_scalars_filters(self):
provider = self.create_provider()
result = provider.list_scalars(
experiment_id="unused",
plugin_name=scalar_metadata.PLUGIN_NAME,
run_tag_filter=base_provider.RunTagFilter(["waves"], ["square"]),
)
self.assertItemsEqual(result.keys(), ["waves"])
self.assertItemsEqual(result["waves"].keys(), ["square"])
result = provider.list_scalars(
experiment_id="unused",
plugin_name=scalar_metadata.PLUGIN_NAME,
run_tag_filter=base_provider.RunTagFilter(
tags=["square", "quartic"]),
)
self.assertItemsEqual(result.keys(), ["polynomials", "waves"])
self.assertItemsEqual(result["polynomials"].keys(), ["square"])
self.assertItemsEqual(result["waves"].keys(), ["square"])
result = provider.list_scalars(
experiment_id="unused",
plugin_name=scalar_metadata.PLUGIN_NAME,
run_tag_filter=base_provider.RunTagFilter(runs=["waves", "hugs"]),
)
self.assertItemsEqual(result.keys(), ["waves"])
self.assertItemsEqual(result["waves"].keys(), ["sine", "square"])
result = provider.list_scalars(
experiment_id="unused",
plugin_name=scalar_metadata.PLUGIN_NAME,
run_tag_filter=base_provider.RunTagFilter(["un"], ["likely"]),
)
self.assertEqual(result, {})
def test_validates_runs_tags(self):
# Accidentally passed scalar strings
with six.assertRaisesRegex(self, TypeError, "runs:.*got.*str.*myrun"):
provider.RunTagFilter(runs="myrun")
with six.assertRaisesRegex(self, TypeError, "tags:.*got.*str.*mytag"):
provider.RunTagFilter(tags="mytag")
# Passed collections with non-string elements
with six.assertRaisesRegex(self, TypeError,
"runs:.*got item of type.*NoneType.*None"):
provider.RunTagFilter(runs=[None])
with six.assertRaisesRegex(self, TypeError,
"tags:.*got item of type.*int.*3"):
provider.RunTagFilter(tags=["one", "two", 3])
def histograms_impl(self, ctx, tag, run, experiment, downsample_to=None):
"""Result of the form `(body, mime_type)`.
At most `downsample_to` events will be returned. If this value is
`None`, then default downsampling will be performed.
Raises:
tensorboard.errors.PublicError: On invalid request.
"""
sample_count = (
downsample_to if downsample_to is not None else self._downsample_to
)
all_histograms = self._data_provider.read_tensors(
ctx,
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=sample_count,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
histograms = all_histograms.get(run, {}).get(tag, None)
if histograms is None:
raise errors.NotFoundError(
"No histogram tag %r for run %r" % (tag, run)
)
events = [(e.wall_time, e.step, e.numpy.tolist()) for e in histograms]
return (events, "application/json")
def scalars_impl(self, ctx, experiment, tag, run):
"""Returns scalar data for the specified tag and run.
For details on how to use tags and runs, see
https://github.com/tensorflow/tensorboard#tags-giving-names-to-data
Args:
tag: string
run: string
Returns:
A list of ScalarEvents - tuples containing 3 numbers describing entries in
the data series.
Raises:
NotFoundError if there are no scalars data for provided `run` and
`tag`.
"""
all_scalars = self._data_provider.read_scalars(
ctx,
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=5000,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
scalars = all_scalars.get(run, {}).get(tag, None)
if scalars is None:
raise errors.NotFoundError("No scalar data for run=%r, tag=%r" %
(run, tag))
return [(x.wall_time, x.step, x.value) for x in scalars]
def test_list_tensors(self):
res = data_provider_pb2.ListTensorsResponse()
run1 = res.runs.add(run_name="val")
tag11 = run1.tags.add(tag_name="weights")
tag11.metadata.max_step = 7
tag11.metadata.max_wall_time = 7.77
tag11.metadata.summary_metadata.plugin_data.content = b"magic"
tag11.metadata.summary_metadata.summary_description = "hey"
tag12 = run1.tags.add(tag_name="other")
tag12.metadata.max_step = 8
tag12.metadata.max_wall_time = 8.88
run2 = res.runs.add(run_name="test")
tag21 = run2.tags.add(tag_name="weights")
tag21.metadata.max_step = 9
tag21.metadata.max_wall_time = 9.99
self.stub.ListTensors.return_value = res
actual = self.provider.list_tensors(
self.ctx,
experiment_id="123",
plugin_name="histograms",
run_tag_filter=provider.RunTagFilter(tags=["weights", "other"]),
)
expected = {
"val": {
"weights":
provider.TensorTimeSeries(
max_step=7,
max_wall_time=7.77,
plugin_content=b"magic",
description="hey",
display_name="",
),
"other":
provider.TensorTimeSeries(
max_step=8,
max_wall_time=8.88,
plugin_content=b"",
description="",
display_name="",
),
},
"test": {
"weights":
provider.TensorTimeSeries(
max_step=9,
max_wall_time=9.99,
plugin_content=b"",
description="",
display_name="",
),
},
}
self.assertEqual(actual, expected)
req = data_provider_pb2.ListTensorsRequest()
req.experiment_id = "123"
req.plugin_filter.plugin_name = "histograms"
req.run_tag_filter.tags.names.extend(["other", "weights"]) # sorted
self.stub.ListTensors.assert_called_once_with(req)
def test_read_tensors(self):
multiplexer = self.create_multiplexer()
provider = data_provider.MultiplexerDataProvider(
multiplexer, self.logdir)
run_tag_filter = base_provider.RunTagFilter(
runs=["lebesgue"],
tags=["uniform", "bimodal"],
)
result = provider.read_tensors(
experiment_id="unused",
plugin_name=histogram_metadata.PLUGIN_NAME,
run_tag_filter=run_tag_filter,
downsample=None, # not yet implemented
)
self.assertItemsEqual(result.keys(), ["lebesgue"])
self.assertItemsEqual(result["lebesgue"].keys(),
["uniform", "bimodal"])
for run in result:
for tag in result[run]:
tensor_events = multiplexer.Tensors(run, tag)
self.assertLen(result[run][tag], len(tensor_events))
for (datum, event) in zip(result[run][tag], tensor_events):
self.assertEqual(datum.step, event.step)
self.assertEqual(datum.wall_time, event.wall_time)
np.testing.assert_equal(
datum.numpy,
tensor_util.make_ndarray(event.tensor_proto),
)
def pr_curves_impl(self, ctx, experiment, runs, tag):
"""Creates the JSON object for the PR curves response for a run-tag
combo.
Arguments:
runs: A list of runs to fetch the curves for.
tag: The tag to fetch the curves for.
Raises:
ValueError: If no PR curves could be fetched for a run and tag.
Returns:
The JSON object for the PR curves route response.
"""
response_mapping = {}
rtf = provider.RunTagFilter(runs, [tag])
read_result = self._data_provider.read_tensors(
ctx,
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
run_tag_filter=rtf,
downsample=self._downsample_to,
)
for run in runs:
data = read_result.get(run, {}).get(tag)
if data is None:
raise ValueError(
"No PR curves could be found for run %r and tag %r" %
(run, tag))
response_mapping[run] = [self._process_datum(d) for d in data]
return response_mapping
def histograms_impl(self, ctx, tag, run, experiment, downsample_to=None):
"""Result of the form `(body, mime_type)`.
At most `downsample_to` events will be returned. If this value is
`None`, then default downsampling will be performed.
Raises:
tensorboard.errors.PublicError: On invalid request.
"""
sample_count = (
downsample_to if downsample_to is not None else self._downsample_to
)
all_histograms = self._data_provider.read_tensors(
ctx,
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=sample_count,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
histograms = all_histograms.get(run, {}).get(tag, None)
if histograms is None:
raise errors.NotFoundError(
"No histogram tag %r for run %r" % (tag, run)
)
# Downsample again, even though the data provider is supposed to,
# because the multiplexer provider currently doesn't. (For
# well-behaved data providers, this is a no-op.)
if downsample_to is not None:
rng = random.Random(0)
histograms = _downsample(rng, histograms, downsample_to)
events = [(e.wall_time, e.step, e.numpy.tolist()) for e in histograms]
return (events, "application/json")
def graph_impl(
self,
run,
tag,
is_conceptual,
experiment=None,
limit_attr_size=None,
large_attrs_key=None,
):
"""Result of the form `(body, mime_type)`, or `None` if no graph
exists."""
if self._data_provider:
graph_blob_sequences = self._data_provider.read_blob_sequences(
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
blob_datum_list = graph_blob_sequences.get(run, {}).get(tag, ())
try:
blob_ref = blob_datum_list[0].values[0]
except IndexError:
return None
# Always use the blob_key approach for now, even if there is a direct url.
graph_raw = self._data_provider.read_blob(blob_ref.blob_key)
# This method ultimately returns pbtxt, but we have to deserialize and
# later reserialize this anyway, because a) this way we accept binary
# protobufs too, and b) below we run `prepare_graph_for_ui` on the graph.
graph = graph_pb2.GraphDef.FromString(graph_raw)
elif is_conceptual:
tensor_events = self._multiplexer.Tensors(run, tag)
# Take the first event if there are multiple events written from different
# steps.
keras_model_config = json.loads(
tensor_events[0].tensor_proto.string_val[0]
)
graph = keras_util.keras_model_to_graph_def(keras_model_config)
elif tag:
tensor_events = self._multiplexer.Tensors(run, tag)
# Take the first event if there are multiple events written from different
# steps.
run_metadata = config_pb2.RunMetadata.FromString(
tensor_events[0].tensor_proto.string_val[0]
)
graph = graph_pb2.GraphDef()
for func_graph in run_metadata.function_graphs:
graph_util.combine_graph_defs(
graph, func_graph.pre_optimization_graph
)
else:
graph = self._multiplexer.Graph(run)
# This next line might raise a ValueError if the limit parameters
# are invalid (size is negative, size present but key absent, etc.).
process_graph.prepare_graph_for_ui(
graph, limit_attr_size, large_attrs_key
)
return (str(graph), "text/x-protobuf") # pbtxt
def _image_response_for_run(self, ctx, experiment, run, tag, sample):
"""Builds a JSON-serializable object with information about images.
Args:
run: The name of the run.
tag: The name of the tag the images all belong to.
sample: The zero-indexed sample of the image for which to retrieve
information. For instance, setting `sample` to `2` will fetch
information about only the third image of each batch. Steps with
fewer than three images will be omitted from the results.
Returns:
A list of dictionaries containing the wall time, step, and URL
for each image.
Raises:
KeyError, NotFoundError: If no image data exists for the given
parameters.
"""
all_images = self._data_provider.read_blob_sequences(
ctx,
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=self._downsample_to,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
images = all_images.get(run, {}).get(tag, None)
if images is None:
raise errors.NotFoundError("No image data for run=%r, tag=%r" %
(run, tag))
return [{
"wall_time": datum.wall_time,
"step": datum.step,
"query": self._data_provider_query(datum.values[sample + 2]),
} for datum in images if len(datum.values) - 2 > sample]
def read_blob_sequences(self,
ctx,
*,
experiment_id,
plugin_name,
downsample=None,
run_tag_filter=None):
self._validate_eid(experiment_id)
if run_tag_filter is None:
run_tag_filter = provider.RunTagFilter()
rtf = run_tag_filter
expected_run = "%s/test" % experiment_id
expected_tag = "input.%s" % plugin_name
if rtf.runs is not None and expected_run not in rtf.runs:
return {}
if rtf.tags is not None and expected_tag not in rtf.tags:
return {}
return {
expected_run: {
expected_tag: [
provider.BlobSequenceDatum(
step=0,
wall_time=0.0,
values=[
self._make_blob_reference("experiment: %s" %
experiment_id),
self._make_blob_reference("name: %s" % self._name),
],
),
]
}
}
def list_blob_sequences(self,
experiment_id,
plugin_name,
run_tag_filter=None):
self._validate_experiment_id(experiment_id)
if run_tag_filter is None:
run_tag_filter = provider.RunTagFilter(runs=None, tags=None)
# TODO(davidsoergel, wchargin): consider images, etc.
# Note this plugin_name can really just be 'graphs' for now; the
# v2 cases are not handled yet.
if plugin_name != graphs_metadata.PLUGIN_NAME:
logger.warn("Directory has no blob data for plugin %r",
plugin_name)
return {}
result = collections.defaultdict(lambda: {})
for (run, run_info) in six.iteritems(self._multiplexer.Runs()):
tag = None
if not self._test_run_tag(run_tag_filter, run, tag):
continue
if not run_info[plugin_event_accumulator.GRAPH]:
continue
result[run][tag] = provider.BlobSequenceTimeSeries(
max_step=0,
max_wall_time=0,
latest_max_index=0, # Graphs are always one blob at a time
plugin_content=None,
description=None,
display_name=None,
)
return result
def list_scalars(self, experiment_id, plugin_name, run_tag_filter=None):
del experiment_id # ignored for now
run_tag_content = self._multiplexer.PluginRunToTagToContent(
plugin_name)
result = {}
if run_tag_filter is None:
run_tag_filter = provider.RunTagFilter(runs=None, tags=None)
for (run, tag_to_content) in six.iteritems(run_tag_content):
result_for_run = {}
for tag in tag_to_content:
if not self._test_run_tag(run_tag_filter, run, tag):
continue
result[run] = result_for_run
max_step = None
max_wall_time = None
for event in self._multiplexer.Tensors(run, tag):
if max_step is None or max_step < event.step:
max_step = event.step
if max_wall_time is None or max_wall_time < event.wall_time:
max_wall_time = event.wall_time
summary_metadata = self._multiplexer.SummaryMetadata(run, tag)
result_for_run[tag] = provider.ScalarTimeSeries(
max_step=max_step,
max_wall_time=max_wall_time,
plugin_content=summary_metadata.plugin_data.content,
description=summary_metadata.summary_description,
display_name=summary_metadata.display_name,
)
return result
def read_scalars(self,
ctx,
*,
experiment_id,
plugin_name,
downsample=None,
run_tag_filter=None):
self._validate_eid(experiment_id)
if run_tag_filter is None:
run_tag_filter = provider.RunTagFilter()
rtf = run_tag_filter
expected_run = "%s/train" % experiment_id
expected_tag = "loss.%s" % plugin_name
if rtf.runs is not None and expected_run not in rtf.runs:
return {}
if rtf.tags is not None and expected_tag not in rtf.tags:
return {}
return {
expected_run: {
expected_tag: [
provider.ScalarDatum(step=0,
wall_time=0.0,
value=float(len(plugin_name))),
provider.ScalarDatum(step=1,
wall_time=0.5,
value=float(len(experiment_id))),
]
}
}
def _get_run_to_histogram_series(self, ctx, experiment, tag, runs):
"""Builds a run-to-histogram-series dict for client consumption.
Args:
ctx: A `tensorboard.context.RequestContext` value.
experiment: a string experiment id.
tag: string of the requested tag.
runs: optional list of run names as strings.
Returns:
A map from string run names to `HistogramStepDatum` (see http_api.md).
"""
mapping = self._data_provider.read_tensors(
ctx,
experiment_id=experiment,
plugin_name=histogram_metadata.PLUGIN_NAME,
downsample=self._plugin_downsampling["histograms"],
run_tag_filter=provider.RunTagFilter(runs=runs, tags=[tag]),
)
run_to_series = {}
for (result_run, tag_data) in mapping.items():
if tag not in tag_data:
continue
values = [{
"wallTime": datum.wall_time,
"step": datum.step,
"bins": self._format_histogram_datum_bins(datum),
} for datum in tag_data[tag]]
run_to_series[result_run] = values
return run_to_series
def _collect_tensor_events(self, request, step=None):
"""Collects list of tensor events based on request."""
ctx = plugin_util.context(request.environ)
experiment = plugin_util.experiment_id(request.environ)
run = request.args.get("run")
tag = request.args.get("tag")
tensor_events = [] # List of tuples (meta, tensor) that contain tag.
for instance_tag in self._instance_tags(ctx, experiment, run, tag):
tensors = self._data_provider.read_tensors(
ctx,
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
run_tag_filter=provider.RunTagFilter(
runs=[run], tags=[instance_tag]
),
downsample=self._downsample_to,
)[run][instance_tag]
meta = self._instance_tag_metadata(
ctx, experiment, run, instance_tag
)
tensor_events += [(meta, tensor) for tensor in tensors]
if step is not None:
tensor_events = [
event for event in tensor_events if event[1].step == step
]
else:
# Make sure tensors sorted by step in ascending order.
tensor_events = sorted(
tensor_events, key=lambda tensor_data: tensor_data[1].step
)
return tensor_events
def scalars_impl(self, tag, run, experiment, output_format):
"""Result of the form `(body, mime_type)`."""
if self._data_provider:
# Downsample reads to 1000 scalars per time series, which is the
# default size guidance for scalars under the multiplexer loading
# logic.
SAMPLE_COUNT = 1000
all_scalars = self._data_provider.read_scalars(
experiment_id=None, # experiment support not yet implemented
plugin_name=metadata.PLUGIN_NAME,
downsample=SAMPLE_COUNT,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
scalars = all_scalars.get(run, {}).get(tag, None)
if scalars is None:
raise ValueError('No scalar data for run=%r, tag=%r' % (run, tag))
values = [(x.wall_time, x.step, x.value) for x in scalars]
elif self._db_connection_provider:
db = self._db_connection_provider()
# We select for steps greater than -1 because the writer inserts
# placeholder rows en masse. The check for step filters out those rows.
cursor = db.execute('''
SELECT
Tensors.step,
Tensors.computed_time,
Tensors.data,
Tensors.dtype
FROM Tensors
JOIN Tags
ON Tensors.series = Tags.tag_id
JOIN Runs
ON Tags.run_id = Runs.run_id
WHERE
/* For backwards compatibility, ignore the experiment id
for matching purposes if it is empty. */
(:exp == '' OR Runs.experiment_id == CAST(:exp AS INT))
AND Runs.run_name = :run
AND Tags.tag_name = :tag
AND Tags.plugin_name = :plugin
AND Tensors.shape = ''
AND Tensors.step > -1
ORDER BY Tensors.step
''', dict(exp=experiment, run=run, tag=tag, plugin=metadata.PLUGIN_NAME))
values = [(wall_time, step, self._get_value(data, dtype_enum))
for (step, wall_time, data, dtype_enum) in cursor]
else:
tensor_events = self._multiplexer.Tensors(run, tag)
values = [(tensor_event.wall_time,
tensor_event.step,
tensor_util.make_ndarray(tensor_event.tensor_proto).item())
for tensor_event in tensor_events]
if output_format == OutputFormat.CSV:
string_io = StringIO()
writer = csv.writer(string_io)
writer.writerow(['Wall time', 'Step', 'Value'])
writer.writerows(values)
return (string_io.getvalue(), 'text/csv')
else:
return (values, 'application/json')
def _get_run_to_image_series(self, ctx, experiment, tag, sample, runs):
"""Builds a run-to-image-series dict for client consumption.
Args:
ctx: A `tensorboard.context.RequestContext` value.
experiment: a string experiment id.
tag: string of the requested tag.
sample: zero-indexed integer for the requested sample.
runs: optional list of run names as strings.
Returns:
A `RunToSeries` dict (see http_api.md).
"""
mapping = self._data_provider.read_blob_sequences(
ctx,
experiment_id=experiment,
plugin_name=image_metadata.PLUGIN_NAME,
downsample=self._plugin_downsampling["images"],
run_tag_filter=provider.RunTagFilter(runs, tags=[tag]),
)
run_to_series = {}
for (result_run, tag_data) in mapping.items():
if tag not in tag_data:
continue
blob_sequence_datum_list = tag_data[tag]
series = _format_image_blob_sequence_datum(
blob_sequence_datum_list, sample)
if series:
run_to_series[result_run] = series
return run_to_series
def test_list_blob_sequences(self):
provider = self.create_provider()
with self.subTest("finds all time series for a plugin"):
result = provider.list_blob_sequences(
self.ctx,
experiment_id="unused",
plugin_name=image_metadata.PLUGIN_NAME,
)
self.assertItemsEqual(result.keys(), ["mondrian"])
self.assertItemsEqual(result["mondrian"].keys(),
["red", "blue", "yellow"])
sample = result["mondrian"]["blue"]
self.assertIsInstance(sample, base_provider.BlobSequenceTimeSeries)
self.assertEqual(sample.max_step, 10)
# nothing to test for wall time, as it can't be mocked out
self.assertEqual(sample.plugin_content, b"")
self.assertEqual(sample.max_length, 6 + 2)
self.assertEqual(sample.description, "bottom-left")
self.assertEqual(sample.display_name, "")
with self.subTest("filters by run/tag"):
result = provider.list_blob_sequences(
self.ctx,
experiment_id="unused",
plugin_name=image_metadata.PLUGIN_NAME,
run_tag_filter=base_provider.RunTagFilter(
runs=["mondrian", "picasso"], tags=["yellow", "green't"]),
)
self.assertItemsEqual(result.keys(), ["mondrian"])
self.assertItemsEqual(result["mondrian"].keys(), ["yellow"])
self.assertIsInstance(
result["mondrian"]["yellow"],
base_provider.BlobSequenceTimeSeries,
)
def graph_execution_digest_run_tag_filter(run, begin, end, trace_id=None):
"""Create a RunTagFilter for GraphExecutionTraceDigests.
This differs from `graph_execution_data_run_tag_filter()` in that it is for
the small-size digest objects for intra-graph execution debug events, instead
of the full-size data objects.
Args:
run: tfdbg2 run name.
begin: Beginning index of GraphExecutionTraceDigests.
end: Ending index of GraphExecutionTraceDigests.
Returns:
`RunTagFilter` for the run and range of GraphExecutionTraceDigests.
"""
# TODO(cais): Implement support for trace_id once joining of eager
# execution and intra-graph execution is supported by DebugDataReader.
if trace_id is not None:
raise NotImplementedError(
"trace_id support for graph_execution_digest_run_tag_filter() is "
"not implemented yet.")
return provider.RunTagFilter(
runs=[run],
tags=[
"%s_%d_%d" % (GRAPH_EXECUTION_DIGESTS_BLOB_TAG_PREFIX, begin, end)
],
)
def test_read_scalars(self):
res = data_provider_pb2.ReadScalarsResponse()
run = res.runs.add(run_name="test")
tag = run.tags.add(tag_name="accuracy")
tag.data.step.extend([0, 1, 2, 4])
tag.data.wall_time.extend([1234.0, 1235.0, 1236.0, 1237.0])
tag.data.value.extend([0.25, 0.50, 0.75, 1.00])
self.stub.ReadScalars.return_value = res
actual = self.provider.read_scalars(
self.ctx,
experiment_id="123",
plugin_name="scalars",
run_tag_filter=provider.RunTagFilter(runs=["test", "nope"]),
downsample=4,
)
expected = {
"test": {
"accuracy": [
provider.ScalarDatum(step=0, wall_time=1234.0, value=0.25),
provider.ScalarDatum(step=1, wall_time=1235.0, value=0.50),
provider.ScalarDatum(step=2, wall_time=1236.0, value=0.75),
provider.ScalarDatum(step=4, wall_time=1237.0, value=1.00),
],
},
}
self.assertEqual(actual, expected)
req = data_provider_pb2.ReadScalarsRequest()
req.experiment_id = "123"
req.plugin_filter.plugin_name = "scalars"
req.run_tag_filter.runs.names.extend(["nope", "test"]) # sorted
req.downsample.num_points = 4
self.stub.ReadScalars.assert_called_once_with(req)
def test_read_scalars(self):
multiplexer = self.create_multiplexer()
provider = data_provider.MultiplexerDataProvider(
multiplexer, self.logdir)
run_tag_filter = base_provider.RunTagFilter(
runs=["waves", "polynomials", "unicorns"],
tags=["sine", "square", "cube", "iridescence"],
)
result = provider.read_scalars(
experiment_id="unused",
plugin_name=scalar_metadata.PLUGIN_NAME,
run_tag_filter=run_tag_filter,
downsample=None, # not yet implemented
)
self.assertItemsEqual(result.keys(), ["polynomials", "waves"])
self.assertItemsEqual(result["polynomials"].keys(), ["square", "cube"])
self.assertItemsEqual(result["waves"].keys(), ["square", "sine"])
for run in result:
for tag in result[run]:
tensor_events = multiplexer.Tensors(run, tag)
self.assertLen(result[run][tag], len(tensor_events))
for (datum, event) in zip(result[run][tag], tensor_events):
self.assertEqual(datum.step, event.step)
self.assertEqual(datum.wall_time, event.wall_time)
self.assertEqual(
datum.value,
tensor_util.make_ndarray(event.tensor_proto).item(),
)
def _serve_greetings(self, request):
"""Serves greeting data for the specified tag and run.
For details on how to use tags and runs, see
https://github.com/tensorflow/tensorboard#tags-giving-names-to-data
"""
run = request.args.get("run")
tag = request.args.get("tag")
ctx = plugin_util.context(request.environ)
experiment = plugin_util.experiment_id(request.environ)
if run is None or tag is None:
raise werkzeug.exceptions.BadRequest("Must specify run and tag")
read_result = self.data_provider.read_tensors(
ctx,
downsample=1000,
plugin_name=metadata.PLUGIN_NAME,
experiment_id=experiment,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
data = read_result.get(run, {}).get(tag, [])
if not data:
raise werkzeug.exceptions.BadRequest("Invalid run or tag")
event_data = [datum.numpy.item().decode("utf-8") for datum in data]
contents = json.dumps(event_data, sort_keys=True)
return werkzeug.Response(contents, content_type="application/json")
def test_defensive_copy(self):
runs = ["r1"]
tags = ["t1"]
f = provider.RunTagFilter(runs, tags)
runs.append("r2")
tags.pop()
self.assertEqual(frozenset(f.runs), frozenset(["r1"]))
self.assertEqual(frozenset(f.tags), frozenset(["t1"]))
def source_file_list_run_tag_filter(run):
"""Create a RunTagFilter for listing source files.
Args:
run: tfdbg2 run name.
Returns:
`RunTagFilter` for listing the source files in the tfdbg2 run.
"""
return provider.RunTagFilter(runs=[run], tags=[SOURCE_FILE_LIST_BLOB_TAG])
def test_read_blob_sequences_and_read_blob(self):
provider = self.create_provider()
with self.subTest("reads all time series for a plugin"):
result = provider.read_blob_sequences(
self.ctx,
experiment_id="unused",
plugin_name=image_metadata.PLUGIN_NAME,
downsample=4,
)
self.assertItemsEqual(result.keys(), ["mondrian"])
self.assertItemsEqual(
result["mondrian"].keys(), ["red", "blue", "yellow"]
)
sample = result["mondrian"]["blue"]
self.assertLen(sample, 4) # downsampled from 10
last = sample[-1]
self.assertIsInstance(last, base_provider.BlobSequenceDatum)
self.assertEqual(last.step, 10)
self.assertLen(last.values, 2 + 2)
blobs = [
provider.read_blob(self.ctx, blob_key=v.blob_key)
for v in last.values
]
self.assertEqual(blobs[0], b"10")
self.assertEqual(blobs[1], b"10")
self.assertStartsWith(blobs[2], b"\x89PNG")
self.assertStartsWith(blobs[3], b"\x89PNG")
blue1 = blobs[2]
blue2 = blobs[3]
red1 = provider.read_blob(
self.ctx,
blob_key=result["mondrian"]["red"][-1].values[2].blob_key,
)
self.assertEqual(blue1, blue2)
self.assertNotEqual(blue1, red1)
with self.subTest("filters by run/tag"):
result = provider.read_blob_sequences(
self.ctx,
experiment_id="unused",
plugin_name=image_metadata.PLUGIN_NAME,
run_tag_filter=base_provider.RunTagFilter(
runs=["mondrian", "picasso"], tags=["yellow", "green't"]
),
downsample=1,
)
self.assertItemsEqual(result.keys(), ["mondrian"])
self.assertItemsEqual(result["mondrian"].keys(), ["yellow"])
self.assertIsInstance(
result["mondrian"]["yellow"][0],
base_provider.BlobSequenceDatum,
)
