def read_blob(self, blob_key):
# note: ignoring nearly all key elements: there is only one graph per run.
(
unused_experiment_id,
plugin_name,
run,
unused_tag,
unused_step,
unused_index,
) = _decode_blob_key(blob_key)
# TODO(davidsoergel, wchargin): consider images, etc.
if plugin_name != graphs_metadata.PLUGIN_NAME:
logger.warn("Directory has no blob data for plugin %r",
plugin_name)
raise errors.NotFoundError()
serialized_graph = self._multiplexer.SerializedGraph(run)
# TODO(davidsoergel): graph_defs have no step attribute so we don't filter
# on it. Other blob types might, though.
if serialized_graph is None:
logger.warn("No blob found for key %r", blob_key)
raise errors.NotFoundError()
# TODO(davidsoergel): consider internal structure of non-graphdef blobs.
# In particular, note we ignore the requested index, since it's always 0.
return serialized_graph
def histograms_impl(self, ctx, tag, run, experiment, downsample_to=None):
"""Result of the form `(body, mime_type)`.
At most `downsample_to` events will be returned. If this value is
`None`, then default downsampling will be performed.
Raises:
tensorboard.errors.PublicError: On invalid request.
"""
if self._data_provider:
sample_count = (
downsample_to
if downsample_to is not None
else self._downsample_to
)
all_histograms = self._data_provider.read_tensors(
ctx,
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=sample_count,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
histograms = all_histograms.get(run, {}).get(tag, None)
if histograms is None:
raise errors.NotFoundError(
"No histogram tag %r for run %r" % (tag, run)
)
# Downsample again, even though the data provider is supposed to,
# because the multiplexer provider currently doesn't. (For
# well-behaved data providers, this is a no-op.)
if downsample_to is not None:
rng = random.Random(0)
histograms = _downsample(rng, histograms, downsample_to)
events = [
(e.wall_time, e.step, e.numpy.tolist()) for e in histograms
]
else:
# Serve data from events files.
try:
tensor_events = self._multiplexer.Tensors(run, tag)
except KeyError:
raise errors.NotFoundError(
"No histogram tag %r for run %r" % (tag, run)
)
if downsample_to is not None:
rng = random.Random(0)
tensor_events = _downsample(rng, tensor_events, downsample_to)
events = [
[
e.wall_time,
e.step,
tensor_util.make_ndarray(e.tensor_proto).tolist(),
]
for e in tensor_events
]
return (events, "application/json")
def GraphOpInfo(self, run, graph_id, op_name):
"""Get the information regarding a graph op's creation.
Args:
run: Name of the run.
graph_id: Debugger-generated ID of the graph that contains
the op in question. This ID is available from other methods
of this class, e.g., the return value of `GraphExecutionDigests()`.
op_name: Name of the op.
Returns:
A JSON-serializable object containing the information regarding
the op's creation and its immediate inputs and consumers.
Raises:
NotFoundError if the graph_id or op_name does not exist.
"""
runs = self.Runs()
if run not in runs:
return None
try:
graph = self._reader.graph_by_id(graph_id)
except KeyError:
raise errors.NotFoundError('There is no graph with ID "%s"' %
graph_id)
try:
op_creation_digest = graph.get_op_creation_digest(op_name)
except KeyError:
raise errors.NotFoundError(
'There is no op named "%s" in graph with ID "%s"' %
(op_name, graph_id))
data_object = self._opCreationDigestToDataObject(
op_creation_digest, graph)
# Populate data about immediate inputs.
for input_spec in data_object["inputs"]:
try:
input_op_digest = graph.get_op_creation_digest(
input_spec["op_name"])
except KeyError:
input_op_digest = None
if input_op_digest:
input_spec["data"] = self._opCreationDigestToDataObject(
input_op_digest, graph)
# Populate data about immediate consuming ops.
for slot_consumer_specs in data_object["consumers"]:
for consumer_spec in slot_consumer_specs:
try:
digest = graph.get_op_creation_digest(
consumer_spec["op_name"])
except KeyError:
digest = None
if digest:
consumer_spec["data"] = self._opCreationDigestToDataObject(
digest, graph)
return data_object
def GraphOpInfo(self, run, graph_id, op_name):
"""Get the information regarding a graph op's creation.
Args:
run: Name of the run.
graph_id: Debugger-generated ID of the graph that contains
the op in question. This ID is available from other methods
of this class, e.g., the return value of `GraphExecutionDigests()`.
op_name: Name of the op.
Returns:
A JSON-serializable object containing the information regarding
the op's creation and its immediate inputs and consumers.
Raises:
NotFoundError if the graph_id or op_name does not exist.
"""
runs = self.Runs()
if run not in runs:
return None
try:
graph = self._reader.graph_by_id(graph_id)
except KeyError:
raise errors.NotFoundError(
'There is no graph with ID "%s"' % graph_id
)
try:
op_creation_digest = graph.get_op_creation_digest(op_name)
except KeyError:
raise errors.NotFoundError(
'There is no op named "%s" in graph with ID "%s"'
% (op_name, graph_id)
)
data_object = self._opCreationDigestToDataObject(op_creation_digest)
# Populate data about immediate inputs.
data_object["inputs"] = None
if op_creation_digest.input_names:
data_object["inputs"] = []
for input_tensor_name in op_creation_digest.input_names:
input_op_name = tensor_name_to_op_name(input_tensor_name)
input_op_digest = graph.get_op_creation_digest(input_op_name)
data_object["inputs"].append(
self._opCreationDigestToDataObject(input_op_digest)
)
# Populate data about immediate consuming ops.
data_object["consumers"] = collections.defaultdict(list)
for src_slot, consumer_op_name, _ in graph.get_op_consumers(op_name):
digest = graph.get_op_creation_digest(consumer_op_name)
data_object["consumers"][src_slot].append(
self._opCreationDigestToDataObject(digest)
)
return data_object
def read_blob(self, ctx, blob_key):
(prefix, sub_key) = _decode_blob_key(blob_key)
if prefix is None:
if self._unprefixed_provider is None:
raise errors.NotFoundError(
"Invalid blob key: no unprefixed provider")
return self._unprefixed_provider.read_blob(ctx, blob_key=sub_key)
sub_provider = self._providers.get(prefix)
if sub_provider is None:
raise errors.NotFoundError(
"Invalid blob key: no such provider: %r; have: %r" %
(prefix, sorted(self._providers)))
return sub_provider.read_blob(ctx, blob_key=sub_key)
def scalars_impl(self, ctx, experiment, tag, run):
"""Returns scalar data for the specified tag and run.
For details on how to use tags and runs, see
https://github.com/tensorflow/tensorboard#tags-giving-names-to-data
Args:
tag: string
run: string
Returns:
A list of ScalarEvents - tuples containing 3 numbers describing entries in
the data series.
Raises:
NotFoundError if there are no scalars data for provided `run` and
`tag`.
"""
all_scalars = self._data_provider.read_scalars(
ctx,
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=5000,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
scalars = all_scalars.get(run, {}).get(tag, None)
if scalars is None:
raise errors.NotFoundError("No scalar data for run=%r, tag=%r" %
(run, tag))
return [(x.wall_time, x.step, x.value) for x in scalars]
def _image_response_for_run(self, ctx, experiment, run, tag, sample):
"""Builds a JSON-serializable object with information about images.
Args:
run: The name of the run.
tag: The name of the tag the images all belong to.
sample: The zero-indexed sample of the image for which to retrieve
information. For instance, setting `sample` to `2` will fetch
information about only the third image of each batch. Steps with
fewer than three images will be omitted from the results.
Returns:
A list of dictionaries containing the wall time, step, and URL
for each image.
Raises:
KeyError, NotFoundError: If no image data exists for the given
parameters.
"""
all_images = self._data_provider.read_blob_sequences(
ctx,
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=self._downsample_to,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
images = all_images.get(run, {}).get(tag, None)
if images is None:
raise errors.NotFoundError("No image data for run=%r, tag=%r" %
(run, tag))
return [{
"wall_time": datum.wall_time,
"step": datum.step,
"query": self._data_provider_query(datum.values[sample + 2]),
} for datum in images if len(datum.values) - 2 > sample]
def GraphInfo(self, run, graph_id):
"""Get the information regarding a TensorFlow graph.
Args:
run: Name of the run.
graph_id: Debugger-generated ID of the graph in question.
This information is available in the return values
of `GraphOpInfo`, `GraphExecution`, etc.
Returns:
A JSON-serializable object containing the information regarding
the TensorFlow graph.
Raises:
NotFoundError if the graph_id is not known to the debugger.
"""
runs = self.Runs()
if run not in runs:
return None
try:
graph = self._reader.graph_by_id(graph_id)
except KeyError:
raise errors.NotFoundError('There is no graph with ID "%s"' %
graph_id)
return graph.to_json()
def _decode_blob_key(key):
"""Decode a prefix (optional) and sub-key from a blob key.
Left inverse of `_encode_blob_key`.
Args:
key: A blob key in the form returned by `_encode_blob_key`.
Returns;
A tuple `(prefix, sub_key)`, where `prefix` is either `None` or a
sub-provider prefix, and `sub_key` is an opaque key from a
sub-provider.
Raises:
errors.NotFoundError: If `key` is invalid and has no preimage.
"""
failure = errors.NotFoundError("Invalid blob key: %r" % key)
b64_str = key + "==" # ensure adequate padding (overpadding is okay)
json_str = base64.urlsafe_b64decode(b64_str).decode("ascii")
payload = json.loads(json_str)
if not isinstance(payload, list) or len(payload) != 2:
raise failure
(prefix, sub_key) = payload
if not (prefix is None or isinstance(prefix, str)):
raise failure
if not isinstance(sub_key, str):
raise failure
return (prefix, sub_key)
def scalars_impl(self, tag, run):
"""Returns scalar data for the specified tag and run.
For details on how to use tags and runs, see
https://github.com/tensorflow/tensorboard#tags-giving-names-to-data
Args:
tag: string
run: string
Returns:
A list of ScalarEvents - tuples containing 3 numbers describing entries in
the data series.
Raises:
errors.NotFoundError: if run+tag pair has no scalar data.
"""
try:
tensor_events = self._multiplexer.Tensors(run, tag)
values = [(
tensor_event.wall_time,
tensor_event.step,
tensor_util.make_ndarray(tensor_event.tensor_proto).item(),
) for tensor_event in tensor_events]
except KeyError:
raise errors.NotFoundError("No scalar data for run=%r, tag=%r" %
(run, tag))
return values
def histograms_impl(self, ctx, tag, run, experiment, downsample_to=None):
"""Result of the form `(body, mime_type)`.
At most `downsample_to` events will be returned. If this value is
`None`, then default downsampling will be performed.
Raises:
tensorboard.errors.PublicError: On invalid request.
"""
sample_count = (
downsample_to if downsample_to is not None else self._downsample_to
)
all_histograms = self._data_provider.read_tensors(
ctx,
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=sample_count,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
histograms = all_histograms.get(run, {}).get(tag, None)
if histograms is None:
raise errors.NotFoundError(
"No histogram tag %r for run %r" % (tag, run)
)
events = [(e.wall_time, e.step, e.numpy.tolist()) for e in histograms]
return (events, "application/json")
def get_dataset_path(self, run, dataset):
try:
tensor_event = self._multiplexer.Tensors(
run, "{}_path".format(dataset))[0]
value = tensor_event.tensor_proto.string_val[0].decode("utf-8")
except KeyError:
raise errors.NotFoundError("No dataset recorded")
return value
def __call__(self, environ, start_response):
path = environ.get("PATH_INFO", "")
if path != self._path_prefix and not path.startswith(
self._strict_prefix):
raise errors.NotFoundError()
environ["PATH_INFO"] = path[len(self._path_prefix):]
environ["SCRIPT_NAME"] = (environ.get("SCRIPT_NAME", "") +
self._path_prefix)
return self._application(environ, start_response)
def _serve_plugin_entry(self, request):
"""Serves a HTML for iframed plugin entry point.
Args:
request: The werkzeug.Request object.
Returns:
A werkzeug.Response object.
"""
name = request.args.get('name')
plugins = [
plugin for plugin in self._plugins if plugin.plugin_name == name
]
if not plugins:
raise errors.NotFoundError(name)
if len(plugins) > 1:
# Technically is not possible as plugin names are unique and is checked
# by the check on __init__.
raise AssertionError(
'Plugin invariant error: multiple plugins with name {name} found: {list}'
.format(
name=name,
list=plugins,
))
plugin = plugins[0]
module_path = plugin.frontend_metadata().es_module_path
if not module_path:
return http_util.Respond(request,
'Plugin is not module loadable',
'text/plain',
code=400)
# non-self origin is blocked by CSP but this is a good invariant checking.
if urlparse.urlparse(module_path).netloc:
raise ValueError('Expected es_module_path to be non-absolute path')
module_json = json.dumps("." + module_path)
script_content = 'import({}).then((m) => void m.render());'.format(
module_json)
digest = hashlib.sha256(script_content.encode('utf-8')).digest()
script_sha = base64.b64encode(digest).decode("ascii")
html = textwrap.dedent("""
<!DOCTYPE html>
<head><base href="plugin/{name}/" /></head>
<body><script type="module">{script_content}</script></body>
""").format(name=name, script_content=script_content)
return http_util.Respond(
request,
html,
'text/html',
csp_scripts_sha256s=[script_sha],
)
def _translate_grpc_error():
try:
yield
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.INVALID_ARGUMENT:
raise errors.InvalidArgumentError(e.details())
if e.code() == grpc.StatusCode.NOT_FOUND:
raise errors.NotFoundError(e.details())
if e.code() == grpc.StatusCode.PERMISSION_DENIED:
raise errors.PermissionDeniedError(e.details())
raise
def _parse_eid(self, experiment_id):
"""Parse an experiment ID into prefix, sub-ID, and sub-provider.
The returned prefix may be `None` if this instance has an
unprefixed data provider registered. If the experiment ID is
invalid, this method may raise an `errors.NotFoundError`.
"""
parts = experiment_id.split(_SEPARATOR, 1)
if len(parts) == 1:
if self._unprefixed_provider is None:
raise errors.NotFoundError(
"No data provider found for unprefixed experiment ID: %r" %
experiment_id)
return (None, experiment_id, self._unprefixed_provider)
(prefix, sub_eid) = parts
sub_provider = self._providers.get(prefix)
if sub_provider is None:
raise errors.NotFoundError("Unknown prefix in experiment ID: %r" %
experiment_id)
return (prefix, sub_eid, sub_provider)
def read_blob(self, blob_key):
(
unused_experiment_id,
plugin_name,
run,
tag,
step,
index,
) = _decode_blob_key(blob_key)
summary_metadata = self._multiplexer.SummaryMetadata(run, tag)
if summary_metadata.data_class != summary_pb2.DATA_CLASS_BLOB_SEQUENCE:
raise errors.NotFoundError(blob_key)
tensor_events = self._multiplexer.Tensors(run, tag)
# In case of multiple events at this step, take first (arbitrary).
matching_step = next((e for e in tensor_events if e.step == step), None)
if not matching_step:
raise errors.NotFoundError("%s: no such step %r" % (blob_key, step))
tensor = tensor_util.make_ndarray(matching_step.tensor_proto)
return tensor[index]
def scalars_impl(self, tag, run, experiment, output_format):
"""Result of the form `(body, mime_type)`."""
if self._data_provider:
all_scalars = self._data_provider.read_scalars(
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=self._downsample_to,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
scalars = all_scalars.get(run, {}).get(tag, None)
if scalars is None:
raise errors.NotFoundError(
"No scalar data for run=%r, tag=%r" % (run, tag)
)
values = [(x.wall_time, x.step, x.value) for x in scalars]
else:
try:
tensor_events = self._multiplexer.Tensors(run, tag)
except KeyError:
raise errors.NotFoundError(
"No scalar data for run=%r, tag=%r" % (run, tag)
)
values = [
(
tensor_event.wall_time,
tensor_event.step,
tensor_util.make_ndarray(tensor_event.tensor_proto).item(),
)
for tensor_event in tensor_events
]
if output_format == OutputFormat.CSV:
string_io = StringIO()
writer = csv.writer(string_io)
writer.writerow(["Wall time", "Step", "Value"])
writer.writerows(values)
return (string_io.getvalue(), "text/csv")
else:
return (values, "application/json")
def serve_dataset_path(self, request):
model = request.args.get("model")
dataset = request.args.get("dataset")
# Get dataset path
try:
tensor_event = self._multiplexer.Tensors(
model, "{}_path".format(dataset))[0]
path = tensor_event.tensor_proto.string_val[0].decode("utd-8")
except KeyError:
raise errors.NotFoundError("No dataset path found")
dataset_path = {'path': path}
return http_util.Respond(request, dataset_path, "application/json")
def _get_mime_type(self, ctx, experiment, run, tag):
# TODO(@wchargin): Move this call from `/audio` (called many
# times) to `/tags` (called few times) to reduce data provider
# calls.
mapping = self._data_provider.list_blob_sequences(
ctx, experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME,
)
time_series = mapping.get(run, {}).get(tag, None)
if time_series is None:
raise errors.NotFoundError(
"No audio data for run=%r, tag=%r" % (run, tag)
)
parsed = metadata.parse_plugin_metadata(time_series.plugin_content)
return _MIME_TYPES.get(parsed.encoding, _DEFAULT_MIME_TYPE)
def SourceLines(self, run, index):
runs = self.Runs()
if run not in runs:
return None
try:
host_name, file_path = self._reader.source_file_list()[index]
except IndexError:
raise errors.NotFoundError(
"There is no source-code file at index %d" % index)
return {
"host_name": host_name,
"file_path": file_path,
"lines": self._reader.source_lines(host_name, file_path),
}
def _audio_response_for_run(self, ctx, experiment, run, tag, sample):
"""Builds a JSON-serializable object with information about audio.
Args:
run: The name of the run.
tag: The name of the tag the audio entries all belong to.
sample: The zero-indexed sample of the audio sample for which to
retrieve information. For instance, setting `sample` to `2` will
fetch information about only the third audio clip of each batch,
and steps with fewer than three audio clips will be omitted from
the results.
Returns:
A list of dictionaries containing the wall time, step, label,
content type, and query string for each audio entry.
"""
all_audio = self._data_provider.read_blob_sequences(
ctx,
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=self._downsample_to,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
audio = all_audio.get(run, {}).get(tag, None)
if audio is None:
raise errors.NotFoundError(
"No audio data for run=%r, tag=%r" % (run, tag)
)
content_type = self._get_mime_type(ctx, experiment, run, tag)
response = []
for datum in audio:
if len(datum.values) < sample:
continue
query = urllib.parse.urlencode(
{
"blob_key": datum.values[sample].blob_key,
"content_type": content_type,
}
)
response.append(
{
"wall_time": datum.wall_time,
"label": "",
"step": datum.step,
"contentType": content_type,
"query": query,
}
)
return response
def StackFrames(self, run, stack_frame_ids):
runs = self.Runs()
if run not in runs:
return None
stack_frames = []
for stack_frame_id in stack_frame_ids:
if stack_frame_id not in self._reader._stack_frame_by_id:
raise errors.NotFoundError(
"Cannot find stack frame with ID %s" % stack_frame_id)
# TODO(cais): Use public method (`stack_frame_by_id()`) when
# available.
# pylint: disable=protected-access
stack_frames.append(
self._reader._stack_frame_by_id[stack_frame_id])
# pylint: enable=protected-access
return {"stack_frames": stack_frames}
def scalars_route(self, request):
tag = request.args.get("tag")
run = request.args.get("run")
try:
events = self._multiplexer.Tensors('.', 'test')
result = { 'tag': tag, 'steps': {} }
print('events', len(events))
for event in events:
print('step', event.step)
tensor = tensor_util.make_ndarray(event.tensor_proto).tolist()
result['steps'][event.step] = tensor
except KeyError:
raise errors.NotFoundError(f'No scalar data for run={run}, tag={tag}')
return http_util.Respond(request, result, "application/json")
def _read_blob(self, ctx, experiment, plugin_names, run, tag):
for plugin_name in plugin_names:
blob_sequences = self._data_provider.read_blob_sequences(
ctx,
experiment_id=experiment,
plugin_name=plugin_name,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
downsample=1,
)
blob_sequence_data = blob_sequences.get(run, {}).get(tag, ())
try:
blob_ref = blob_sequence_data[0].values[0]
except IndexError:
continue
return self._data_provider.read_blob(ctx,
blob_key=blob_ref.blob_key)
raise errors.NotFoundError()
def scalars_impl(self, ctx, tag, run, experiment, output_format):
"""Result of the form `(body, mime_type)`."""
all_scalars = self._data_provider.read_scalars(
ctx,
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=self._downsample_to,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
scalars = all_scalars.get(run, {}).get(tag, None)
if scalars is None:
raise errors.NotFoundError("No scalar data for run=%r, tag=%r" %
(run, tag))
values = [(x.wall_time, x.step, x.value) for x in scalars]
if output_format == OutputFormat.CSV:
string_io = io.StringIO()
writer = csv.writer(string_io)
writer.writerow(["Wall time", "Step", "Value"])
writer.writerows(values)
return (string_io.getvalue(), "text/csv")
else:
return (values, "application/json")
def app(request):
raise errors.NotFoundError('no scalar data for run=foo, tag=bar')
def histograms_impl(self, tag, run, experiment, downsample_to=None):
"""Result of the form `(body, mime_type)`.
At most `downsample_to` events will be returned. If this value is
`None`, then no downsampling will be performed.
Raises:
tensorboard.errors.PublicError: On invalid request.
"""
if self._data_provider:
# Downsample reads to 500 histograms per time series, which is
# the default size guidance for histograms under the multiplexer
# loading logic.
SAMPLE_COUNT = downsample_to if downsample_to is not None else 500
all_histograms = self._data_provider.read_tensors(
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=SAMPLE_COUNT,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
histograms = all_histograms.get(run, {}).get(tag, None)
if histograms is None:
raise errors.NotFoundError("No histogram tag %r for run %r" %
(tag, run))
# Downsample again, even though the data provider is supposed to,
# because the multiplexer provider currently doesn't. (For
# well-behaved data providers, this is a no-op.)
if downsample_to is not None:
rng = random.Random(0)
histograms = _downsample(rng, histograms, downsample_to)
events = [(e.wall_time, e.step, e.numpy.tolist())
for e in histograms]
elif self._db_connection_provider:
# Serve data from the database.
db = self._db_connection_provider()
cursor = db.cursor()
# Prefetch the tag ID matching this run and tag.
cursor.execute(
"""
SELECT
tag_id
FROM Tags
JOIN Runs USING (run_id)
WHERE
Runs.run_name = :run
AND Tags.tag_name = :tag
AND Tags.plugin_name = :plugin
""",
{
"run": run,
"tag": tag,
"plugin": metadata.PLUGIN_NAME
},
)
row = cursor.fetchone()
if not row:
raise errors.NotFoundError("No histogram tag %r for run %r" %
(tag, run))
(tag_id, ) = row
# Fetch tensor values, optionally with linear-spaced sampling by step.
# For steps ranging from s_min to s_max and sample size k, this query
# divides the range into k - 1 equal-sized intervals and returns the
# lowest step at or above each of the k interval boundaries (which always
# includes s_min and s_max, and may be fewer than k results if there are
# intervals where no steps are present). For contiguous steps the results
# can be formally expressed as the following:
# [s_min + math.ceil(i / k * (s_max - s_min)) for i in range(0, k + 1)]
cursor.execute(
"""
SELECT
MIN(step) AS step,
computed_time,
data,
dtype,
shape
FROM Tensors
INNER JOIN (
SELECT
MIN(step) AS min_step,
MAX(step) AS max_step
FROM Tensors
/* Filter out NULL so we can use TensorSeriesStepIndex. */
WHERE series = :tag_id AND step IS NOT NULL
)
/* Ensure we omit reserved rows, which have NULL step values. */
WHERE series = :tag_id AND step IS NOT NULL
/* Bucket rows into sample_size linearly spaced buckets, or do
no sampling if sample_size is NULL. */
GROUP BY
IFNULL(:sample_size - 1, max_step - min_step)
* (step - min_step) / (max_step - min_step)
ORDER BY step
""",
{
"tag_id": tag_id,
"sample_size": downsample_to
},
)
events = [(computed_time, step,
self._get_values(data, dtype, shape))
for step, computed_time, data, dtype, shape in cursor]
else:
# Serve data from events files.
try:
tensor_events = self._multiplexer.Tensors(run, tag)
except KeyError:
raise errors.NotFoundError("No histogram tag %r for run %r" %
(tag, run))
if downsample_to is not None:
rng = random.Random(0)
tensor_events = _downsample(rng, tensor_events, downsample_to)
events = [[
e.wall_time,
e.step,
tensor_util.make_ndarray(e.tensor_proto).tolist(),
] for e in tensor_events]
return (events, "application/json")
def scalars_impl(self, tag, run, experiment, output_format):
"""Result of the form `(body, mime_type)`."""
if self._data_provider:
# Downsample reads to 1000 scalars per time series, which is the
# default size guidance for scalars under the multiplexer loading
# logic.
SAMPLE_COUNT = 1000
all_scalars = self._data_provider.read_scalars(
experiment_id=experiment,
plugin_name=metadata.PLUGIN_NAME,
downsample=SAMPLE_COUNT,
run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
)
scalars = all_scalars.get(run, {}).get(tag, None)
if scalars is None:
raise errors.NotFoundError(
'No scalar data for run=%r, tag=%r' % (run, tag))
values = [(x.wall_time, x.step, x.value) for x in scalars]
elif self._db_connection_provider:
db = self._db_connection_provider()
# We select for steps greater than -1 because the writer inserts
# placeholder rows en masse. The check for step filters out those rows.
cursor = db.execute(
'''
SELECT
Tensors.step,
Tensors.computed_time,
Tensors.data,
Tensors.dtype
FROM Tensors
JOIN Tags
ON Tensors.series = Tags.tag_id
JOIN Runs
ON Tags.run_id = Runs.run_id
WHERE
/* For backwards compatibility, ignore the experiment id
for matching purposes if it is empty. */
(:exp == '' OR Runs.experiment_id == CAST(:exp AS INT))
AND Runs.run_name = :run
AND Tags.tag_name = :tag
AND Tags.plugin_name = :plugin
AND Tensors.shape = ''
AND Tensors.step > -1
ORDER BY Tensors.step
''', dict(exp=experiment, run=run, tag=tag, plugin=metadata.PLUGIN_NAME))
values = [(wall_time, step, self._get_value(data, dtype_enum))
for (step, wall_time, data, dtype_enum) in cursor]
else:
try:
tensor_events = self._multiplexer.Tensors(run, tag)
except KeyError:
raise errors.NotFoundError(
'No scalar data for run=%r, tag=%r' % (run, tag))
values = [
(tensor_event.wall_time, tensor_event.step,
tensor_util.make_ndarray(tensor_event.tensor_proto).item())
for tensor_event in tensor_events
]
if output_format == OutputFormat.CSV:
string_io = StringIO()
writer = csv.writer(string_io)
writer.writerow(['Wall time', 'Step', 'Value'])
writer.writerows(values)
return (string_io.getvalue(), 'text/csv')
else:
return (values, 'application/json')
def _validate_eid(self, eid):
if eid not in self._eids:
raise errors.NotFoundError("%r not in %r" % (eid, self._eids))