def scalar_pb(tag, tensor, description=None):
"""Create a scalar tf.Summary protobuf.
Arguments:
tag: String tag for the summary.
tensor: A rank-0 `np.array` or a compatible python number type.
description: Optional long-form description for this summary, as a
`str`. Markdown is supported. Defaults to empty.
Returns:
A `tf.Summary` protobuf object.
"""
arr = np.array(tensor)
if arr.shape != ():
raise ValueError('Expected scalar shape for tensor, got shape: %s.'
% arr.shape)
if arr.dtype.kind not in ('b', 'i', 'u', 'f'): # bool, int, uint, float
raise ValueError('Cast %s to float is not supported' % arr.dtype.name)
tensor_proto = tf.make_tensor_proto(arr.astype(np.float32))
summary_metadata = metadata.create_summary_metadata(
display_name=None, description=description)
summary = tf.Summary()
summary.value.add(tag=tag,
metadata=summary_metadata,
tensor=tensor_proto)
return summary
def AddScalar(self, tag, wall_time=0, step=0, value=0):
event = tf.Event(
wall_time=wall_time,
step=step,
summary=tf.Summary(
value=[tf.Summary.Value(tag=tag, simple_value=value)]))
self.AddEvent(event)
def pb(name, data, display_name=None, description=None):
"""Create a legacy scalar summary protobuf.
Arguments:
name: A unique name for the generated summary, including any desired
name scopes.
data: A rank-0 `np.array` or array-like form (so raw `int`s and
`float`s are fine, too).
display_name: Optional name for this summary in TensorBoard, as a
`str`. Defaults to `name`.
description: Optional long-form description for this summary, as a
`str`. Markdown is supported. Defaults to empty.
Returns:
A `tf.Summary` protobuf object.
"""
data = np.array(data)
if data.shape != ():
raise ValueError('Expected scalar shape for data, saw shape: %s.'
% data.shape)
if data.dtype.kind not in ('b', 'i', 'u', 'f'): # bool, int, uint, float
raise ValueError('Cast %s to float is not supported' % data.dtype.name)
tensor = tf.make_tensor_proto(data.astype(np.float32))
if display_name is None:
display_name = name
summary_metadata = metadata.create_summary_metadata(
display_name=display_name, description=description)
summary = tf.Summary()
summary.value.add(tag='%s/scalar_summary' % name,
metadata=summary_metadata,
tensor=tensor)
return summary
def test_session_start_pb(self):
start_time_secs = 314160
session_start_info = plugin_data_pb2.SessionStartInfo(
model_uri="//model/uri",
group_name="session_group",
start_time_secs=start_time_secs)
session_start_info.hparams["param1"].string_value = "string"
# TODO: Fix nondeterminism.
# session_start_info.hparams["param2"].number_value = 5.0
# session_start_info.hparams["param3"].bool_value = False
self.assertEqual(
summary.session_start_pb(
hparams={
"param1": "string",
# "param2":5,
# "param3":False,
},
model_uri="//model/uri",
group_name="session_group",
start_time_secs=start_time_secs),
tf.Summary(value=[
tf.Summary.Value(
tag="_hparams_/session_start_info",
metadata=tf.SummaryMetadata(
plugin_data=tf.SummaryMetadata.PluginData(
plugin_name="hparams",
content=(plugin_data_pb2.HParamsPluginData(
version=0,
session_start_info=session_start_info).
SerializeToString()))))
]))
def _summary(tag, hparams_plugin_data):
"""Helper function for creating a summary holding the given HParamsPluginData
message.
"""
summary = tf.Summary()
summary.value.add(
tag=tag,
metadata=metadata.create_summary_metadata(hparams_plugin_data))
return summary
def _GenerateEventsData(self):
fw = tf.summary.FileWriter(self.log_dir)
event = tf.Event(
wall_time=1,
step=1,
summary=tf.Summary(
value=[tf.Summary.Value(tag='s1', simple_value=0)]))
fw.add_event(event)
fw.close()
def _value_from_op(self, op):
with tf.Session() as sess:
summary_pbtxt = sess.run(op)
summary = tf.Summary()
summary.ParseFromString(summary_pbtxt)
# There may be multiple values (e.g., for an image summary that emits
# multiple images in one batch). That's fine; we'll choose any
# representative value, assuming that they're homogeneous.
assert summary.value
return summary.value[0]
def WriteHistogramSeries(writer, tag, mu_sigma_tuples, n=20):
"""Write a sequence of normally distributed histograms to writer."""
step = 0
wall_time = _start_time
for [mean, stddev] in mu_sigma_tuples:
data = [random.normalvariate(mean, stddev) for _ in xrange(n)]
histo = _MakeHistogram(data)
summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=histo)])
event = tf.Event(wall_time=wall_time, step=step, summary=summary)
writer.add_event(event)
step += 10
wall_time += 100
def WriteScalarSeries(writer, tag, f, n=5):
"""Write a series of scalar events to writer, using f to create values."""
step = 0
wall_time = _start_time
for i in xrange(n):
v = f(i)
value = tf.Summary.Value(tag=tag, simple_value=v)
summary = tf.Summary(value=[value])
event = tf.Event(wall_time=wall_time, step=step, summary=summary)
writer.add_event(event)
step += 1
wall_time += 10
def raw_data_pb(name,
true_positive_counts,
false_positive_counts,
true_negative_counts,
false_negative_counts,
precision,
recall,
num_thresholds=None,
display_name=None,
description=None):
"""Create a PR curves summary protobuf from raw data values.
Args:
name: A tag attached to the summary. Used by TensorBoard for organization.
true_positive_counts: A rank-1 numpy array of true positive counts. Must
contain `num_thresholds` elements and be castable to float32.
false_positive_counts: A rank-1 numpy array of false positive counts. Must
contain `num_thresholds` elements and be castable to float32.
true_negative_counts: A rank-1 numpy array of true negative counts. Must
contain `num_thresholds` elements and be castable to float32.
false_negative_counts: A rank-1 numpy array of false negative counts. Must
contain `num_thresholds` elements and be castable to float32.
precision: A rank-1 numpy array of precision values. Must contain
`num_thresholds` elements and be castable to float32.
recall: A rank-1 numpy array of recall values. Must contain `num_thresholds`
elements and be castable to float32.
num_thresholds: Number of thresholds, evenly distributed in `[0, 1]`, to
compute PR metrics for. Should be an int `>= 2`.
display_name: Optional name for this summary in TensorBoard, as a `str`.
Defaults to `name`.
description: Optional long-form description for this summary, as a `str`.
Markdown is supported. Defaults to empty.
Returns:
A summary operation for use in a TensorFlow graph. See docs for the `op`
method for details on the float32 tensor produced by this summary.
"""
if display_name is None:
display_name = name
summary_metadata = metadata.create_summary_metadata(
display_name=display_name if display_name is not None else name,
description=description or '',
num_thresholds=num_thresholds)
summary = tf.Summary()
data = np.stack(
(true_positive_counts, false_positive_counts, true_negative_counts,
false_negative_counts, precision, recall))
tensor = tf.make_tensor_proto(np.float32(data), dtype=tf.float32)
summary.value.add(tag='%s/pr_curves' % name,
metadata=summary_metadata,
tensor=tensor)
return summary
def AddScalarTensor(self, tag, wall_time=0, step=0, value=0):
"""Add a rank-0 tensor event.
Note: This is not related to the scalar plugin; it's just a
convenience function to add an event whose contents aren't
important.
"""
tensor = tf.make_tensor_proto(float(value))
event = tf.Event(
wall_time=wall_time,
step=step,
summary=tf.Summary(
value=[tf.Summary.Value(tag=tag, tensor=tensor)]))
self.AddEvent(event)
def AddImage(self,
tag,
wall_time=0,
step=0,
encoded_image_string=b'imgstr',
width=150,
height=100):
image = tf.Summary.Image(encoded_image_string=encoded_image_string,
width=width,
height=height)
event = tf.Event(
wall_time=wall_time,
step=step,
summary=tf.Summary(value=[tf.Summary.Value(tag=tag, image=image)]))
self.AddEvent(event)
def normalize_summary_pb(self, pb):
"""Pass `pb`'s `TensorProto` through a marshalling roundtrip.
`TensorProto`s can be equal in value even if they are not identical
in representation, because data can be stored in either the
`tensor_content` field or the `${dtype}_value` field. This
normalization ensures a canonical form, and should be used before
comparing two `Summary`s for equality.
"""
result = tf.Summary()
result.MergeFrom(pb)
for value in result.value:
if value.HasField('tensor'):
new_tensor = tf.make_tensor_proto(tf.make_ndarray(
value.tensor))
value.ClearField('tensor')
value.tensor.MergeFrom(new_tensor)
return result
def AddAudio(self,
tag,
wall_time=0,
step=0,
encoded_audio_string=b'sndstr',
content_type='audio/wav',
sample_rate=44100,
length_frames=22050):
audio = tf.Summary.Audio(encoded_audio_string=encoded_audio_string,
content_type=content_type,
sample_rate=sample_rate,
length_frames=length_frames)
event = tf.Event(
wall_time=wall_time,
step=step,
summary=tf.Summary(value=[tf.Summary.Value(tag=tag, audio=audio)]))
self.AddEvent(event)
def pb(name, images, max_outputs=3, display_name=None, description=None):
"""Create an image summary protobuf.
This behaves as if you were to create an `op` with the same arguments
(wrapped with constant tensors where appropriate) and then execute
that summary op in a TensorFlow session.
Arguments:
name: A unique name for the generated summary, including any desired
name scopes.
images: An `np.array` representing pixel data with shape
`[k, h, w, c]`, where `k` is the number of images, `w` and `h` are
the width and height of the images, and `c` is the number of
channels, which should be 1, 3, or 4.
max_outputs: Optional `int`. At most this many images will be
emitted. If more than this many images are provided, the first
`max_outputs` many images will be used and the rest silently
discarded.
display_name: Optional name for this summary in TensorBoard, as a
`str`. Defaults to `name`.
description: Optional long-form description for this summary, as a
`str`. Markdown is supported. Defaults to empty.
Returns:
A `tf.Summary` protobuf object.
"""
images = np.array(images).astype(np.uint8)
if images.ndim != 4:
raise ValueError('Shape %r must have rank 4' % (images.shape, ))
limited_images = images[:max_outputs]
encoded_images = [util.encode_png(image) for image in limited_images]
(width, height) = (images.shape[2], images.shape[1])
content = [str(width), str(height)] + encoded_images
tensor = tf.make_tensor_proto(content, dtype=tf.string)
if display_name is None:
display_name = name
summary_metadata = metadata.create_summary_metadata(
display_name=display_name, description=description)
summary = tf.Summary()
summary.value.add(tag='%s/image_summary' % name,
metadata=summary_metadata,
tensor=tensor)
return summary
def test_session_end_pb(self):
end_time_secs = 1234.0
self.assertEqual(
summary.session_end_pb(api_pb2.STATUS_SUCCESS, end_time_secs),
tf.Summary(value=[
tf.Summary.Value(
tag="_hparams_/session_end_info",
metadata=tf.SummaryMetadata(
plugin_data=tf.SummaryMetadata.PluginData(
plugin_name="hparams",
content=(plugin_data_pb2.HParamsPluginData(
version=0,
session_end_info=(
plugin_data_pb2.SessionEndInfo(
status=api_pb2.STATUS_SUCCESS,
end_time_secs=end_time_secs,
))).SerializeToString()))))
]))
def _writeMetadata(self, logdir, summary_metadata, nonce=''):
"""Write to disk a summary with the given metadata.
Arguments:
logdir: a string
summary_metadata: a `SummaryMetadata` protobuf object
nonce: optional; will be added to the end of the event file name
to guarantee that multiple calls to this function do not stomp the
same file
"""
summary = tf.Summary()
summary.value.add(tensor=tf.make_tensor_proto(['po', 'ta', 'to'],
dtype=tf.string),
tag='you_are_it',
metadata=summary_metadata)
writer = tf.summary.FileWriter(logdir, filename_suffix=nonce)
writer.add_summary(summary.SerializeToString())
writer.close()
def test_experiment_pb(self):
hparam_infos = [
api_pb2.HParamInfo(name="param1",
display_name="display_name1",
description="foo",
type=api_pb2.DATA_TYPE_STRING,
domain_discrete=struct_pb2.ListValue(values=[
struct_pb2.Value(string_value='a'),
struct_pb2.Value(string_value='b')
])),
api_pb2.HParamInfo(name="param2",
display_name="display_name2",
description="bar",
type=api_pb2.DATA_TYPE_FLOAT64,
domain_interval=api_pb2.Interval(
min_value=-100.0, max_value=100.0))
]
metric_infos = [
api_pb2.MetricInfo(name=api_pb2.MetricName(tag="loss"),
dataset_type=api_pb2.DATASET_VALIDATION),
api_pb2.MetricInfo(name=api_pb2.MetricName(group="train/",
tag="acc"),
dataset_type=api_pb2.DATASET_TRAINING),
]
time_created_secs = 314159.0
self.assertEqual(
summary.experiment_pb(hparam_infos,
metric_infos,
time_created_secs=time_created_secs),
tf.Summary(value=[
tf.Summary.Value(
tag="_hparams_/experiment",
metadata=tf.SummaryMetadata(
plugin_data=tf.SummaryMetadata.PluginData(
plugin_name="hparams",
content=(plugin_data_pb2.HParamsPluginData(
version=0,
experiment=api_pb2.Experiment(
time_created_secs=time_created_secs,
hparam_infos=hparam_infos,
metric_infos=metric_infos)).
SerializeToString()))))
]))
def _WriteScalarSummaries(self, data, subdirs=('',)):
# Writes data to a tempfile in subdirs, and returns generator for the data.
# If subdirs is given, writes data identically to all subdirectories.
for subdir_ in subdirs:
subdir = os.path.join(self.logdir, subdir_)
self._MakeDirectoryIfNotExists(subdir)
sw = tf.summary.FileWriter(subdir)
for datum in data:
summary = tf.Summary()
if 'simple_value' in datum:
summary.value.add(tag=datum['tag'],
simple_value=datum['simple_value'])
sw.add_summary(summary, global_step=datum['step'])
elif 'histo' in datum:
summary.value.add(tag=datum['tag'], histo=tf.HistogramProto())
sw.add_summary(summary, global_step=datum['step'])
elif 'session_log' in datum:
sw.add_session_log(datum['session_log'], global_step=datum['step'])
sw.close()
def test_only_1_summary_generated(self):
"""Tests that the streaming op only generates 1 summary for PR curves.
This test was made in response to a bug in which calling the streaming op
actually introduced 2 tags.
"""
predictions = tf.constant([0.2, 0.4, 0.5, 0.6, 0.8], dtype=tf.float32)
labels = tf.constant([False, True, True, False, True], dtype=tf.bool)
_, update_op = summary.streaming_op(name='pr_curve',
predictions=predictions,
labels=labels,
num_thresholds=10)
with self.test_session() as sess:
sess.run(tf.local_variables_initializer())
sess.run(update_op)
summary_proto = tf.Summary()
summary_proto.ParseFromString(sess.run(tf.summary.merge_all()))
tags = [v.tag for v in summary_proto.value]
# Only 1 tag should have been introduced.
self.assertEqual(['pr_curve/pr_curves'], tags)
def pb(scalars_layout):
"""Creates a summary that contains a layout.
When users navigate to the custom scalars dashboard, they will see a layout
based on the proto provided to this function.
Args:
scalars_layout: The scalars_layout_pb2.Layout proto that specifies the
layout.
Returns:
A summary proto containing the layout.
"""
assert isinstance(scalars_layout, layout_pb2.Layout)
tensor = tf.make_tensor_proto(
scalars_layout.SerializeToString(), dtype=tf.string)
summary = tf.Summary()
summary.value.add(tag=metadata.CONFIG_SUMMARY_TAG,
metadata=_create_summary_metadata(),
tensor=tensor)
return summary
def AddHistogram(self,
tag,
wall_time=0,
step=0,
hmin=1,
hmax=2,
hnum=3,
hsum=4,
hsum_squares=5,
hbucket_limit=None,
hbucket=None):
histo = tf.HistogramProto(min=hmin,
max=hmax,
num=hnum,
sum=hsum,
sum_squares=hsum_squares,
bucket_limit=hbucket_limit,
bucket=hbucket)
event = tf.Event(
wall_time=wall_time,
step=step,
summary=tf.Summary(value=[tf.Summary.Value(tag=tag, histo=histo)]))
self.AddEvent(event)
def FakeScalarSummary(tag, value):
value = tf.Summary.Value(tag=tag, simple_value=value)
summary = tf.Summary(value=[value])
return summary
def pb(name, data, bucket_count=None, display_name=None, description=None):
"""Create a histogram summary protobuf.
Arguments:
name: A unique name for the generated summary, including any desired
name scopes.
data: A `np.array` or array-like form of any shape. Must have type
castable to `float`.
bucket_count: Optional positive `int`. The output will have this
many buckets, except in two edge cases. If there is no data, then
there are no buckets. If there is data but all points have the
same value, then there is one bucket whose left and right
endpoints are the same.
display_name: Optional name for this summary in TensorBoard, as a
`str`. Defaults to `name`.
description: Optional long-form description for this summary, as a
`str`. Markdown is supported. Defaults to empty.
Returns:
A `tf.Summary` protobuf object.
"""
if bucket_count is None:
bucket_count = DEFAULT_BUCKET_COUNT
data = np.array(data).flatten().astype(float)
if data.size == 0:
buckets = np.array([]).reshape((0, 3))
else:
min_ = np.min(data)
max_ = np.max(data)
range_ = max_ - min_
if range_ == 0:
center = min_
buckets = np.array([[center - 0.5, center + 0.5,
float(data.size)]])
else:
bucket_width = range_ / bucket_count
offsets = data - min_
bucket_indices = np.floor(offsets / bucket_width).astype(int)
clamped_indices = np.minimum(bucket_indices, bucket_count - 1)
one_hots = (np.array([clamped_indices
]).transpose() == np.arange(0, bucket_count)
) # broadcast
assert one_hots.shape == (data.size,
bucket_count), (one_hots.shape,
(data.size,
bucket_count))
bucket_counts = np.sum(one_hots, axis=0)
edges = np.linspace(min_, max_, bucket_count + 1)
left_edges = edges[:-1]
right_edges = edges[1:]
buckets = np.array([left_edges, right_edges,
bucket_counts]).transpose()
tensor = tf.make_tensor_proto(buckets, dtype=tf.float64)
if display_name is None:
display_name = name
summary_metadata = metadata.create_summary_metadata(
display_name=display_name, description=description)
summary = tf.Summary()
summary.value.add(tag='%s/histogram_summary' % name,
metadata=summary_metadata,
tensor=tensor)
return summary
def pb_via_op(self, summary_op, feed_dict=None):
with tf.Session() as sess:
actual_pbtxt = sess.run(summary_op, feed_dict=feed_dict or {})
actual_proto = tf.Summary()
actual_proto.ParseFromString(actual_pbtxt)
return actual_proto
def pb(name,
audio,
sample_rate,
labels=None,
max_outputs=3,
encoding=None,
display_name=None,
description=None):
"""Create an audio summary protobuf.
This behaves as if you were to create an `op` with the same arguments
(wrapped with constant tensors where appropriate) and then execute
that summary op in a TensorFlow session.
Arguments:
name: A unique name for the generated summary node.
audio: An `np.array` representing audio data with shape `[k, t, c]`,
where `k` is the number of audio clips, `t` is the number of
frames, and `c` is the number of channels. Elements should be
floating-point values in `[-1.0, 1.0]`.
sample_rate: An `int` that represents the sample rate, in Hz.
Must be positive.
labels: Optional list (or rank-1 `np.array`) of textstrings or UTF-8
bytestrings whose length is the first dimension of `audio`, where
`labels[i]` contains arbitrary textual information about
`audio[i]`. (For instance, this could be some text that a TTS
system was supposed to produce.) Markdown is supported.
max_outputs: Optional `int`. At most this many audio clips will be
emitted. When more than `max_outputs` many clips are provided, the
first `max_outputs` many clips will be used and the rest silently
discarded.
encoding: A constant `str` indicating the desired encoding. You
can choose any format you like, as long as it's "wav". Please see
the "API compatibility note" below.
display_name: Optional name for this summary in TensorBoard, as a
`str`. Defaults to `name`.
description: Optional long-form description for this summary, as a
`str`. Markdown is supported. Defaults to empty.
Returns:
A `tf.Summary` protobuf object.
API compatibility note: The default value of the `encoding`
argument is _not_ guaranteed to remain unchanged across TensorBoard
versions. In the future, we will by default encode as FLAC instead of
as WAV. If the specific format is important to you, please provide a
file format explicitly.
"""
audio = np.array(audio)
if audio.ndim != 3:
raise ValueError('Shape %r must have rank 3' % (audio.shape, ))
if encoding is None:
encoding = 'wav'
if encoding == 'wav':
encoding = metadata.Encoding.Value('WAV')
encoder = functools.partial(util.encode_wav,
samples_per_second=sample_rate)
else:
raise ValueError('Unknown encoding: %r' % encoding)
limited_audio = audio[:max_outputs]
if labels is None:
limited_labels = [b''] * len(limited_audio)
else:
limited_labels = [
tf.compat.as_bytes(label) for label in labels[:max_outputs]
]
encoded_audio = [encoder(a) for a in limited_audio]
content = np.array([encoded_audio, limited_labels]).transpose()
tensor = tf.make_tensor_proto(content, dtype=tf.string)
if display_name is None:
display_name = name
summary_metadata = metadata.create_summary_metadata(
display_name=display_name, description=description, encoding=encoding)
summary = tf.Summary()
summary.value.add(tag='%s/audio_summary' % name,
metadata=summary_metadata,
tensor=tensor)
return summary
def tensor_via_op(self, summary_op):
actual_pbtxt = summary_op.eval()
actual_proto = tf.Summary()
actual_proto.ParseFromString(actual_pbtxt)
return actual_proto
