def _histogram_summary(self, tf_name, value, step=None):
"""
Args:
tf_name (str): name of tensorflow variable
value (tuple or list): either a tuple of bin_edges and bincounts or
a list of values to summarize in a histogram.
References:
https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/04-utils/tensorboard/logger.py#L45
Example:
>>> tf_name = 'foo'
>>> value = ([0, 1, 2, 3, 4, 5], [1, 20, 10, 22, 11])
>>> self = Logger(None, is_dummy=True)
>>> summary = self._histogram_summary(tf_name, value, step=None)
>>> assert summary.value[0].histo.max == 5
Example:
>>> tf_name = 'foo'
>>> value = [0.72, 0.18, 0.34, 0.66, 0.11, 0.70, 0.23]
>>> self = Logger(None, is_dummy=True)
>>> summary = self._histogram_summary(tf_name, value, step=None)
>>> assert summary.value[0].histo.num == 7.0
"""
if isinstance(value, tuple):
bin_edges, bincounts = value
assert len(bin_edges) == len(bincounts) + 1, (
'must have one more edge than count')
hist = summary_pb2.HistogramProto()
hist.min = float(min(bin_edges))
hist.max = float(max(bin_edges))
else:
values = np.array(value)
bincounts, bin_edges = np.histogram(values)
hist = summary_pb2.HistogramProto()
hist.min = float(np.min(values))
hist.max = float(np.max(values))
hist.num = int(np.prod(values.shape))
hist.sum = float(np.sum(values))
hist.sum_squares = float(np.sum(values**2))
# Add bin edges and counts
for edge in bin_edges[1:]:
hist.bucket_limit.append(edge)
for v in bincounts:
hist.bucket.append(v)
summary = summary_pb2.Summary()
summary.value.add(tag=tf_name, histo=hist)
return summary
def AddHistogram(self,
tag,
wall_time=0,
step=0,
hmin=1,
hmax=2,
hnum=3,
hsum=4,
hsum_squares=5,
hbucket_limit=None,
hbucket=None):
histo = summary_pb2.HistogramProto(
min=hmin,
max=hmax,
num=hnum,
sum=hsum,
sum_squares=hsum_squares,
bucket_limit=hbucket_limit,
bucket=hbucket)
event = event_pb2.Event(
wall_time=wall_time,
step=step,
summary=summary_pb2.Summary(
value=[summary_pb2.Summary.Value(
tag=tag, histo=histo)]))
self.AddEvent(event)
def value(self):
"""Retrieves the current distribution of samples.
Returns:
A HistogramProto describing the distribution of samples.
"""
with c_api_util.tf_buffer() as buffer_:
pywrap_tfe.TFE_MonitoringSamplerCellValue(self._cell, buffer_)
proto_data = pywrap_tf_session.TF_GetBuffer(buffer_)
histogram_proto = summary_pb2.HistogramProto()
histogram_proto.ParseFromString(compat.as_bytes(proto_data))
return histogram_proto
def _MakeHistogram(values):
"""Convert values into a histogram proto using logic from histogram.cc."""
limits = _MakeHistogramBuckets()
counts = [0] * len(limits)
for v in values:
idx = bisect.bisect_left(limits, v)
counts[idx] += 1
limit_counts = [(limits[i], counts[i]) for i in xrange(len(limits))
if counts[i]]
bucket_limit = [lc[0] for lc in limit_counts]
bucket = [lc[1] for lc in limit_counts]
sum_sq = sum(v * v for v in values)
return summary_pb2.HistogramProto(min=min(values),
max=max(values),
num=len(values),
sum=sum(values),
sum_squares=sum_sq,
bucket_limit=bucket_limit,
bucket=bucket)
def _WriteScalarSummaries(self, data, subdirs=('', )):
# Writes data to a tempfile in subdirs, and returns generator for the data.
# If subdirs is given, writes data identically to all subdirectories.
for subdir_ in subdirs:
subdir = os.path.join(self.logdir, subdir_)
self._MakeDirectoryIfNotExists(subdir)
sw = summary_lib.summary.FileWriter(subdir)
for datum in data:
summary = summary_pb2.Summary()
if 'simple_value' in datum:
summary.value.add(tag=datum['tag'],
simple_value=datum['simple_value'])
sw.add_summary(summary, global_step=datum['step'])
elif 'histo' in datum:
summary.value.add(tag=datum['tag'],
histo=summary_pb2.HistogramProto())
sw.add_summary(summary, global_step=datum['step'])
elif 'session_log' in datum:
sw.add_session_log(datum['session_log'],
global_step=datum['step'])
sw.close()
def _get_histogram_proto(self, proto_bytes):
histogram_proto = summary_pb2.HistogramProto()
histogram_proto.ParseFromString(proto_bytes)
return histogram_proto
def _get_read_histogram_proto(self, api_label): proto_bytes = metrics.GetCheckpointReadDurations(api_label=api_label) histogram_proto = summary_pb2.HistogramProto() histogram_proto.ParseFromString(proto_bytes) return histogram_proto
def _GenerateTestData(self):
"""Generates the test data directory.
The test data has a single run named run1 which contains:
- a histogram
- an image at timestamp and step 0
- scalar events containing the value i at step 10 * i and wall time
100 * i, for i in [1, _SCALAR_COUNT).
- a graph definition
Returns:
temp_dir: The directory the test data is generated under.
"""
temp_dir = tempfile.mkdtemp(prefix=self.get_temp_dir())
self.addCleanup(shutil.rmtree, temp_dir)
run1_path = os.path.join(temp_dir, 'run1')
os.makedirs(run1_path)
writer = writer_lib.FileWriter(run1_path)
histogram_value = summary_pb2.HistogramProto(
min=0,
max=2,
num=3,
sum=6,
sum_squares=5,
bucket_limit=[0, 1, 2],
bucket=[1, 1, 1])
# Add a simple graph event.
graph_def = graph_pb2.GraphDef()
node1 = graph_def.node.add()
node1.name = 'a'
node2 = graph_def.node.add()
node2.name = 'b'
node2.attr['very_large_attr'].s = b'a' * 2048 # 2 KB attribute
meta_graph_def = meta_graph_pb2.MetaGraphDef(graph_def=graph_def)
if self._only_use_meta_graph:
writer.add_meta_graph(meta_graph_def)
else:
writer.add_graph(graph_def)
# Add a simple run metadata event.
run_metadata = config_pb2.RunMetadata()
device_stats = run_metadata.step_stats.dev_stats.add()
device_stats.device = 'test device'
writer.add_run_metadata(run_metadata, 'test run')
# 1x1 transparent GIF.
encoded_image = base64.b64decode(
'R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7')
image_value = summary_pb2.Summary.Image(
height=1, width=1, colorspace=1, encoded_image_string=encoded_image)
audio_value = summary_pb2.Summary.Audio(
sample_rate=44100,
length_frames=22050,
num_channels=2,
encoded_audio_string=b'',
content_type='audio/wav')
writer.add_event(
event_pb2.Event(
wall_time=0,
step=0,
summary=summary_pb2.Summary(value=[
summary_pb2.Summary.Value(
tag='histogram', histo=histogram_value),
summary_pb2.Summary.Value(
tag='image', image=image_value), summary_pb2.Summary.Value(
tag='audio', audio=audio_value)
])))
# Write 100 simple values.
for i in xrange(1, self._SCALAR_COUNT + 1):
writer.add_event(
event_pb2.Event(
# We use different values for wall time, step, and the value so we
# can tell them apart.
wall_time=100 * i,
step=10 * i,
summary=summary_pb2.Summary(value=[
summary_pb2.Summary.Value(
tag='simple_values', simple_value=i)
])))
writer.flush()
writer.close()
return temp_dir
