def custom_scalars(layout):
categories = []
for k, v in layout.items():
charts = []
for chart_name, chart_meatadata in v.items():
tags = chart_meatadata[1]
if chart_meatadata[0] == "Margin":
assert len(tags) == 3
mgcc = layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value=tags[0], lower=tags[1], upper=tags[2])
])
chart = layout_pb2.Chart(title=chart_name, margin=mgcc)
else:
mlcc = layout_pb2.MultilineChartContent(tag=tags)
chart = layout_pb2.Chart(title=chart_name, multiline=mlcc)
charts.append(chart)
categories.append(layout_pb2.Category(title=k, chart=charts))
layout = layout_pb2.Layout(category=categories)
plugin_data = SummaryMetadata.PluginData(plugin_name="custom_scalars")
smd = SummaryMetadata(plugin_data=plugin_data)
tensor = TensorProto(
dtype="DT_STRING",
string_val=[layout.SerializeToString()],
tensor_shape=TensorShapeProto(),
)
return Summary(value=[
Summary.Value(
tag="custom_scalars__config__", tensor=tensor, metadata=smd)
])
def testMergedLayout(self):
ctx = context.RequestContext()
parsed_layout = layout_pb2.Layout()
json_format.Parse(self.plugin.layout_impl(ctx, "exp_id"), parsed_layout)
correct_layout = layout_pb2.Layout(
category=[
# A category with this name is also present in a layout for a
# different run (the logdir run)
layout_pb2.Category(
title="cross entropy",
chart=[
layout_pb2.Chart(
title="cross entropy",
multiline=layout_pb2.MultilineChartContent(
tag=[r"cross entropy"],
),
),
layout_pb2.Chart(
title="cross entropy margin chart",
margin=layout_pb2.MarginChartContent(
series=[
layout_pb2.MarginChartContent.Series(
value="cross entropy",
lower="cross entropy lower",
upper="cross entropy upper",
),
],
),
),
],
closed=True,
),
layout_pb2.Category(
title="mean biases",
chart=[
layout_pb2.Chart(
title="mean layer biases",
multiline=layout_pb2.MultilineChartContent(
tag=[
r"mean/layer0/biases",
r"mean/layer1/biases",
],
),
),
],
),
layout_pb2.Category(
title="std weights",
chart=[
layout_pb2.Chart(
title="stddev layer weights",
multiline=layout_pb2.MultilineChartContent(
tag=[r"stddev/layer\d+/weights"],
),
),
],
),
]
)
self.assertProtoEquals(correct_layout, parsed_layout)
def testSetLayout(self):
layout_proto_to_write = layout_pb2.Layout(category=[
layout_pb2.Category(
title='mean biases',
chart=[
layout_pb2.Chart(
title='mean layer biases',
multiline=layout_pb2.MultilineChartContent(
tag=[r'mean/layer\d+/biases'], )),
]),
layout_pb2.Category(
title='std weights',
chart=[
layout_pb2.Chart(
title='stddev layer weights',
multiline=layout_pb2.MultilineChartContent(
tag=[r'stddev/layer\d+/weights'], )),
]),
layout_pb2.Category(
title='cross entropy ... and maybe some other values',
chart=[
layout_pb2.Chart(
title='cross entropy',
multiline=layout_pb2.MultilineChartContent(
tag=[r'cross entropy'], )),
layout_pb2.Chart(
title='accuracy',
margin=layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value='accuracy',
lower='accuracy_lower_margin',
upper='accuracy_upper_margin')
])),
layout_pb2.Chart(
title='max layer weights',
multiline=layout_pb2.MultilineChartContent(
tag=[r'max/layer1/.*', r'max/layer2/.*'], )),
],
closed=True)
])
# Write the data as a summary for the '.' run.
with tf.Session() as s, tf.summary.FileWriter(self.logdir) as writer:
writer.add_summary(s.run(summary.op(layout_proto_to_write)))
# Read the data from disk.
multiplexer = event_multiplexer.EventMultiplexer()
multiplexer.AddRunsFromDirectory(self.logdir)
multiplexer.Reload()
tensor_events = multiplexer.Tensors('.', metadata.CONFIG_SUMMARY_TAG)
self.assertEqual(1, len(tensor_events))
# Parse the data.
string_array = tf.make_ndarray(tensor_events[0].tensor_proto)
content = np.asscalar(string_array)
layout_proto_from_disk = layout_pb2.Layout()
layout_proto_from_disk.ParseFromString(tf.compat.as_bytes(content))
# Verify the content.
self.assertProtoEquals(layout_proto_to_write, layout_proto_from_disk)
def class_score_mmm(name='class_score'):
# Create a custom scalar summary as a margin chart.
# Displays the mean value and the min and max values as the upper and lower limits.
tf_class_scores = tf.placeholder(tf.float32, shape=(None, ))
tf_mean_op = summary_lib.scalar(name, tf.math.reduce_mean(tf_class_scores))
tf_min_op = summary_lib.scalar(name + '/min',
tf.math.reduce_min(tf_class_scores))
tf_max_op = summary_lib.scalar(name + '/max',
tf.math.reduce_max(tf_class_scores))
tf_class_score_chart = layout_pb2.Chart(
title=name,
margin=layout_pb2.MarginChartContent(
series=[
layout_pb2.MarginChartContent.Series(
value=tf_mean_op.name.split(
':')[0], #name + '/scalar_summary',
lower=tf_min_op.name.split(':')
[0], #name + '/min/scalar_summary',
upper=tf_max_op.name.split(':')[
0] #name + '/max/scalar_summary'
),
], ))
return tf_class_scores, tf_class_score_chart
def finalize_autosummaries() -> None:
"""Create the necessary ops to include autosummaries in TensorBoard report.
Note: This should be done only once per graph.
"""
global _finalized
tfutil.assert_tf_initialized()
if _finalized:
return None
_finalized = True
tfutil.init_uninitialized_vars([var for vars_list in _vars.values() for var in vars_list])
# Create summary ops.
with tf.device(None), tf.control_dependencies(None):
for name, vars_list in _vars.items():
name_id = name.replace("/", "_")
with tfutil.absolute_name_scope("Autosummary/" + name_id):
moments = tf.add_n(vars_list)
moments /= moments[0]
with tf.control_dependencies([moments]): # read before resetting
reset_ops = [tf.assign(var, tf.zeros(3, dtype=_dtype)) for var in vars_list]
with tf.name_scope(None), tf.control_dependencies(reset_ops): # reset before reporting
mean = moments[1]
std = tf.sqrt(moments[2] - tf.square(moments[1]))
tf.summary.scalar(name, mean)
if enable_custom_scalars:
tf.summary.scalar("xCustomScalars/" + name + "/margin_lo", mean - std)
tf.summary.scalar("xCustomScalars/" + name + "/margin_hi", mean + std)
# Setup layout for custom scalars.
layout = None
if enable_custom_scalars:
cat_dict = OrderedDict()
for series_name in sorted(_vars.keys()):
p = series_name.split("/")
cat = p[0] if len(p) >= 2 else ""
chart = "/".join(p[1:-1]) if len(p) >= 3 else p[-1]
if cat not in cat_dict:
cat_dict[cat] = OrderedDict()
if chart not in cat_dict[cat]:
cat_dict[cat][chart] = []
cat_dict[cat][chart].append(series_name)
categories = []
for cat_name, chart_dict in cat_dict.items():
charts = []
for chart_name, series_names in chart_dict.items():
series = []
for series_name in series_names:
series.append(layout_pb2.MarginChartContent.Series(
value=series_name,
lower="xCustomScalars/" + series_name + "/margin_lo",
upper="xCustomScalars/" + series_name + "/margin_hi"))
margin = layout_pb2.MarginChartContent(series=series)
charts.append(layout_pb2.Chart(title=chart_name, margin=margin))
categories.append(layout_pb2.Category(title=cat_name, chart=charts))
layout = summary_lib.custom_scalar_pb(layout_pb2.Layout(category=categories))
return layout
def get_layout_summary(
*,
metrics: Tuple[DictConfig, ...],
tasks: Tuple[DictConfig, ...],
closed: bool = False,
):
"""Builds a summary that describes custom scalars layout for TensorBoard.
At each evaluation step, we compute metrics for multiple tasks sampled from
the same distribution and estimate the mean and std value of the metric.
The layout specified by this function nicely groups metrics and defines
margin charts that use fill area to visualize lower and upper bounds.
Parameters
----------
metrics : tuple of DictConfigs
tasks : tuple of DictConfigs
closed : bool (default: False)
Returns
-------
A summary proto containing the layout.
"""
set_names = sorted(set(t.set_name for t in tasks))
task_regimes = sorted(set(t.regime for t in tasks))
layout_summary = summary_lib.custom_scalar_pb(
layout_pb2.Layout(category=[
# Category for each metric.
layout_pb2.Category(
title=m.name,
chart=[
# A chart for each type of the eval distribution.
layout_pb2.Chart(
title=f"{t_regime}/{m.name} (CI {m.ci:.0f}%)",
margin=layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value=
f"{s_name}/{t_regime}/{m.name}_mean/scalar_summary",
lower=
f"{s_name}/{t_regime}/{m.name}_lower/scalar_summary",
upper=
f"{s_name}/{t_regime}/{m.name}_upper/scalar_summary",
) for s_name in set_names
]),
) for t_regime in task_regimes
],
closed=closed,
) for m in metrics
]))
return layout_summary
def layout_dashboard(writer):
from tensorboard import summary
from tensorboard.plugins.custom_scalar import layout_pb2
# This action does not have to be performed at every step, so the action is not
# taken care of by an op in the graph. We only need to specify the layout once.
# We only need to specify the layout once (instead of per step).
layout_summary = summary.custom_scalar_pb(layout_pb2.Layout(
category=[
layout_pb2.Category(
title='losses',
chart=[
layout_pb2.Chart(
title='losses',
multiline=layout_pb2.MultilineChartContent(
tag=[r'loss.*'],
)),
layout_pb2.Chart(
title='baz',
margin=layout_pb2.MarginChartContent(
series=[
layout_pb2.MarginChartContent.Series(
value='loss/baz/scalar_summary',
lower='baz_lower/baz/scalar_summary',
upper='baz_upper/baz/scalar_summary'),
],
)),
]),
layout_pb2.Category(
title='trig functions',
chart=[
layout_pb2.Chart(
title='wave trig functions',
multiline=layout_pb2.MultilineChartContent(
tag=[r'trigFunctions/cosine', r'trigFunctions/sine'],
)),
# The range of tangent is different. Let's give it its own chart.
layout_pb2.Chart(
title='tan',
multiline=layout_pb2.MultilineChartContent(
tag=[r'trigFunctions/tangent'],
)),
],
# This category we care less about. Let's make it initially closed.
closed=True),
]))
writer.add_summary(layout_summary)
def create_layout_summary():
return cs_summary.pb(
layout_pb2.Layout(category=[
layout_pb2.Category(
title="losses",
chart=[
layout_pb2.Chart(
title="losses",
multiline=layout_pb2.MultilineChartContent(
tag=[r"loss(?!.*margin.*)"], ),
),
layout_pb2.Chart(
title="baz",
margin=layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value="loss/baz",
lower="loss/baz_lower",
upper="loss/baz_upper",
),
], ),
),
],
),
layout_pb2.Category(
title="trig functions",
chart=[
layout_pb2.Chart(
title="wave trig functions",
multiline=layout_pb2.MultilineChartContent(tag=[
r"trigFunctions/cosine",
r"trigFunctions/sine",
], ),
),
# The range of tangent is different. Give it its own chart.
layout_pb2.Chart(
title="tan",
multiline=layout_pb2.MultilineChartContent(
tag=[r"trigFunctions/tangent"], ),
),
],
# This category we care less about. Make it initially closed.
closed=True,
),
]))
def testMergedLayout(self):
parsed_layout = layout_pb2.Layout()
json_format.Parse(self.plugin.layout_impl(), parsed_layout)
correct_layout = layout_pb2.Layout(category=[
# A category with this name is also present in a layout for a
# different run (the logdir run)
layout_pb2.Category(
title='cross entropy',
chart=[
layout_pb2.Chart(
title='cross entropy',
multiline=layout_pb2.MultilineChartContent(
tag=[r'cross entropy'], )),
layout_pb2.Chart(
title='cross entropy margin chart',
margin=layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value='cross entropy',
lower='cross entropy lower',
upper='cross entropy upper'),
], )),
],
closed=True,
),
layout_pb2.Category(
title='mean biases',
chart=[
layout_pb2.Chart(
title='mean layer biases',
multiline=layout_pb2.MultilineChartContent(tag=[
r'mean/layer0/biases', r'mean/layer1/biases'
], )),
]),
layout_pb2.Category(
title='std weights',
chart=[
layout_pb2.Chart(
title='stddev layer weights',
multiline=layout_pb2.MultilineChartContent(
tag=[r'stddev/layer\d+/weights'], )),
]),
])
self.assertProtoEquals(correct_layout, parsed_layout)
def __init__(self, *args, **kwargs):
super(CustomScalarsPluginTest, self).__init__(*args, **kwargs)
self.logdir = os.path.join(self.get_temp_dir(), "logdir")
os.makedirs(self.logdir)
self.logdir_layout = layout_pb2.Layout(category=[
layout_pb2.Category(
title="cross entropy",
chart=[
layout_pb2.Chart(
title="cross entropy",
multiline=layout_pb2.MultilineChartContent(
tag=[r"cross entropy"], ),
),
],
closed=True,
)
])
self.foo_layout = layout_pb2.Layout(category=[
layout_pb2.Category(
title="mean biases",
chart=[
layout_pb2.Chart(
title="mean layer biases",
multiline=layout_pb2.MultilineChartContent(tag=[
r"mean/layer0/biases",
r"mean/layer1/biases",
], ),
),
],
),
layout_pb2.Category(
title="std weights",
chart=[
layout_pb2.Chart(
title="stddev layer weights",
multiline=layout_pb2.MultilineChartContent(
tag=[r"stddev/layer\d+/weights"], ),
),
],
),
# A category with this name is also present in a layout for a
# different run (the logdir run) and also contains a duplicate chart
layout_pb2.Category(
title="cross entropy",
chart=[
layout_pb2.Chart(
title="cross entropy margin chart",
margin=layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value="cross entropy",
lower="cross entropy lower",
upper="cross entropy upper",
),
], ),
),
layout_pb2.Chart(
title="cross entropy",
multiline=layout_pb2.MultilineChartContent(
tag=[r"cross entropy"], ),
),
],
),
])
# Generate test data.
with test_util.FileWriterCache.get(os.path.join(self.logdir,
"foo")) as writer:
writer.add_summary(
test_util.ensure_tb_summary_proto(summary.pb(self.foo_layout)))
for step in range(4):
writer.add_summary(
test_util.ensure_tb_summary_proto(
scalar_summary.pb("squares", step * step)),
step,
)
with test_util.FileWriterCache.get(os.path.join(self.logdir,
"bar")) as writer:
for step in range(3):
writer.add_summary(
test_util.ensure_tb_summary_proto(
scalar_summary.pb("increments", step + 1)),
step,
)
# The '.' run lacks scalar data but has a layout.
with test_util.FileWriterCache.get(self.logdir) as writer:
writer.add_summary(
test_util.ensure_tb_summary_proto(
summary.pb(self.logdir_layout)))
self.plugin = self.createPlugin(self.logdir)
def run():
"""Run custom scalar demo and generate event files."""
step = tf.placeholder(tf.float32, shape=[])
with tf.name_scope('loss'):
# Specify 2 different loss values, each tagged differently.
summary_lib.scalar('foo', tf.pow(0.9, step))
summary_lib.scalar('bar', tf.pow(0.85, step + 2))
# Log metric baz as well as upper and lower bounds for a margin chart.
middle_baz_value = step + 4 * tf.random_uniform([]) - 2
summary_lib.scalar('baz', middle_baz_value)
summary_lib.scalar('baz_lower',
middle_baz_value - 6.42 - tf.random_uniform([]))
summary_lib.scalar('baz_upper',
middle_baz_value + 6.42 + tf.random_uniform([]))
with tf.name_scope('trigFunctions'):
summary_lib.scalar('cosine', tf.cos(step))
summary_lib.scalar('sine', tf.sin(step))
summary_lib.scalar('tangent', tf.tan(step))
merged_summary = tf.summary.merge_all()
with tf.Session() as sess, tf.summary.FileWriter(LOGDIR) as writer:
# We only need to specify the layout once (instead of per step).
layout_summary = summary_lib.custom_scalar_pb(
layout_pb2.Layout(category=[
layout_pb2.Category(
title='losses',
chart=[
layout_pb2.Chart(
title='losses',
multiline=layout_pb2.MultilineChartContent(
tag=[r'loss(?!.*margin.*)'], )),
layout_pb2.Chart(
title='baz',
margin=layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value='loss/baz/scalar_summary',
lower='loss/baz_lower/scalar_summary',
upper='loss/baz_upper/scalar_summary'),
], )),
]),
layout_pb2.Category(
title='trig functions',
chart=[
layout_pb2.Chart(
title='wave trig functions',
multiline=layout_pb2.MultilineChartContent(tag=[
r'trigFunctions/cosine', r'trigFunctions/sine'
], )),
# The range of tangent is different. Give it its own chart.
layout_pb2.Chart(
title='tan',
multiline=layout_pb2.MultilineChartContent(
tag=[r'trigFunctions/tangent'], )),
],
# This category we care less about. Make it initially closed.
closed=True),
]))
writer.add_summary(layout_summary)
for i in xrange(42):
summary = sess.run(merged_summary, feed_dict={step: i})
writer.add_summary(summary, global_step=i)
def write_combined_events(dpath, d_combined, dname='combined'):
# ['reward', 'floor', 'reward.std', 'floor.std', 'steps', 'FPS', 'value_loss', 'action_loss_', 'dist_entropy_']
fpath = os.path.join(dpath, dname)
writer = tf.summary.FileWriter(fpath)
tags, values = zip(*d_combined.items())
# We only need to specify the layout once (instead of per step).
layout_summary = summary_lib.custom_scalar_pb(
layout_pb2.Layout(category=[
layout_pb2.Category(
title='losses',
chart=[
# layout_pb2.Chart(
# title='losses',
# multiline=layout_pb2.MultilineChartContent(
# tag=[r'loss(?!.*margin.*)'],)),
layout_pb2.Chart(
title='floor',
margin=layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value='floor/mean',
lower='floor/std_lower',
upper='floor/std_upper'),
], )),
# layout_pb2.Chart(
# title='floor',
# margin=layout_pb2.MarginChartContent(
# series=[
# layout_pb2.MarginChartContent.Series(
# value='floor/mean/scalar_summary',
# lower='floor/std_lower/scalar_summary',
# upper='floor/std_upper/scalar_summary'
# ),
# ],)),
]),
# layout_pb2.Category(
# title='trig functions',
# chart=[
# layout_pb2.Chart(
# title='wave trig functions',
# multiline=layout_pb2.MultilineChartContent(
# tag=[
# r'trigFunctions/cosine', r'trigFunctions/sine'
# ],)),
# # The range of tangent is different. Give it its own chart.
# layout_pb2.Chart(
# title='tan',
# multiline=layout_pb2.MultilineChartContent(
# tag=[r'trigFunctions/tangent'],)),
# ],
# # This category we care less about. Make it initially closed.
# closed=True),
]))
writer.add_summary(layout_summary)
floor = d_combined['floor']
floor_std = d_combined['floor.std']
# for tag, tuples in zip(tags, values):
# for t in tuples:
for i, t in enumerate(floor):
step = t[0]
mean = t[1]
lower = mean - floor_std[i][1]
upper = mean + floor_std[i][1]
summary = tf.Summary(
value=[tf.Summary.Value(tag='floor/mean', simple_value=mean)])
writer.add_summary(summary, global_step=step)
summary = tf.Summary(value=[
tf.Summary.Value(tag='floor/std_lower', simple_value=lower)
])
writer.add_summary(summary, global_step=step)
summary = tf.Summary(value=[
tf.Summary.Value(tag='floor/std_upper', simple_value=upper)
])
writer.add_summary(summary, global_step=step)
# for tag, tuples in zip(tags, values):
# for t in tuples:
# step = t[0]
# value = t[1]
# summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)])
# writer.add_summary(summary, global_step=step)
writer.flush()
def __init__(self, *args, **kwargs):
super(CustomScalarsPluginTest, self).__init__(*args, **kwargs)
self.logdir = os.path.join(self.get_temp_dir(), 'logdir')
os.makedirs(self.logdir)
self.logdir_layout = layout_pb2.Layout(category=[
layout_pb2.Category(
title='cross entropy',
chart=[
layout_pb2.Chart(
title='cross entropy',
multiline=layout_pb2.MultilineChartContent(
tag=[r'cross entropy'], )),
],
closed=True)
])
self.foo_layout = layout_pb2.Layout(category=[
layout_pb2.Category(
title='mean biases',
chart=[
layout_pb2.Chart(
title='mean layer biases',
multiline=layout_pb2.MultilineChartContent(tag=[
r'mean/layer0/biases', r'mean/layer1/biases'
], )),
]),
layout_pb2.Category(
title='std weights',
chart=[
layout_pb2.Chart(
title='stddev layer weights',
multiline=layout_pb2.MultilineChartContent(
tag=[r'stddev/layer\d+/weights'], )),
]),
# A category with this name is also present in a layout for a
# different run (the logdir run)
layout_pb2.Category(
title='cross entropy',
chart=[
layout_pb2.Chart(
title='cross entropy margin chart',
margin=layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value='cross entropy',
lower='cross entropy lower',
upper='cross entropy upper'),
], )),
]),
])
# Generate test data.
with tf.summary.FileWriter(os.path.join(self.logdir, 'foo')) as writer:
writer.add_summary(summary.pb(self.foo_layout))
for step in range(4):
writer.add_summary(scalar_summary.pb('squares', step * step),
step)
with tf.summary.FileWriter(os.path.join(self.logdir, 'bar')) as writer:
for step in range(3):
writer.add_summary(scalar_summary.pb('increments', step + 1),
step)
# The '.' run lacks scalar data but has a layout.
with tf.summary.FileWriter(self.logdir) as writer:
writer.add_summary(summary.pb(self.logdir_layout))
self.plugin = self.createPlugin(self.logdir)
layout_summary = summary_lib.custom_scalar_pb(layout_pb2.Layout(
category=[
layout_pb2.Category(
title='losses',
chart=[
layout_pb2.Chart(
title='losses',
multiline=layout_pb2.MultilineChartContent(
tag=[r'loss(?!.*margin.*)'],
)),
layout_pb2.Chart(
title='baz',
margin=layout_pb2.MarginChartContent(
series=[
layout_pb2.MarginChartContent.Series(
value='loss/baz/scalar_summary',
lower='loss/baz_lower_margin/scalar_summary',
upper='loss/baz_upper_margin/scalar_summary'),
],
)),
]),
layout_pb2.Category(
title='trig functions',
chart=[
layout_pb2.Chart(
title='wave trig functions',
multiline=layout_pb2.MultilineChartContent(
tag=[r'trigFunctions/cosine', r'trigFunctions/sine'],
)),
# The range of tangent is different. Give it its own chart.
layout_pb2.Chart(
title='tan',
