def finalize_autosummaries() -> None:
"""Create the necessary ops to include autosummaries in TensorBoard report.
Note: This should be done only once per graph.
"""
global _finalized
tfutil.assert_tf_initialized()
if _finalized:
return None
_finalized = True
tfutil.init_uninitialized_vars([var for vars_list in _vars.values() for var in vars_list])
# Create summary ops.
with tf.device(None), tf.control_dependencies(None):
for name, vars_list in _vars.items():
name_id = name.replace("/", "_")
with tfutil.absolute_name_scope("Autosummary/" + name_id):
moments = tf.add_n(vars_list)
moments /= moments[0]
with tf.control_dependencies([moments]): # read before resetting
reset_ops = [tf.assign(var, tf.zeros(3, dtype=_dtype)) for var in vars_list]
with tf.name_scope(None), tf.control_dependencies(reset_ops): # reset before reporting
mean = moments[1]
std = tf.sqrt(moments[2] - tf.square(moments[1]))
tf.summary.scalar(name, mean)
tf.summary.scalar("xCustomScalars/" + name + "/margin_lo", mean - std)
tf.summary.scalar("xCustomScalars/" + name + "/margin_hi", mean + std)
# Group by category and chart name.
cat_dict = OrderedDict()
for series_name in sorted(_vars.keys()):
p = series_name.split("/")
cat = p[0] if len(p) >= 2 else ""
chart = "/".join(p[1:-1]) if len(p) >= 3 else p[-1]
if cat not in cat_dict:
cat_dict[cat] = OrderedDict()
if chart not in cat_dict[cat]:
cat_dict[cat][chart] = []
cat_dict[cat][chart].append(series_name)
# Setup custom_scalar layout.
categories = []
for cat_name, chart_dict in cat_dict.items():
charts = []
for chart_name, series_names in chart_dict.items():
series = []
for series_name in series_names:
series.append(layout_pb2.MarginChartContent.Series(
value=series_name,
lower="xCustomScalars/" + series_name + "/margin_lo",
upper="xCustomScalars/" + series_name + "/margin_hi"))
margin = layout_pb2.MarginChartContent(series=series)
charts.append(layout_pb2.Chart(title=chart_name, margin=margin))
categories.append(layout_pb2.Category(title=cat_name, chart=charts))
layout = custom_scalar_pb(layout_pb2.Layout(category=categories))
return layout
def get_layout_summary(
*,
metrics: Tuple[DictConfig, ...],
tasks: Tuple[DictConfig, ...],
closed: bool = False,
):
"""Builds a summary that describes custom scalars layout for TensorBoard.
At each evaluation step, we compute metrics for multiple tasks sampled from
the same distribution and estimate the mean and std value of the metric.
The layout specified by this function nicely groups metrics and defines
margin charts that use fill area to visualize lower and upper bounds.
Parameters
----------
metrics : tuple of DictConfigs
tasks : tuple of DictConfigs
closed : bool (default: False)
Returns
-------
A summary proto containing the layout.
"""
set_names = sorted(set(t.set_name for t in tasks))
task_regimes = sorted(set(t.regime for t in tasks))
layout_summary = summary_lib.custom_scalar_pb(
layout_pb2.Layout(category=[
# Category for each metric.
layout_pb2.Category(
title=m.name,
chart=[
# A chart for each type of the eval distribution.
layout_pb2.Chart(
title=f"{t_regime}/{m.name} (CI {m.ci:.0f}%)",
margin=layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value=
f"{s_name}/{t_regime}/{m.name}_mean/scalar_summary",
lower=
f"{s_name}/{t_regime}/{m.name}_lower/scalar_summary",
upper=
f"{s_name}/{t_regime}/{m.name}_upper/scalar_summary",
) for s_name in set_names
]),
) for t_regime in task_regimes
],
closed=closed,
) for m in metrics
]))
return layout_summary
def run():
"""Run custom scalar demo and generate event files."""
step = tf.compat.v1.placeholder(tf.float32, shape=[])
with tf.name_scope('loss'):
# Specify 2 different loss values, each tagged differently.
summary_lib.scalar('foo', tf.pow(0.9, step))
summary_lib.scalar('bar', tf.pow(0.85, step + 2))
# Log metric baz as well as upper and lower bounds for a margin chart.
middle_baz_value = step + 4 * tf.random.uniform([]) - 2
summary_lib.scalar('baz', middle_baz_value)
summary_lib.scalar('baz_lower',
middle_baz_value - 6.42 - tf.random.uniform([]))
summary_lib.scalar('baz_upper',
middle_baz_value + 6.42 + tf.random.uniform([]))
with tf.name_scope('trigFunctions'):
summary_lib.scalar('cosine', tf.cos(step))
summary_lib.scalar('sine', tf.sin(step))
summary_lib.scalar('tangent', tf.tan(step))
merged_summary = tf.compat.v1.summary.merge_all()
with tf.compat.v1.Session() as sess, tf.summary.FileWriter(
LOGDIR) as writer:
# We only need to specify the layout once (instead of per step).
layout_summary = summary_lib.custom_scalar_pb(
layout_pb2.Layout(category=[
layout_pb2.Category(
title='losses',
chart=[
layout_pb2.Chart(
title='losses',
multiline=layout_pb2.MultilineChartContent(
tag=[r'loss(?!.*margin.*)'], )),
layout_pb2.Chart(
title='baz',
margin=layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value='loss/baz/scalar_summary',
lower='loss/baz_lower/scalar_summary',
upper='loss/baz_upper/scalar_summary'),
], )),
]),
layout_pb2.Category(
title='trig functions',
chart=[
layout_pb2.Chart(
title='wave trig functions',
multiline=layout_pb2.MultilineChartContent(tag=[
r'trigFunctions/cosine', r'trigFunctions/sine'
], )),
# The range of tangent is different. Give it its own chart.
layout_pb2.Chart(
title='tan',
multiline=layout_pb2.MultilineChartContent(
tag=[r'trigFunctions/tangent'], )),
],
# This category we care less about. Make it initially closed.
closed=True),
]))
writer.add_summary(layout_summary)
for i in xrange(42):
summary = sess.run(merged_summary, feed_dict={step: i})
writer.add_summary(summary, global_step=i)
def write_combined_events(dpath, d_combined, dname='combined'):
# ['reward', 'floor', 'reward.std', 'floor.std', 'steps', 'FPS', 'value_loss', 'action_loss_', 'dist_entropy_']
fpath = os.path.join(dpath, dname)
writer = tf.summary.FileWriter(fpath)
tags, values = zip(*d_combined.items())
# We only need to specify the layout once (instead of per step).
layout_summary = summary_lib.custom_scalar_pb(
layout_pb2.Layout(category=[
layout_pb2.Category(
title='losses',
chart=[
# layout_pb2.Chart(
# title='losses',
# multiline=layout_pb2.MultilineChartContent(
# tag=[r'loss(?!.*margin.*)'],)),
layout_pb2.Chart(
title='floor',
margin=layout_pb2.MarginChartContent(series=[
layout_pb2.MarginChartContent.Series(
value='floor/mean',
lower='floor/std_lower',
upper='floor/std_upper'),
], )),
# layout_pb2.Chart(
# title='floor',
# margin=layout_pb2.MarginChartContent(
# series=[
# layout_pb2.MarginChartContent.Series(
# value='floor/mean/scalar_summary',
# lower='floor/std_lower/scalar_summary',
# upper='floor/std_upper/scalar_summary'
# ),
# ],)),
]),
# layout_pb2.Category(
# title='trig functions',
# chart=[
# layout_pb2.Chart(
# title='wave trig functions',
# multiline=layout_pb2.MultilineChartContent(
# tag=[
# r'trigFunctions/cosine', r'trigFunctions/sine'
# ],)),
# # The range of tangent is different. Give it its own chart.
# layout_pb2.Chart(
# title='tan',
# multiline=layout_pb2.MultilineChartContent(
# tag=[r'trigFunctions/tangent'],)),
# ],
# # This category we care less about. Make it initially closed.
# closed=True),
]))
writer.add_summary(layout_summary)
floor = d_combined['floor']
floor_std = d_combined['floor.std']
# for tag, tuples in zip(tags, values):
# for t in tuples:
for i, t in enumerate(floor):
step = t[0]
mean = t[1]
lower = mean - floor_std[i][1]
upper = mean + floor_std[i][1]
summary = tf.Summary(
value=[tf.Summary.Value(tag='floor/mean', simple_value=mean)])
writer.add_summary(summary, global_step=step)
summary = tf.Summary(value=[
tf.Summary.Value(tag='floor/std_lower', simple_value=lower)
])
writer.add_summary(summary, global_step=step)
summary = tf.Summary(value=[
tf.Summary.Value(tag='floor/std_upper', simple_value=upper)
])
writer.add_summary(summary, global_step=step)
# for tag, tuples in zip(tags, values):
# for t in tuples:
# step = t[0]
# value = t[1]
# summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)])
# writer.add_summary(summary, global_step=step)
writer.flush()