def generate_run(self, run_name):
tf.compat.v1.reset_default_graph()
sess = tf.compat.v1.Session()
placeholder = tf.compat.v1.placeholder(tf.float32, shape=[3])
if run_name == self._RUN_WITH_LEGACY_SCALARS:
tf.compat.v1.summary.scalar(
self._LEGACY_SCALAR_TAG,
tf.reduce_mean(input_tensor=placeholder))
elif run_name == self._RUN_WITH_SCALARS:
summary.op(self._SCALAR_TAG,
tf.reduce_sum(input_tensor=placeholder),
display_name=self._DISPLAY_NAME,
description=self._DESCRIPTION)
elif run_name == self._RUN_WITH_HISTOGRAM:
tf.compat.v1.summary.histogram(self._HISTOGRAM_TAG, placeholder)
else:
assert False, 'Invalid run name: %r' % run_name
summ = tf.compat.v1.summary.merge_all()
subdir = os.path.join(self.logdir, run_name)
with test_util.FileWriterCache.get(subdir) as writer:
writer.add_graph(sess.graph)
for step in xrange(self._STEPS):
feed_dict = {placeholder: [1 + step, 2 + step, 3 + step]}
s = sess.run(summ, feed_dict=feed_dict)
writer.add_summary(s, global_step=step)
def testNewStyleScalarSummary(self):
"""Verify processing of tensorboard.plugins.scalar.summary."""
event_sink = _EventGenerator(self, zero_out_timestamps=True)
writer = tf.summary.FileWriter(self.get_temp_dir())
writer.event_writer = event_sink
with self.test_session() as sess:
step = tf.placeholder(tf.float32, shape=[])
scalar_summary.op('accuracy', 1.0 - 1.0 / (step + tf.constant(1.0)))
scalar_summary.op('xent', 1.0 / (step + tf.constant(1.0)))
merged = tf.summary.merge_all()
writer.add_graph(sess.graph)
for i in xrange(10):
summ = sess.run(merged, feed_dict={step: float(i)})
writer.add_summary(summ, global_step=i)
accumulator = ea.EventAccumulator(event_sink)
accumulator.Reload()
tags = [
u'accuracy/scalar_summary',
u'xent/scalar_summary',
]
self.assertTagsEqual(accumulator.Tags(), {
ea.TENSORS: tags,
ea.GRAPH: True,
ea.META_GRAPH: False,
})
def testNewStyleScalarSummary(self):
"""Verify processing of tensorboard.plugins.scalar.summary."""
event_sink = _EventGenerator(self, zero_out_timestamps=True)
writer = test_util.FileWriter(self.get_temp_dir())
writer.event_writer = event_sink
with tf.compat.v1.Graph().as_default():
with self.test_session() as sess:
step = tf.compat.v1.placeholder(tf.float32, shape=[])
scalar_summary.op(
"accuracy", 1.0 - 1.0 / (step + tf.constant(1.0))
)
scalar_summary.op("xent", 1.0 / (step + tf.constant(1.0)))
merged = tf.compat.v1.summary.merge_all()
writer.add_graph(sess.graph)
for i in xrange(10):
summ = sess.run(merged, feed_dict={step: float(i)})
writer.add_summary(summ, global_step=i)
accumulator = ea.EventAccumulator(event_sink)
accumulator.Reload()
tags = [
u"accuracy/scalar_summary",
u"xent/scalar_summary",
]
self.assertTagsEqual(
accumulator.Tags(),
{ea.TENSORS: tags, ea.GRAPH: True, ea.META_GRAPH: False,},
)
def run(logdir, run_name, tag_value_map, parameter_map):
"""Create a dummy set of events data by logging some fake metrics to the runs directory."""
tf.reset_default_graph()
placeholders = {
tag: tf.placeholder(shape=(), dtype=tf.float32)
for tag in tag_value_map
}
summary_ops = {
tag: summary.op(tag, placeholders[tag])
for tag in tag_value_map
}
run_path = os.path.join(logdir, run_name)
writer = tf.summary.FileWriter(run_path)
config_writer = ParamPlotConfigWriter(run_path)
# Add the parameters to the run config
config_writer.SetParameters(parameter_map)
# Write the value under the final_loss summary for that particular run
with tf.Session() as session:
for tag_name in tag_value_map:
for _ in range(tag_value_map[tag_name]["samples"]):
summary_data = session.run(
summary_ops[tag_name],
feed_dict={
placeholders[tag_name]:
tag_value_map[tag_name]["func"]()
})
writer.add_summary(summary_data)
config_writer.Save()
writer.close()
def test_new_style_scalar(self):
op = scalar_summary.op('important_constants', tf.constant(0x5f3759df),
display_name='Important constants',
description='evil floating point bit magic')
value = self._value_from_op(op)
assert value.HasField('tensor'), value
self._assert_noop(value)
def test_new_style_scalar(self):
op = scalar_summary.op('important_constants', tf.constant(0x5f3759df),
display_name='Important constants',
description='evil floating point bit magic')
value = self._value_from_op(op)
assert value.HasField('tensor'), value
self._assert_noop(value)
def compute_and_check_summary_pb(self,
name,
data,
display_name=None,
description=None,
data_tensor=None,
feed_dict=None):
"""Use both `op` and `pb` to get a summary, asserting equality.
Returns:
a `Summary` protocol buffer
"""
if data_tensor is None:
data_tensor = tf.constant(data)
op = summary.op(name,
data,
display_name=display_name,
description=description)
pb = self.normalize_summary_pb(
summary.pb(name,
data,
display_name=display_name,
description=description))
pb_via_op = self.normalize_summary_pb(
self.pb_via_op(op, feed_dict=feed_dict))
self.assertProtoEquals(pb, pb_via_op)
return pb
def test_new_style_scalar(self):
with tf.compat.v1.Graph().as_default():
op = scalar_summary.op(
"important_constants",
tf.constant(0x5F3759DF),
display_name="Important constants",
description="evil floating point bit magic",
)
value = self._value_from_op(op)
assert value.HasField("tensor"), value
self._assert_noop(value)
def generate_run(self, logdir, run_name):
subdir = os.path.join(logdir, run_name)
with test_util.FileWriterCache.get(subdir) as writer:
for step in range(self._STEPS):
data = [1 + step, 2 + step, 3 + step]
if run_name == self._RUN_WITH_LEGACY_SCALARS:
summ = tf.compat.v1.summary.scalar(
self._LEGACY_SCALAR_TAG,
tf.reduce_mean(data),
).numpy()
elif run_name == self._RUN_WITH_SCALARS:
summ = summary.op(
self._SCALAR_TAG,
tf.reduce_sum(data),
display_name=self._DISPLAY_NAME,
description=self._DESCRIPTION,
).numpy()
elif run_name == self._RUN_WITH_SCALARS_2:
summ = summary.op(
self._SCALAR_TAG,
2 * tf.reduce_sum(data),
display_name=self._DISPLAY_NAME,
description=self._DESCRIPTION,
).numpy()
elif run_name == self._RUN_WITH_SCALARS_3:
summ = summary.op(
self._SCALAR_TAG,
3 * tf.reduce_sum(data),
display_name=self._DISPLAY_NAME,
description=self._DESCRIPTION,
).numpy()
elif run_name == self._RUN_WITH_HISTOGRAM:
summ = tf.compat.v1.summary.histogram(
self._HISTOGRAM_TAG, data
).numpy()
else:
assert False, "Invalid run name: %r" % run_name
writer.add_summary(summ, global_step=step)
def compute_and_check_summary_pb(self, name, data,
display_name=None, description=None,
data_tensor=None, feed_dict=None):
"""Use both `op` and `pb` to get a summary, asserting equality.
Returns:
a `Summary` protocol buffer
"""
if data_tensor is None:
data_tensor = tf.constant(data)
op = summary.op(
name, data, display_name=display_name, description=description)
pb = self.normalize_summary_pb(summary.pb(
name, data, display_name=display_name, description=description))
pb_via_op = self.normalize_summary_pb(
self.pb_via_op(op, feed_dict=feed_dict))
self.assertProtoEquals(pb, pb_via_op)
return pb
def run(run_path, parameter_dict, values):
tf.reset_default_graph()
tag_value_placeholder = tf.placeholder(dtype=tf.float32, shape=())
summary_op = summary.op("random_tag", tag_value_placeholder)
writer = tf.summary.FileWriter(run_path)
config_writer = ParamPlotConfigWriter(run_path)
for parameter in parameter_dict:
config_writer.AddParameter(parameter, parameter_dict[parameter])
with tf.Session() as session:
for value in values:
summary_data = session.run(
summary_op, feed_dict={tag_value_placeholder: value})
writer.add_summary(summary_data)
writer.close()
config_writer.Save()
def set_model(self, model):
super().set_model(model)
if self.layout_summary:
self.writer.add_summary(self.layout_summary)
if self.pr_curve:
for class_idx in range(self.num_classes):
predictions = self.model._feed_outputs[0][..., class_idx]
labels = tf.cast(self.model._feed_targets[0][..., class_idx], tf.bool)
summary_op = pr_summary.op(
name='pr_curve_class_' + str(class_idx),
predictions=predictions,
labels=labels,
display_name='Precision-Recall (Class ' + str(class_idx) + ')')
self.pr_summary.append(summary_op)
if self.tfpn:
predictions = self.model._feed_outputs[0]
labels = tf.cast(self.model._feed_targets[0], tf.bool)
_, precision = tf.metrics.precision(labels, predictions)
_, recall = tf.metrics.recall(labels, predictions)
_, tp = tf.metrics.true_positives(labels, predictions)
_, fn = tf.metrics.false_negatives(labels, predictions)
_, fp = tf.metrics.false_positives(labels, predictions)
_, f1 = tf.contrib.metrics.f1_score(labels, predictions) # TODO this recalcs tp,fn,fp. TODO utilise existing results
self.precision_summary = scalar_summary.op(name='precision', data=precision)
self.recall_summary = scalar_summary.op(name='recall', data=recall)
self.f1_summary = scalar_summary.op(name='f1', data=f1)
self.fp_summary = scalar_summary.op(name='fp', data=fp)
self.tp_summary = scalar_summary.op(name='tp', data=tp)
self.fn_summary = scalar_summary.op(name='fn', data=fn)
self.merged = tf.summary.merge_all()
def scalar(self, *args, **kwargs):
return tf.Summary.FromString(summary.op(*args, **kwargs).numpy())
def test_string_value_in_op(self):
with six.assertRaisesRegex(self, Exception, r'Cast str.*float'):
with tf.Session() as sess:
sess.run(summary.op('a', tf.constant("113")))
def run(logdir, session_id, hparams, group_name):
"""Runs a temperature simulation.
This will simulate an object at temperature `initial_temperature`
sitting at rest in a large room at temperature `ambient_temperature`.
The object has some intrinsic `heat_coefficient`, which indicates
how much thermal conductivity it has: for instance, metals have high
thermal conductivity, while the thermal conductivity of water is low.
Over time, the object's temperature will adjust to match the
temperature of its environment. We'll track the object's temperature,
how far it is from the room's temperature, and how much it changes at
each time step.
Arguments:
logdir: the top-level directory into which to write summary data
session_id: an id for the session.
hparams: A dictionary mapping an hyperparameter name to its value.
group_name: an id for the session group this session belongs to.
"""
tf.reset_default_graph()
tf.set_random_seed(0)
initial_temperature = hparams['initial_temperature']
ambient_temperature = hparams['ambient_temperature']
heat_coefficient = hparams['heat_coefficient']
session_dir = os.path.join(logdir, session_id)
writer = tf.summary.FileWriter(session_dir)
writer.add_summary(summary.session_start_pb(hparams=hparams,
group_name=group_name))
writer.flush()
with tf.name_scope('temperature'):
# Create a mutable variable to hold the object's temperature, and
# create a scalar summary to track its value over time. The name of
# the summary will appear as "temperature/current" due to the
# name-scope above.
temperature = tf.Variable(tf.constant(initial_temperature),
name='temperature')
scalar_summary.op('current', temperature,
display_name='Temperature',
description='The temperature of the object under '
'simulation, in Kelvins.')
# Compute how much the object's temperature differs from that of its
# environment, and track this, too: likewise, as
# "temperature/difference_to_ambient".
ambient_difference = temperature - ambient_temperature
scalar_summary.op('difference_to_ambient', ambient_difference,
display_name='Difference to ambient temperature',
description=('The difference between the ambient '
'temperature and the temperature of the '
'object under simulation, in Kelvins.'))
# Newton suggested that the rate of change of the temperature of an
# object is directly proportional to this `ambient_difference` above,
# where the proportionality constant is what we called the heat
# coefficient. But in real life, not everything is quite so clean, so
# we'll add in some noise. (The value of 50 is arbitrary, chosen to
# make the data look somewhat interesting. :-) )
noise = 50 * tf.random_normal([])
delta = -heat_coefficient * (ambient_difference + noise)
scalar_summary.op('delta', delta,
description='The change in temperature from the previous '
'step, in Kelvins.')
# Collect all the scalars that we want to keep track of.
summ = tf.summary.merge_all()
# Now, augment the current temperature by this delta that we computed,
# blocking the assignment on summary collection to avoid race conditions
# and ensure that the summary always reports the pre-update value.
with tf.control_dependencies([summ]):
update_step = temperature.assign_add(delta)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for step in xrange(STEPS):
# By asking TensorFlow to compute the update step, we force it to
# change the value of the temperature variable. We don't actually
# care about this value, so we discard it; instead, we grab the
# summary data computed along the way.
(s, _) = sess.run([summ, update_step])
writer.add_summary(s, global_step=step)
writer.add_summary(summary.session_end_pb(api_pb2.STATUS_SUCCESS))
writer.close()
def test_requires_rank_0_in_op(self):
with six.assertRaisesRegex(self, Exception, r'Expected scalar shape'):
with tf.Session() as sess:
sess.run(summary.op('a', tf.constant([1, 1, 3])))
def run(logdir, session_id, hparams, group_name):
"""Runs a temperature simulation.
This will simulate an object at temperature `initial_temperature`
sitting at rest in a large room at temperature `ambient_temperature`.
The object has some intrinsic `heat_coefficient`, which indicates
how much thermal conductivity it has: for instance, metals have high
thermal conductivity, while the thermal conductivity of water is low.
Over time, the object's temperature will adjust to match the
temperature of its environment. We'll track the object's temperature,
how far it is from the room's temperature, and how much it changes at
each time step.
Arguments:
logdir: the top-level directory into which to write summary data
session_id: an id for the session.
hparams: A dictionary mapping a hyperparameter name to its value.
group_name: an id for the session group this session belongs to.
"""
tf.reset_default_graph()
tf.set_random_seed(0)
initial_temperature = hparams["initial_temperature"]
ambient_temperature = hparams["ambient_temperature"]
heat_coefficient = HEAT_COEFFICIENTS[hparams["material"]]
session_dir = os.path.join(logdir, session_id)
writer = tf.summary.FileWriter(session_dir)
writer.add_summary(
summary.session_start_pb(hparams=hparams, group_name=group_name))
writer.flush()
with tf.name_scope("temperature"):
# Create a mutable variable to hold the object's temperature, and
# create a scalar summary to track its value over time. The name of
# the summary will appear as 'temperature/current' due to the
# name-scope above.
temperature = tf.Variable(tf.constant(initial_temperature),
name="temperature")
scalar_summary.op(
"current",
temperature,
display_name="Temperature",
description="The temperature of the object under "
"simulation, in Kelvins.",
)
# Compute how much the object's temperature differs from that of its
# environment, and track this, too: likewise, as
# 'temperature/difference_to_ambient'.
ambient_difference = temperature - ambient_temperature
scalar_summary.op(
"difference_to_ambient",
ambient_difference,
display_name="Difference to ambient temperature",
description=("The difference between the ambient "
"temperature and the temperature of the "
"object under simulation, in Kelvins."),
)
# Newton suggested that the rate of change of the temperature of an
# object is directly proportional to this `ambient_difference` above,
# where the proportionality constant is what we called the heat
# coefficient. But in real life, not everything is quite so clean, so
# we'll add in some noise. (The value of 50 is arbitrary, chosen to
# make the data look somewhat interesting. :-) )
noise = 50 * tf.random.normal([])
delta = -heat_coefficient * (ambient_difference + noise)
scalar_summary.op(
"delta",
delta,
description="The change in temperature from the previous "
"step, in Kelvins.",
)
# Collect all the scalars that we want to keep track of.
summ = tf.summary.merge_all()
# Now, augment the current temperature by this delta that we computed,
# blocking the assignment on summary collection to avoid race conditions
# and ensure that the summary always reports the pre-update value.
with tf.control_dependencies([summ]):
update_step = temperature.assign_add(delta)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for step in range(FLAGS.num_steps):
# By asking TensorFlow to compute the update step, we force it to
# change the value of the temperature variable. We don't actually
# care about this value, so we discard it; instead, we grab the
# summary data computed along the way.
(s, _) = sess.run([summ, update_step])
if (step % FLAGS.summary_freq) == 0:
writer.add_summary(s, global_step=step)
writer.add_summary(summary.session_end_pb(api_pb2.STATUS_SUCCESS))
writer.close()
def test_requires_rank_0_in_op(self):
with six.assertRaisesRegex(self, Exception, r'Expected scalar shape'):
with tf.Session() as sess:
sess.run(summary.op('a', tf.constant([1, 1, 3])))
def run(logdir, run_name, initial_temperature, ambient_temperature,
heat_coefficient):
"""Run a temperature simulation.
This will simulate an object at temperature `initial_temperature`
sitting at rest in a large room at temperature `ambient_temperature`.
The object has some intrinsic `heat_coefficient`, which indicates
how much thermal conductivity it has: for instance, metals have high
thermal conductivity, while the thermal conductivity of water is low.
Over time, the object's temperature will adjust to match the
temperature of its environment. We'll track the object's temperature,
how far it is from the room's temperature, and how much it changes at
each time step.
Arguments:
logdir: the top-level directory into which to write summary data
run_name: the name of this run; will be created as a subdirectory
under logdir
initial_temperature: float; the object's initial temperature
ambient_temperature: float; the temperature of the enclosing room
heat_coefficient: float; a measure of the object's thermal
conductivity
"""
tf.reset_default_graph()
tf.set_random_seed(0)
with tf.name_scope('temperature'):
# Create a mutable variable to hold the object's temperature, and
# create a scalar summary to track its value over time. The name of
# the summary will appear as "temperature/current" due to the
# name-scope above.
temperature = tf.Variable(tf.constant(initial_temperature),
name='temperature')
summary.op('current',
temperature,
display_name='Temperature',
description='The temperature of the object under '
'simulation, in Kelvins.')
# Compute how much the object's temperature differs from that of its
# environment, and track this, too: likewise, as
# "temperature/difference_to_ambient".
ambient_difference = temperature - ambient_temperature
summary.op('difference_to_ambient',
ambient_difference,
display_name='Difference to ambient temperature',
description='The difference between the ambient '
'temperature and the temperature of the '
'object under simulation, in Kelvins.')
# Newton suggested that the rate of change of the temperature of an
# object is directly proportional to this `ambient_difference` above,
# where the proportionality constant is what we called the heat
# coefficient. But in real life, not everything is quite so clean, so
# we'll add in some noise. (The value of 50 is arbitrary, chosen to
# make the data look somewhat interesting. :-) )
noise = 50 * tf.random_normal([])
delta = -heat_coefficient * (ambient_difference + noise)
summary.op('delta',
delta,
description='The change in temperature from the previous '
'step, in Kelvins.')
# Now, augment the current temperature by this delta that we computed.
update_step = temperature.assign_add(delta)
# Collect all the scalars that we want to keep track of.
summ = tf.summary.merge_all()
sess = tf.Session()
writer = tf.summary.FileWriter(os.path.join(logdir, run_name))
writer.add_graph(sess.graph)
sess.run(tf.global_variables_initializer())
for step in xrange(STEPS):
# By asking TensorFlow to compute the update step, we force it to
# change the value of the temperature variable. We don't actually
# care about this value, so we discard it; instead, we grab the
# summary data computed along the way.
(s, _) = sess.run([summ, update_step])
writer.add_summary(s, global_step=step)
writer.close()
def test_string_value_in_op(self):
with six.assertRaisesRegex(self, Exception, r'Cast str.*float'):
with tf.Session() as sess:
sess.run(summary.op('a', tf.constant("113")))
