def pr_curves_impl(self, runs, tag):
"""Creates the JSON object for the PR curves response for a run-tag
combo.
Arguments:
runs: A list of runs to fetch the curves for.
tag: The tag to fetch the curves for.
Raises:
ValueError: If no PR curves could be fetched for a run and tag.
Returns:
The JSON object for the PR curves route response.
"""
response_mapping = {}
for run in runs:
try:
tensor_events = self._multiplexer.Tensors(run, tag)
except KeyError:
raise ValueError(
"No PR curves could be found for run %r and tag %r" %
(run, tag))
content = self._multiplexer.SummaryMetadata(
run, tag).plugin_data.content
pr_curve_data = metadata.parse_plugin_metadata(content)
thresholds = self._compute_thresholds(pr_curve_data.num_thresholds)
response_mapping[run] = [
self._process_tensor_event(e, thresholds)
for e in tensor_events
]
return response_mapping
def pr_curves_impl(self, runs, tag):
"""Creates the JSON object for the PR curves response for a run-tag combo.
Arguments:
runs: A list of runs to fetch the curves for.
tag: The tag to fetch the curves for.
Raises:
ValueError: If no PR curves could be fetched for a run and tag.
Returns:
The JSON object for the PR curves route response.
"""
response_mapping = {}
for run in runs:
try:
tensor_events = self._multiplexer.Tensors(run, tag)
except KeyError:
raise ValueError(
'No PR curves could be fetched for run %r and tag %r' % (run, tag))
content = self._multiplexer.SummaryMetadata(run, tag).plugin_data.content
pr_curve_data = metadata.parse_plugin_metadata(content)
thresholds = [
float(v) / pr_curve_data.num_thresholds
for v in range(1, pr_curve_data.num_thresholds + 1)]
response_mapping[run] = [
self._process_tensor_event(e, thresholds) for e in tensor_events]
return response_mapping
def testRawDataOp(self):
with tf.summary.FileWriter(
self.logdir) as writer, tf.Session() as sess:
# We pass raw counts and precision/recall values.
writer.add_summary(
sess.run(
summary.raw_data_op(
tag='foo',
true_positive_counts=tf.constant([75, 64, 21, 5, 0]),
false_positive_counts=tf.constant([150, 105, 18, 0,
0]),
true_negative_counts=tf.constant(
[0, 45, 132, 150, 150]),
false_negative_counts=tf.constant([0, 11, 54, 70, 75]),
precision=tf.constant(
[0.3333333, 0.3786982, 0.5384616, 1.0, 0.0]),
recall=tf.constant(
[1.0, 0.8533334, 0.28, 0.0666667, 0.0]),
num_thresholds=5,
display_name='some_raw_values',
description='We passed raw values into a summary op.'))
)
multiplexer = self.createMultiplexer()
accumulator = multiplexer.GetAccumulator('.')
tag_content_dict = accumulator.PluginTagToContent('pr_curves')
self.assertItemsEqual(['foo/pr_curves'], list(tag_content_dict.keys()))
# Test the metadata.
summary_metadata = multiplexer.SummaryMetadata('.', 'foo/pr_curves')
self.assertEqual('some_raw_values', summary_metadata.display_name)
self.assertEqual('We passed raw values into a summary op.',
summary_metadata.summary_description)
# Test the stored plugin data.
plugin_data = metadata.parse_plugin_metadata(
tag_content_dict['foo/pr_curves'])
self.assertEqual(5, plugin_data.num_thresholds)
# Test the summary contents.
tensor_events = accumulator.Tensors('foo/pr_curves')
self.assertEqual(1, len(tensor_events))
self.validateTensorEvent(
0,
[
[75.0, 64.0, 21.0, 5.0, 0.0], # True positives.
[150.0, 105.0, 18.0, 0.0, 0.0], # False positives.
[0.0, 45.0, 132.0, 150.0, 150.0], # True negatives.
[0.0, 11.0, 54.0, 70.0, 75.0], # False negatives.
[0.3333333, 0.3786982, 0.5384616, 1.0, 0.0], # Precision.
[1.0, 0.8533334, 0.28, 0.0666667, 0.0], # Recall.
],
tensor_events[0])
def test_metadata(self):
pb = self.compute_and_check_summary_pb(name='foo',
labels=np.array([True]),
predictions=np.float32([0.42]),
num_thresholds=3)
summary_metadata = pb.value[0].metadata
plugin_data = summary_metadata.plugin_data
self.assertEqual('foo', summary_metadata.display_name)
self.assertEqual('', summary_metadata.summary_description)
self.assertEqual(metadata.PLUGIN_NAME, plugin_data.plugin_name)
plugin_data = metadata.parse_plugin_metadata(
summary_metadata.plugin_data.content)
self.assertEqual(3, plugin_data.num_thresholds)
def tags_impl(self, ctx, experiment):
"""Creates the JSON object for the tags route response.
Returns:
The JSON object for the tags route response.
"""
mapping = self._data_provider.list_tensors(
ctx, experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME)
result = {run: {} for run in mapping}
for (run, tag_to_time_series) in mapping.items():
for (tag, time_series) in tag_to_time_series.items():
md = metadata.parse_plugin_metadata(time_series.plugin_content)
if not self._version_checker.ok(md.version, run, tag):
continue
result[run][tag] = {
"displayName":
time_series.display_name,
"description":
plugin_util.markdown_to_safe_html(time_series.description),
}
return result
def testWeight1(self):
self.generateDemoData()
multiplexer = self.createMultiplexer()
# Verify that the metadata was correctly written.
accumulator = multiplexer.GetAccumulator('colors')
tag_content_dict = accumulator.PluginTagToContent('pr_curves')
# Test the summary contents.
expected_tags = ['red/pr_curves', 'green/pr_curves', 'blue/pr_curves']
self.assertItemsEqual(expected_tags, list(tag_content_dict.keys()))
for tag in expected_tags:
# Parse the data within the JSON string and set the proto's fields.
plugin_data = metadata.parse_plugin_metadata(tag_content_dict[tag])
self.assertEqual(5, plugin_data.num_thresholds)
# Test the summary contents.
tensor_events = accumulator.Tensors(tag)
self.assertEqual(3, len(tensor_events))
# Test the output for the red classifier. The red classifier has the
# narrowest standard deviation.
tensor_events = accumulator.Tensors('red/pr_curves')
self.validateTensorEvent(
0,
[
[100.0, 45.0, 11.0, 2.0, 0.0], # True positives.
[350.0, 50.0, 11.0, 2.0, 0.0], # False positives.
[0.0, 300.0, 339.0, 348.0, 350.0], # True negatives.
[0.0, 55.0, 89.0, 98.0, 100.0], # False negatives.
[0.2222222, 0.4736842, 0.5, 0.5, 0.0], # Precision.
[1.0, 0.45, 0.11, 0.02, 0.0], # Recall.
],
tensor_events[0])
self.validateTensorEvent(
1,
[
[100.0, 41.0, 11.0, 1.0, 0.0], # True positives.
[350.0, 48.0, 7.0, 1.0, 0.0], # False positives.
[0.0, 302.0, 343.0, 349.0, 350.0], # True negatives.
[0.0, 59.0, 89.0, 99.0, 100.0], # False negatives.
[0.2222222, 0.4606742, 0.6111111, 0.5, 0.0], # Precision.
[1.0, 0.41, 0.11, 0.01, 0.0], # Recall.
],
tensor_events[1])
self.validateTensorEvent(
2,
[
[100.0, 39.0, 11.0, 2.0, 0.0], # True positives.
[350.0, 54.0, 13.0, 1.0, 0.0], # False positives.
[0.0, 296.0, 337.0, 349.0, 350.0], # True negatives.
[0.0, 61.0, 89.0, 98.0, 100.0], # False negatives.
[0.2222222, 0.4193548, 0.4583333, 0.6666667, 0.0
], # Precision.
[1.0, 0.39, 0.11, 0.02, 0.0], # Recall.
],
tensor_events[2])
# Test the output for the green classifier.
tensor_events = accumulator.Tensors('green/pr_curves')
self.validateTensorEvent(
0,
[
[200.0, 125.0, 48.0, 7.0, 0.0], # True positives.
[250.0, 100.0, 13.0, 2.0, 0.0], # False positives.
[0.0, 150.0, 237.0, 248.0, 250.0], # True negatives.
[0.0, 75.0, 152.0, 193.0, 200.0], # False negatives.
[0.4444444, 0.5555556, 0.7868853, 0.7777778, 0.0
], # Precision.
[1.0, 0.625, 0.24, 0.035, 0.0], # Recall.
],
tensor_events[0])
self.validateTensorEvent(
1,
[
[200.0, 123.0, 36.0, 7.0, 0.0], # True positives.
[250.0, 91.0, 18.0, 2.0, 0.0], # False positives.
[0.0, 159.0, 232.0, 248.0, 250.0], # True negatives.
[0.0, 77.0, 164.0, 193.0, 200.0], # False negatives.
[0.4444444, 0.5747663, 0.6666667, 0.7777778, 0.0
], # Precision.
[1.0, 0.615, 0.18, 0.035, 0.0], # Recall.
],
tensor_events[1])
self.validateTensorEvent(
2,
[
[200.0, 116.0, 40.0, 5.0, 0.0], # True positives.
[250.0, 87.0, 18.0, 1.0, 0.0], # False positives.
[0.0, 163.0, 232.0, 249.0, 250.0], # True negatives.
[0.0, 84.0, 160.0, 195.0, 200.0], # False negatives.
[0.4444444, 0.5714286, 0.6896552, 0.8333333, 0.0
], # Precision.
[1.0, 0.58, 0.2, 0.025, 0.0], # Recall.
],
tensor_events[2])
# Test the output for the blue classifier. The normal distribution that is
# the blue classifier has the widest standard deviation.
tensor_events = accumulator.Tensors('blue/pr_curves')
self.validateTensorEvent(
0,
[
[150.0, 126.0, 45.0, 6.0, 0.0], # True positives.
[300.0, 201.0, 38.0, 2.0, 0.0], # False positives.
[0.0, 99.0, 262.0, 298.0, 300.0], # True negatives.
[0.0, 24.0, 105.0, 144.0, 150.0], # False negatives.
[0.3333333, 0.3853211, 0.5421687, 0.75, 0.0], # Precision.
[1.0, 0.84, 0.3, 0.04, 0.0], # Recall.
],
tensor_events[0])
self.validateTensorEvent(
1,
[
[150.0, 128.0, 45.0, 4.0, 0.0], # True positives.
[300.0, 204.0, 39.0, 6.0, 0.0], # False positives.
[0.0, 96.0, 261.0, 294.0, 300.0], # True negatives.
[0.0, 22.0, 105.0, 146.0, 150.0], # False negatives.
[0.3333333, 0.3855422, 0.5357143, 0.4, 0.0], # Precision.
[1.0, 0.8533334, 0.3, 0.0266667, 0.0], # Recall.
],
tensor_events[1])
self.validateTensorEvent(
2,
[
[150.0, 120.0, 39.0, 4.0, 0.0], # True positives.
[300.0, 185.0, 38.0, 2.0, 0.0], # False positives.
[0.0, 115.0, 262.0, 298.0, 300.0], # True negatives.
[0.0, 30.0, 111.0, 146.0, 150.0], # False negatives.
[0.3333333, 0.3934426, 0.5064935, 0.6666667, 0.0
], # Precision.
[1.0, 0.8, 0.26, 0.0266667, 0.0], # Recall.
],
tensor_events[2])
def testExplicitWeights(self):
self.generateDemoData()
multiplexer = self.createMultiplexer()
# Verify that the metadata was correctly written.
accumulator = multiplexer.GetAccumulator('mask_every_other_prediction')
tag_content_dict = accumulator.PluginTagToContent('pr_curves')
# Test the summary contents.
expected_tags = ['red/pr_curves', 'green/pr_curves', 'blue/pr_curves']
self.assertItemsEqual(expected_tags, list(tag_content_dict.keys()))
for tag in expected_tags:
# Parse the data within the JSON string and set the proto's fields.
plugin_data = metadata.parse_plugin_metadata(tag_content_dict[tag])
self.assertEqual(5, plugin_data.num_thresholds)
# Test the summary contents.
tensor_events = accumulator.Tensors(tag)
self.assertEqual(3, len(tensor_events))
# Test the output for the red classifier. The red classifier has the
# narrowest standard deviation.
tensor_events = accumulator.Tensors('red/pr_curves')
self.validateTensorEvent(
0,
[
[50.0, 22.0, 4.0, 0.0, 0.0], # True positives.
[175.0, 22.0, 6.0, 1.0, 0.0], # False positives.
[0.0, 153.0, 169.0, 174.0, 175.0], # True negatives.
[0.0, 28.0, 46.0, 50.0, 50.0], # False negatives.
[0.2222222, 0.5, 0.4, 0.0, 0.0], # Precision.
[1.0, 0.44, 0.08, 0.0, 0.0], # Recall.
],
tensor_events[0])
self.validateTensorEvent(
1,
[
[50.0, 17.0, 5.0, 1.0, 0.0], # True positives.
[175.0, 28.0, 1.0, 0.0, 0.0], # False positives.
[0.0, 147.0, 174.0, 175.0, 175.0], # True negatives.
[0.0, 33.0, 45.0, 49.0, 50.0], # False negatives.
[0.2222222, 0.3777778, 0.8333333, 1.0, 0.0], # Precision.
[1.0, 0.34, 0.1, 0.02, 0.0], # Recall.
],
tensor_events[1])
self.validateTensorEvent(
2,
[
[50.0, 18.0, 6.0, 1.0, 0.0], # True positives.
[175.0, 27.0, 6.0, 0.0, 0.0], # False positives.
[0.0, 148.0, 169.0, 175.0, 175.0], # True negatives.
[0.0, 32.0, 44.0, 49.0, 50.0], # False negatives.
[0.2222222, 0.4, 0.5, 1.0, 0.0], # Precision.
[1.0, 0.36, 0.12, 0.02, 0.0], # Recall.
],
tensor_events[2])
# Test the output for the green classifier.
tensor_events = accumulator.Tensors('green/pr_curves')
self.validateTensorEvent(
0,
[
[100.0, 71.0, 24.0, 2.0, 0.0], # True positives.
[125.0, 51.0, 5.0, 2.0, 0.0], # False positives.
[0.0, 74.0, 120.0, 123.0, 125.0], # True negatives.
[0.0, 29.0, 76.0, 98.0, 100.0], # False negatives.
[0.4444444, 0.5819672, 0.8275862, 0.5, 0.0], # Precision.
[1.0, 0.71, 0.24, 0.02, 0.0], # Recall.
],
tensor_events[0])
self.validateTensorEvent(
1,
[
[100.0, 63.0, 20.0, 5.0, 0.0], # True positives.
[125.0, 42.0, 7.0, 1.0, 0.0], # False positives.
[0.0, 83.0, 118.0, 124.0, 125.0], # True negatives.
[0.0, 37.0, 80.0, 95.0, 100.0], # False negatives.
[0.4444444, 0.6, 0.7407407, 0.8333333, 0.0], # Precision.
[1.0, 0.63, 0.2, 0.05, 0.0], # Recall.
],
tensor_events[1])
self.validateTensorEvent(
2,
[
[100.0, 58.0, 19.0, 2.0, 0.0], # True positives.
[125.0, 40.0, 7.0, 0.0, 0.0], # False positives.
[0.0, 85.0, 118.0, 125.0, 125.0], # True negatives.
[0.0, 42.0, 81.0, 98.0, 100.0], # False negatives.
[0.4444444, 0.5918368, 0.7307692, 1.0, 0.0], # Precision.
[1.0, 0.58, 0.19, 0.02, 0.0], # Recall.
],
tensor_events[2])
# Test the output for the blue classifier. The normal distribution that is
# the blue classifier has the widest standard deviation.
tensor_events = accumulator.Tensors('blue/pr_curves')
self.validateTensorEvent(
0,
[
[75.0, 64.0, 21.0, 5.0, 0.0], # True positives.
[150.0, 105.0, 18.0, 0.0, 0.0], # False positives.
[0.0, 45.0, 132.0, 150.0, 150.0], # True negatives.
[0.0, 11.0, 54.0, 70.0, 75.0], # False negatives.
[0.3333333, 0.3786982, 0.5384616, 1.0, 0.0], # Precision.
[1.0, 0.8533334, 0.28, 0.0666667, 0.0], # Recall.
],
tensor_events[0])
self.validateTensorEvent(
1,
[
[75.0, 62.0, 21.0, 1.0, 0.0], # True positives.
[150.0, 99.0, 21.0, 3.0, 0.0], # False positives.
[0.0, 51.0, 129.0, 147.0, 150.0], # True negatives.
[0.0, 13.0, 54.0, 74.0, 75.0], # False negatives.
[0.3333333, 0.3850932, 0.5, 0.25, 0.0], # Precision.
[1.0, 0.8266667, 0.28, 0.0133333, 0.0], # Recall.
],
tensor_events[1])
self.validateTensorEvent(
2,
[
[75.0, 61.0, 16.0, 2.0, 0.0], # True positives.
[150.0, 92.0, 20.0, 1.0, 0.0], # False positives.
[0.0, 58.0, 130.0, 149.0, 150.0], # True negatives.
[0.0, 14.0, 59.0, 73.0, 75.0], # False negatives.
[0.3333333, 0.3986928, 0.4444444, 0.6666667, 0.0
], # Precision.
[1.0, 0.8133333, 0.2133333, 0.0266667, 0.0], # Recall.
],
tensor_events[2])
def test_raw_data(self):
# We pass these raw counts and precision/recall values.
name = "foo"
true_positive_counts = [75, 64, 21, 5, 0]
false_positive_counts = [150, 105, 18, 0, 0]
true_negative_counts = [0, 45, 132, 150, 150]
false_negative_counts = [0, 11, 54, 70, 75]
precision = [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0]
recall = [1.0, 0.8533334, 0.28, 0.0666667, 0.0]
num_thresholds = 5
display_name = "some_raw_values"
description = "We passed raw values into a summary op."
op = summary.raw_data_op(
name=name,
true_positive_counts=tf.constant(true_positive_counts),
false_positive_counts=tf.constant(false_positive_counts),
true_negative_counts=tf.constant(true_negative_counts),
false_negative_counts=tf.constant(false_negative_counts),
precision=tf.constant(precision),
recall=tf.constant(recall),
num_thresholds=num_thresholds,
display_name=display_name,
description=description,
)
pb_via_op = self.normalize_summary_pb(self.pb_via_op(op))
# Call the corresponding method that is decoupled from TensorFlow.
pb = self.normalize_summary_pb(
summary.raw_data_pb(
name=name,
true_positive_counts=true_positive_counts,
false_positive_counts=false_positive_counts,
true_negative_counts=true_negative_counts,
false_negative_counts=false_negative_counts,
precision=precision,
recall=recall,
num_thresholds=num_thresholds,
display_name=display_name,
description=description,
))
# The 2 methods above should write summaries with the same data.
self.assertProtoEquals(pb, pb_via_op)
# Test the metadata.
summary_metadata = pb.value[0].metadata
self.assertEqual("some_raw_values", summary_metadata.display_name)
self.assertEqual(
"We passed raw values into a summary op.",
summary_metadata.summary_description,
)
self.assertEqual(metadata.PLUGIN_NAME,
summary_metadata.plugin_data.plugin_name)
plugin_data = metadata.parse_plugin_metadata(
summary_metadata.plugin_data.content)
self.assertEqual(5, plugin_data.num_thresholds)
# Test the summary contents.
values = tensor_util.make_ndarray(pb.value[0].tensor)
self.verify_float_arrays_are_equal(
[
[75.0, 64.0, 21.0, 5.0, 0.0], # True positives.
[150.0, 105.0, 18.0, 0.0, 0.0], # False positives.
[0.0, 45.0, 132.0, 150.0, 150.0], # True negatives.
[0.0, 11.0, 54.0, 70.0, 75.0], # False negatives.
[0.3333333, 0.3786982, 0.5384616, 1.0, 0.0], # Precision.
[1.0, 0.8533334, 0.28, 0.0666667, 0.0], # Recall.
],
values,
)
def pr_curves_impl(self, runs, tag):
"""Creates the JSON object for the PR curves response for a run-tag combo.
Arguments:
runs: A list of runs to fetch the curves for.
tag: The tag to fetch the curves for.
Raises:
ValueError: If no PR curves could be fetched for a run and tag.
Returns:
The JSON object for the PR curves route response.
"""
if self._db_connection_provider:
# Serve data from the database.
db = self._db_connection_provider()
# We select for steps greater than -1 because the writer inserts
# placeholder rows en masse. The check for step filters out those rows.
cursor = db.execute('''
SELECT
Runs.run_name,
Tensors.step,
Tensors.computed_time,
Tensors.data,
Tensors.dtype,
Tensors.shape,
Tags.plugin_data
FROM Tensors
JOIN Tags
ON Tensors.series = Tags.tag_id
JOIN Runs
ON Tags.run_id = Runs.run_id
WHERE
Runs.run_name IN (%s)
AND Tags.tag_name = ?
AND Tags.plugin_name = ?
AND Tensors.step > -1
ORDER BY Tensors.step
''' % ','.join(['?'] * len(runs)), runs + [tag, metadata.PLUGIN_NAME])
response_mapping = {}
for (run, step, wall_time, data, dtype, shape, plugin_data) in cursor:
if run not in response_mapping:
response_mapping[run] = []
dtype_for_buf = tf.DType(dtype) if USE_TF else tf.dtypes.DType(dtype)
buf = np.frombuffer(data, dtype=dtype_for_buf.as_numpy_dtype)
data_array = buf.reshape([int(i) for i in shape.split(',')])
plugin_data_proto = plugin_data_pb2.PrCurvePluginData()
string_buffer = np.frombuffer(plugin_data, dtype=np.dtype('b'))
plugin_data_proto.ParseFromString(tf.compat.as_bytes(
string_buffer.tostring()))
thresholds = self._compute_thresholds(plugin_data_proto.num_thresholds)
entry = self._make_pr_entry(step, wall_time, data_array, thresholds)
response_mapping[run].append(entry)
else:
# Serve data from events files.
response_mapping = {}
for run in runs:
try:
tensor_events = self._multiplexer.Tensors(run, tag)
except KeyError:
raise ValueError(
'No PR curves could be found for run %r and tag %r' % (run, tag))
content = self._multiplexer.SummaryMetadata(
run, tag).plugin_data.content
pr_curve_data = metadata.parse_plugin_metadata(content)
thresholds = self._compute_thresholds(pr_curve_data.num_thresholds)
response_mapping[run] = [
self._process_tensor_event(e, thresholds) for e in tensor_events]
return response_mapping
