def pr_curves_impl(self, runs, tag): """Creates the JSON object for the PR curves response for a run-tag combo. Arguments: runs: A list of runs to fetch the curves for. tag: The tag to fetch the curves for. Raises: ValueError: If no PR curves could be fetched for a run and tag. Returns: The JSON object for the PR curves route response. """ response_mapping = {} for run in runs: try: tensor_events = self._multiplexer.Tensors(run, tag) except KeyError: raise ValueError( "No PR curves could be found for run %r and tag %r" % (run, tag)) content = self._multiplexer.SummaryMetadata( run, tag).plugin_data.content pr_curve_data = metadata.parse_plugin_metadata(content) thresholds = self._compute_thresholds(pr_curve_data.num_thresholds) response_mapping[run] = [ self._process_tensor_event(e, thresholds) for e in tensor_events ] return response_mapping
def pr_curves_impl(self, runs, tag): """Creates the JSON object for the PR curves response for a run-tag combo. Arguments: runs: A list of runs to fetch the curves for. tag: The tag to fetch the curves for. Raises: ValueError: If no PR curves could be fetched for a run and tag. Returns: The JSON object for the PR curves route response. """ response_mapping = {} for run in runs: try: tensor_events = self._multiplexer.Tensors(run, tag) except KeyError: raise ValueError( 'No PR curves could be fetched for run %r and tag %r' % (run, tag)) content = self._multiplexer.SummaryMetadata(run, tag).plugin_data.content pr_curve_data = metadata.parse_plugin_metadata(content) thresholds = [ float(v) / pr_curve_data.num_thresholds for v in range(1, pr_curve_data.num_thresholds + 1)] response_mapping[run] = [ self._process_tensor_event(e, thresholds) for e in tensor_events] return response_mapping
def testRawDataOp(self): with tf.summary.FileWriter( self.logdir) as writer, tf.Session() as sess: # We pass raw counts and precision/recall values. writer.add_summary( sess.run( summary.raw_data_op( tag='foo', true_positive_counts=tf.constant([75, 64, 21, 5, 0]), false_positive_counts=tf.constant([150, 105, 18, 0, 0]), true_negative_counts=tf.constant( [0, 45, 132, 150, 150]), false_negative_counts=tf.constant([0, 11, 54, 70, 75]), precision=tf.constant( [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0]), recall=tf.constant( [1.0, 0.8533334, 0.28, 0.0666667, 0.0]), num_thresholds=5, display_name='some_raw_values', description='We passed raw values into a summary op.')) ) multiplexer = self.createMultiplexer() accumulator = multiplexer.GetAccumulator('.') tag_content_dict = accumulator.PluginTagToContent('pr_curves') self.assertItemsEqual(['foo/pr_curves'], list(tag_content_dict.keys())) # Test the metadata. summary_metadata = multiplexer.SummaryMetadata('.', 'foo/pr_curves') self.assertEqual('some_raw_values', summary_metadata.display_name) self.assertEqual('We passed raw values into a summary op.', summary_metadata.summary_description) # Test the stored plugin data. plugin_data = metadata.parse_plugin_metadata( tag_content_dict['foo/pr_curves']) self.assertEqual(5, plugin_data.num_thresholds) # Test the summary contents. tensor_events = accumulator.Tensors('foo/pr_curves') self.assertEqual(1, len(tensor_events)) self.validateTensorEvent( 0, [ [75.0, 64.0, 21.0, 5.0, 0.0], # True positives. [150.0, 105.0, 18.0, 0.0, 0.0], # False positives. [0.0, 45.0, 132.0, 150.0, 150.0], # True negatives. [0.0, 11.0, 54.0, 70.0, 75.0], # False negatives. [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0], # Precision. [1.0, 0.8533334, 0.28, 0.0666667, 0.0], # Recall. ], tensor_events[0])
def test_metadata(self): pb = self.compute_and_check_summary_pb(name='foo', labels=np.array([True]), predictions=np.float32([0.42]), num_thresholds=3) summary_metadata = pb.value[0].metadata plugin_data = summary_metadata.plugin_data self.assertEqual('foo', summary_metadata.display_name) self.assertEqual('', summary_metadata.summary_description) self.assertEqual(metadata.PLUGIN_NAME, plugin_data.plugin_name) plugin_data = metadata.parse_plugin_metadata( summary_metadata.plugin_data.content) self.assertEqual(3, plugin_data.num_thresholds)
def tags_impl(self, ctx, experiment): """Creates the JSON object for the tags route response. Returns: The JSON object for the tags route response. """ mapping = self._data_provider.list_tensors( ctx, experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME) result = {run: {} for run in mapping} for (run, tag_to_time_series) in mapping.items(): for (tag, time_series) in tag_to_time_series.items(): md = metadata.parse_plugin_metadata(time_series.plugin_content) if not self._version_checker.ok(md.version, run, tag): continue result[run][tag] = { "displayName": time_series.display_name, "description": plugin_util.markdown_to_safe_html(time_series.description), } return result
def testWeight1(self): self.generateDemoData() multiplexer = self.createMultiplexer() # Verify that the metadata was correctly written. accumulator = multiplexer.GetAccumulator('colors') tag_content_dict = accumulator.PluginTagToContent('pr_curves') # Test the summary contents. expected_tags = ['red/pr_curves', 'green/pr_curves', 'blue/pr_curves'] self.assertItemsEqual(expected_tags, list(tag_content_dict.keys())) for tag in expected_tags: # Parse the data within the JSON string and set the proto's fields. plugin_data = metadata.parse_plugin_metadata(tag_content_dict[tag]) self.assertEqual(5, plugin_data.num_thresholds) # Test the summary contents. tensor_events = accumulator.Tensors(tag) self.assertEqual(3, len(tensor_events)) # Test the output for the red classifier. The red classifier has the # narrowest standard deviation. tensor_events = accumulator.Tensors('red/pr_curves') self.validateTensorEvent( 0, [ [100.0, 45.0, 11.0, 2.0, 0.0], # True positives. [350.0, 50.0, 11.0, 2.0, 0.0], # False positives. [0.0, 300.0, 339.0, 348.0, 350.0], # True negatives. [0.0, 55.0, 89.0, 98.0, 100.0], # False negatives. [0.2222222, 0.4736842, 0.5, 0.5, 0.0], # Precision. [1.0, 0.45, 0.11, 0.02, 0.0], # Recall. ], tensor_events[0]) self.validateTensorEvent( 1, [ [100.0, 41.0, 11.0, 1.0, 0.0], # True positives. [350.0, 48.0, 7.0, 1.0, 0.0], # False positives. [0.0, 302.0, 343.0, 349.0, 350.0], # True negatives. [0.0, 59.0, 89.0, 99.0, 100.0], # False negatives. [0.2222222, 0.4606742, 0.6111111, 0.5, 0.0], # Precision. [1.0, 0.41, 0.11, 0.01, 0.0], # Recall. ], tensor_events[1]) self.validateTensorEvent( 2, [ [100.0, 39.0, 11.0, 2.0, 0.0], # True positives. [350.0, 54.0, 13.0, 1.0, 0.0], # False positives. [0.0, 296.0, 337.0, 349.0, 350.0], # True negatives. [0.0, 61.0, 89.0, 98.0, 100.0], # False negatives. [0.2222222, 0.4193548, 0.4583333, 0.6666667, 0.0 ], # Precision. [1.0, 0.39, 0.11, 0.02, 0.0], # Recall. ], tensor_events[2]) # Test the output for the green classifier. tensor_events = accumulator.Tensors('green/pr_curves') self.validateTensorEvent( 0, [ [200.0, 125.0, 48.0, 7.0, 0.0], # True positives. [250.0, 100.0, 13.0, 2.0, 0.0], # False positives. [0.0, 150.0, 237.0, 248.0, 250.0], # True negatives. [0.0, 75.0, 152.0, 193.0, 200.0], # False negatives. [0.4444444, 0.5555556, 0.7868853, 0.7777778, 0.0 ], # Precision. [1.0, 0.625, 0.24, 0.035, 0.0], # Recall. ], tensor_events[0]) self.validateTensorEvent( 1, [ [200.0, 123.0, 36.0, 7.0, 0.0], # True positives. [250.0, 91.0, 18.0, 2.0, 0.0], # False positives. [0.0, 159.0, 232.0, 248.0, 250.0], # True negatives. [0.0, 77.0, 164.0, 193.0, 200.0], # False negatives. [0.4444444, 0.5747663, 0.6666667, 0.7777778, 0.0 ], # Precision. [1.0, 0.615, 0.18, 0.035, 0.0], # Recall. ], tensor_events[1]) self.validateTensorEvent( 2, [ [200.0, 116.0, 40.0, 5.0, 0.0], # True positives. [250.0, 87.0, 18.0, 1.0, 0.0], # False positives. [0.0, 163.0, 232.0, 249.0, 250.0], # True negatives. [0.0, 84.0, 160.0, 195.0, 200.0], # False negatives. [0.4444444, 0.5714286, 0.6896552, 0.8333333, 0.0 ], # Precision. [1.0, 0.58, 0.2, 0.025, 0.0], # Recall. ], tensor_events[2]) # Test the output for the blue classifier. The normal distribution that is # the blue classifier has the widest standard deviation. tensor_events = accumulator.Tensors('blue/pr_curves') self.validateTensorEvent( 0, [ [150.0, 126.0, 45.0, 6.0, 0.0], # True positives. [300.0, 201.0, 38.0, 2.0, 0.0], # False positives. [0.0, 99.0, 262.0, 298.0, 300.0], # True negatives. [0.0, 24.0, 105.0, 144.0, 150.0], # False negatives. [0.3333333, 0.3853211, 0.5421687, 0.75, 0.0], # Precision. [1.0, 0.84, 0.3, 0.04, 0.0], # Recall. ], tensor_events[0]) self.validateTensorEvent( 1, [ [150.0, 128.0, 45.0, 4.0, 0.0], # True positives. [300.0, 204.0, 39.0, 6.0, 0.0], # False positives. [0.0, 96.0, 261.0, 294.0, 300.0], # True negatives. [0.0, 22.0, 105.0, 146.0, 150.0], # False negatives. [0.3333333, 0.3855422, 0.5357143, 0.4, 0.0], # Precision. [1.0, 0.8533334, 0.3, 0.0266667, 0.0], # Recall. ], tensor_events[1]) self.validateTensorEvent( 2, [ [150.0, 120.0, 39.0, 4.0, 0.0], # True positives. [300.0, 185.0, 38.0, 2.0, 0.0], # False positives. [0.0, 115.0, 262.0, 298.0, 300.0], # True negatives. [0.0, 30.0, 111.0, 146.0, 150.0], # False negatives. [0.3333333, 0.3934426, 0.5064935, 0.6666667, 0.0 ], # Precision. [1.0, 0.8, 0.26, 0.0266667, 0.0], # Recall. ], tensor_events[2])
def testExplicitWeights(self): self.generateDemoData() multiplexer = self.createMultiplexer() # Verify that the metadata was correctly written. accumulator = multiplexer.GetAccumulator('mask_every_other_prediction') tag_content_dict = accumulator.PluginTagToContent('pr_curves') # Test the summary contents. expected_tags = ['red/pr_curves', 'green/pr_curves', 'blue/pr_curves'] self.assertItemsEqual(expected_tags, list(tag_content_dict.keys())) for tag in expected_tags: # Parse the data within the JSON string and set the proto's fields. plugin_data = metadata.parse_plugin_metadata(tag_content_dict[tag]) self.assertEqual(5, plugin_data.num_thresholds) # Test the summary contents. tensor_events = accumulator.Tensors(tag) self.assertEqual(3, len(tensor_events)) # Test the output for the red classifier. The red classifier has the # narrowest standard deviation. tensor_events = accumulator.Tensors('red/pr_curves') self.validateTensorEvent( 0, [ [50.0, 22.0, 4.0, 0.0, 0.0], # True positives. [175.0, 22.0, 6.0, 1.0, 0.0], # False positives. [0.0, 153.0, 169.0, 174.0, 175.0], # True negatives. [0.0, 28.0, 46.0, 50.0, 50.0], # False negatives. [0.2222222, 0.5, 0.4, 0.0, 0.0], # Precision. [1.0, 0.44, 0.08, 0.0, 0.0], # Recall. ], tensor_events[0]) self.validateTensorEvent( 1, [ [50.0, 17.0, 5.0, 1.0, 0.0], # True positives. [175.0, 28.0, 1.0, 0.0, 0.0], # False positives. [0.0, 147.0, 174.0, 175.0, 175.0], # True negatives. [0.0, 33.0, 45.0, 49.0, 50.0], # False negatives. [0.2222222, 0.3777778, 0.8333333, 1.0, 0.0], # Precision. [1.0, 0.34, 0.1, 0.02, 0.0], # Recall. ], tensor_events[1]) self.validateTensorEvent( 2, [ [50.0, 18.0, 6.0, 1.0, 0.0], # True positives. [175.0, 27.0, 6.0, 0.0, 0.0], # False positives. [0.0, 148.0, 169.0, 175.0, 175.0], # True negatives. [0.0, 32.0, 44.0, 49.0, 50.0], # False negatives. [0.2222222, 0.4, 0.5, 1.0, 0.0], # Precision. [1.0, 0.36, 0.12, 0.02, 0.0], # Recall. ], tensor_events[2]) # Test the output for the green classifier. tensor_events = accumulator.Tensors('green/pr_curves') self.validateTensorEvent( 0, [ [100.0, 71.0, 24.0, 2.0, 0.0], # True positives. [125.0, 51.0, 5.0, 2.0, 0.0], # False positives. [0.0, 74.0, 120.0, 123.0, 125.0], # True negatives. [0.0, 29.0, 76.0, 98.0, 100.0], # False negatives. [0.4444444, 0.5819672, 0.8275862, 0.5, 0.0], # Precision. [1.0, 0.71, 0.24, 0.02, 0.0], # Recall. ], tensor_events[0]) self.validateTensorEvent( 1, [ [100.0, 63.0, 20.0, 5.0, 0.0], # True positives. [125.0, 42.0, 7.0, 1.0, 0.0], # False positives. [0.0, 83.0, 118.0, 124.0, 125.0], # True negatives. [0.0, 37.0, 80.0, 95.0, 100.0], # False negatives. [0.4444444, 0.6, 0.7407407, 0.8333333, 0.0], # Precision. [1.0, 0.63, 0.2, 0.05, 0.0], # Recall. ], tensor_events[1]) self.validateTensorEvent( 2, [ [100.0, 58.0, 19.0, 2.0, 0.0], # True positives. [125.0, 40.0, 7.0, 0.0, 0.0], # False positives. [0.0, 85.0, 118.0, 125.0, 125.0], # True negatives. [0.0, 42.0, 81.0, 98.0, 100.0], # False negatives. [0.4444444, 0.5918368, 0.7307692, 1.0, 0.0], # Precision. [1.0, 0.58, 0.19, 0.02, 0.0], # Recall. ], tensor_events[2]) # Test the output for the blue classifier. The normal distribution that is # the blue classifier has the widest standard deviation. tensor_events = accumulator.Tensors('blue/pr_curves') self.validateTensorEvent( 0, [ [75.0, 64.0, 21.0, 5.0, 0.0], # True positives. [150.0, 105.0, 18.0, 0.0, 0.0], # False positives. [0.0, 45.0, 132.0, 150.0, 150.0], # True negatives. [0.0, 11.0, 54.0, 70.0, 75.0], # False negatives. [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0], # Precision. [1.0, 0.8533334, 0.28, 0.0666667, 0.0], # Recall. ], tensor_events[0]) self.validateTensorEvent( 1, [ [75.0, 62.0, 21.0, 1.0, 0.0], # True positives. [150.0, 99.0, 21.0, 3.0, 0.0], # False positives. [0.0, 51.0, 129.0, 147.0, 150.0], # True negatives. [0.0, 13.0, 54.0, 74.0, 75.0], # False negatives. [0.3333333, 0.3850932, 0.5, 0.25, 0.0], # Precision. [1.0, 0.8266667, 0.28, 0.0133333, 0.0], # Recall. ], tensor_events[1]) self.validateTensorEvent( 2, [ [75.0, 61.0, 16.0, 2.0, 0.0], # True positives. [150.0, 92.0, 20.0, 1.0, 0.0], # False positives. [0.0, 58.0, 130.0, 149.0, 150.0], # True negatives. [0.0, 14.0, 59.0, 73.0, 75.0], # False negatives. [0.3333333, 0.3986928, 0.4444444, 0.6666667, 0.0 ], # Precision. [1.0, 0.8133333, 0.2133333, 0.0266667, 0.0], # Recall. ], tensor_events[2])
def test_raw_data(self): # We pass these raw counts and precision/recall values. name = "foo" true_positive_counts = [75, 64, 21, 5, 0] false_positive_counts = [150, 105, 18, 0, 0] true_negative_counts = [0, 45, 132, 150, 150] false_negative_counts = [0, 11, 54, 70, 75] precision = [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0] recall = [1.0, 0.8533334, 0.28, 0.0666667, 0.0] num_thresholds = 5 display_name = "some_raw_values" description = "We passed raw values into a summary op." op = summary.raw_data_op( name=name, true_positive_counts=tf.constant(true_positive_counts), false_positive_counts=tf.constant(false_positive_counts), true_negative_counts=tf.constant(true_negative_counts), false_negative_counts=tf.constant(false_negative_counts), precision=tf.constant(precision), recall=tf.constant(recall), num_thresholds=num_thresholds, display_name=display_name, description=description, ) pb_via_op = self.normalize_summary_pb(self.pb_via_op(op)) # Call the corresponding method that is decoupled from TensorFlow. pb = self.normalize_summary_pb( summary.raw_data_pb( name=name, true_positive_counts=true_positive_counts, false_positive_counts=false_positive_counts, true_negative_counts=true_negative_counts, false_negative_counts=false_negative_counts, precision=precision, recall=recall, num_thresholds=num_thresholds, display_name=display_name, description=description, )) # The 2 methods above should write summaries with the same data. self.assertProtoEquals(pb, pb_via_op) # Test the metadata. summary_metadata = pb.value[0].metadata self.assertEqual("some_raw_values", summary_metadata.display_name) self.assertEqual( "We passed raw values into a summary op.", summary_metadata.summary_description, ) self.assertEqual(metadata.PLUGIN_NAME, summary_metadata.plugin_data.plugin_name) plugin_data = metadata.parse_plugin_metadata( summary_metadata.plugin_data.content) self.assertEqual(5, plugin_data.num_thresholds) # Test the summary contents. values = tensor_util.make_ndarray(pb.value[0].tensor) self.verify_float_arrays_are_equal( [ [75.0, 64.0, 21.0, 5.0, 0.0], # True positives. [150.0, 105.0, 18.0, 0.0, 0.0], # False positives. [0.0, 45.0, 132.0, 150.0, 150.0], # True negatives. [0.0, 11.0, 54.0, 70.0, 75.0], # False negatives. [0.3333333, 0.3786982, 0.5384616, 1.0, 0.0], # Precision. [1.0, 0.8533334, 0.28, 0.0666667, 0.0], # Recall. ], values, )
def pr_curves_impl(self, runs, tag): """Creates the JSON object for the PR curves response for a run-tag combo. Arguments: runs: A list of runs to fetch the curves for. tag: The tag to fetch the curves for. Raises: ValueError: If no PR curves could be fetched for a run and tag. Returns: The JSON object for the PR curves route response. """ if self._db_connection_provider: # Serve data from the database. db = self._db_connection_provider() # We select for steps greater than -1 because the writer inserts # placeholder rows en masse. The check for step filters out those rows. cursor = db.execute(''' SELECT Runs.run_name, Tensors.step, Tensors.computed_time, Tensors.data, Tensors.dtype, Tensors.shape, Tags.plugin_data FROM Tensors JOIN Tags ON Tensors.series = Tags.tag_id JOIN Runs ON Tags.run_id = Runs.run_id WHERE Runs.run_name IN (%s) AND Tags.tag_name = ? AND Tags.plugin_name = ? AND Tensors.step > -1 ORDER BY Tensors.step ''' % ','.join(['?'] * len(runs)), runs + [tag, metadata.PLUGIN_NAME]) response_mapping = {} for (run, step, wall_time, data, dtype, shape, plugin_data) in cursor: if run not in response_mapping: response_mapping[run] = [] dtype_for_buf = tf.DType(dtype) if USE_TF else tf.dtypes.DType(dtype) buf = np.frombuffer(data, dtype=dtype_for_buf.as_numpy_dtype) data_array = buf.reshape([int(i) for i in shape.split(',')]) plugin_data_proto = plugin_data_pb2.PrCurvePluginData() string_buffer = np.frombuffer(plugin_data, dtype=np.dtype('b')) plugin_data_proto.ParseFromString(tf.compat.as_bytes( string_buffer.tostring())) thresholds = self._compute_thresholds(plugin_data_proto.num_thresholds) entry = self._make_pr_entry(step, wall_time, data_array, thresholds) response_mapping[run].append(entry) else: # Serve data from events files. response_mapping = {} for run in runs: try: tensor_events = self._multiplexer.Tensors(run, tag) except KeyError: raise ValueError( 'No PR curves could be found for run %r and tag %r' % (run, tag)) content = self._multiplexer.SummaryMetadata( run, tag).plugin_data.content pr_curve_data = metadata.parse_plugin_metadata(content) thresholds = self._compute_thresholds(pr_curve_data.num_thresholds) response_mapping[run] = [ self._process_tensor_event(e, thresholds) for e in tensor_events] return response_mapping