def _test_distributions(self, run_name, tag_name, should_work=True): self.set_up_with_runs( [ self._RUN_WITH_SCALARS, self._RUN_WITH_LEGACY_DISTRIBUTION, self._RUN_WITH_DISTRIBUTION, ] ) if should_work: (data, mime_type) = self.plugin.distributions_impl( context.RequestContext(), tag_name, run_name, experiment="exp" ) self.assertEqual("application/json", mime_type) self.assertEqual(len(data), self._STEPS) for i in xrange(self._STEPS): [_unused_wall_time, step, bps_and_icdfs] = data[i] self.assertEqual(i, step) (bps, _unused_icdfs) = zip(*bps_and_icdfs) self.assertEqual(bps, compressor.NORMAL_HISTOGRAM_BPS) else: with self.assertRaises(errors.NotFoundError): self.plugin.distributions_impl( context.RequestContext(), self._DISTRIBUTION_TAG, run_name, experiment="exp", )
def test_image_data_from_time_series_query(self): self._write_image("run1", "images/tagA", samples=3) self._multiplexer.Reload() requests = [ { "plugin": "images", "tag": "images/tagA", "run": "run1", "sample": 2, } ] original_response = self._plugin._time_series_impl( context.RequestContext(), "expid", requests ) response = self._plugin._time_series_impl( context.RequestContext(), "expid", requests ) clean_response = self._clean_time_series_responses(response) self.assertEqual( [ { "plugin": "images", "tag": "images/tagA", "run": "run1", "sample": 2, "runToSeries": { "run1": [ { "wallTime": "<wall_time>", "step": 0, "imageId": "<image_id>", } ] }, } ], clean_response, ) image_id = original_response[0]["runToSeries"]["run1"][0]["imageId"] (data, content_type) = self._plugin._image_data_impl( context.RequestContext(), image_id ) self.assertIsInstance(data, bytes) self.assertGreater(len(data), 0)
def test_index(self): self.set_up_with_runs( [ self._RUN_WITH_SCALARS, self._RUN_WITH_LEGACY_DISTRIBUTION, self._RUN_WITH_DISTRIBUTION, ] ) self.assertEqual( { # _RUN_WITH_SCALARS omitted: No distribution data. self._RUN_WITH_LEGACY_DISTRIBUTION: { self._LEGACY_DISTRIBUTION_TAG: { "displayName": self._LEGACY_DISTRIBUTION_TAG, "description": "", }, }, self._RUN_WITH_DISTRIBUTION: { "%s/histogram_summary" % self._DISTRIBUTION_TAG: { "displayName": self._DISPLAY_NAME, "description": self._HTML_DESCRIPTION, }, }, }, self.plugin.index_impl(context.RequestContext(), experiment="exp"), )
def testPluginIndexImpl(self): plugin = self.load_plugin() run_to_tags = plugin.index_impl(context.RequestContext(), experiment="123") self.assertItemsEqual(["fry", "leela"], run_to_tags.keys()) self.assertItemsEqual(["message", "vector"], run_to_tags["fry"]) self.assertItemsEqual(["message", "vector"], run_to_tags["leela"])
def testAnnotations(self): plugin = self.create_plugin() annotations = plugin.annotations_impl( context.RequestContext(), experiment="exp", ) self.assertItemsEqual(["name_1", "name_2"], annotations["run_1"]) self.assertItemsEqual(["name_1", "name_2"], annotations["run_2"])
def test_index(self): plugin = self.load_plugin( [ self._RUN_WITH_SCALARS, self._RUN_WITH_LEGACY_HISTOGRAM, self._RUN_WITH_HISTOGRAM, ] ) self.assertEqual( { # _RUN_WITH_SCALARS omitted: No histogram data. self._RUN_WITH_LEGACY_HISTOGRAM: { self._LEGACY_HISTOGRAM_TAG: { "displayName": self._LEGACY_HISTOGRAM_TAG, "description": "", }, }, self._RUN_WITH_HISTOGRAM: { "%s/histogram_summary" % self._HISTOGRAM_TAG: { "displayName": self._DISPLAY_NAME, "description": self._HTML_DESCRIPTION, }, }, }, plugin.index_impl(context.RequestContext(), experiment="exp"), )
def test_time_series_scalar(self): self._write_scalar_data("run1", "scalars/tagA", [0, 100, -200]) self._multiplexer.Reload() requests = [{"plugin": "scalars", "tag": "scalars/tagA"}] response = self._plugin._time_series_impl(context.RequestContext(), "", requests) clean_response = self._clean_time_series_responses(response) self.assertEqual( [{ "plugin": "scalars", "tag": "scalars/tagA", "runToSeries": { "run1": [ { "wallTime": "<wall_time>", "step": 0, "value": 0.0, }, { "wallTime": "<wall_time>", "step": 1, "value": 100.0, }, { "wallTime": "<wall_time>", "step": 2, "value": -200.0, }, ] }, }], clean_response, )
def test_time_series_single_request_specific_run(self): self._write_scalar_data("run1", "scalars/tagA", [0]) self._write_scalar_data("run2", "scalars/tagA", [1]) self._multiplexer.Reload() requests = [{ "plugin": "scalars", "tag": "scalars/tagA", "run": "run2" }] response = self._plugin._time_series_impl(context.RequestContext(), "", requests) clean_response = self._clean_time_series_responses(response) self.assertEqual( [{ "plugin": "scalars", "runToSeries": { "run2": [ { "step": 0, "value": 1.0, "wallTime": "<wall_time>", }, ], }, "tag": "scalars/tagA", "run": "run2", }], clean_response, )
def test_tags_conflicting_description(self): self._write_scalar("run1", "scalars/tagA", None) self._write_scalar("run2", "scalars/tagA", "tagA is hot") self._write_scalar("run3", "scalars/tagA", "tagA is cold") self._write_scalar("run4", "scalars/tagA", "tagA is cold") self._write_histogram("run1", "histograms/tagA", None) self._write_histogram("run2", "histograms/tagA", "tagA is hot") self._write_histogram("run3", "histograms/tagA", "tagA is cold") self._write_histogram("run4", "histograms/tagA", "tagA is cold") self._multiplexer.Reload() response = self._plugin._tags_impl(context.RequestContext(), "eid") expected_composite_description = ("<h1>Multiple descriptions</h1>\n" "<h2>For runs: run3, run4</h2>\n" "<p>tagA is cold</p>\n" "<h2>For run: run2</h2>\n" "<p>tagA is hot</p>") self.assertEqual( {"scalars/tagA": expected_composite_description}, response["scalars"]["tagDescriptions"], ) self.assertEqual( {"histograms/tagA": expected_composite_description}, response["histograms"]["tagDescriptions"], )
def test_tags_unsafe_conflicting_description(self): self._write_scalar("<&#run1>", "scalars/<&#tag>", None) self._write_scalar("<&#run2>", "scalars/<&#tag>", "<&# is hot>") self._write_scalar("<&#run3>", "scalars/<&#tag>", "<&# is cold>") self._write_scalar("<&#run4>", "scalars/<&#tag>", "<&# is cold>") self._write_histogram("<&#run1>", "histograms/<&#tag>", None) self._write_histogram("<&#run2>", "histograms/<&#tag>", "<&# is hot>") self._write_histogram("<&#run3>", "histograms/<&#tag>", "<&# is cold>") self._write_histogram("<&#run4>", "histograms/<&#tag>", "<&# is cold>") self._multiplexer.Reload() response = self._plugin._tags_impl(context.RequestContext(), "eid") expected_composite_description = ( "<h1>Multiple descriptions</h1>\n" "<h2>For runs: <&#run3>, <&#run4></h2>\n" "<p><&# is cold></p>\n" "<h2>For run: <&#run2></h2>\n" "<p><&# is hot></p>") self.assertEqual( {"scalars/<&#tag>": expected_composite_description}, response["scalars"]["tagDescriptions"], ) self.assertEqual( {"histograms/<&#tag>": expected_composite_description}, response["histograms"]["tagDescriptions"], )
def testValues(self): plugin = self.create_plugin() values = plugin.values_impl(context.RequestContext(), experiment="exp") self.assertItemsEqual([1.0, -1.0], values["run_1"][0]) self.assertItemsEqual([0.5, -0.5], values["run_1"][1]) self.assertItemsEqual([1.0, -1.0], values["run_2"][0]) self.assertItemsEqual([-0.5, None], values["run_2"][1])
def is_active(self): """The graphs plugin is active iff any run has a graph or metadata.""" if self._data_provider: return False # `list_plugins` as called by TB core suffices empty_context = context.RequestContext() # not used return bool(self.info_impl(empty_context))
def testMergedLayout(self): ctx = context.RequestContext() parsed_layout = layout_pb2.Layout() json_format.Parse(self.plugin.layout_impl(ctx, "exp_id"), parsed_layout) correct_layout = layout_pb2.Layout( category=[ # A category with this name is also present in a layout for a # different run (the logdir run) layout_pb2.Category( title="cross entropy", chart=[ layout_pb2.Chart( title="cross entropy", multiline=layout_pb2.MultilineChartContent( tag=[r"cross entropy"], ), ), layout_pb2.Chart( title="cross entropy margin chart", margin=layout_pb2.MarginChartContent( series=[ layout_pb2.MarginChartContent.Series( value="cross entropy", lower="cross entropy lower", upper="cross entropy upper", ), ], ), ), ], closed=True, ), layout_pb2.Category( title="mean biases", chart=[ layout_pb2.Chart( title="mean layer biases", multiline=layout_pb2.MultilineChartContent( tag=[ r"mean/layer0/biases", r"mean/layer1/biases", ], ), ), ], ), layout_pb2.Category( title="std weights", chart=[ layout_pb2.Chart( title="stddev layer weights", multiline=layout_pb2.MultilineChartContent( tag=[r"stddev/layer\d+/weights"], ), ), ], ), ] ) self.assertProtoEquals(correct_layout, parsed_layout)
def testMetrics(self): plugin = self.create_plugin() metrics = plugin.metrics_impl( context.RequestContext(), experiment="exp", ) self.assertItemsEqual(["A", "B"], metrics["run_1"]) self.assertItemsEqual(["A", "B"], metrics["run_2"])
def test_experiment_with_experiment_tag(self): experiment = """ description: 'Test experiment' metric_infos: [ { name: { tag: 'current_temp' } } ] """ run = "exp" tag = metadata.EXPERIMENT_TAG m = summary_pb2.SummaryMetadata() m.data_class = summary_pb2.DATA_CLASS_TENSOR m.plugin_data.plugin_name = metadata.PLUGIN_NAME m.plugin_data.content = self._serialized_plugin_data( DATA_TYPE_EXPERIMENT, experiment) self._mock_multiplexer.AllSummaryMetadata.side_effect = None self._mock_multiplexer.AllSummaryMetadata.return_value = { run: { tag: m } } ctxt = backend_context.Context(self._mock_tb_context) request_ctx = context.RequestContext() self.assertProtoEquals( experiment, ctxt.experiment_from_metadata( request_ctx, "123", ctxt.hparams_metadata(request_ctx, "123")), )
def test_info(self, plugin): expected = { "_RUN_WITH_GRAPH_WITH_METADATA": { "run": "_RUN_WITH_GRAPH_WITH_METADATA", "run_graph": True, "tags": { "secret-stats": { "conceptual_graph": False, "profile": True, "tag": "secret-stats", "op_graph": False, }, }, }, "_RUN_WITH_GRAPH_WITHOUT_METADATA": { "run": "_RUN_WITH_GRAPH_WITHOUT_METADATA", "run_graph": True, "tags": {}, }, "_RUN_WITHOUT_GRAPH_WITH_METADATA": { "run": "_RUN_WITHOUT_GRAPH_WITH_METADATA", "run_graph": False, "tags": { "secret-stats": { "conceptual_graph": False, "profile": True, "tag": "secret-stats", "op_graph": False, }, }, }, } actual = plugin.info_impl(context.RequestContext(), "eid") self.assertEqual(expected, actual)
def _update_configs(self): """Updates `self._configs` and `self._run_paths`.""" if self.data_provider and self.logdir: # Create a background context; we may not be in a request. ctx = context.RequestContext() run_paths = { run.run_name: os.path.join(self.logdir, run.run_name) for run in self.data_provider.list_runs(ctx, experiment_id="") } else: run_paths = {} run_paths_changed = run_paths != self._run_paths self._run_paths = run_paths run_path_pairs = list(self._run_paths.items()) self._append_plugin_asset_directories(run_path_pairs) # Also accept the root logdir as a model checkpoint directory, # so that the projector still works when there are no runs. # (Case on `run` rather than `path` to avoid issues with # absolute/relative paths on any filesystems.) if "." not in self._run_paths: run_path_pairs.append((".", self.logdir)) if run_paths_changed or _latest_checkpoints_changed( self._configs, run_path_pairs ): self.readers = {} self._configs, self.config_fpaths = self._read_latest_config_files( run_path_pairs ) self._augment_configs_with_checkpoint_info()
def test_image_bad_request(self): self._write_image("run1", "images/tagA", 1, None) self._multiplexer.Reload() invalid_sample = 999 requests = [ { "plugin": "images", "tag": "images/tagA", "sample": invalid_sample, "run": "run1", }, { "plugin": "images", "tag": "images/tagA", "run": "run1" }, { "plugin": "images", "tag": "images/tagA", }, ] response = self._plugin._time_series_impl(context.RequestContext(), "expid", requests) errors = [ series_response.get("error", "") for series_response in response ] self.assertEqual(errors, ["", "Missing sample", "Missing run"])
def test_tags(self): self._write_scalar("run1", "scalars/tagA", None) self._write_scalar("run1", "scalars/tagA", None) self._write_scalar("run1", "scalars/tagB", None) self._write_scalar("run2", "scalars/tagB", None) self._write_histogram("run1", "histograms/tagA", None) self._write_histogram("run1", "histograms/tagA", None) self._write_histogram("run1", "histograms/tagB", None) self._write_histogram("run2", "histograms/tagB", None) self._write_image("run1", "images/tagA", 1, None) self._write_image("run1", "images/tagA", 2, None) self._write_image("run1", "images/tagB", 3, None) self._write_image("run2", "images/tagB", 4, None) self._multiplexer.Reload() response = self._plugin._tags_impl(context.RequestContext(), "eid") self.assertEqual( { "runTagInfo": { "run1": ["scalars/tagA", "scalars/tagB"], "run2": ["scalars/tagB"], }, "tagDescriptions": {}, }, response["scalars"], ) self.assertEqual( { "runTagInfo": { "run1": ["histograms/tagA", "histograms/tagB"], "run2": ["histograms/tagB"], }, "tagDescriptions": {}, }, response["histograms"], ) self.assertEqual( { "tagDescriptions": {}, "tagRunSampledInfo": { "images/tagA": { "run1": { "maxSamplesPerStep": 2 } }, "images/tagB": { "run1": { "maxSamplesPerStep": 3 }, "run2": { "maxSamplesPerStep": 4 }, }, }, }, response["images"], )
def test_experiment_without_experiment_tag_many_distinct_values(self): self.session_1_start_info_ = """ hparams:[ {key: 'batch_size' value: {number_value: 100}}, {key: 'lr' value: {string_value: '0.01'}} ] """ self.session_2_start_info_ = """ hparams:[ {key: 'lr' value: {number_value: 0.02}}, {key: 'model_type' value: {string_value: 'CNN'}} ] """ self.session_3_start_info_ = """ hparams:[ {key: 'batch_size' value: {bool_value: true}}, {key: 'model_type' value: {string_value: 'CNN'}} ] """ expected_exp = """ hparam_infos: { name: 'batch_size' type: DATA_TYPE_STRING } hparam_infos: { name: 'lr' type: DATA_TYPE_STRING } hparam_infos: { name: 'model_type' type: DATA_TYPE_STRING domain_discrete: { values: [{string_value: 'CNN'}] } } metric_infos: { name: {group: '', tag: 'accuracy'} } metric_infos: { name: {group: '', tag: 'loss'} } metric_infos: { name: {group: 'eval', tag: 'loss'} } metric_infos: { name: {group: 'train', tag: 'loss'} } """ ctxt = backend_context.Context( self._mock_tb_context, max_domain_discrete_len=1 ) request_ctx = context.RequestContext() actual_exp = ctxt.experiment_from_metadata( request_ctx, "123", ctxt.hparams_metadata(request_ctx, "123"), ) _canonicalize_experiment(actual_exp) self.assertProtoEquals(expected_exp, actual_exp)
def testEmbeddings(self): plugin = self.create_plugin() embeddings = plugin.embeddings_impl(context.RequestContext(), experiment="exp") self.assertItemsEqual([1.0, 0.5], embeddings["run_1"][0]) self.assertItemsEqual([-0.5, 0.5], embeddings["run_1"][1]) self.assertItemsEqual([1.0, 0.5], embeddings["run_2"][0]) self.assertItemsEqual([-0.5, 0.5], embeddings["run_2"][1])
def test_args(self): auth = auth_lib.AuthContext({}, {"REQUEST_METHOD": "GET"}) ctx = context.RequestContext(auth=auth, remote_ip=REMOTE_IP, x_forwarded_for=X_FORWARDED_FOR_IPS) self.assertEqual(ctx.auth, auth) self.assertEqual(ctx.remote_ip, REMOTE_IP) self.assertEqual(ctx.x_forwarded_for, X_FORWARDED_FOR_IPS)
def test_run_metadata(self, plugin): ctx = context.RequestContext() result = plugin.run_metadata_impl(ctx, "123", _RUN_WITH_GRAPH_WITH_METADATA[0], self._METADATA_TAG) (metadata_pbtxt, mime_type) = result self.assertEqual(mime_type, "text/x-protobuf") text_format.Parse(metadata_pbtxt, config_pb2.RunMetadata())
def testLayoutFromSingleRun(self): # The foo directory contains 1 single layout. ctx = context.RequestContext() local_plugin = self.createPlugin(os.path.join(self.logdir, "foo")) parsed_layout = layout_pb2.Layout() json_format.Parse( local_plugin.layout_impl(ctx, "exp_id"), parsed_layout ) self.assertProtoEquals(self.foo_layout, parsed_layout)
def _get_graph(self, plugin, run=None, **kwargs): """Set up runs, then fetch and return the graph as a proto.""" (graph_pbtxt, mime_type) = plugin.graph_impl( context.RequestContext(), run if run is not None else _RUN_WITH_GRAPH_WITH_METADATA[0], **kwargs, ) self.assertEqual(mime_type, "text/x-protobuf") return text_format.Parse(graph_pbtxt, tf.compat.v1.GraphDef())
def testTags(self): plugin = self.create_plugin() tags = plugin.tags_impl(context.RequestContext(), experiment="exp") tags = json.loads(tags) gt_runs = ["run_1", "run_2"] gt_tags = ["_npmi_/annotations", "_npmi_/metrics", "_npmi_/values"] self.assertItemsEqual(gt_runs, tags.keys()) self.assertItemsEqual(gt_tags, tags["run_1"]) self.assertItemsEqual(gt_tags, tags["run_2"])
def _run_handler(self, request): request_proto = api_pb2.ListMetricEvalsRequest() text_format.Merge(request, request_proto) handler = list_metric_evals.Handler( context.RequestContext(), request_proto, self._mock_scalars_plugin, "exp_id", ) return handler.run()
def test_scalars_with_histogram(self): plugin = self.load_plugin([self._RUN_WITH_HISTOGRAM]) with self.assertRaises(errors.NotFoundError): plugin.scalars_impl( context.RequestContext(), self._HISTOGRAM_TAG, self._RUN_WITH_HISTOGRAM, "eid", scalars_plugin.OutputFormat.JSON, )
def testPrCurvesRaisesValueErrorWhenNoData(self): """Tests that the method for obtaining PR curve data raises a ValueError. The handler should raise a ValueError when no PR curve data can be found for a certain run-tag combination. """ with six.assertRaisesRegex(self, ValueError, r"No PR curves could be found"): self.plugin.pr_curves_impl(context.RequestContext(), "123", ["colors"], "non_existent_tag") with six.assertRaisesRegex(self, ValueError, r"No PR curves could be found"): self.plugin.pr_curves_impl( context.RequestContext(), "123", ["non_existent_run"], "blue/pr_curves", )
def is_active(self): """This plugin is active iff any run has at least one histograms tag.""" if self._data_provider: return False # `list_plugins` as called by TB core suffices if self._multiplexer: empty_context = context.RequestContext() # not used return any(self.index_impl(empty_context, experiment="").values()) return False