def test_session_start_pb(self): start_time_secs = 314160 session_start_info = plugin_data_pb2.SessionStartInfo( model_uri="//model/uri", group_name="session_group", start_time_secs=start_time_secs) session_start_info.hparams["param1"].string_value = "string" # TODO: Fix nondeterminism. # session_start_info.hparams["param2"].number_value = 5.0 # session_start_info.hparams["param3"].bool_value = False self.assertEqual( summary.session_start_pb( hparams={ "param1": "string", # "param2":5, # "param3":False, }, model_uri="//model/uri", group_name="session_group", start_time_secs=start_time_secs), tf.Summary(value=[ tf.Summary.Value( tag="_hparams_/session_start_info", metadata=tf.SummaryMetadata( plugin_data=tf.SummaryMetadata.PluginData( plugin_name="hparams", content=(plugin_data_pb2.HParamsPluginData( version=0, session_start_info=session_start_info). SerializeToString())))) ]))
def setUp(self): self.logdir = os.path.join(self.get_temp_dir(), "logs") self.hparams = { hp.HParam("learning_rate", hp.RealInterval(1e-2, 1e-1)): 0.02, hp.HParam("dense_layers", hp.IntInterval(2, 7)): 5, hp.HParam("optimizer", hp.Discrete(["adam", "sgd"])): "adam", hp.HParam("who_knows_what"): "???", hp.HParam( "magic", hp.Discrete([False, True]), display_name="~*~ Magic ~*~", description="descriptive", ): True, "dropout": 0.3, } self.normalized_hparams = { "learning_rate": 0.02, "dense_layers": 5, "optimizer": "adam", "who_knows_what": "???", "magic": True, "dropout": 0.3, } self.start_time_secs = 123.45 self.trial_id = "psl27" self.expected_session_start_pb = plugin_data_pb2.SessionStartInfo() text_format.Merge( """ hparams { key: "learning_rate" value { number_value: 0.02 } } hparams { key: "dense_layers" value { number_value: 5 } } hparams { key: "optimizer" value { string_value: "adam" } } hparams { key: "who_knows_what" value { string_value: "???" } } hparams { key: "magic" value { bool_value: true } } hparams { key: "dropout" value { number_value: 0.3 } } """, self.expected_session_start_pb, ) self.expected_session_start_pb.group_name = self.trial_id self.expected_session_start_pb.start_time_secs = self.start_time_secs
def hparams_pb(hparams, trial_id=None, start_time_secs=None): # NOTE: Keep docs in sync with `hparams` above. """Create a summary encoding hyperparameter values for a single trial. Args: hparams: A `dict` mapping hyperparameters to the values used in this trial. Keys should be the names of `HParam` objects used in an experiment, or the `HParam` objects themselves. Values should be Python `bool`, `int`, `float`, or `string` values, depending on the type of the hyperparameter. trial_id: An optional `str` ID for the set of hyperparameter values used in this trial. Defaults to a hash of the hyperparameters. start_time_secs: The time that this trial started training, as seconds since epoch. Defaults to the current time. Returns: A TensorBoard `summary_pb2.Summary` message. """ if start_time_secs is None: start_time_secs = time.time() hparams = _normalize_hparams(hparams) group_name = _derive_session_group_name(trial_id, hparams) session_start_info = plugin_data_pb2.SessionStartInfo( group_name=group_name, start_time_secs=start_time_secs, ) for hp_name in sorted(hparams): hp_value = hparams[hp_name] if isinstance(hp_value, bool): session_start_info.hparams[hp_name].bool_value = hp_value elif isinstance(hp_value, (float, int)): session_start_info.hparams[hp_name].number_value = hp_value elif isinstance(hp_value, six.string_types): session_start_info.hparams[hp_name].string_value = hp_value else: raise TypeError( "hparams[%r] = %r, of unsupported type %r" % (hp_name, hp_value, type(hp_value)) ) return _summary_pb( metadata.SESSION_START_INFO_TAG, plugin_data_pb2.HParamsPluginData( session_start_info=session_start_info ), )
def session_start_pb(hparams, model_uri="", monitor_url="", group_name="", start_time_secs=None): """Creates a summary that contains a training session metadata information. One such summary per training session should be created. Each should have a different run. Arguments: hparams: A dictionary with string keys. Describes the hyperparameter values used in the session mappng each hyperparameter name to its value. Supported value types are bool, int, float, or str. model_uri: See the comment for the field with the same name of plugin_data_pb2.SessionStartInfo. monitor_url: See the comment for the field with the same name of plugin_data_pb2.SessionStartInfo. group_name: See the comment for the field with the same name of plugin_data_pb2.SessionStartInfo. start_time_secs: float. The time to use as the session start time. Represented as seconds since the UNIX epoch. If None uses the current time. Returns: Returns the summary protobuffer mentioned above. """ if start_time_secs is None: start_time_secs = time.time() session_start_info = plugin_data_pb2.SessionStartInfo( model_uri=model_uri, monitor_url=monitor_url, group_name=group_name, start_time_secs=start_time_secs) for (hp_name, hp_val) in six.iteritems(hparams): if isinstance(hp_val, (float, int)): session_start_info.hparams[hp_name].number_value = hp_val elif isinstance(hp_val, six.string_types): session_start_info.hparams[hp_name].string_value = hp_val elif isinstance(hp_val, bool): session_start_info.hparams[hp_name].bool_value = hp_val else: raise TypeError('hparams[%s]=%s has type: %s which is not supported' % (hp_name, hp_val, type(hp_val))) return _summary(metadata.SESSION_START_INFO_TAG, plugin_data_pb2.HParamsPluginData( session_start_info=session_start_info))
def session_start_pb(hparams, model_uri="", monitor_url="", group_name="", start_time_secs=None): """Constructs a SessionStartInfo protobuffer. Creates a summary that contains a training session metadata information. One such summary per training session should be created. Each should have a different run. Args: hparams: A dictionary with string keys. Describes the hyperparameter values used in the session, mapping each hyperparameter name to its value. Supported value types are `bool`, `int`, `float`, `str`, `list`, `tuple`. The type of value must correspond to the type of hyperparameter (defined in the corresponding api_pb2.HParamInfo member of the Experiment protobuf) as follows: +-----------------+---------------------------------+ |Hyperparameter | Allowed (Python) value types | |type | | +-----------------+---------------------------------+ |DATA_TYPE_BOOL | bool | |DATA_TYPE_FLOAT64| int, float | |DATA_TYPE_STRING | six.string_types, tuple, list | +-----------------+---------------------------------+ Tuple and list instances will be converted to their string representation. model_uri: See the comment for the field with the same name of plugin_data_pb2.SessionStartInfo. monitor_url: See the comment for the field with the same name of plugin_data_pb2.SessionStartInfo. group_name: See the comment for the field with the same name of plugin_data_pb2.SessionStartInfo. start_time_secs: float. The time to use as the session start time. Represented as seconds since the UNIX epoch. If None uses the current time. Returns: The summary protobuffer mentioned above. """ if start_time_secs is None: start_time_secs = time.time() session_start_info = plugin_data_pb2.SessionStartInfo( model_uri=model_uri, monitor_url=monitor_url, group_name=group_name, start_time_secs=start_time_secs, ) for (hp_name, hp_val) in six.iteritems(hparams): if isinstance(hp_val, (float, int)): session_start_info.hparams[hp_name].number_value = hp_val elif isinstance(hp_val, six.string_types): session_start_info.hparams[hp_name].string_value = hp_val elif isinstance(hp_val, bool): session_start_info.hparams[hp_name].bool_value = hp_val elif isinstance(hp_val, (list, tuple)): session_start_info.hparams[hp_name].string_value = str(hp_val) else: raise TypeError( "hparams[%s]=%s has type: %s which is not supported" % (hp_name, hp_val, type(hp_val))) return _summary( metadata.SESSION_START_INFO_TAG, plugin_data_pb2.HParamsPluginData( session_start_info=session_start_info), )
def test_eager(self): def mock_time(): mock_time.time += 1 return mock_time.time mock_time.time = 1556227801.875 initial_time = mock_time.time with mock.patch("time.time", mock_time): self._initialize_model(writer=self.logdir) self.model.fit(x=[(1,)], y=[(2,)], callbacks=[self.callback]) final_time = mock_time.time files = os.listdir(self.logdir) self.assertEqual(len(files), 1, files) events_file = os.path.join(self.logdir, files[0]) plugin_data = [] for event in tf.compat.v1.train.summary_iterator(events_file): if event.WhichOneof("what") != "summary": continue self.assertEqual(len(event.summary.value), 1, event.summary.value) value = event.summary.value[0] self.assertEqual( value.metadata.plugin_data.plugin_name, metadata.PLUGIN_NAME, ) plugin_data.append(value.metadata.plugin_data.content) self.assertEqual(len(plugin_data), 2, plugin_data) (start_plugin_data, end_plugin_data) = plugin_data start_pb = metadata.parse_session_start_info_plugin_data(start_plugin_data) end_pb = metadata.parse_session_end_info_plugin_data(end_plugin_data) # We're not the only callers of `time.time`; Keras calls it # internally an unspecified number of times, so we're not guaranteed # to know the exact values. Instead, we perform relative checks... self.assertGreater(start_pb.start_time_secs, initial_time) self.assertLess(start_pb.start_time_secs, end_pb.end_time_secs) self.assertLessEqual(start_pb.start_time_secs, final_time) # ...and then stub out the times for proto equality checks below. start_pb.start_time_secs = 1234.5 end_pb.end_time_secs = 6789.0 expected_start_pb = plugin_data_pb2.SessionStartInfo() text_format.Merge( """ start_time_secs: 1234.5 group_name: "my_trial" hparams { key: "optimizer" value { string_value: "adam" } } hparams { key: "dense_neurons" value { number_value: 8.0 } } """, expected_start_pb, ) self.assertEqual(start_pb, expected_start_pb) expected_end_pb = plugin_data_pb2.SessionEndInfo() text_format.Merge( """ end_time_secs: 6789.0 status: STATUS_SUCCESS """, expected_end_pb, ) self.assertEqual(end_pb, expected_end_pb)