def _make_session_end_summary(status: str, end_time_secs: Optional[int] = None): """ Args: status: outcome of this run, one of of 'UNKNOWN', 'SUCCESS', 'FAILURE', 'RUNNING' end_time_secs: optional ending time in seconds Returns: """ status = Status.DESCRIPTOR.values_by_name[ f"STATUS_{status.upper()}"].number if end_time_secs is None: end_time_secs = int(time.time()) session_end_summary = SessionEndInfo(status=status, end_time_secs=end_time_secs) session_end_content = HParamsPluginData( session_end_info=session_end_summary, version=PLUGIN_DATA_VERSION) session_end_summary_metadata = SummaryMetadata( plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=session_end_content.SerializeToString())) session_end_summary = Summary(value=[ Summary.Value(tag=SESSION_END_INFO_TAG, metadata=session_end_summary_metadata) ]) return session_end_summary
def _make_session_start_summary( hparam_values, group_name: Optional[str] = None, start_time_secs: Optional[int] = None, ): """Assign values to the hyperparameters in the context of this session. Args: hparam_values: a dict of ``hp_name`` -> ``hp_value`` mappings group_name: optional group name for this session start_time_secs: optional starting time in seconds Returns: """ if start_time_secs is None: start_time_secs = int(time.time()) session_start_info = SessionStartInfo(group_name=group_name, start_time_secs=start_time_secs) for hp_name, hp_value in hparam_values.items(): # Logging a None would raise an exception when setting session_start_info.hparams[hp_name].number_value = None. # Logging a float.nan instead would work, but that run would not show at all in the tensorboard hparam plugin. # The best thing to do here is to skip that value, it will show as a blank cell in the table view of the # tensorboard plugin. However, that run would not be shown in the parallel coord or in the scatter plot view. if hp_value is None: logger.warning( f"Hyper parameter {hp_name} is `None`: the tensorboard hp plugin " f"will show this run in table view, but not in parallel coordinates " f"view or in scatter plot matrix view") continue if isinstance(hp_value, (str, list, tuple)): session_start_info.hparams[hp_name].string_value = str(hp_value) continue if isinstance(hp_value, bool): session_start_info.hparams[hp_name].bool_value = hp_value continue if not isinstance(hp_value, (int, float)): hp_value = make_np(hp_value)[0] session_start_info.hparams[hp_name].number_value = hp_value session_start_content = HParamsPluginData( session_start_info=session_start_info, version=PLUGIN_DATA_VERSION) session_start_summary_metadata = SummaryMetadata( plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=session_start_content.SerializeToString())) session_start_summary = Summary(value=[ Summary.Value(tag=SESSION_START_INFO_TAG, metadata=session_start_summary_metadata) ]) return session_start_summary
def hparams(hparam_dict=None, metric_dict=None, hparam_domain_discrete=None): """Outputs three `Summary` protocol buffers needed by hparams plugin. `Experiment` keeps the metadata of an experiment, such as the name of the hyperparameters and the name of the metrics. `SessionStartInfo` keeps key-value pairs of the hyperparameters `SessionEndInfo` describes status of the experiment e.g. STATUS_SUCCESS Args: hparam_dict: A dictionary that contains names of the hyperparameters and their values. metric_dict: A dictionary that contains names of the metrics and their values. hparam_domain_discrete: (Optional[Dict[str, List[Any]]]) A dictionary that contains names of the hyperparameters and all discrete values they can hold Returns: The `Summary` protobufs for Experiment, SessionStartInfo and SessionEndInfo """ import torch from six import string_types from tensorboard.plugins.hparams.api_pb2 import ( Experiment, HParamInfo, MetricInfo, MetricName, Status, DataType, ) from tensorboard.plugins.hparams.metadata import ( PLUGIN_NAME, PLUGIN_DATA_VERSION, EXPERIMENT_TAG, SESSION_START_INFO_TAG, SESSION_END_INFO_TAG, ) from tensorboard.plugins.hparams.plugin_data_pb2 import ( HParamsPluginData, SessionEndInfo, SessionStartInfo, ) # TODO: expose other parameters in the future. # hp = HParamInfo(name='lr',display_name='learning rate', # type=DataType.DATA_TYPE_FLOAT64, domain_interval=Interval(min_value=10, # max_value=100)) # mt = MetricInfo(name=MetricName(tag='accuracy'), display_name='accuracy', # description='', dataset_type=DatasetType.DATASET_VALIDATION) # exp = Experiment(name='123', description='456', time_created_secs=100.0, # hparam_infos=[hp], metric_infos=[mt], user='******') if not isinstance(hparam_dict, dict): logger.warning( "parameter: hparam_dict should be a dictionary, nothing logged.") raise TypeError( "parameter: hparam_dict should be a dictionary, nothing logged.") if not isinstance(metric_dict, dict): logger.warning( "parameter: metric_dict should be a dictionary, nothing logged.") raise TypeError( "parameter: metric_dict should be a dictionary, nothing logged.") hparam_domain_discrete = hparam_domain_discrete or {} if not isinstance(hparam_domain_discrete, dict): raise TypeError( "parameter: hparam_domain_discrete should be a dictionary, nothing logged." ) for k, v in hparam_domain_discrete.items(): if (k not in hparam_dict or not isinstance(v, list) or not all(isinstance(d, type(hparam_dict[k])) for d in v)): raise TypeError( "parameter: hparam_domain_discrete[{}] should be a list of same type as " "hparam_dict[{}].".format(k, k)) hps = [] ssi = SessionStartInfo() for k, v in hparam_dict.items(): if v is None: continue if isinstance(v, int) or isinstance(v, float): ssi.hparams[k].number_value = v if k in hparam_domain_discrete: domain_discrete: Optional[ struct_pb2.ListValue] = struct_pb2.ListValue(values=[ struct_pb2.Value(number_value=d) for d in hparam_domain_discrete[k] ]) else: domain_discrete = None hps.append( HParamInfo( name=k, type=DataType.Value("DATA_TYPE_FLOAT64"), domain_discrete=domain_discrete, )) continue if isinstance(v, string_types): ssi.hparams[k].string_value = v if k in hparam_domain_discrete: domain_discrete = struct_pb2.ListValue(values=[ struct_pb2.Value(string_value=d) for d in hparam_domain_discrete[k] ]) else: domain_discrete = None hps.append( HParamInfo( name=k, type=DataType.Value("DATA_TYPE_STRING"), domain_discrete=domain_discrete, )) continue if isinstance(v, bool): ssi.hparams[k].bool_value = v if k in hparam_domain_discrete: domain_discrete = struct_pb2.ListValue(values=[ struct_pb2.Value(bool_value=d) for d in hparam_domain_discrete[k] ]) else: domain_discrete = None hps.append( HParamInfo( name=k, type=DataType.Value("DATA_TYPE_BOOL"), domain_discrete=domain_discrete, )) continue if isinstance(v, torch.Tensor): v = make_np(v)[0] ssi.hparams[k].number_value = v hps.append( HParamInfo(name=k, type=DataType.Value("DATA_TYPE_FLOAT64"))) continue raise ValueError( "value should be one of int, float, str, bool, or torch.Tensor") content = HParamsPluginData(session_start_info=ssi, version=PLUGIN_DATA_VERSION) smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=content.SerializeToString())) ssi = Summary( value=[Summary.Value(tag=SESSION_START_INFO_TAG, metadata=smd)]) mts = [MetricInfo(name=MetricName(tag=k)) for k in metric_dict.keys()] exp = Experiment(hparam_infos=hps, metric_infos=mts) content = HParamsPluginData(experiment=exp, version=PLUGIN_DATA_VERSION) smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=content.SerializeToString())) exp = Summary(value=[Summary.Value(tag=EXPERIMENT_TAG, metadata=smd)]) sei = SessionEndInfo(status=Status.Value("STATUS_SUCCESS")) content = HParamsPluginData(session_end_info=sei, version=PLUGIN_DATA_VERSION) smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=content.SerializeToString())) sei = Summary( value=[Summary.Value(tag=SESSION_END_INFO_TAG, metadata=smd)]) return exp, ssi, sei
def hparams(hparam_dict=None, metric_dict=None): """Outputs three `Summary` protocol buffers needed by hparams plugin. `Experiment` keeps the metadata of an experiment, such as the name of the hyperparameters and the name of the metrics. `SessionStartInfo` keeps key-value pairs of the hyperparameters `SessionEndInfo` describes status of the experiment e.g. STATUS_SUCCESS Args: hparam_dict: A dictionary that contains names of the hyperparameters and their values. metric_dict: A dictionary that contains names of the metrics and their values. Returns: The `Summary` protobufs for Experiment, SessionStartInfo and SessionEndInfo """ import torch from six import string_types from tensorboard.plugins.hparams.api_pb2 import (Experiment, HParamInfo, MetricInfo, MetricName, Status) from tensorboard.plugins.hparams.metadata import (PLUGIN_NAME, PLUGIN_DATA_VERSION, EXPERIMENT_TAG, SESSION_START_INFO_TAG, SESSION_END_INFO_TAG) from tensorboard.plugins.hparams.plugin_data_pb2 import (HParamsPluginData, SessionEndInfo, SessionStartInfo) # TODO: expose other parameters in the future. # hp = HParamInfo(name='lr',display_name='learning rate', # type=DataType.DATA_TYPE_FLOAT64, domain_interval=Interval(min_value=10, # max_value=100)) # mt = MetricInfo(name=MetricName(tag='accuracy'), display_name='accuracy', # description='', dataset_type=DatasetType.DATASET_VALIDATION) # exp = Experiment(name='123', description='456', time_created_secs=100.0, # hparam_infos=[hp], metric_infos=[mt], user='******') if not isinstance(hparam_dict, dict): logging.warning( 'parameter: hparam_dict should be a dictionary, nothing logged.') raise TypeError( 'parameter: hparam_dict should be a dictionary, nothing logged.') if not isinstance(metric_dict, dict): logging.warning( 'parameter: metric_dict should be a dictionary, nothing logged.') raise TypeError( 'parameter: metric_dict should be a dictionary, nothing logged.') hps = [HParamInfo(name=k) for k in hparam_dict.keys()] mts = [MetricInfo(name=MetricName(tag=k)) for k in metric_dict.keys()] exp = Experiment(hparam_infos=hps, metric_infos=mts) content = HParamsPluginData(experiment=exp, version=PLUGIN_DATA_VERSION) smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=content.SerializeToString())) exp = Summary(value=[Summary.Value(tag=EXPERIMENT_TAG, metadata=smd)]) ssi = SessionStartInfo() for k, v in hparam_dict.items(): if v is None: continue if isinstance(v, int) or isinstance(v, float): ssi.hparams[k].number_value = v continue if isinstance(v, string_types): ssi.hparams[k].string_value = v continue if isinstance(v, bool): ssi.hparams[k].bool_value = v continue if isinstance(v, torch.Tensor): v = make_np(v)[0] ssi.hparams[k].number_value = v continue raise ValueError( 'value should be one of int, float, str, bool, or torch.Tensor') content = HParamsPluginData(session_start_info=ssi, version=PLUGIN_DATA_VERSION) smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=content.SerializeToString())) ssi = Summary( value=[Summary.Value(tag=SESSION_START_INFO_TAG, metadata=smd)]) sei = SessionEndInfo(status=Status.Value('STATUS_SUCCESS')) content = HParamsPluginData(session_end_info=sei, version=PLUGIN_DATA_VERSION) smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=content.SerializeToString())) sei = Summary( value=[Summary.Value(tag=SESSION_END_INFO_TAG, metadata=smd)]) return exp, ssi, sei
def _make_experiment_summary(hparam_infos, metric_infos, experiment): """Define hyperparameters and metrics. Args: hparam_infos: information about all hyperparameters (name, description, type etc.), list of dicts containing 'name' (required), 'type', 'description', 'display_name', 'domain_discrete', 'domain_interval' metric_infos: information about all metrics (tag, description etc.), list of dicts containing 'tag' (required), 'dataset_type', 'description', 'display_name' experiment: dict containing 'name' (required), 'description', 'time_created_secs', 'user' Returns: """ def make_hparam_info(hparam): data_types = { None: DataType.DATA_TYPE_UNSET, str: DataType.DATA_TYPE_STRING, list: DataType.DATA_TYPE_STRING, tuple: DataType.DATA_TYPE_STRING, bool: DataType.DATA_TYPE_BOOL, int: DataType.DATA_TYPE_FLOAT64, float: DataType.DATA_TYPE_FLOAT64, } return HParamInfo( name=hparam["name"], type=data_types[hparam.get("type")], description=hparam.get("description"), display_name=hparam.get("display_name"), domain_discrete=hparam.get("domain_discrete"), domain_interval=hparam.get("domain_interval"), ) def make_metric_info(metric): return MetricInfo( name=MetricName(tag=metric["tag"]), dataset_type=DatasetType.Value( f'DATASET_{metric.get("dataset_type", "UNKNOWN").upper()}'), description=metric.get("description"), display_name=metric.get("display_name"), ) def make_experiment_info(experiment, metric_infos, hparam_infos): return Experiment( name=experiment["name"], description=experiment.get("description"), time_created_secs=experiment.get("time_created_secs"), user=experiment.get("user"), metric_infos=metric_infos, hparam_infos=hparam_infos, ) metric_infos = [make_metric_info(m) for m in metric_infos] hparam_infos = [make_hparam_info(hp) for hp in hparam_infos] experiment = make_experiment_info(experiment, metric_infos, hparam_infos) experiment_content = HParamsPluginData(experiment=experiment, version=PLUGIN_DATA_VERSION) experiment_summary_metadata = SummaryMetadata( plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=experiment_content.SerializeToString())) experiment_summary = Summary(value=[ Summary.Value(tag=EXPERIMENT_TAG, metadata=experiment_summary_metadata) ]) return experiment_summary