def pr_curve_raw(tag, tp, fp, tn, fn, precision, recall, num_thresholds=127, weights=None): if num_thresholds > 127: # weird, value > 127 breaks protobuf num_thresholds = 127 data = np.stack((tp, fp, tn, fn, precision, recall)) pr_curve_plugin_data = PrCurvePluginData( version=0, num_thresholds=num_thresholds).SerializeToString() plugin_data = SummaryMetadata.PluginData(plugin_name="pr_curves", content=pr_curve_plugin_data) smd = SummaryMetadata(plugin_data=plugin_data) tensor = TensorProto( dtype="DT_FLOAT", float_val=data.reshape(-1).tolist(), tensor_shape=TensorShapeProto(dim=[ TensorShapeProto.Dim(size=data.shape[0]), TensorShapeProto.Dim(size=data.shape[1]), ]), ) return Summary(value=[Summary.Value(tag=tag, metadata=smd, tensor=tensor)])
def scalar(name, scalar, collections=None, new_style=False): """Outputs a `Summary` protocol buffer containing a single scalar value. The generated Summary has a Tensor.proto containing the input Tensor. Args: name: A name for the generated node. Will also serve as the series name in TensorBoard. tensor: A real numeric Tensor containing a single value. collections: Optional list of graph collections keys. The new summary op is added to these collections. Defaults to `[GraphKeys.SUMMARIES]`. new_style: Whether to use new style (tensor field) or old style (simple_value field). New style could lead to faster data loading. Returns: A scalar `Tensor` of type `string`. Which contains a `Summary` protobuf. Raises: ValueError: If tensor has the wrong shape or type. """ scalar = make_np(scalar) assert scalar.squeeze().ndim == 0, "scalar should be 0D" scalar = float(scalar) if new_style: plugin_data = SummaryMetadata.PluginData(plugin_name="scalars") smd = SummaryMetadata(plugin_data=plugin_data) return Summary(value=[ Summary.Value( tag=name, tensor=TensorProto(float_val=[scalar], dtype="DT_FLOAT"), metadata=smd, ) ]) else: return Summary(value=[Summary.Value(tag=name, simple_value=scalar)])
def custom_scalars(layout): categories = [] for k, v in layout.items(): charts = [] for chart_name, chart_meatadata in v.items(): tags = chart_meatadata[1] if chart_meatadata[0] == "Margin": assert len(tags) == 3 mgcc = layout_pb2.MarginChartContent(series=[ layout_pb2.MarginChartContent.Series( value=tags[0], lower=tags[1], upper=tags[2]) ]) chart = layout_pb2.Chart(title=chart_name, margin=mgcc) else: mlcc = layout_pb2.MultilineChartContent(tag=tags) chart = layout_pb2.Chart(title=chart_name, multiline=mlcc) charts.append(chart) categories.append(layout_pb2.Category(title=k, chart=charts)) layout = layout_pb2.Layout(category=categories) plugin_data = SummaryMetadata.PluginData(plugin_name="custom_scalars") smd = SummaryMetadata(plugin_data=plugin_data) tensor = TensorProto( dtype="DT_STRING", string_val=[layout.SerializeToString()], tensor_shape=TensorShapeProto(), ) return Summary(value=[ Summary.Value( tag="custom_scalars__config__", tensor=tensor, metadata=smd) ])
def _make_session_end_summary(status: str, end_time_secs: Optional[int] = None): """ Args: status: outcome of this run, one of of 'UNKNOWN', 'SUCCESS', 'FAILURE', 'RUNNING' end_time_secs: optional ending time in seconds Returns: """ status = Status.DESCRIPTOR.values_by_name[ f"STATUS_{status.upper()}"].number if end_time_secs is None: end_time_secs = int(time.time()) session_end_summary = SessionEndInfo(status=status, end_time_secs=end_time_secs) session_end_content = HParamsPluginData( session_end_info=session_end_summary, version=PLUGIN_DATA_VERSION) session_end_summary_metadata = SummaryMetadata( plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=session_end_content.SerializeToString())) session_end_summary = Summary(value=[ Summary.Value(tag=SESSION_END_INFO_TAG, metadata=session_end_summary_metadata) ]) return session_end_summary
def text(tag, text): plugin_data = SummaryMetadata.PluginData( plugin_name='text', content=TextPluginData(version=0).SerializeToString()) smd = SummaryMetadata(plugin_data=plugin_data) tensor = TensorProto(dtype='DT_STRING', string_val=[text.encode(encoding='utf_8')], tensor_shape=TensorShapeProto(dim=[TensorShapeProto.Dim(size=1)])) return Summary(value=[Summary.Value(tag=tag + '/text_summary', metadata=smd, tensor=tensor)])
def _make_session_start_summary( hparam_values, group_name: Optional[str] = None, start_time_secs: Optional[int] = None, ): """Assign values to the hyperparameters in the context of this session. Args: hparam_values: a dict of ``hp_name`` -> ``hp_value`` mappings group_name: optional group name for this session start_time_secs: optional starting time in seconds Returns: """ if start_time_secs is None: start_time_secs = int(time.time()) session_start_info = SessionStartInfo(group_name=group_name, start_time_secs=start_time_secs) for hp_name, hp_value in hparam_values.items(): # Logging a None would raise an exception when setting session_start_info.hparams[hp_name].number_value = None. # Logging a float.nan instead would work, but that run would not show at all in the tensorboard hparam plugin. # The best thing to do here is to skip that value, it will show as a blank cell in the table view of the # tensorboard plugin. However, that run would not be shown in the parallel coord or in the scatter plot view. if hp_value is None: logger.warning( f"Hyper parameter {hp_name} is `None`: the tensorboard hp plugin " f"will show this run in table view, but not in parallel coordinates " f"view or in scatter plot matrix view") continue if isinstance(hp_value, (str, list, tuple)): session_start_info.hparams[hp_name].string_value = str(hp_value) continue if isinstance(hp_value, bool): session_start_info.hparams[hp_name].bool_value = hp_value continue if not isinstance(hp_value, (int, float)): hp_value = make_np(hp_value)[0] session_start_info.hparams[hp_name].number_value = hp_value session_start_content = HParamsPluginData( session_start_info=session_start_info, version=PLUGIN_DATA_VERSION) session_start_summary_metadata = SummaryMetadata( plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=session_start_content.SerializeToString())) session_start_summary = Summary(value=[ Summary.Value(tag=SESSION_START_INFO_TAG, metadata=session_start_summary_metadata) ]) return session_start_summary
def pr_curve(tag, labels, predictions, num_thresholds=127, weights=None): # weird, value > 127 breaks protobuf num_thresholds = min(num_thresholds, 127) data = compute_curve(labels, predictions, num_thresholds=num_thresholds, weights=weights) pr_curve_plugin_data = PrCurvePluginData( version=0, num_thresholds=num_thresholds).SerializeToString() plugin_data = SummaryMetadata.PluginData( plugin_name='pr_curves', content=pr_curve_plugin_data) smd = SummaryMetadata(plugin_data=plugin_data) tensor = TensorProto(dtype='DT_FLOAT', float_val=data.reshape(-1).tolist(), tensor_shape=TensorShapeProto( dim=[TensorShapeProto.Dim(size=data.shape[0]), TensorShapeProto.Dim(size=data.shape[1])])) return Summary(value=[Summary.Value(tag=tag, metadata=smd, tensor=tensor)])
def create_summary_metadata(description): """Creates summary metadata. Reserved for future use. Required by TensorBoard. Arguments: description: The description to show in TensorBoard. Returns: A `SummaryMetadata` protobuf object. """ return SummaryMetadata( summary_description=description, plugin_data=SummaryMetadata.PluginData(plugin_name=PLUGIN_NAME, content=b''), )
def create_summary_metadata(description, metadata): """Creates summary metadata. Reserved for future use. Required by TensorBoard. Args: description: The description to show in TensorBoard. Returns: A `SummaryMetadata` protobuf object. """ ln_proto = LabelToNames() if 'label_to_names' in metadata: ln_proto.label_to_names.update(metadata['label_to_names']) return SummaryMetadata( summary_description=description, plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=ln_proto.SerializeToString()), )
def scalar(name, tensor, collections=None, new_style=False, double_precision=False): """Outputs a `Summary` protocol buffer containing a single scalar value. The generated Summary has a Tensor.proto containing the input Tensor. Args: name: A name for the generated node. Will also serve as the series name in TensorBoard. tensor: A real numeric Tensor containing a single value. collections: Optional list of graph collections keys. The new summary op is added to these collections. Defaults to `[GraphKeys.SUMMARIES]`. new_style: Whether to use new style (tensor field) or old style (simple_value field). New style could lead to faster data loading. Returns: A scalar `Tensor` of type `string`. Which contains a `Summary` protobuf. Raises: ValueError: If tensor has the wrong shape or type. """ tensor = make_np(tensor).squeeze() assert ( tensor.ndim == 0 ), f"Tensor should contain one element (0 dimensions). Was given size: {tensor.size} and {tensor.ndim} dimensions." # python float is double precision in numpy scalar = float(tensor) if new_style: tensor_proto = TensorProto(float_val=[scalar], dtype="DT_FLOAT") if double_precision: tensor_proto = TensorProto(double_val=[scalar], dtype="DT_DOUBLE") plugin_data = SummaryMetadata.PluginData(plugin_name="scalars") smd = SummaryMetadata(plugin_data=plugin_data) return Summary( value=[ Summary.Value( tag=name, tensor=tensor_proto, metadata=smd, ) ] ) else: return Summary(value=[Summary.Value(tag=name, simple_value=scalar)])
def hparams(hparam_dict=None, metric_dict=None, hparam_domain_discrete=None): """Outputs three `Summary` protocol buffers needed by hparams plugin. `Experiment` keeps the metadata of an experiment, such as the name of the hyperparameters and the name of the metrics. `SessionStartInfo` keeps key-value pairs of the hyperparameters `SessionEndInfo` describes status of the experiment e.g. STATUS_SUCCESS Args: hparam_dict: A dictionary that contains names of the hyperparameters and their values. metric_dict: A dictionary that contains names of the metrics and their values. hparam_domain_discrete: (Optional[Dict[str, List[Any]]]) A dictionary that contains names of the hyperparameters and all discrete values they can hold Returns: The `Summary` protobufs for Experiment, SessionStartInfo and SessionEndInfo """ import torch from six import string_types from tensorboard.plugins.hparams.api_pb2 import ( Experiment, HParamInfo, MetricInfo, MetricName, Status, DataType, ) from tensorboard.plugins.hparams.metadata import ( PLUGIN_NAME, PLUGIN_DATA_VERSION, EXPERIMENT_TAG, SESSION_START_INFO_TAG, SESSION_END_INFO_TAG, ) from tensorboard.plugins.hparams.plugin_data_pb2 import ( HParamsPluginData, SessionEndInfo, SessionStartInfo, ) # TODO: expose other parameters in the future. # hp = HParamInfo(name='lr',display_name='learning rate', # type=DataType.DATA_TYPE_FLOAT64, domain_interval=Interval(min_value=10, # max_value=100)) # mt = MetricInfo(name=MetricName(tag='accuracy'), display_name='accuracy', # description='', dataset_type=DatasetType.DATASET_VALIDATION) # exp = Experiment(name='123', description='456', time_created_secs=100.0, # hparam_infos=[hp], metric_infos=[mt], user='******') if not isinstance(hparam_dict, dict): logger.warning( "parameter: hparam_dict should be a dictionary, nothing logged.") raise TypeError( "parameter: hparam_dict should be a dictionary, nothing logged.") if not isinstance(metric_dict, dict): logger.warning( "parameter: metric_dict should be a dictionary, nothing logged.") raise TypeError( "parameter: metric_dict should be a dictionary, nothing logged.") hparam_domain_discrete = hparam_domain_discrete or {} if not isinstance(hparam_domain_discrete, dict): raise TypeError( "parameter: hparam_domain_discrete should be a dictionary, nothing logged." ) for k, v in hparam_domain_discrete.items(): if (k not in hparam_dict or not isinstance(v, list) or not all(isinstance(d, type(hparam_dict[k])) for d in v)): raise TypeError( "parameter: hparam_domain_discrete[{}] should be a list of same type as " "hparam_dict[{}].".format(k, k)) hps = [] ssi = SessionStartInfo() for k, v in hparam_dict.items(): if v is None: continue if isinstance(v, int) or isinstance(v, float): ssi.hparams[k].number_value = v if k in hparam_domain_discrete: domain_discrete: Optional[ struct_pb2.ListValue] = struct_pb2.ListValue(values=[ struct_pb2.Value(number_value=d) for d in hparam_domain_discrete[k] ]) else: domain_discrete = None hps.append( HParamInfo( name=k, type=DataType.Value("DATA_TYPE_FLOAT64"), domain_discrete=domain_discrete, )) continue if isinstance(v, string_types): ssi.hparams[k].string_value = v if k in hparam_domain_discrete: domain_discrete = struct_pb2.ListValue(values=[ struct_pb2.Value(string_value=d) for d in hparam_domain_discrete[k] ]) else: domain_discrete = None hps.append( HParamInfo( name=k, type=DataType.Value("DATA_TYPE_STRING"), domain_discrete=domain_discrete, )) continue if isinstance(v, bool): ssi.hparams[k].bool_value = v if k in hparam_domain_discrete: domain_discrete = struct_pb2.ListValue(values=[ struct_pb2.Value(bool_value=d) for d in hparam_domain_discrete[k] ]) else: domain_discrete = None hps.append( HParamInfo( name=k, type=DataType.Value("DATA_TYPE_BOOL"), domain_discrete=domain_discrete, )) continue if isinstance(v, torch.Tensor): v = make_np(v)[0] ssi.hparams[k].number_value = v hps.append( HParamInfo(name=k, type=DataType.Value("DATA_TYPE_FLOAT64"))) continue raise ValueError( "value should be one of int, float, str, bool, or torch.Tensor") content = HParamsPluginData(session_start_info=ssi, version=PLUGIN_DATA_VERSION) smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=content.SerializeToString())) ssi = Summary( value=[Summary.Value(tag=SESSION_START_INFO_TAG, metadata=smd)]) mts = [MetricInfo(name=MetricName(tag=k)) for k in metric_dict.keys()] exp = Experiment(hparam_infos=hps, metric_infos=mts) content = HParamsPluginData(experiment=exp, version=PLUGIN_DATA_VERSION) smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=content.SerializeToString())) exp = Summary(value=[Summary.Value(tag=EXPERIMENT_TAG, metadata=smd)]) sei = SessionEndInfo(status=Status.Value("STATUS_SUCCESS")) content = HParamsPluginData(session_end_info=sei, version=PLUGIN_DATA_VERSION) smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=content.SerializeToString())) sei = Summary( value=[Summary.Value(tag=SESSION_END_INFO_TAG, metadata=smd)]) return exp, ssi, sei
def hparams(hparam_dict=None, metric_dict=None): """Outputs three `Summary` protocol buffers needed by hparams plugin. `Experiment` keeps the metadata of an experiment, such as the name of the hyperparameters and the name of the metrics. `SessionStartInfo` keeps key-value pairs of the hyperparameters `SessionEndInfo` describes status of the experiment e.g. STATUS_SUCCESS Args: hparam_dict: A dictionary that contains names of the hyperparameters and their values. metric_dict: A dictionary that contains names of the metrics and their values. Returns: The `Summary` protobufs for Experiment, SessionStartInfo and SessionEndInfo """ import torch from six import string_types from tensorboard.plugins.hparams.api_pb2 import (Experiment, HParamInfo, MetricInfo, MetricName, Status) from tensorboard.plugins.hparams.metadata import (PLUGIN_NAME, PLUGIN_DATA_VERSION, EXPERIMENT_TAG, SESSION_START_INFO_TAG, SESSION_END_INFO_TAG) from tensorboard.plugins.hparams.plugin_data_pb2 import (HParamsPluginData, SessionEndInfo, SessionStartInfo) # TODO: expose other parameters in the future. # hp = HParamInfo(name='lr',display_name='learning rate', # type=DataType.DATA_TYPE_FLOAT64, domain_interval=Interval(min_value=10, # max_value=100)) # mt = MetricInfo(name=MetricName(tag='accuracy'), display_name='accuracy', # description='', dataset_type=DatasetType.DATASET_VALIDATION) # exp = Experiment(name='123', description='456', time_created_secs=100.0, # hparam_infos=[hp], metric_infos=[mt], user='******') if not isinstance(hparam_dict, dict): logging.warning( 'parameter: hparam_dict should be a dictionary, nothing logged.') raise TypeError( 'parameter: hparam_dict should be a dictionary, nothing logged.') if not isinstance(metric_dict, dict): logging.warning( 'parameter: metric_dict should be a dictionary, nothing logged.') raise TypeError( 'parameter: metric_dict should be a dictionary, nothing logged.') hps = [HParamInfo(name=k) for k in hparam_dict.keys()] mts = [MetricInfo(name=MetricName(tag=k)) for k in metric_dict.keys()] exp = Experiment(hparam_infos=hps, metric_infos=mts) content = HParamsPluginData(experiment=exp, version=PLUGIN_DATA_VERSION) smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=content.SerializeToString())) exp = Summary(value=[Summary.Value(tag=EXPERIMENT_TAG, metadata=smd)]) ssi = SessionStartInfo() for k, v in hparam_dict.items(): if v is None: continue if isinstance(v, int) or isinstance(v, float): ssi.hparams[k].number_value = v continue if isinstance(v, string_types): ssi.hparams[k].string_value = v continue if isinstance(v, bool): ssi.hparams[k].bool_value = v continue if isinstance(v, torch.Tensor): v = make_np(v)[0] ssi.hparams[k].number_value = v continue raise ValueError( 'value should be one of int, float, str, bool, or torch.Tensor') content = HParamsPluginData(session_start_info=ssi, version=PLUGIN_DATA_VERSION) smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=content.SerializeToString())) ssi = Summary( value=[Summary.Value(tag=SESSION_START_INFO_TAG, metadata=smd)]) sei = SessionEndInfo(status=Status.Value('STATUS_SUCCESS')) content = HParamsPluginData(session_end_info=sei, version=PLUGIN_DATA_VERSION) smd = SummaryMetadata(plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=content.SerializeToString())) sei = Summary( value=[Summary.Value(tag=SESSION_END_INFO_TAG, metadata=smd)]) return exp, ssi, sei
def _make_experiment_summary(hparam_infos, metric_infos, experiment): """Define hyperparameters and metrics. Args: hparam_infos: information about all hyperparameters (name, description, type etc.), list of dicts containing 'name' (required), 'type', 'description', 'display_name', 'domain_discrete', 'domain_interval' metric_infos: information about all metrics (tag, description etc.), list of dicts containing 'tag' (required), 'dataset_type', 'description', 'display_name' experiment: dict containing 'name' (required), 'description', 'time_created_secs', 'user' Returns: """ def make_hparam_info(hparam): data_types = { None: DataType.DATA_TYPE_UNSET, str: DataType.DATA_TYPE_STRING, list: DataType.DATA_TYPE_STRING, tuple: DataType.DATA_TYPE_STRING, bool: DataType.DATA_TYPE_BOOL, int: DataType.DATA_TYPE_FLOAT64, float: DataType.DATA_TYPE_FLOAT64, } return HParamInfo( name=hparam["name"], type=data_types[hparam.get("type")], description=hparam.get("description"), display_name=hparam.get("display_name"), domain_discrete=hparam.get("domain_discrete"), domain_interval=hparam.get("domain_interval"), ) def make_metric_info(metric): return MetricInfo( name=MetricName(tag=metric["tag"]), dataset_type=DatasetType.Value( f'DATASET_{metric.get("dataset_type", "UNKNOWN").upper()}'), description=metric.get("description"), display_name=metric.get("display_name"), ) def make_experiment_info(experiment, metric_infos, hparam_infos): return Experiment( name=experiment["name"], description=experiment.get("description"), time_created_secs=experiment.get("time_created_secs"), user=experiment.get("user"), metric_infos=metric_infos, hparam_infos=hparam_infos, ) metric_infos = [make_metric_info(m) for m in metric_infos] hparam_infos = [make_hparam_info(hp) for hp in hparam_infos] experiment = make_experiment_info(experiment, metric_infos, hparam_infos) experiment_content = HParamsPluginData(experiment=experiment, version=PLUGIN_DATA_VERSION) experiment_summary_metadata = SummaryMetadata( plugin_data=SummaryMetadata.PluginData( plugin_name=PLUGIN_NAME, content=experiment_content.SerializeToString())) experiment_summary = Summary(value=[ Summary.Value(tag=EXPERIMENT_TAG, metadata=experiment_summary_metadata) ]) return experiment_summary