def create_experiment_summary(num_units_list, dropout_rate_list, optimizer_list):
from tensorboard.plugins.hparams import api_pb2
from tensorboard.plugins.hparams import summary as hparams_summary
from google.protobuf import struct_pb2
num_units_list_val = struct_pb2.ListValue()
num_units_list_val.extend(num_units_list)
dropout_rate_list_val = struct_pb2.ListValue()
dropout_rate_list_val.extend(dropout_rate_list)
optimizer_list_val = struct_pb2.ListValue()
optimizer_list_val.extend(optimizer_list)
return hparams_summary.experiment_pb(
# The hyperparameters being changed
hparam_infos=[
api_pb2.HParamInfo(name='num_units',
display_name='Number of units',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=num_units_list_val),
api_pb2.HParamInfo(name='dropout_rate',
display_name='Dropout rate',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=dropout_rate_list_val),
api_pb2.HParamInfo(name='optimizer',
display_name='Optimizer',
type=api_pb2.DATA_TYPE_STRING,
domain_discrete=optimizer_list_val)
],
# The metrics being tracked
metric_infos=[
api_pb2.MetricInfo(
name=api_pb2.MetricName(
tag='epoch_accuracy'),
display_name='Accuracy'),
]
)
def _to_summary_pb(num_units_list, dropout_rate_list, optimizer_list):
nus_val = struct_pb2.ListValue()
nus_val.extend(num_units_list)
drs_val = struct_pb2.ListValue()
drs_val.extend(dropout_rate_list)
opts_val = struct_pb2.ListValue()
opts_val.extend(optimizer_list)
return hparams.experiment_pb(
hparam_infos=[
api_pb2.HParamInfo(
name="num_units",
display_name="Number of units",
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=nus_val,
),
api_pb2.HParamInfo(
name="drop_rate",
display_name="Dropout rate",
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=drs_val,
),
api_pb2.HParamInfo(
name="optimizer",
display_name="Optimizer",
type=api_pb2.DATA_TYPE_STRING,
domain_discrete=opts_val,
),
],
metric_infos=[
api_pb2.MetricInfo(name=api_pb2.MetricName(tag="accuracy"), display_name="Accuracy"),
],
)
def create_experiment_summary():
"""Returns a summary proto buffer holding this experiment."""
# Convert TEMPERATURE_LIST to google.protobuf.ListValue
temperature_list = struct_pb2.ListValue()
temperature_list.extend(TEMPERATURE_LIST)
materials = struct_pb2.ListValue()
materials.extend(HEAT_COEFFICIENTS.keys())
return summary.experiment_pb(
hparam_infos=[
api_pb2.HParamInfo(name='initial_temperature',
display_name='Initial temperature',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=temperature_list),
api_pb2.HParamInfo(name='ambient_temperature',
display_name='Ambient temperature',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=temperature_list),
api_pb2.HParamInfo(name='material',
display_name='Material',
type=api_pb2.DATA_TYPE_STRING,
domain_discrete=materials)
],
metric_infos=[
api_pb2.MetricInfo(name=api_pb2.MetricName(
tag='temperature/current/scalar_summary'),
display_name='Current Temp.'),
api_pb2.MetricInfo(name=api_pb2.MetricName(
tag='temperature/difference_to_ambient/scalar_summary'),
display_name='Difference To Ambient Temp.'),
api_pb2.MetricInfo(
name=api_pb2.MetricName(tag='delta/scalar_summary'),
display_name='Delta T')
])
def create_experiment_summary(optimizer_list, num_units_list, dropout_list):
optimizer_list_val = struct_pb2.ListValue()
optimizer_list_val.extend(optimizer_list)
num_units_list_val = struct_pb2.ListValue()
num_units_list_val.extend(num_units_list)
dropout_list_val = struct_pb2.ListValue()
dropout_list_val.extend(dropout_list)
return hparams_summary.experiment_pb(
hparam_infos=[
api_pb2.HParamInfo(
name='optimizer',
display_name='Optimizer',
type=api_pb2.DATA_TYPE_STRING,
domain_discrete=optimizer_list_val),
api_pb2.HParamInfo(
name='num_units',
display_name='Number of units',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=num_units_list_val),
api_pb2.HParamInfo(
name='dropout',
display_name='Dropout',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=dropout_list_val),
],
metric_infos=[
api_pb2.MetricInfo(
name=api_pb2.MetricName(tag='accuracy'), display_name='Accuracy'),
api_pb2.MetricInfo(
name=api_pb2.MetricName(tag='elapse'), display_name='Elapse (s)')
])
def _to_summary_pb(num_units_list, dropout_rate_list, optimizer_list):
nus_val = struct_pb2.ListValue()
nus_val.extend(num_units_list)
drs_val = struct_pb2.ListValue()
drs_val.extend(dropout_rate_list)
opts_val = struct_pb2.ListValue()
opts_val.extend(optimizer_list)
return hparams.experiment_pb(
hparam_infos=[
api_pb2.HParamInfo(
name='num_units',
display_name='Number of units',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=nus_val),
api_pb2.HParamInfo(
name='dropout_rate',
display_name='Dropout rate',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=drs_val),
api_pb2.HParamInfo(
name='optimizer',
display_name='Optimizer',
type=api_pb2.DATA_TYPE_STRING,
domain_discrete=opts_val)
],
metric_infos=[
api_pb2.MetricInfo(
name=api_pb2.MetricName(tag='accuracy'),
display_name='Accuracy'),
])
def create_experiment_summary():
"""Returns a summary proto buffer holding this experiment"""
# Convert TEMPERATURE_LIST to google.protobuf.ListValue
temperature_list = struct_pb2.ListValue()
temperature_list.extend(TEMPERATURE_LIST)
return summary.experiment_pb(
hparam_infos=[
api_pb2.HParamInfo(name="initial_temperature",
display_name="initial temperature",
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=temperature_list),
api_pb2.HParamInfo(name="ambient_temperature",
display_name="ambient temperature",
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=temperature_list),
api_pb2.HParamInfo(name="heat_coefficient",
display_name="heat coefficient",
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=temperature_list)
],
metric_infos=[
api_pb2.MetricInfo(name=api_pb2.MetricName(
tag="temperature/current/scalar_summary"),
display_name="Current Temp."),
api_pb2.MetricInfo(name=api_pb2.MetricName(
tag="temperature/difference_to_ambient/scalar_summary"),
display_name="Difference To Ambient Temp."),
api_pb2.MetricInfo(
name=api_pb2.MetricName(tag="delta/scalar_summary"),
display_name="Delta T")
])
def create_experiment_summary():
"""Create an `api_pb2.Experiment` proto describing the experiment."""
def discrete_domain(values):
domain = struct_pb2.ListValue()
domain.extend(values)
return domain
hparams = [
api_pb2.HParamInfo(
name="conv_layers",
type=api_pb2.DATA_TYPE_FLOAT64, # actually int
domain_discrete=discrete_domain([1, 2, 3]),
),
api_pb2.HParamInfo(
name="conv_kernel_size",
type=api_pb2.DATA_TYPE_FLOAT64, # actually int
domain_discrete=discrete_domain([3, 5]),
),
api_pb2.HParamInfo(
name="dense_layers",
type=api_pb2.DATA_TYPE_FLOAT64, # actually int
domain_discrete=discrete_domain([1, 2, 3]),
),
api_pb2.HParamInfo(
name="dropout",
type=api_pb2.DATA_TYPE_FLOAT64,
domain_interval=api_pb2.Interval(min_value=0.1, max_value=0.4),
),
api_pb2.HParamInfo(
name="optimizer",
type=api_pb2.DATA_TYPE_STRING,
domain_discrete=discrete_domain(["adam", "adagrad"]),
),
]
metrics = [
api_pb2.MetricInfo(
name=api_pb2.MetricName(group="validation", tag="epoch_accuracy"),
display_name="accuracy (val.)",
),
api_pb2.MetricInfo(
name=api_pb2.MetricName(group="validation", tag="epoch_loss"),
display_name="loss (val.)",
),
api_pb2.MetricInfo(
name=api_pb2.MetricName(group="train", tag="batch_accuracy"),
display_name="accuracy (train)",
),
api_pb2.MetricInfo(
name=api_pb2.MetricName(group="train", tag="batch_loss"),
display_name="loss (train)",
),
]
return hparams_summary.experiment_pb(
hparam_infos=hparams,
metric_infos=metrics,
)
def test_experiment_pb(self):
hparam_infos = [
api_pb2.HParamInfo(
name="param1",
display_name="display_name1",
description="foo",
type=api_pb2.DATA_TYPE_STRING,
domain_discrete=struct_pb2.ListValue(values=[
struct_pb2.Value(string_value="a"),
struct_pb2.Value(string_value="b"),
]),
),
api_pb2.HParamInfo(
name="param2",
display_name="display_name2",
description="bar",
type=api_pb2.DATA_TYPE_FLOAT64,
domain_interval=api_pb2.Interval(min_value=-100.0,
max_value=100.0),
),
]
metric_infos = [
api_pb2.MetricInfo(
name=api_pb2.MetricName(tag="loss"),
dataset_type=api_pb2.DATASET_VALIDATION,
),
api_pb2.MetricInfo(
name=api_pb2.MetricName(group="train/", tag="acc"),
dataset_type=api_pb2.DATASET_TRAINING,
),
]
time_created_secs = 314159.0
self.assertEqual(
summary.experiment_pb(hparam_infos,
metric_infos,
time_created_secs=time_created_secs),
tf.compat.v1.Summary(value=[
tf.compat.v1.Summary.Value(
tag="_hparams_/experiment",
tensor=summary._TF_NULL_TENSOR,
metadata=tf.compat.v1.SummaryMetadata(
plugin_data=tf.compat.v1.SummaryMetadata.PluginData(
plugin_name="hparams",
content=(plugin_data_pb2.HParamsPluginData(
version=0,
experiment=api_pb2.Experiment(
time_created_secs=time_created_secs,
hparam_infos=hparam_infos,
metric_infos=metric_infos,
),
).SerializeToString()),
)),
)
]),
)
def summary_pb(self):
"""Create a top-level experiment summary describing this experiment.
The resulting summary should be written to a log directory that
encloses all the individual sessions' log directories.
Analogous to the low-level `experiment_pb` function in the
`hparams.summary` module.
"""
hparam_infos = []
for hparam in self._hparams:
info = api_pb2.HParamInfo(
name=hparam.name,
description=hparam.description,
display_name=hparam.display_name,
)
domain = hparam.domain
if domain is not None:
domain.update_hparam_info(info)
hparam_infos.append(info)
metric_infos = [metric.as_proto() for metric in self._metrics]
return summary.experiment_pb(
hparam_infos=hparam_infos,
metric_infos=metric_infos,
user=self._user,
description=self._description,
time_created_secs=self._time_created_secs,
)
def _create_experiment_summary(self):
alpha_list_val = struct_pb2.ListValue()
alpha_list_val.extend(self._alpha_list)
alpha_decay_list_val = struct_pb2.ListValue()
alpha_decay_list_val.extend(self._alpha_decay_list)
gamma_list_val = struct_pb2.ListValue()
gamma_list_val.extend(self._gamma_list)
init_epsilon_list_val = struct_pb2.ListValue()
init_epsilon_list_val.extend(self._init_epsilon_list)
n_exploration_episodes_val = struct_pb2.ListValue()
n_exploration_episodes_val.extend(self._n_exploration_episodes)
return hparams_summary.experiment_pb(
# The hyperparameters being changed
hparam_infos=[
api_pb2.HParamInfo(name='alpha',
display_name='Learning rate',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=alpha_list_val),
api_pb2.HParamInfo(name='alpha_decay',
display_name='Learning rate decay',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=alpha_decay_list_val),
api_pb2.HParamInfo(name='gamma',
display_name='Reward discount factor',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=gamma_list_val),
api_pb2.HParamInfo(name='init_epsilon',
display_name='Initial exploration',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=init_epsilon_list_val),
api_pb2.HParamInfo(name='n_exploration_episodes',
display_name='Initial exploration',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=n_exploration_episodes_val)
],
# The metrics being tracked
metric_infos=[
api_pb2.MetricInfo(
name=api_pb2.MetricName(
tag='sum_reward'),
display_name='SumReward'),
]
)
def _create_experiment_summary(self):
alpha_list_val = struct_pb2.ListValue()
alpha_list_val.extend(self._alpha_list)
gamma_list_val = struct_pb2.ListValue()
gamma_list_val.extend(self._gamma_list)
return hparams_summary.experiment_pb(
# The hyperparameters being changed
hparam_infos=[
api_pb2.HParamInfo(name='alpha',
display_name='Learning rate',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=alpha_list_val),
api_pb2.HParamInfo(name='gamma',
display_name='Reward discount factor',
type=api_pb2.DATA_TYPE_FLOAT64,
domain_discrete=gamma_list_val)
],
# The metrics being tracked
metric_infos=[
api_pb2.MetricInfo(
name=api_pb2.MetricName(tag='cummulative_reward'),
display_name='CumReward'),
])
def _compute_hparam_info_from_values(self, name, values):
"""Builds an HParamInfo message from the hparam name and list of
values.
Args:
name: string. The hparam name.
values: list of google.protobuf.Value messages. The list of values for the
hparam.
Returns:
An api_pb2.HParamInfo message.
"""
# Figure out the type from the values.
# Ignore values whose type is not listed in api_pb2.DataType
# If all values have the same type, then that is the type used.
# Otherwise, the returned type is DATA_TYPE_STRING.
result = api_pb2.HParamInfo(name=name, type=api_pb2.DATA_TYPE_UNSET)
distinct_values = set(
_protobuf_value_to_string(v)
for v in values
if _protobuf_value_type(v)
)
for v in values:
v_type = _protobuf_value_type(v)
if not v_type:
continue
if result.type == api_pb2.DATA_TYPE_UNSET:
result.type = v_type
elif result.type != v_type:
result.type = api_pb2.DATA_TYPE_STRING
if result.type == api_pb2.DATA_TYPE_STRING:
# A string result.type does not change, so we can exit the loop.
break
# If we couldn't figure out a type, then we can't compute the hparam_info.
if result.type == api_pb2.DATA_TYPE_UNSET:
return None
# If the result is a string, set the domain to be the distinct values if
# there aren't too many of them.
if (
result.type == api_pb2.DATA_TYPE_STRING
and len(distinct_values) <= self._max_domain_discrete_len
):
result.domain_discrete.extend(distinct_values)
return result
def hparams_config_pb(hparams, metrics, time_created_secs=None):
# NOTE: Keep docs in sync with `hparams_config` above.
"""Create a top-level experiment configuration.
This configuration describes the hyperparameters and metrics that will
be tracked in the experiment, but does not record any actual values of
those hyperparameters and metrics. It can be created before any models
are actually trained.
Args:
hparams: A list of `HParam` values.
metrics: A list of `Metric` values.
time_created_secs: The time that this experiment was created, as
seconds since epoch. Defaults to the current time.
Returns:
A TensorBoard `summary_pb2.Summary` message.
"""
hparam_infos = []
for hparam in hparams:
info = api_pb2.HParamInfo(
name=hparam.name,
description=hparam.description,
display_name=hparam.display_name,
)
domain = hparam.domain
if domain is not None:
domain.update_hparam_info(info)
hparam_infos.append(info)
metric_infos = [metric.as_proto() for metric in metrics]
experiment = api_pb2.Experiment(
hparam_infos=hparam_infos,
metric_infos=metric_infos,
time_created_secs=time_created_secs,
)
return _summary_pb(
metadata.EXPERIMENT_TAG,
plugin_data_pb2.HParamsPluginData(experiment=experiment),
)
def hparams_config(hparams, metrics, time_created_secs=None):
"""Write a top-level experiment configuration.
This configuration describes the hyperparameters and metrics that will
be tracked in the experiment, but does not record any actual values of
those hyperparameters and metrics. It can be created before any models
are actually trained.
Args:
hparams: A list of `HParam` values.
metrics: A list of `Metric` values.
time_created_secs: The time that this experiment was created, as
seconds since epoch. Defaults to the current time.
"""
hparam_infos = []
for hparam in hparams:
info = api_pb2.HParamInfo(
name=hparam.name,
description=hparam.description,
display_name=hparam.display_name,
)
domain = hparam.domain
if domain is not None:
domain.update_hparam_info(info)
hparam_infos.append(info)
metric_infos = [metric.as_proto() for metric in metrics]
experiment_pb = summary.experiment_pb(
hparam_infos=hparam_infos,
metric_infos=metric_infos,
time_created_secs=time_created_secs,
)
raw_pb = experiment_pb.SerializeToString()
summary_scope = (getattr(tf.compat.v2.summary.experimental,
"summary_scope", None)
or tf.summary.summary_scope)
with summary_scope("hparams_summary"):
return tf.compat.v2.summary.experimental.write_raw_pb(raw_pb, step=0)
def create_hparam_info(name, display_name, domain_discrete):
result = api_pb2.HParamInfo(name=name, display_name=display_name)
result.domain_discrete.extend(domain_discrete)
return result
