class PipelineSpecBuilderTest(parameterized.TestCase):
def setUp(self):
self.maxDiff = None
@parameterized.parameters(
{
'channel':
pipeline_channel.PipelineParameterChannel(
name='output1', task_name='task1', channel_type='String'),
'expected':
'pipelinechannel--task1-output1',
},
{
'channel':
pipeline_channel.PipelineArtifactChannel(
name='output1', task_name='task1', channel_type='Artifact'),
'expected':
'pipelinechannel--task1-output1',
},
{
'channel':
pipeline_channel.PipelineParameterChannel(name='param1',
channel_type='String'),
'expected':
'pipelinechannel--param1',
},
)
def test_additional_input_name_for_pipeline_channel(
self, channel, expected):
self.assertEqual(
expected,
pipeline_spec_builder._additional_input_name_for_pipeline_channel(
channel))
@parameterized.parameters(
{
'parameter_type': pipeline_spec_pb2.ParameterType.NUMBER_INTEGER,
'default_value': None,
'expected': struct_pb2.Value(),
},
{
'parameter_type': pipeline_spec_pb2.ParameterType.NUMBER_INTEGER,
'default_value': 1,
'expected': struct_pb2.Value(number_value=1),
},
{
'parameter_type': pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE,
'default_value': 1.2,
'expected': struct_pb2.Value(number_value=1.2),
},
{
'parameter_type': pipeline_spec_pb2.ParameterType.STRING,
'default_value': 'text',
'expected': struct_pb2.Value(string_value='text'),
},
{
'parameter_type': pipeline_spec_pb2.ParameterType.BOOLEAN,
'default_value': True,
'expected': struct_pb2.Value(bool_value=True),
},
{
'parameter_type': pipeline_spec_pb2.ParameterType.BOOLEAN,
'default_value': False,
'expected': struct_pb2.Value(bool_value=False),
},
{
'parameter_type':
pipeline_spec_pb2.ParameterType.STRUCT,
'default_value': {
'a': 1,
'b': 2,
},
'expected':
struct_pb2.Value(struct_value=struct_pb2.Struct(
fields={
'a': struct_pb2.Value(number_value=1),
'b': struct_pb2.Value(number_value=2),
})),
},
{
'parameter_type':
pipeline_spec_pb2.ParameterType.LIST,
'default_value': ['a', 'b'],
'expected':
struct_pb2.Value(list_value=struct_pb2.ListValue(values=[
struct_pb2.Value(string_value='a'),
struct_pb2.Value(string_value='b'),
])),
},
{
'parameter_type':
pipeline_spec_pb2.ParameterType.LIST,
'default_value': [{
'a': 1,
'b': 2
}, {
'a': 10,
'b': 20
}],
'expected':
struct_pb2.Value(list_value=struct_pb2.ListValue(values=[
struct_pb2.Value(struct_value=struct_pb2.Struct(
fields={
'a': struct_pb2.Value(number_value=1),
'b': struct_pb2.Value(number_value=2),
})),
struct_pb2.Value(struct_value=struct_pb2.Struct(
fields={
'a': struct_pb2.Value(number_value=10),
'b': struct_pb2.Value(number_value=20),
})),
])),
},
)
def test_fill_in_component_input_default_value(self, parameter_type,
default_value, expected):
component_spec = pipeline_spec_pb2.ComponentSpec(
input_definitions=pipeline_spec_pb2.ComponentInputsSpec(
parameters={
'input1':
pipeline_spec_pb2.ComponentInputsSpec.ParameterSpec(
parameter_type=parameter_type)
}))
pipeline_spec_builder._fill_in_component_input_default_value(
component_spec=component_spec,
input_name='input1',
default_value=default_value)
self.assertEqual(
expected,
component_spec.input_definitions.parameters['input1'].
default_value,
)
def testStruct(self):
struct = struct_pb2.Struct()
self.assertIsInstance(struct, collections.Mapping)
self.assertEqual(0, len(struct))
struct_class = struct.__class__
struct['key1'] = 5
struct['key2'] = 'abc'
struct['key3'] = True
struct.get_or_create_struct('key4')['subkey'] = 11.0
struct_list = struct.get_or_create_list('key5')
self.assertIsInstance(struct_list, collections.Sequence)
struct_list.extend([6, 'seven', True, False, None])
struct_list.add_struct()['subkey2'] = 9
struct['key6'] = {'subkey': {}}
struct['key7'] = [2, False]
self.assertEqual(7, len(struct))
self.assertTrue(isinstance(struct, well_known_types.Struct))
self.assertEqual(5, struct['key1'])
self.assertEqual('abc', struct['key2'])
self.assertIs(True, struct['key3'])
self.assertEqual(11, struct['key4']['subkey'])
inner_struct = struct_class()
inner_struct['subkey2'] = 9
self.assertEqual([6, 'seven', True, False, None, inner_struct],
list(struct['key5'].items()))
self.assertEqual({}, dict(struct['key6']['subkey'].fields))
self.assertEqual([2, False], list(struct['key7'].items()))
serialized = struct.SerializeToString()
struct2 = struct_pb2.Struct()
struct2.ParseFromString(serialized)
self.assertEqual(struct, struct2)
for key, value in struct.items():
self.assertIn(key, struct)
self.assertIn(key, struct2)
self.assertEqual(value, struct2[key])
self.assertEqual(7, len(struct.keys()))
self.assertEqual(7, len(struct.values()))
for key in struct.keys():
self.assertIn(key, struct)
self.assertIn(key, struct2)
self.assertEqual(struct[key], struct2[key])
item = (next(iter(struct.keys())), next(iter(struct.values())))
self.assertEqual(item, next(iter(struct.items())))
self.assertTrue(isinstance(struct2, well_known_types.Struct))
self.assertEqual(5, struct2['key1'])
self.assertEqual('abc', struct2['key2'])
self.assertIs(True, struct2['key3'])
self.assertEqual(11, struct2['key4']['subkey'])
self.assertEqual([6, 'seven', True, False, None, inner_struct],
list(struct2['key5'].items()))
struct_list = struct2['key5']
self.assertEqual(6, struct_list[0])
self.assertEqual('seven', struct_list[1])
self.assertEqual(True, struct_list[2])
self.assertEqual(False, struct_list[3])
self.assertEqual(None, struct_list[4])
self.assertEqual(inner_struct, struct_list[5])
struct_list[1] = 7
self.assertEqual(7, struct_list[1])
struct_list.add_list().extend([1, 'two', True, False, None])
self.assertEqual([1, 'two', True, False, None],
list(struct_list[6].items()))
struct_list.extend([{
'nested_struct': 30
}, ['nested_list', 99], {}, []])
self.assertEqual(11, len(struct_list.values))
self.assertEqual(30, struct_list[7]['nested_struct'])
self.assertEqual('nested_list', struct_list[8][0])
self.assertEqual(99, struct_list[8][1])
self.assertEqual({}, dict(struct_list[9].fields))
self.assertEqual([], list(struct_list[10].items()))
struct_list[0] = {'replace': 'set'}
struct_list[1] = ['replace', 'set']
self.assertEqual('set', struct_list[0]['replace'])
self.assertEqual(['replace', 'set'], list(struct_list[1].items()))
text_serialized = str(struct)
struct3 = struct_pb2.Struct()
text_format.Merge(text_serialized, struct3)
self.assertEqual(struct, struct3)
struct.get_or_create_struct('key3')['replace'] = 12
self.assertEqual(12, struct['key3']['replace'])
# Tests empty list.
struct.get_or_create_list('empty_list')
empty_list = struct['empty_list']
self.assertEqual([], list(empty_list.items()))
list2 = struct_pb2.ListValue()
list2.add_list()
empty_list = list2[0]
self.assertEqual([], list(empty_list.items()))
# Tests empty struct.
struct.get_or_create_struct('empty_struct')
empty_struct = struct['empty_struct']
self.assertEqual({}, dict(empty_struct.fields))
list2.add_struct()
empty_struct = list2[1]
self.assertEqual({}, dict(empty_struct.fields))
self.assertEqual(9, len(struct))
del struct['key3']
del struct['key4']
self.assertEqual(7, len(struct))
self.assertEqual(6, len(struct['key5']))
del struct['key5'][1]
self.assertEqual(5, len(struct['key5']))
self.assertEqual([6, True, False, None, inner_struct],
list(struct['key5'].items()))
def discrete_domain(values):
domain = struct_pb2.ListValue()
domain.extend(values)
return domain
def hparams(hparam_dict=None, metric_dict=None, hparam_domain_discrete=None):
"""Outputs three `Summary` protocol buffers needed by hparams plugin.
`Experiment` keeps the metadata of an experiment, such as the name of the
hyperparameters and the name of the metrics.
`SessionStartInfo` keeps key-value pairs of the hyperparameters
`SessionEndInfo` describes status of the experiment e.g. STATUS_SUCCESS
Args:
hparam_dict: A dictionary that contains names of the hyperparameters
and their values.
metric_dict: A dictionary that contains names of the metrics
and their values.
hparam_domain_discrete: (Optional[Dict[str, List[Any]]]) A dictionary that
contains names of the hyperparameters and all discrete values they can hold
Returns:
The `Summary` protobufs for Experiment, SessionStartInfo and
SessionEndInfo
"""
import torch
from six import string_types
from tensorboard.plugins.hparams.api_pb2 import (
Experiment, HParamInfo, MetricInfo, MetricName, Status, DataType
)
from tensorboard.plugins.hparams.metadata import (
PLUGIN_NAME,
PLUGIN_DATA_VERSION,
EXPERIMENT_TAG,
SESSION_START_INFO_TAG,
SESSION_END_INFO_TAG
)
from tensorboard.plugins.hparams.plugin_data_pb2 import (
HParamsPluginData, SessionEndInfo, SessionStartInfo
)
# TODO: expose other parameters in the future.
# hp = HParamInfo(name='lr',display_name='learning rate',
# type=DataType.DATA_TYPE_FLOAT64, domain_interval=Interval(min_value=10,
# max_value=100))
# mt = MetricInfo(name=MetricName(tag='accuracy'), display_name='accuracy',
# description='', dataset_type=DatasetType.DATASET_VALIDATION)
# exp = Experiment(name='123', description='456', time_created_secs=100.0,
# hparam_infos=[hp], metric_infos=[mt], user='******')
if not isinstance(hparam_dict, dict):
logging.warning('parameter: hparam_dict should be a dictionary, nothing logged.')
raise TypeError('parameter: hparam_dict should be a dictionary, nothing logged.')
if not isinstance(metric_dict, dict):
logging.warning('parameter: metric_dict should be a dictionary, nothing logged.')
raise TypeError('parameter: metric_dict should be a dictionary, nothing logged.')
hparam_domain_discrete = hparam_domain_discrete or {}
if not isinstance(hparam_domain_discrete, dict):
raise TypeError(
"parameter: hparam_domain_discrete should be a dictionary, nothing logged."
)
for k, v in hparam_domain_discrete.items():
if (
k not in hparam_dict
or not isinstance(v, list)
or not all(isinstance(d, type(hparam_dict[k])) for d in v)
):
raise TypeError(
"parameter: hparam_domain_discrete[{}] should be a list of same type as "
"hparam_dict[{}].".format(k, k)
)
hps = []
ssi = SessionStartInfo()
for k, v in hparam_dict.items():
if v is None:
continue
if isinstance(v, int) or isinstance(v, float):
ssi.hparams[k].number_value = v
if k in hparam_domain_discrete:
domain_discrete: Optional[struct_pb2.ListValue] = struct_pb2.ListValue(
values=[
struct_pb2.Value(number_value=d)
for d in hparam_domain_discrete[k]
]
)
else:
domain_discrete = None
hps.append(
HParamInfo(
name=k,
type=DataType.Value("DATA_TYPE_FLOAT64"),
domain_discrete=domain_discrete,
)
)
continue
if isinstance(v, string_types):
ssi.hparams[k].string_value = v
if k in hparam_domain_discrete:
domain_discrete = struct_pb2.ListValue(
values=[
struct_pb2.Value(string_value=d)
for d in hparam_domain_discrete[k]
]
)
else:
domain_discrete = None
hps.append(
HParamInfo(
name=k,
type=DataType.Value("DATA_TYPE_STRING"),
domain_discrete=domain_discrete,
)
)
continue
if isinstance(v, bool):
ssi.hparams[k].bool_value = v
if k in hparam_domain_discrete:
domain_discrete = struct_pb2.ListValue(
values=[
struct_pb2.Value(bool_value=d)
for d in hparam_domain_discrete[k]
]
)
else:
domain_discrete = None
hps.append(
HParamInfo(
name=k,
type=DataType.Value("DATA_TYPE_BOOL"),
domain_discrete=domain_discrete,
)
)
continue
if isinstance(v, torch.Tensor):
v = make_np(v)[0]
ssi.hparams[k].number_value = v
hps.append(HParamInfo(name=k, type=DataType.Value("DATA_TYPE_FLOAT64")))
continue
raise ValueError('value should be one of int, float, str, bool, or torch.Tensor')
content = HParamsPluginData(session_start_info=ssi,
version=PLUGIN_DATA_VERSION)
smd = SummaryMetadata(
plugin_data=SummaryMetadata.PluginData(
plugin_name=PLUGIN_NAME,
content=content.SerializeToString()
)
)
ssi = Summary(value=[Summary.Value(tag=SESSION_START_INFO_TAG, metadata=smd)])
mts = [MetricInfo(name=MetricName(tag=k)) for k in metric_dict.keys()]
exp = Experiment(hparam_infos=hps, metric_infos=mts)
content = HParamsPluginData(experiment=exp, version=PLUGIN_DATA_VERSION)
smd = SummaryMetadata(
plugin_data=SummaryMetadata.PluginData(
plugin_name=PLUGIN_NAME,
content=content.SerializeToString()
)
)
exp = Summary(value=[Summary.Value(tag=EXPERIMENT_TAG, metadata=smd)])
sei = SessionEndInfo(status=Status.Value('STATUS_SUCCESS'))
content = HParamsPluginData(session_end_info=sei, version=PLUGIN_DATA_VERSION)
smd = SummaryMetadata(
plugin_data=SummaryMetadata.PluginData(
plugin_name=PLUGIN_NAME,
content=content.SerializeToString()
)
)
sei = Summary(value=[Summary.Value(tag=SESSION_END_INFO_TAG, metadata=smd)])
return exp, ssi, sei
def test_experiment_pb(self):
hparam_infos = [
api_pb2.HParamInfo(
name="param1",
display_name="display_name1",
description="foo",
type=api_pb2.DATA_TYPE_STRING,
domain_discrete=struct_pb2.ListValue(
values=[
struct_pb2.Value(string_value="a"),
struct_pb2.Value(string_value="b"),
]
),
),
api_pb2.HParamInfo(
name="param2",
display_name="display_name2",
description="bar",
type=api_pb2.DATA_TYPE_FLOAT64,
domain_interval=api_pb2.Interval(
min_value=-100.0, max_value=100.0
),
),
]
metric_infos = [
api_pb2.MetricInfo(
name=api_pb2.MetricName(tag="loss"),
dataset_type=api_pb2.DATASET_VALIDATION,
),
api_pb2.MetricInfo(
name=api_pb2.MetricName(group="train/", tag="acc"),
dataset_type=api_pb2.DATASET_TRAINING,
),
]
time_created_secs = 314159.0
self.assertEqual(
summary.experiment_pb(
hparam_infos, metric_infos, time_created_secs=time_created_secs
),
tf.compat.v1.Summary(
value=[
tf.compat.v1.Summary.Value(
tag="_hparams_/experiment",
tensor=summary._TF_NULL_TENSOR,
metadata=tf.compat.v1.SummaryMetadata(
plugin_data=tf.compat.v1.SummaryMetadata.PluginData(
plugin_name="hparams",
content=(
plugin_data_pb2.HParamsPluginData(
version=0,
experiment=api_pb2.Experiment(
time_created_secs=time_created_secs,
hparam_infos=hparam_infos,
metric_infos=metric_infos,
),
).SerializeToString()
),
)
),
)
]
),
)
