def _verify_full_response(self, request, expected_response):
request_proto = api_pb2.ListSessionGroupsRequest()
text_format.Merge(request, request_proto)
handler = list_session_groups.Handler(
backend_context.Context(self._mock_tb_context), request_proto)
response = handler.run()
self.assertProtoEquals(expected_response, response)
def __init__(self, context):
"""Instantiates HParams plugin via TensorBoard core.
Args:
context: A base_plugin.TBContext instance.
"""
self._context = backend_context.Context(context)
def test_experiment_with_experiment_tag(self):
experiment = """
description: 'Test experiment'
metric_infos: [
{ name: { tag: 'current_temp' } }
]
"""
run = "exp"
tag = metadata.EXPERIMENT_TAG
m = summary_pb2.SummaryMetadata()
m.data_class = summary_pb2.DATA_CLASS_TENSOR
m.plugin_data.plugin_name = metadata.PLUGIN_NAME
m.plugin_data.content = self._serialized_plugin_data(
DATA_TYPE_EXPERIMENT, experiment)
self._mock_multiplexer.AllSummaryMetadata.side_effect = None
self._mock_multiplexer.AllSummaryMetadata.return_value = {
run: {
tag: m
}
}
ctxt = backend_context.Context(self._mock_tb_context)
request_ctx = context.RequestContext()
self.assertProtoEquals(
experiment,
ctxt.experiment_from_metadata(
request_ctx, "123", ctxt.hparams_metadata(request_ctx, "123")),
)
def test_experiment_without_experiment_tag_many_distinct_values(self):
self.session_1_start_info_ = """
hparams:[
{key: 'batch_size' value: {number_value: 100}},
{key: 'lr' value: {string_value: '0.01'}}
]
"""
self.session_2_start_info_ = """
hparams:[
{key: 'lr' value: {number_value: 0.02}},
{key: 'model_type' value: {string_value: 'CNN'}}
]
"""
self.session_3_start_info_ = """
hparams:[
{key: 'batch_size' value: {bool_value: true}},
{key: 'model_type' value: {string_value: 'CNN'}}
]
"""
expected_exp = """
hparam_infos: {
name: 'batch_size'
type: DATA_TYPE_STRING
}
hparam_infos: {
name: 'lr'
type: DATA_TYPE_STRING
}
hparam_infos: {
name: 'model_type'
type: DATA_TYPE_STRING
domain_discrete: {
values: [{string_value: 'CNN'}]
}
}
metric_infos: {
name: {group: '', tag: 'accuracy'}
}
metric_infos: {
name: {group: '', tag: 'loss'}
}
metric_infos: {
name: {group: 'eval', tag: 'loss'}
}
metric_infos: {
name: {group: 'train', tag: 'loss'}
}
"""
ctxt = backend_context.Context(
self._mock_tb_context, max_domain_discrete_len=1
)
request_ctx = context.RequestContext()
actual_exp = ctxt.experiment_from_metadata(
request_ctx,
"123",
ctxt.hparams_metadata(request_ctx, "123"),
)
_canonicalize_experiment(actual_exp)
self.assertProtoEquals(expected_exp, actual_exp)
def test_experiment_without_experiment_tag(self):
self.session_1_start_info_ = """
hparams: [
{key: 'batch_size' value: {number_value: 100}},
{key: 'lr' value: {number_value: 0.01}},
{key: 'model_type' value: {string_value: 'CNN'}}
]
"""
self.session_2_start_info_ = """
hparams:[
{key: 'batch_size' value: {number_value: 200}},
{key: 'lr' value: {number_value: 0.02}},
{key: 'model_type' value: {string_value: 'LATTICE'}}
]
"""
self.session_3_start_info_ = """
hparams:[
{key: 'batch_size' value: {number_value: 300}},
{key: 'lr' value: {number_value: 0.05}},
{key: 'model_type' value: {string_value: 'CNN'}}
]
"""
expected_exp = """
hparam_infos: {
name: 'batch_size'
type: DATA_TYPE_FLOAT64
},
hparam_infos: {
name: 'lr'
type: DATA_TYPE_FLOAT64
},
hparam_infos: {
name: 'model_type'
type: DATA_TYPE_STRING
domain_discrete: {
values: [{string_value: 'CNN'},
{string_value: 'LATTICE'}]
}
}
metric_infos: {
name: {group: '', tag: 'accuracy'}
}
metric_infos: {
name: {group: '', tag: 'loss'}
}
metric_infos: {
name: {group: 'eval', tag: 'loss'}
}
metric_infos: {
name: {group: 'train', tag: 'loss'}
}
"""
ctxt = backend_context.Context(self._mock_tb_context)
actual_exp = ctxt.experiment_from_metadata(
"123", ctxt.hparams_metadata("123"))
_canonicalize_experiment(actual_exp)
self.assertProtoEquals(expected_exp, actual_exp)
def _verify_handler(self, request, expected_session_group_names,
expected_total_size):
request_proto = api_pb2.ListSessionGroupsRequest()
text_format.Merge(request, request_proto)
handler = list_session_groups.Handler(
backend_context.Context(self._mock_tb_context), request_proto)
response = handler.run()
self.assertEqual(expected_session_group_names,
[sg.name for sg in response.session_groups])
self.assertEqual(expected_total_size, response.total_size)
def _run_handler(self, request):
request_proto = api_pb2.ListSessionGroupsRequest()
text_format.Merge(request, request_proto)
handler = list_session_groups.Handler(
backend_context.Context(self._mock_tb_context), request_proto)
response = handler.run()
# Sort the metric values repeated field in each session group to
# canonicalize the response.
for group in response.session_groups:
group.metric_values.sort(key=operator.attrgetter("name.tag"))
return response
def test_experiment_without_any_hparams_summaries(self):
ctxt = backend_context.Context(self._mock_tb_context,
max_domain_discrete_len=1)
request_ctx = context.RequestContext()
actual_exp = ctxt.experiment_from_metadata(
request_ctx,
"123",
ctxt.hparams_metadata(request_ctx, "123"),
)
self.assertIsInstance(actual_exp, api_pb2.Experiment)
self.assertProtoEquals("", actual_exp)
def _run_handler(self, experiment, session_groups, response_format):
experiment_proto = text_format.Merge(experiment, api_pb2.Experiment())
session_groups_proto = text_format.Merge(
session_groups, api_pb2.ListSessionGroupsResponse())
num_columns = len(experiment_proto.hparam_infos) + len(
experiment_proto.metric_infos)
handler = download_data.Handler(
backend_context.Context(self._mock_tb_context),
experiment_proto,
session_groups_proto,
response_format,
[True] * num_columns,
)
return handler.run()
def test_experiment_with_experiment_tag(self):
experiment = """
description: 'Test experiment'
metric_infos: [
{ name: { tag: 'current_temp' } }
]
"""
self._mock_multiplexer.PluginRunToTagToContent.side_effect = None
self._mock_multiplexer.PluginRunToTagToContent.return_value = {
"exp": {
metadata.EXPERIMENT_TAG:
self._serialized_plugin_data(DATA_TYPE_EXPERIMENT, experiment)
}
}
ctxt = backend_context.Context(self._mock_tb_context)
self.assertProtoEquals(experiment, ctxt.experiment())
def test_experiment_without_experiment_tag_different_hparam_types(self):
self.session_1_start_info_ = """
hparams:[
{key: 'batch_size' value: {number_value: 100}},
{key: 'lr' value: {string_value: '0.01'}}
]
"""
self.session_2_start_info_ = """
hparams:[
{key: 'lr' value: {number_value: 0.02}},
{key: 'model_type' value: {string_value: 'LATTICE'}}
]
"""
self.session_3_start_info_ = """
hparams:[
{key: 'batch_size' value: {bool_value: true}},
{key: 'model_type' value: {string_value: 'CNN'}}
]
"""
expected_exp = """
hparam_infos: {
name: 'batch_size'
type: DATA_TYPE_STRING
domain_discrete: {
values: [{string_value: '100.0'},
{string_value: 'true'}]
}
}
hparam_infos: {
name: 'lr'
type: DATA_TYPE_STRING
domain_discrete: {
values: [{string_value: '0.01'},
{string_value: '0.02'}]
}
}
hparam_infos: {
name: 'model_type'
type: DATA_TYPE_STRING
domain_discrete: {
values: [{string_value: 'CNN'},
{string_value: 'LATTICE'}]
}
}
metric_infos: {
name: {group: '', tag: 'accuracy'}
}
metric_infos: {
name: {group: '', tag: 'loss'}
}
metric_infos: {
name: {group: 'eval', tag: 'loss'}
}
metric_infos: {
name: {group: 'train', tag: 'loss'}
}
"""
ctxt = backend_context.Context(self._mock_tb_context)
actual_exp = ctxt.experiment()
_canonicalize_experiment(actual_exp)
self.assertProtoEquals(expected_exp, actual_exp)
