def _find_experiment(self):
mapping = self.multiplexer.PluginRunToTagToContent(
metadata.PLUGIN_NAME)
for tag_to_content in mapping.values():
if metadata.EXPERIMENT_TAG in tag_to_content:
return metadata.parse_experiment_plugin_data(
tag_to_content[metadata.EXPERIMENT_TAG])
return None
def _find_experiment(self):
mapping = self.multiplexer.PluginRunToTagToContent(
metadata.PLUGIN_NAME)
for tag_to_content in mapping.values():
if metadata.EXPERIMENT_TAG in tag_to_content:
return metadata.parse_experiment_plugin_data(
tag_to_content[metadata.EXPERIMENT_TAG])
return None
def _check_summary(self, summary_pb):
"""Test that a summary contains exactly the expected experiment PB."""
values = summary_pb.value
self.assertEqual(len(values), 1, values)
actual_value = values[0]
self.assertEqual(
actual_value.metadata.plugin_data.plugin_name, metadata.PLUGIN_NAME,
)
plugin_content = actual_value.metadata.plugin_data.content
self.assertEqual(
metadata.parse_experiment_plugin_data(plugin_content),
self.expected_experiment_pb,
)
def _find_experiment_tag(self, hparams_run_to_tag_to_content):
"""Finds the experiment associcated with the metadata.EXPERIMENT_TAG
tag.
Returns:
The experiment or None if no such experiment is found.
"""
# We expect only one run to have an `EXPERIMENT_TAG`; look
# through all of them and arbitrarily pick the first one.
for tags in hparams_run_to_tag_to_content.values():
maybe_content = tags.get(metadata.EXPERIMENT_TAG)
if maybe_content is not None:
return metadata.parse_experiment_plugin_data(maybe_content)
return None
def _find_experiment_tag(self):
"""Finds the experiment associcated with the metadata.EXPERIMENT_TAG tag.
Caches the experiment if it was found.
Returns:
The experiment or None if no such experiment is found.
"""
with self._experiment_from_tag_lock:
if self._experiment_from_tag is None:
mapping = self.multiplexer.PluginRunToTagToContent(
metadata.PLUGIN_NAME)
for tag_to_content in mapping.values():
if metadata.EXPERIMENT_TAG in tag_to_content:
self._experiment_from_tag = metadata.parse_experiment_plugin_data(
tag_to_content[metadata.EXPERIMENT_TAG])
break
return self._experiment_from_tag
def _find_experiment_tag(self, experiment_id):
"""Finds the experiment associcated with the metadata.EXPERIMENT_TAG
tag.
Returns:
The experiment or None if no such experiment is found.
"""
mapping = self.hparams_metadata(
experiment_id,
run_tag_filter=provider.RunTagFilter(
tags=[metadata.EXPERIMENT_TAG]),
)
if not mapping:
return None
# We expect only one run to have an `EXPERIMENT_TAG`; pick
# arbitrarily.
tag_to_content = next(iter(mapping.values()))
content = next(iter(tag_to_content.values()))
return metadata.parse_experiment_plugin_data(content)
def test_summary_pb(self):
hparams = [
hp.HParam("learning_rate", hp.RealInterval(1e-2, 1e-1)),
hp.HParam("dense_layers", hp.IntInterval(2, 7)),
hp.HParam("optimizer", hp.Discrete(["adam", "sgd"])),
hp.HParam("who_knows_what"),
hp.HParam(
"magic",
hp.Discrete([False, True]),
display_name="~*~ Magic ~*~",
description="descriptive",
),
]
metrics = [
hp.Metric("samples_per_second"),
hp.Metric(group="train", tag="batch_loss", display_name="loss (train)"),
hp.Metric(
group="validation",
tag="epoch_accuracy",
display_name="accuracy (val.)",
description="Accuracy on the _validation_ dataset.",
dataset_type=hp.Metric.VALIDATION,
),
]
experiment = hp.Experiment(
hparams=hparams,
metrics=metrics,
user="******",
description="nothing to see here; move along",
time_created_secs=1555624767,
)
self.assertEqual(experiment.hparams, hparams)
self.assertEqual(experiment.metrics, metrics)
self.assertEqual(experiment.user, "zalgo"),
self.assertEqual(experiment.description, "nothing to see here; move along")
self.assertEqual(experiment.time_created_secs, 1555624767)
expected_experiment_pb = api_pb2.Experiment()
text_format.Merge(
"""
description: "nothing to see here; move along"
user: "******"
time_created_secs: 1555624767.0
hparam_infos {
name: "learning_rate"
type: DATA_TYPE_FLOAT64
domain_interval {
min_value: 0.01
max_value: 0.1
}
}
hparam_infos {
name: "dense_layers"
type: DATA_TYPE_FLOAT64
domain_interval {
min_value: 2
max_value: 7
}
}
hparam_infos {
name: "optimizer"
type: DATA_TYPE_STRING
domain_discrete {
values {
string_value: "adam"
}
values {
string_value: "sgd"
}
}
}
hparam_infos {
name: "who_knows_what"
}
hparam_infos {
name: "magic"
type: DATA_TYPE_BOOL
display_name: "~*~ Magic ~*~"
description: "descriptive"
domain_discrete {
values {
bool_value: false
}
values {
bool_value: true
}
}
}
metric_infos {
name {
tag: "samples_per_second"
}
}
metric_infos {
name {
group: "train"
tag: "batch_loss"
}
display_name: "loss (train)"
}
metric_infos {
name {
group: "validation"
tag: "epoch_accuracy"
}
display_name: "accuracy (val.)"
description: "Accuracy on the _validation_ dataset."
dataset_type: DATASET_VALIDATION
}
""",
expected_experiment_pb,
)
actual_summary_pb = experiment.summary_pb()
plugin_content = actual_summary_pb.value[0].metadata.plugin_data.content
self.assertEqual(
metadata.parse_experiment_plugin_data(plugin_content),
expected_experiment_pb,
)
