def _create_hparams_config(searchspace):
hparams = []
for key, val in searchspace.names().items():
if val == "DOUBLE":
hparams.append(
hp.HParam(
key,
hp.RealInterval(
float(searchspace.get(key)[0]), float(searchspace.get(key)[1])
),
)
)
elif val == "INTEGER":
hparams.append(
hp.HParam(
key,
hp.IntInterval(searchspace.get(key)[0], searchspace.get(key)[1]),
)
)
elif val == "DISCRETE":
hparams.append(hp.HParam(key, hp.Discrete(searchspace.get(key))))
elif val == "CATEGORICAL":
hparams.append(hp.HParam(key, hp.Discrete(searchspace.get(key))))
return hparams
def setUp(self):
self.logdir = os.path.join(self.get_temp_dir(), "logs")
self.hparams = {
hp.HParam("learning_rate", hp.RealInterval(1e-2, 1e-1)):
0.02,
hp.HParam("dense_layers", hp.IntInterval(2, 7)):
5,
hp.HParam("optimizer", hp.Discrete(["adam", "sgd"])):
"adam",
hp.HParam("who_knows_what"):
"???",
hp.HParam(
"magic",
hp.Discrete([False, True]),
display_name="~*~ Magic ~*~",
description="descriptive",
):
True,
"dropout":
0.3,
}
self.normalized_hparams = {
"learning_rate": 0.02,
"dense_layers": 5,
"optimizer": "adam",
"who_knows_what": "???",
"magic": True,
"dropout": 0.3,
}
self.start_time_secs = 123.45
self.trial_id = "psl27"
self.expected_session_start_pb = plugin_data_pb2.SessionStartInfo()
text_format.Merge(
"""
hparams { key: "learning_rate" value { number_value: 0.02 } }
hparams { key: "dense_layers" value { number_value: 5 } }
hparams { key: "optimizer" value { string_value: "adam" } }
hparams { key: "who_knows_what" value { string_value: "???" } }
hparams { key: "magic" value { bool_value: true } }
hparams { key: "dropout" value { number_value: 0.3 } }
""",
self.expected_session_start_pb,
)
self.expected_session_start_pb.group_name = self.trial_id
self.expected_session_start_pb.start_time_secs = self.start_time_secs
def test_sample_uniform_unseeded(self):
domain = hp.Discrete(["red", "green", "blue"])
with mock.patch.object(random, "choice") as m:
sentinel = object()
m.return_value = sentinel
result = domain.sample_uniform()
self.assertIs(result, sentinel)
# Call to `sorted` is an implementation detail of `sample_uniform`.
m.assert_called_once_with(sorted(["red", "green", "blue"]))
def test_sample_uniform(self):
domain = hp.Discrete(["red", "green", "blue"])
rng = mock.Mock()
sentinel = object()
rng.choice.return_value = sentinel
result = domain.sample_uniform(rng)
self.assertIs(result, sentinel)
# Call to `sorted` is an implementation detail of `sample_uniform`.
rng.choice.assert_called_once_with(sorted(["red", "green", "blue"]))
def _HParam(name, vals):
from tensorboard.plugins.hparams import summary_v2 as hp
if _all_numbers(vals):
min_val = float(min(vals))
max_val = float(max(vals))
return hp.HParam(name, hp.RealInterval(min_val, max_val))
else:
legal_vals = [_valid_param_val(val) for val in vals]
return hp.HParam(name, hp.Discrete(legal_vals))
def test_consistency_across_string_key_and_object_key(self):
hparams_1 = {
hp.HParam("optimizer", hp.Discrete(["adam", "sgd"])): "adam",
"learning_rate": 0.02,
}
hparams_2 = {
"optimizer": "adam",
hp.HParam("learning_rate", hp.RealInterval(1e-2, 1e-1)): 0.02,
}
self.assert_hparams_summaries_equal(
hp.hparams_pb(hparams_1, start_time_secs=self.start_time_secs),
hp.hparams_pb(hparams_2, start_time_secs=self.start_time_secs),
)
def _HParam(name, vals):
"""Returns an `HParam` object for flag name and known values.
Uses real interval domain iff all values are numeric.
Uses discrete interval iff all values are strings.
Otherwise does not use a domain.
"""
from tensorboard.plugins.hparams import summary_v2 as hp
type = hparam_type(vals)
if type == HPARAM_TYPE_NUMBER:
return hp.HParam(name, hp.RealInterval(float("-inf"), float("inf")))
elif type == HPARAM_TYPE_BOOL:
return hp.HParam(name, hp.Discrete((True, False)))
elif type == HPARAM_TYPE_STRING:
return hp.HParam(name, hp.Discrete(vals))
elif type == HPARAM_TYPE_NONE:
return hp.HParam(name)
else:
assert False, type
def test_dtype_mismatch(self):
with six.assertRaisesRegex(
self, TypeError, r"dtype mismatch: not isinstance\(2, str\)"):
hp.Discrete(["one", 2])
def test_empty_with_unspecified_dtype(self):
with six.assertRaisesRegex(self, ValueError,
"Empty domain with no dtype specified"):
hp.Discrete([])
def test_empty_with_explicit_dtype(self):
domain = hp.Discrete([], dtype=bool)
self.assertIs(domain.dtype, bool)
self.assertEqual(domain.values, [])
def test_values_sorted(self):
domain = hp.Discrete([2, 3, 1])
self.assertEqual(domain.values, [1, 2, 3])
self.assertEqual(domain.dtype, int)
def test_simple(self):
domain = hp.Discrete([1, 2, 5])
self.assertEqual(domain.values, [1, 2, 5])
self.assertEqual(domain.dtype, int)
def setUp(self):
self.logdir = os.path.join(self.get_temp_dir(), "logs")
self.hparams = [
hp.HParam("learning_rate", hp.RealInterval(1e-2, 1e-1)),
hp.HParam("dense_layers", hp.IntInterval(2, 7)),
hp.HParam("optimizer", hp.Discrete(["adam", "sgd"])),
hp.HParam("who_knows_what"),
hp.HParam(
"magic",
hp.Discrete([False, True]),
display_name="~*~ Magic ~*~",
description="descriptive",
),
]
self.metrics = [
hp.Metric("samples_per_second"),
hp.Metric(group="train",
tag="batch_loss",
display_name="loss (train)"),
hp.Metric(
group="validation",
tag="epoch_accuracy",
display_name="accuracy (val.)",
description="Accuracy on the _validation_ dataset.",
dataset_type=hp.Metric.VALIDATION,
),
]
self.time_created_secs = 1555624767.0
self.expected_experiment_pb = api_pb2.Experiment()
text_format.Merge(
"""
time_created_secs: 1555624767.0
hparam_infos {
name: "learning_rate"
type: DATA_TYPE_FLOAT64
domain_interval {
min_value: 0.01
max_value: 0.1
}
}
hparam_infos {
name: "dense_layers"
type: DATA_TYPE_FLOAT64
domain_interval {
min_value: 2
max_value: 7
}
}
hparam_infos {
name: "optimizer"
type: DATA_TYPE_STRING
domain_discrete {
values {
string_value: "adam"
}
values {
string_value: "sgd"
}
}
}
hparam_infos {
name: "who_knows_what"
}
hparam_infos {
name: "magic"
type: DATA_TYPE_BOOL
display_name: "~*~ Magic ~*~"
description: "descriptive"
domain_discrete {
values {
bool_value: false
}
values {
bool_value: true
}
}
}
metric_infos {
name {
tag: "samples_per_second"
}
}
metric_infos {
name {
group: "train"
tag: "batch_loss"
}
display_name: "loss (train)"
}
metric_infos {
name {
group: "validation"
tag: "epoch_accuracy"
}
display_name: "accuracy (val.)"
description: "Accuracy on the _validation_ dataset."
dataset_type: DATASET_VALIDATION
}
""",
self.expected_experiment_pb,
)
def HParam(name, vals):
if all_numbers(vals):
return hp.HParam(name,
hp.RealInterval(float(min(vals)), float(max(vals))))
else:
return hp.HParam(name, hp.Discrete(vals))