def test_validate_algorithm_settings(self):
# Valid case.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="grid",
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]),
),
api_pb2.ParameterSpec(
name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["cat1", "cat2", "cat3"])
),
api_pb2.ParameterSpec(
name="param-3",
parameter_type=api_pb2.DISCRETE,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "2", "6"])
),
api_pb2.ParameterSpec(
name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="2.9", min="1", list=[], step="0.5")
)
]
),
max_trial_count=12,
parallel_trial_count=3,
)
_, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.OK)
# Invalid cases.
# Empty step.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="grid",
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="3", min="1", list=[])
)
]
),
)
_, _, code, details = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'Param: param-1 step is nil')
# Max trial count > search space combinations.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="grid",
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="2", min="1", list=[]),
),
api_pb2.ParameterSpec(
name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["cat1", "cat2"])
),
api_pb2.ParameterSpec(
name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="2", min="1", list=[], step="0.5")
)
]
),
max_trial_count=15,
)
_, _, code, details = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'Max Trial Count: 15 > all possible search space combinations: 12')
def test_validate_algorithm_settings(self):
# Valid cases.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state", value="10")
],
))
_, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.OK)
# Invalid cases.
# Unknown algorithm name.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown"))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, "unknown algorithm name unknown")
# Unknown config name.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")
]))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details,
"unknown setting unknown_conf for algorithm bayesianoptimization")
# Unknown base_estimator
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="base_estimator",
value="unknown estimator")
]))
wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[
0]
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details,
"{name} {value} is not supported in Bayesian optimization".format(
name=wrong_algorithm_setting.name,
value=wrong_algorithm_setting.value))
# Wrong n_initial_points
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="n_initial_points",
value="-1")
]))
wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[
0]
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details, "{name} should be great or equal than zero".format(
name=wrong_algorithm_setting.name))
# Unknown acq_func
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="acq_func", value="unknown")
]))
wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[
0]
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details,
"{name} {value} is not supported in Bayesian optimization".format(
name=wrong_algorithm_setting.name,
value=wrong_algorithm_setting.value))
# Unknown acq_optimizer
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="acq_optimizer",
value="unknown")
]))
wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[
0]
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details,
"{name} {value} is not supported in Bayesian optimization".format(
name=wrong_algorithm_setting.name,
value=wrong_algorithm_setting.value))
# Wrong random_state
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state", value="-1")
]))
wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[
0]
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details, "{name} should be great or equal than zero".format(
name=wrong_algorithm_setting.name))
def test_validate_early_stopping_settings(self):
# Valid cases
early_stopping = api_pb2.EarlyStoppingSpec(
algorithm_name="medianstop",
algorithm_settings=[
api_pb2.EarlyStoppingSetting(
name="min_trials_required",
value="2",
),
api_pb2.EarlyStoppingSetting(
name="start_step",
value="5",
),
],
)
_, _, code, _ = utils.call_validate(self.test_server, early_stopping)
self.assertEqual(code, grpc.StatusCode.OK)
# Invalid cases
# Unknown algorithm name
early_stopping = api_pb2.EarlyStoppingSpec(algorithm_name="unknown")
_, _, code, details = utils.call_validate(self.test_server,
early_stopping)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, "unknown algorithm name unknown")
# Unknown config name
early_stopping = api_pb2.EarlyStoppingSpec(
algorithm_name="medianstop",
algorithm_settings=[
api_pb2.EarlyStoppingSetting(
name="unknown_conf",
value="100",
),
],
)
_, _, code, details = utils.call_validate(self.test_server,
early_stopping)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details, "unknown setting unknown_conf for algorithm medianstop")
# Wrong min_trials_required
early_stopping = api_pb2.EarlyStoppingSpec(
algorithm_name="medianstop",
algorithm_settings=[
api_pb2.EarlyStoppingSetting(
name="min_trials_required",
value="0",
),
],
)
_, _, code, details = utils.call_validate(self.test_server,
early_stopping)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details,
"min_trials_required must be greater than zero (>0)")
# Wrong start_step
early_stopping = api_pb2.EarlyStoppingSpec(
algorithm_name="medianstop",
algorithm_settings=[
api_pb2.EarlyStoppingSetting(
name="start_step",
value="0",
),
],
)
_, _, code, details = utils.call_validate(self.test_server,
early_stopping)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details,
"start_step must be greater or equal than one (>=1)")
def test_validate_algorithm_settings(self):
# Valid cases.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state", value="10"),
api_pb2.AlgorithmSetting(name="gamma", value="0.25"),
api_pb2.AlgorithmSetting(name="prior_weight", value="1.0"),
api_pb2.AlgorithmSetting(name="n_EI_candidates",
value="24"),
]))
_, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.OK)
# Invalid cases.
# Unknown algorithm name.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown"))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, "unknown algorithm name unknown")
# Unknown algorithm setting name.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="random",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")
]))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details,
"unknown setting unknown_conf for algorithm random")
# Invalid gamma value.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(
name="gamma", value="1.5")
]))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, "gamma should be in the range of (0, 1)")
# Invalid n_EI_candidates value.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="n_EI_candidates", value="0")
]))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, "n_EI_candidates should be great than zero")
# Invalid random_state value.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state", value="-1")
]))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details,
"random_state should be great or equal than zero")
# Invalid prior_weight value.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="prior_weight", value="aaa")
]))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertTrue(
details.startswith("failed to validate prior_weight(aaa)"))