def generate(param):
algorithm_settings = [
api_pb2.AlgorithmSetting(name="eta", value=str(param.eta)),
api_pb2.AlgorithmSetting(name="s_max", value=str(param.s_max)),
api_pb2.AlgorithmSetting(name="r_l", value=str(param.r_l)),
api_pb2.AlgorithmSetting(name="b_l", value=str(param.b_l)),
api_pb2.AlgorithmSetting(name="r", value=str(param.r)),
api_pb2.AlgorithmSetting(name="n", value=str(param.n)),
api_pb2.AlgorithmSetting(name="current_s",
value=str(param.current_s)),
api_pb2.AlgorithmSetting(name="current_i",
value=str(param.current_i)),
api_pb2.AlgorithmSetting(name="resource_name",
value=param.resource_name),
api_pb2.AlgorithmSetting(name="evaluating_trials",
value=str(param.evaluating_trials))
]
return api_pb2.AlgorithmSpec(algorithm_settings=algorithm_settings)
def test_get_suggestion(self):
experiment = api_pb2.Experiment(
name="darts-experiment",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="darts",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="num_epoch", value="10")
],
),
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="Best-Genotype"),
parallel_trial_count=1,
max_trial_count=1,
nas_config=api_pb2.NasConfig(
graph_config=api_pb2.GraphConfig(num_layers=3, ),
operations=api_pb2.NasConfig.Operations(operation=[
api_pb2.Operation(
operation_type="separable_convolution",
parameter_specs=api_pb2.Operation.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="filter_size",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "5"])),
])),
], ))))
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
request_number=1,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['Suggestion'].
methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request,
timeout=100)
response, metadata, code, details = get_suggestion.termination()
print(response.parameter_assignments)
self.assertEqual(code, grpc.StatusCode.OK)
self.assertEqual(1, len(response.parameter_assignments))
exp_algorithm_settings = {}
for setting in experiment.spec.algorithm.algorithm_settings:
exp_algorithm_settings[setting.name] = setting.value
exp_num_layers = experiment.spec.nas_config.graph_config.num_layers
exp_search_space = [
"separable_convolution_3x3", "separable_convolution_5x5"
]
for pa in response.parameter_assignments[0].assignments:
if (pa.name == "algorithm-settings"):
algorithm_settings = pa.value.replace("\'", "\"")
algorithm_settings = json.loads(algorithm_settings)
self.assertDictContainsSubset(exp_algorithm_settings,
algorithm_settings)
elif (pa.name == "num-layers"):
self.assertEqual(exp_num_layers, int(pa.value))
elif (pa.name == "search-space"):
search_space = pa.value.replace("\'", "\"")
search_space = json.loads(search_space)
self.assertEqual(exp_search_space, search_space)
def test_get_suggestion(self):
trials = [
api_pb2.Trial(
name="test-asfjh",
spec=api_pb2.TrialSpec(objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="param-1",
value="2",
),
api_pb2.ParameterAssignment(
name="param-2",
value="cat1",
),
api_pb2.ParameterAssignment(
name="param-3",
value="2",
),
api_pb2.ParameterAssignment(
name="param-4",
value="3.44",
)
])),
status=api_pb2.TrialStatus(observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(name="metric=1", value="435"),
api_pb2.Metric(name="metric=2", value="5643"),
]))),
api_pb2.Trial(
name="test-234hs",
spec=api_pb2.TrialSpec(objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="param-1",
value="3",
),
api_pb2.ParameterAssignment(
name="param-2",
value="cat2",
),
api_pb2.ParameterAssignment(
name="param-3",
value="6",
),
api_pb2.ParameterAssignment(
name="param-4",
value="4.44",
)
])),
status=api_pb2.TrialStatus(observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(name="metric=1", value="123"),
api_pb2.Metric(name="metric=2", value="3028"),
])))
]
experiment = api_pb2.Experiment(
name="test",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state",
value="10"),
api_pb2.AlgorithmSetting(name="gamma", value="0.25"),
api_pb2.AlgorithmSetting(name="prior_weight",
value="1.0"),
api_pb2.AlgorithmSetting(name="n_EI_candidates",
value="24"),
],
),
objective=api_pb2.ObjectiveSpec(type=api_pb2.MAXIMIZE,
goal=0.9),
parameter_specs=api_pb2.ExperimentSpec.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]),
),
api_pb2.ParameterSpec(name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None,
min=None,
list=["cat1", "cat2", "cat3"])),
api_pb2.ParameterSpec(
name="param-3",
parameter_type=api_pb2.DISCRETE,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "2", "6"])),
api_pb2.ParameterSpec(name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]))
])))
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=trials,
request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['Suggestion'].
methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request,
timeout=1)
response, metadata, code, details = get_suggestion.termination()
print(response.parameter_assignments)
self.assertEqual(code, grpc.StatusCode.OK)
self.assertEqual(2, len(response.parameter_assignments))
def test_validate_algorithm_settings(self):
experiment_spec = [None]
def call_validate():
experiment = api_pb2.Experiment(name="test",
spec=experiment_spec[0])
request = api_pb2.ValidateAlgorithmSettingsRequest(
experiment=experiment)
validate_algorithm_settings = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['Suggestion'].
methods_by_name['ValidateAlgorithmSettings']),
invocation_metadata={},
request=request,
timeout=1)
return validate_algorithm_settings.termination()
# valid cases
algorithm_spec = api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state", value="10"),
api_pb2.AlgorithmSetting(name="gamma", value="0.25"),
api_pb2.AlgorithmSetting(name="prior_weight", value="1.0"),
api_pb2.AlgorithmSetting(name="n_EI_candidates", value="24"),
],
)
experiment_spec[0] = api_pb2.ExperimentSpec(algorithm=algorithm_spec)
self.assertEqual(call_validate()[2], grpc.StatusCode.OK)
# invalid cases
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown"))
_, _, code, details = call_validate()
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'unknown algorithm name unknown')
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="random",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")
]))
_, _, code, details = call_validate()
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details,
'unknown setting unknown_conf for algorithm random')
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(
name="gamma", value="1.5")
]))
_, _, code, details = call_validate()
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'gamma should be in the range of (0, 1)')
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="n_EI_candidates", value="0")
]))
_, _, code, details = call_validate()
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'n_EI_candidates should be great than zero')
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state", value="-1")
]))
_, _, code, details = call_validate()
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details,
'random_state should be great or equal than zero')
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="prior_weight", value="aaa")
]))
_, _, code, details = call_validate()
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertTrue(
details.startswith('failed to validate prior_weight(aaa)'))
def test_validate_algorithm_settings(self):
# Valid case.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="grid",
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]),
),
api_pb2.ParameterSpec(
name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["cat1", "cat2", "cat3"])
),
api_pb2.ParameterSpec(
name="param-3",
parameter_type=api_pb2.DISCRETE,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "2", "6"])
),
api_pb2.ParameterSpec(
name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="2.9", min="1", list=[], step="0.5")
)
]
),
max_trial_count=12,
parallel_trial_count=3,
)
_, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.OK)
# Invalid cases.
# Empty step.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="grid",
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="3", min="1", list=[])
)
]
),
)
_, _, code, details = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'Param: param-1 step is nil')
# Max trial count > search space combinations.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="grid",
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="2", min="1", list=[]),
),
api_pb2.ParameterSpec(
name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["cat1", "cat2"])
),
api_pb2.ParameterSpec(
name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="2", min="1", list=[], step="0.5")
)
]
),
max_trial_count=15,
)
_, _, code, details = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'Max Trial Count: 15 > all possible search space combinations: 12')
def test_get_suggestion(self):
trials = [
api_pb2.Trial(
name="first-trial",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="Validation-Accuracy",
goal=0.99),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="architecture",
value="[[3], [0, 1], [0, 0, 1], [2, 1, 0, 0]]",
),
api_pb2.ParameterAssignment(
name="nn_config",
value="{'num_layers': 4}",
),
])),
status=api_pb2.TrialStatus(
observation=api_pb2.Observation(metrics=[
api_pb2.Metric(name="Validation-Accuracy",
value="0.88"),
]),
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
)),
api_pb2.Trial(
name="second-trial",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="Validation-Accuracy",
goal=0.99),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="architecture",
value="[[1], [0, 1], [2, 1, 1], [2, 1, 1, 0]]",
),
api_pb2.ParameterAssignment(
name="nn_config",
value="{'num_layers': 4}",
),
], )),
status=api_pb2.TrialStatus(
observation=api_pb2.Observation(metrics=[
api_pb2.Metric(name="Validation-Accuracy",
value="0.84"),
]),
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
))
]
experiment = api_pb2.Experiment(
name="enas-experiment",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="enas", ),
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
goal=0.9,
objective_metric_name="Validation-Accuracy"),
parallel_trial_count=2,
max_trial_count=10,
nas_config=api_pb2.NasConfig(
graph_config=api_pb2.GraphConfig(num_layers=4,
input_sizes=[32, 32, 8],
output_sizes=[10]),
operations=api_pb2.NasConfig.Operations(operation=[
api_pb2.Operation(
operation_type="convolution",
parameter_specs=api_pb2.Operation.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="filter_size",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["5"])),
api_pb2.ParameterSpec(
name="num_filter",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["128"])),
api_pb2.ParameterSpec(
name="stride",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["1", "2"])),
])),
api_pb2.Operation(
operation_type="reduction",
parameter_specs=api_pb2.Operation.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="reduction_type",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None,
min=None,
list=["max_pooling"])),
api_pb2.ParameterSpec(
name="pool_size",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
min="2", max="3", step="1", list=[])),
])),
], ))))
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=trials,
request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['Suggestion'].
methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request,
timeout=100)
response, metadata, code, details = get_suggestion.termination()
print(response.parameter_assignments)
self.assertEqual(code, grpc.StatusCode.OK)
self.assertEqual(2, len(response.parameter_assignments))
def test_validate_algorithm_settings(self):
# Valid cases.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state", value="10")
],
))
_, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.OK)
# Invalid cases.
# Unknown algorithm name.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown"))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, "unknown algorithm name unknown")
# Unknown config name.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")
]))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details,
"unknown setting unknown_conf for algorithm bayesianoptimization")
# Unknown base_estimator
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="base_estimator",
value="unknown estimator")
]))
wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[
0]
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details,
"{name} {value} is not supported in Bayesian optimization".format(
name=wrong_algorithm_setting.name,
value=wrong_algorithm_setting.value))
# Wrong n_initial_points
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="n_initial_points",
value="-1")
]))
wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[
0]
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details, "{name} should be great or equal than zero".format(
name=wrong_algorithm_setting.name))
# Unknown acq_func
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="acq_func", value="unknown")
]))
wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[
0]
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details,
"{name} {value} is not supported in Bayesian optimization".format(
name=wrong_algorithm_setting.name,
value=wrong_algorithm_setting.value))
# Unknown acq_optimizer
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="acq_optimizer",
value="unknown")
]))
wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[
0]
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details,
"{name} {value} is not supported in Bayesian optimization".format(
name=wrong_algorithm_setting.name,
value=wrong_algorithm_setting.value))
# Wrong random_state
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state", value="-1")
]))
wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[
0]
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(
details, "{name} should be great or equal than zero".format(
name=wrong_algorithm_setting.name))
def test_get_suggestion(self, algorithm_name, algorithm_settings):
experiment = api_pb2.Experiment(
name="test",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name=algorithm_name,
algorithm_settings=[
api_pb2.AlgorithmSetting(
name=name,
value=value
) for name, value in algorithm_settings.items()
],
),
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
goal=0.9
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]),
),
api_pb2.ParameterSpec(
name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["cat1", "cat2", "cat3"])
),
api_pb2.ParameterSpec(
name="param-3",
parameter_type=api_pb2.DISCRETE,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "2", "6"])
),
api_pb2.ParameterSpec(
name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[])
)
]
)
)
)
# Run the first suggestion with no previous trials in the request
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=[],
current_request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(api_pb2.DESCRIPTOR
.services_by_name['Suggestion']
.methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request, timeout=1)
response, metadata, code, details = get_suggestion.termination()
assert code == grpc.StatusCode.OK
assert 2 == len(response.parameter_assignments)
# Run the second suggestion with trials whose parameters are assigned in the first request
trials = [
api_pb2.Trial(
name="test-asfjh",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9
),
parameter_assignments=api_pb2.TrialSpec.ParameterAssignments(
assignments=response.parameter_assignments[0].assignments
)
),
status=api_pb2.TrialStatus(
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(
name="metric-1",
value="435"
),
api_pb2.Metric(
name="metric-2",
value="5643"
),
]
)
)
),
api_pb2.Trial(
name="test-234hs",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9
),
parameter_assignments=api_pb2.TrialSpec.ParameterAssignments(
assignments=response.parameter_assignments[1].assignments
)
),
status=api_pb2.TrialStatus(
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(
name="metric-1",
value="123"
),
api_pb2.Metric(
name="metric-2",
value="3028"
),
]
)
)
)
]
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=trials,
current_request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(api_pb2.DESCRIPTOR
.services_by_name['Suggestion']
.methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request, timeout=1)
response, metadata, code, details = get_suggestion.termination()
assert code == grpc.StatusCode.OK
assert 2 == len(response.parameter_assignments)
def test_validate_algorithm_settings(self):
# Valid cases.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state", value="10"),
api_pb2.AlgorithmSetting(name="gamma", value="0.25"),
api_pb2.AlgorithmSetting(name="prior_weight", value="1.0"),
api_pb2.AlgorithmSetting(name="n_EI_candidates",
value="24"),
]))
_, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
self.assertEqual(code, grpc.StatusCode.OK)
# Invalid cases.
# Unknown algorithm name.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown"))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, "unknown algorithm name unknown")
# Unknown algorithm setting name.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="random",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")
]))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details,
"unknown setting unknown_conf for algorithm random")
# Invalid gamma value.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(
name="gamma", value="1.5")
]))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, "gamma should be in the range of (0, 1)")
# Invalid n_EI_candidates value.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="n_EI_candidates", value="0")
]))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, "n_EI_candidates should be great than zero")
# Invalid random_state value.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state", value="-1")
]))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details,
"random_state should be great or equal than zero")
# Invalid prior_weight value.
experiment_spec = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="prior_weight", value="aaa")
]))
_, _, code, details = utils.call_validate(self.test_server,
experiment_spec)
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertTrue(
details.startswith("failed to validate prior_weight(aaa)"))
