def test_get_suggestion(self):
trials = [
api_pb2.Trial(
name="test-asfjh",
spec=api_pb2.TrialSpec(objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="param-1",
value="2",
),
api_pb2.ParameterAssignment(
name="param-2",
value="cat1",
),
api_pb2.ParameterAssignment(
name="param-3",
value="2",
),
api_pb2.ParameterAssignment(
name="param-4",
value="3.44",
)
])),
status=api_pb2.TrialStatus(observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(name="metric=1", value="435"),
api_pb2.Metric(name="metric=2", value="5643"),
]))),
api_pb2.Trial(
name="test-234hs",
spec=api_pb2.TrialSpec(objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="param-1",
value="3",
),
api_pb2.ParameterAssignment(
name="param-2",
value="cat2",
),
api_pb2.ParameterAssignment(
name="param-3",
value="6",
),
api_pb2.ParameterAssignment(
name="param-4",
value="4.44",
)
])),
status=api_pb2.TrialStatus(observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(name="metric=1", value="123"),
api_pb2.Metric(name="metric=2", value="3028"),
])))
]
experiment = api_pb2.Experiment(
name="test",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="bayesianoptimization",
algorithm_setting=[
api_pb2.AlgorithmSetting(name="random_state",
value="10")
],
),
objective=api_pb2.ObjectiveSpec(type=api_pb2.MAXIMIZE,
goal=0.9),
parameter_specs=api_pb2.ExperimentSpec.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]),
),
api_pb2.ParameterSpec(name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None,
min=None,
list=["cat1", "cat2", "cat3"])),
api_pb2.ParameterSpec(
name="param-3",
parameter_type=api_pb2.DISCRETE,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "2", "6"])),
api_pb2.ParameterSpec(name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]))
])))
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=trials,
request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['Suggestion'].
methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request,
timeout=1)
response, metadata, code, details = get_suggestion.termination()
print(response.parameter_assignments)
self.assertEqual(code, grpc.StatusCode.OK)
self.assertEqual(2, len(response.parameter_assignments))
def test_get_suggestion(self):
trials = [
api_pb2.Trial(
name="first-trial",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="Validation-Accuracy",
goal=0.99),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="architecture",
value="[[3], [0, 1], [0, 0, 1], [2, 1, 0, 0]]",
),
api_pb2.ParameterAssignment(
name="nn_config",
value="{'num_layers': 4}",
),
])),
status=api_pb2.TrialStatus(
observation=api_pb2.Observation(metrics=[
api_pb2.Metric(name="Validation-Accuracy",
value="0.88"),
]),
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
)),
api_pb2.Trial(
name="second-trial",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="Validation-Accuracy",
goal=0.99),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="architecture",
value="[[1], [0, 1], [2, 1, 1], [2, 1, 1, 0]]",
),
api_pb2.ParameterAssignment(
name="nn_config",
value="{'num_layers': 4}",
),
], )),
status=api_pb2.TrialStatus(
observation=api_pb2.Observation(metrics=[
api_pb2.Metric(name="Validation-Accuracy",
value="0.84"),
]),
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
))
]
experiment = api_pb2.Experiment(
name="enas-experiment",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="enas", ),
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
goal=0.9,
objective_metric_name="Validation-Accuracy"),
parallel_trial_count=2,
max_trial_count=10,
nas_config=api_pb2.NasConfig(
graph_config=api_pb2.GraphConfig(num_layers=4,
input_sizes=[32, 32, 8],
output_sizes=[10]),
operations=api_pb2.NasConfig.Operations(operation=[
api_pb2.Operation(
operation_type="convolution",
parameter_specs=api_pb2.Operation.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="filter_size",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["5"])),
api_pb2.ParameterSpec(
name="num_filter",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["128"])),
api_pb2.ParameterSpec(
name="stride",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["1", "2"])),
])),
api_pb2.Operation(
operation_type="reduction",
parameter_specs=api_pb2.Operation.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="reduction_type",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None,
min=None,
list=["max_pooling"])),
api_pb2.ParameterSpec(
name="pool_size",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
min="2", max="3", step="1", list=[])),
])),
], ))))
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=trials,
request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['Suggestion'].
methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request,
timeout=100)
response, metadata, code, details = get_suggestion.termination()
print(response.parameter_assignments)
self.assertEqual(code, grpc.StatusCode.OK)
self.assertEqual(2, len(response.parameter_assignments))
def test_validate_algorithm_settings(self):
experiment_spec = [None]
def call_validate():
experiment = api_pb2.Experiment(name="test", spec=experiment_spec[0])
request = api_pb2.ValidateAlgorithmSettingsRequest(experiment=experiment)
validate_algorithm_settings = self.test_server.invoke_unary_unary(
method_descriptor=(api_pb2.DESCRIPTOR
.services_by_name['Suggestion']
.methods_by_name['ValidateAlgorithmSettings']),
invocation_metadata={},
request=request, timeout=1)
return validate_algorithm_settings.termination()
# valid cases
algorithm_spec = api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_setting=[
api_pb2.AlgorithmSetting(
name="random_state",
value="10"
),
api_pb2.AlgorithmSetting(
name="gamma",
value="0.25"
),
api_pb2.AlgorithmSetting(
name="prior_weight",
value="1.0"
),
api_pb2.AlgorithmSetting(
name="n_EI_candidates",
value="24"
),
],
)
experiment_spec[0] = api_pb2.ExperimentSpec(algorithm=algorithm_spec)
self.assertEqual(call_validate()[2], grpc.StatusCode.OK)
# invalid cases
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown"))
_, _, code, details = call_validate()
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'unknown algorithm name unknown')
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="random",
algorithm_setting=[
api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")]
))
_, _, code, details = call_validate()
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'unknown setting unknown_conf for algorithm random')
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_setting=[
api_pb2.AlgorithmSetting(name="gamma", value="1.5")]
))
_, _, code, details = call_validate()
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'gamma should be in the range of (0, 1)')
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_setting=[
api_pb2.AlgorithmSetting(name="n_EI_candidates", value="0")]
))
_, _, code, details = call_validate()
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'n_EI_candidates should be great than zero')
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_setting=[
api_pb2.AlgorithmSetting(name="random_state", value="-1")]
))
_, _, code, details = call_validate()
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertEqual(details, 'random_state should be great or equal than zero')
experiment_spec[0] = api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_setting=[
api_pb2.AlgorithmSetting(name="prior_weight", value="aaa")]
))
_, _, code, details = call_validate()
self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
self.assertTrue(details.startswith('failed to validate prior_weight(aaa)'))
def test_get_suggestion(self):
experiment = api_pb2.Experiment(
name="darts-experiment",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="darts",
algorithm_setting=[
api_pb2.AlgorithmSetting(
name="num_epoch",
value="10"
)
],
),
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="Best-Genotype"
),
parallel_trial_count=1,
max_trial_count=1,
nas_config=api_pb2.NasConfig(
graph_config=api_pb2.GraphConfig(
num_layers=3,
),
operations=api_pb2.NasConfig.Operations(
operation=[
api_pb2.Operation(
operation_type="separable_convolution",
parameter_specs=api_pb2.Operation.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="filter_size",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "5"])
),
]
)
),
],
)
)
)
)
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
request_number=1,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(api_pb2.DESCRIPTOR
.services_by_name['Suggestion']
.methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request, timeout=100)
response, metadata, code, details = get_suggestion.termination()
print(response.parameter_assignments)
self.assertEqual(code, grpc.StatusCode.OK)
self.assertEqual(1, len(response.parameter_assignments))
exp_algorithm_settings = {}
for setting in experiment.spec.algorithm.algorithm_setting:
exp_algorithm_settings[setting.name] = setting.value
exp_num_layers = experiment.spec.nas_config.graph_config.num_layers
exp_search_space = ["separable_convolution_3x3", "separable_convolution_5x5"]
for pa in response.parameter_assignments[0].assignments:
if (pa.name == "algorithm-settings"):
algorithm_settings = pa.value.replace("\'", "\"")
algorithm_settings = json.loads(algorithm_settings)
self.assertDictContainsSubset(exp_algorithm_settings, algorithm_settings)
elif (pa.name == "num-layers"):
self.assertEqual(exp_num_layers, int(pa.value))
elif (pa.name == "search-space"):
search_space = pa.value.replace("\'", "\"")
search_space = json.loads(search_space)
self.assertEqual(exp_search_space, search_space)