def test_get_earlystopping_rules(self):
# TODO (andreyvelich): Add more informative tests.
trials = [
api_pb2.Trial(name="test-asfjh", ),
api_pb2.Trial(name="test-234hs", )
]
experiment = api_pb2.Experiment(name="test", )
request = api_pb2.GetEarlyStoppingRulesRequest(
experiment=experiment,
trials=trials,
db_manager_address="katib-db-manager.kubeflow:6789")
get_earlystopping_rules = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['EarlyStopping'].
methods_by_name['GetEarlyStoppingRules']),
invocation_metadata={},
request=request,
timeout=1)
_, _, code, _ = get_earlystopping_rules.termination()
self.assertEqual(code, grpc.StatusCode.OK)
def call_validate():
experiment = api_pb2.Experiment(name="test",
spec=experiment_spec[0])
request = api_pb2.ValidateAlgorithmSettingsRequest(
experiment=experiment)
validate_algorithm_settings = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['Suggestion'].
methods_by_name['ValidateAlgorithmSettings']),
invocation_metadata={},
request=request,
timeout=1)
return validate_algorithm_settings.termination()
def test_get_suggestion(self):
experiment = api_pb2.Experiment(
name="darts-experiment",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="darts",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="num_epoch", value="10")
],
),
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="Best-Genotype"),
parallel_trial_count=1,
max_trial_count=1,
nas_config=api_pb2.NasConfig(
graph_config=api_pb2.GraphConfig(num_layers=3, ),
operations=api_pb2.NasConfig.Operations(operation=[
api_pb2.Operation(
operation_type="separable_convolution",
parameter_specs=api_pb2.Operation.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="filter_size",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "5"])),
])),
], ))))
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
request_number=1,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['Suggestion'].
methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request,
timeout=100)
response, metadata, code, details = get_suggestion.termination()
print(response.parameter_assignments)
self.assertEqual(code, grpc.StatusCode.OK)
self.assertEqual(1, len(response.parameter_assignments))
exp_algorithm_settings = {}
for setting in experiment.spec.algorithm.algorithm_settings:
exp_algorithm_settings[setting.name] = setting.value
exp_num_layers = experiment.spec.nas_config.graph_config.num_layers
exp_search_space = [
"separable_convolution_3x3", "separable_convolution_5x5"
]
for pa in response.parameter_assignments[0].assignments:
if (pa.name == "algorithm-settings"):
algorithm_settings = pa.value.replace("\'", "\"")
algorithm_settings = json.loads(algorithm_settings)
self.assertDictContainsSubset(exp_algorithm_settings,
algorithm_settings)
elif (pa.name == "num-layers"):
self.assertEqual(exp_num_layers, int(pa.value))
elif (pa.name == "search-space"):
search_space = pa.value.replace("\'", "\"")
search_space = json.loads(search_space)
self.assertEqual(exp_search_space, search_space)
def test_get_suggestion(self):
trials = [
api_pb2.Trial(
name="test-asfjh",
spec=api_pb2.TrialSpec(objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="param-1",
value="2",
),
api_pb2.ParameterAssignment(
name="param-2",
value="cat1",
),
api_pb2.ParameterAssignment(
name="param-3",
value="2",
),
api_pb2.ParameterAssignment(
name="param-4",
value="3.44",
)
])),
status=api_pb2.TrialStatus(observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(name="metric=1", value="435"),
api_pb2.Metric(name="metric=2", value="5643"),
]))),
api_pb2.Trial(
name="test-234hs",
spec=api_pb2.TrialSpec(objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="param-1",
value="3",
),
api_pb2.ParameterAssignment(
name="param-2",
value="cat2",
),
api_pb2.ParameterAssignment(
name="param-3",
value="6",
),
api_pb2.ParameterAssignment(
name="param-4",
value="4.44",
)
])),
status=api_pb2.TrialStatus(observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(name="metric=1", value="123"),
api_pb2.Metric(name="metric=2", value="3028"),
])))
]
experiment = api_pb2.Experiment(
name="test",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state",
value="10"),
api_pb2.AlgorithmSetting(name="gamma", value="0.25"),
api_pb2.AlgorithmSetting(name="prior_weight",
value="1.0"),
api_pb2.AlgorithmSetting(name="n_EI_candidates",
value="24"),
],
),
objective=api_pb2.ObjectiveSpec(type=api_pb2.MAXIMIZE,
goal=0.9),
parameter_specs=api_pb2.ExperimentSpec.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]),
),
api_pb2.ParameterSpec(name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None,
min=None,
list=["cat1", "cat2", "cat3"])),
api_pb2.ParameterSpec(
name="param-3",
parameter_type=api_pb2.DISCRETE,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "2", "6"])),
api_pb2.ParameterSpec(name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]))
])))
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=trials,
request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['Suggestion'].
methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request,
timeout=1)
response, metadata, code, details = get_suggestion.termination()
print(response.parameter_assignments)
self.assertEqual(code, grpc.StatusCode.OK)
self.assertEqual(2, len(response.parameter_assignments))
def test_get_suggestion(self):
trials = [
api_pb2.Trial(
name="first-trial",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="Validation-Accuracy",
goal=0.99),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="architecture",
value="[[3], [0, 1], [0, 0, 1], [2, 1, 0, 0]]",
),
api_pb2.ParameterAssignment(
name="nn_config",
value="{'num_layers': 4}",
),
])),
status=api_pb2.TrialStatus(
observation=api_pb2.Observation(metrics=[
api_pb2.Metric(name="Validation-Accuracy",
value="0.88"),
]),
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
)),
api_pb2.Trial(
name="second-trial",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="Validation-Accuracy",
goal=0.99),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="architecture",
value="[[1], [0, 1], [2, 1, 1], [2, 1, 1, 0]]",
),
api_pb2.ParameterAssignment(
name="nn_config",
value="{'num_layers': 4}",
),
], )),
status=api_pb2.TrialStatus(
observation=api_pb2.Observation(metrics=[
api_pb2.Metric(name="Validation-Accuracy",
value="0.84"),
]),
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
))
]
experiment = api_pb2.Experiment(
name="enas-experiment",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="enas", ),
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
goal=0.9,
objective_metric_name="Validation-Accuracy"),
parallel_trial_count=2,
max_trial_count=10,
nas_config=api_pb2.NasConfig(
graph_config=api_pb2.GraphConfig(num_layers=4,
input_sizes=[32, 32, 8],
output_sizes=[10]),
operations=api_pb2.NasConfig.Operations(operation=[
api_pb2.Operation(
operation_type="convolution",
parameter_specs=api_pb2.Operation.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="filter_size",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["5"])),
api_pb2.ParameterSpec(
name="num_filter",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["128"])),
api_pb2.ParameterSpec(
name="stride",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["1", "2"])),
])),
api_pb2.Operation(
operation_type="reduction",
parameter_specs=api_pb2.Operation.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="reduction_type",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None,
min=None,
list=["max_pooling"])),
api_pb2.ParameterSpec(
name="pool_size",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
min="2", max="3", step="1", list=[])),
])),
], ))))
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=trials,
request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['Suggestion'].
methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request,
timeout=100)
response, metadata, code, details = get_suggestion.termination()
print(response.parameter_assignments)
self.assertEqual(code, grpc.StatusCode.OK)
self.assertEqual(2, len(response.parameter_assignments))
def test_get_suggestion(self, algorithm_name, algorithm_settings):
experiment = api_pb2.Experiment(
name="test",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name=algorithm_name,
algorithm_settings=[
api_pb2.AlgorithmSetting(
name=name,
value=value
) for name, value in algorithm_settings.items()
],
),
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
goal=0.9
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]),
),
api_pb2.ParameterSpec(
name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["cat1", "cat2", "cat3"])
),
api_pb2.ParameterSpec(
name="param-3",
parameter_type=api_pb2.DISCRETE,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "2", "6"])
),
api_pb2.ParameterSpec(
name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[])
)
]
)
)
)
# Run the first suggestion with no previous trials in the request
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=[],
current_request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(api_pb2.DESCRIPTOR
.services_by_name['Suggestion']
.methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request, timeout=1)
response, metadata, code, details = get_suggestion.termination()
assert code == grpc.StatusCode.OK
assert 2 == len(response.parameter_assignments)
# Run the second suggestion with trials whose parameters are assigned in the first request
trials = [
api_pb2.Trial(
name="test-asfjh",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9
),
parameter_assignments=api_pb2.TrialSpec.ParameterAssignments(
assignments=response.parameter_assignments[0].assignments
)
),
status=api_pb2.TrialStatus(
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(
name="metric-1",
value="435"
),
api_pb2.Metric(
name="metric-2",
value="5643"
),
]
)
)
),
api_pb2.Trial(
name="test-234hs",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9
),
parameter_assignments=api_pb2.TrialSpec.ParameterAssignments(
assignments=response.parameter_assignments[1].assignments
)
),
status=api_pb2.TrialStatus(
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(
name="metric-1",
value="123"
),
api_pb2.Metric(
name="metric-2",
value="3028"
),
]
)
)
)
]
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=trials,
current_request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(api_pb2.DESCRIPTOR
.services_by_name['Suggestion']
.methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request, timeout=1)
response, metadata, code, details = get_suggestion.termination()
assert code == grpc.StatusCode.OK
assert 2 == len(response.parameter_assignments)
