def _make_master_bracket(self, experiment, param):
n = param.n
r = int(param.r)
parameter_config = parsing_util.parse_parameter_configs(
experiment.spec.parameter_specs.parameters)
trial_specs = []
for _ in range(n):
sample = parameter_config.random_sample()
suggestion = parsing_util.parse_x_next_vector(
sample,
parameter_config.parameter_types,
parameter_config.names,
parameter_config.discrete_info,
parameter_config.categorical_info)
trial_spec = api_pb2.TrialSpec()
trial_spec.experiment_name = experiment.name
for hp in suggestion:
if hp['name'] == param.resource_name:
hp['value'] = str(r)
trial_spec.parameter_assignments.assignments.add(name=hp['name'],
value=str(hp['value']))
trial_specs.append(trial_spec)
logger.info("Generate %d trials by master bracket.",
n, extra={"experiment_name": experiment.name})
return trial_specs
def _copy_trials(self, trials, r_i, resourceName):
trialSpecs = []
for t in trials:
trial_spec = api_pb2.TrialSpec()
for assignment in t.spec.parameter_assignments.assignments:
if assignment.name == resourceName:
value = str(r_i)
else:
value = assignment.value
trial_spec.parameter_assignments.assignments.add(
name=assignment.name, value=value)
trialSpecs.append(trial_spec)
return trialSpecs
def test_get_suggestion(self):
trials = [
api_pb2.Trial(
name="test-asfjh",
spec=api_pb2.TrialSpec(objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="param-1",
value="2",
),
api_pb2.ParameterAssignment(
name="param-2",
value="cat1",
),
api_pb2.ParameterAssignment(
name="param-3",
value="2",
),
api_pb2.ParameterAssignment(
name="param-4",
value="3.44",
)
])),
status=api_pb2.TrialStatus(observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(name="metric=1", value="435"),
api_pb2.Metric(name="metric=2", value="5643"),
]))),
api_pb2.Trial(
name="test-234hs",
spec=api_pb2.TrialSpec(objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="param-1",
value="3",
),
api_pb2.ParameterAssignment(
name="param-2",
value="cat2",
),
api_pb2.ParameterAssignment(
name="param-3",
value="6",
),
api_pb2.ParameterAssignment(
name="param-4",
value="4.44",
)
])),
status=api_pb2.TrialStatus(observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(name="metric=1", value="123"),
api_pb2.Metric(name="metric=2", value="3028"),
])))
]
experiment = api_pb2.Experiment(
name="test",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name="tpe",
algorithm_settings=[
api_pb2.AlgorithmSetting(name="random_state",
value="10"),
api_pb2.AlgorithmSetting(name="gamma", value="0.25"),
api_pb2.AlgorithmSetting(name="prior_weight",
value="1.0"),
api_pb2.AlgorithmSetting(name="n_EI_candidates",
value="24"),
],
),
objective=api_pb2.ObjectiveSpec(type=api_pb2.MAXIMIZE,
goal=0.9),
parameter_specs=api_pb2.ExperimentSpec.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]),
),
api_pb2.ParameterSpec(name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None,
min=None,
list=["cat1", "cat2", "cat3"])),
api_pb2.ParameterSpec(
name="param-3",
parameter_type=api_pb2.DISCRETE,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "2", "6"])),
api_pb2.ParameterSpec(name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]))
])))
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=trials,
request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['Suggestion'].
methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request,
timeout=1)
response, metadata, code, details = get_suggestion.termination()
print(response.parameter_assignments)
self.assertEqual(code, grpc.StatusCode.OK)
self.assertEqual(2, len(response.parameter_assignments))
def test_get_suggestion(self):
trials = [
api_pb2.Trial(
name="first-trial",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="Validation-Accuracy",
goal=0.99),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="architecture",
value="[[3], [0, 1], [0, 0, 1], [2, 1, 0, 0]]",
),
api_pb2.ParameterAssignment(
name="nn_config",
value="{'num_layers': 4}",
),
])),
status=api_pb2.TrialStatus(
observation=api_pb2.Observation(metrics=[
api_pb2.Metric(name="Validation-Accuracy",
value="0.88"),
]),
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
)),
api_pb2.Trial(
name="second-trial",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="Validation-Accuracy",
goal=0.99),
parameter_assignments=api_pb2.TrialSpec.
ParameterAssignments(assignments=[
api_pb2.ParameterAssignment(
name="architecture",
value="[[1], [0, 1], [2, 1, 1], [2, 1, 1, 0]]",
),
api_pb2.ParameterAssignment(
name="nn_config",
value="{'num_layers': 4}",
),
], )),
status=api_pb2.TrialStatus(
observation=api_pb2.Observation(metrics=[
api_pb2.Metric(name="Validation-Accuracy",
value="0.84"),
]),
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
))
]
experiment = api_pb2.Experiment(
name="enas-experiment",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(algorithm_name="enas", ),
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
goal=0.9,
objective_metric_name="Validation-Accuracy"),
parallel_trial_count=2,
max_trial_count=10,
nas_config=api_pb2.NasConfig(
graph_config=api_pb2.GraphConfig(num_layers=4,
input_sizes=[32, 32, 8],
output_sizes=[10]),
operations=api_pb2.NasConfig.Operations(operation=[
api_pb2.Operation(
operation_type="convolution",
parameter_specs=api_pb2.Operation.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="filter_size",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["5"])),
api_pb2.ParameterSpec(
name="num_filter",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["128"])),
api_pb2.ParameterSpec(
name="stride",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["1", "2"])),
])),
api_pb2.Operation(
operation_type="reduction",
parameter_specs=api_pb2.Operation.
ParameterSpecs(parameters=[
api_pb2.ParameterSpec(
name="reduction_type",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None,
min=None,
list=["max_pooling"])),
api_pb2.ParameterSpec(
name="pool_size",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
min="2", max="3", step="1", list=[])),
])),
], ))))
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=trials,
request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(
api_pb2.DESCRIPTOR.services_by_name['Suggestion'].
methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request,
timeout=100)
response, metadata, code, details = get_suggestion.termination()
print(response.parameter_assignments)
self.assertEqual(code, grpc.StatusCode.OK)
self.assertEqual(2, len(response.parameter_assignments))
def test_get_suggestion(self, algorithm_name, algorithm_settings):
experiment = api_pb2.Experiment(
name="test",
spec=api_pb2.ExperimentSpec(
algorithm=api_pb2.AlgorithmSpec(
algorithm_name=algorithm_name,
algorithm_settings=[
api_pb2.AlgorithmSetting(
name=name,
value=value
) for name, value in algorithm_settings.items()
],
),
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
goal=0.9
),
parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
parameters=[
api_pb2.ParameterSpec(
name="param-1",
parameter_type=api_pb2.INT,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[]),
),
api_pb2.ParameterSpec(
name="param-2",
parameter_type=api_pb2.CATEGORICAL,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["cat1", "cat2", "cat3"])
),
api_pb2.ParameterSpec(
name="param-3",
parameter_type=api_pb2.DISCRETE,
feasible_space=api_pb2.FeasibleSpace(
max=None, min=None, list=["3", "2", "6"])
),
api_pb2.ParameterSpec(
name="param-4",
parameter_type=api_pb2.DOUBLE,
feasible_space=api_pb2.FeasibleSpace(
max="5", min="1", list=[])
)
]
)
)
)
# Run the first suggestion with no previous trials in the request
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=[],
current_request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(api_pb2.DESCRIPTOR
.services_by_name['Suggestion']
.methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request, timeout=1)
response, metadata, code, details = get_suggestion.termination()
assert code == grpc.StatusCode.OK
assert 2 == len(response.parameter_assignments)
# Run the second suggestion with trials whose parameters are assigned in the first request
trials = [
api_pb2.Trial(
name="test-asfjh",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9
),
parameter_assignments=api_pb2.TrialSpec.ParameterAssignments(
assignments=response.parameter_assignments[0].assignments
)
),
status=api_pb2.TrialStatus(
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(
name="metric-1",
value="435"
),
api_pb2.Metric(
name="metric-2",
value="5643"
),
]
)
)
),
api_pb2.Trial(
name="test-234hs",
spec=api_pb2.TrialSpec(
objective=api_pb2.ObjectiveSpec(
type=api_pb2.MAXIMIZE,
objective_metric_name="metric-2",
goal=0.9
),
parameter_assignments=api_pb2.TrialSpec.ParameterAssignments(
assignments=response.parameter_assignments[1].assignments
)
),
status=api_pb2.TrialStatus(
condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED,
observation=api_pb2.Observation(
metrics=[
api_pb2.Metric(
name="metric-1",
value="123"
),
api_pb2.Metric(
name="metric-2",
value="3028"
),
]
)
)
)
]
request = api_pb2.GetSuggestionsRequest(
experiment=experiment,
trials=trials,
current_request_number=2,
)
get_suggestion = self.test_server.invoke_unary_unary(
method_descriptor=(api_pb2.DESCRIPTOR
.services_by_name['Suggestion']
.methods_by_name['GetSuggestions']),
invocation_metadata={},
request=request, timeout=1)
response, metadata, code, details = get_suggestion.termination()
assert code == grpc.StatusCode.OK
assert 2 == len(response.parameter_assignments)
