def _make_master_bracket(self, experiment, param): n = param.n r = int(param.r) parameter_config = parsing_util.parse_parameter_configs( experiment.spec.parameter_specs.parameters) trial_specs = [] for _ in range(n): sample = parameter_config.random_sample() suggestion = parsing_util.parse_x_next_vector( sample, parameter_config.parameter_types, parameter_config.names, parameter_config.discrete_info, parameter_config.categorical_info) trial_spec = api_pb2.TrialSpec() trial_spec.experiment_name = experiment.name for hp in suggestion: if hp['name'] == param.resource_name: hp['value'] = str(r) trial_spec.parameter_assignments.assignments.add(name=hp['name'], value=str(hp['value'])) trial_specs.append(trial_spec) logger.info("Generate %d trials by master bracket.", n, extra={"experiment_name": experiment.name}) return trial_specs
def _copy_trials(self, trials, r_i, resourceName): trialSpecs = [] for t in trials: trial_spec = api_pb2.TrialSpec() for assignment in t.spec.parameter_assignments.assignments: if assignment.name == resourceName: value = str(r_i) else: value = assignment.value trial_spec.parameter_assignments.assignments.add( name=assignment.name, value=value) trialSpecs.append(trial_spec) return trialSpecs
def test_get_suggestion(self): trials = [ api_pb2.Trial( name="test-asfjh", spec=api_pb2.TrialSpec(objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, objective_metric_name="metric-2", goal=0.9), parameter_assignments=api_pb2.TrialSpec. ParameterAssignments(assignments=[ api_pb2.ParameterAssignment( name="param-1", value="2", ), api_pb2.ParameterAssignment( name="param-2", value="cat1", ), api_pb2.ParameterAssignment( name="param-3", value="2", ), api_pb2.ParameterAssignment( name="param-4", value="3.44", ) ])), status=api_pb2.TrialStatus(observation=api_pb2.Observation( metrics=[ api_pb2.Metric(name="metric=1", value="435"), api_pb2.Metric(name="metric=2", value="5643"), ]))), api_pb2.Trial( name="test-234hs", spec=api_pb2.TrialSpec(objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, objective_metric_name="metric-2", goal=0.9), parameter_assignments=api_pb2.TrialSpec. ParameterAssignments(assignments=[ api_pb2.ParameterAssignment( name="param-1", value="3", ), api_pb2.ParameterAssignment( name="param-2", value="cat2", ), api_pb2.ParameterAssignment( name="param-3", value="6", ), api_pb2.ParameterAssignment( name="param-4", value="4.44", ) ])), status=api_pb2.TrialStatus(observation=api_pb2.Observation( metrics=[ api_pb2.Metric(name="metric=1", value="123"), api_pb2.Metric(name="metric=2", value="3028"), ]))) ] experiment = api_pb2.Experiment( name="test", spec=api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec( algorithm_name="tpe", algorithm_settings=[ api_pb2.AlgorithmSetting(name="random_state", value="10"), api_pb2.AlgorithmSetting(name="gamma", value="0.25"), api_pb2.AlgorithmSetting(name="prior_weight", value="1.0"), api_pb2.AlgorithmSetting(name="n_EI_candidates", value="24"), ], ), objective=api_pb2.ObjectiveSpec(type=api_pb2.MAXIMIZE, goal=0.9), parameter_specs=api_pb2.ExperimentSpec. ParameterSpecs(parameters=[ api_pb2.ParameterSpec( name="param-1", parameter_type=api_pb2.INT, feasible_space=api_pb2.FeasibleSpace( max="5", min="1", list=[]), ), api_pb2.ParameterSpec(name="param-2", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["cat1", "cat2", "cat3"])), api_pb2.ParameterSpec( name="param-3", parameter_type=api_pb2.DISCRETE, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["3", "2", "6"])), api_pb2.ParameterSpec(name="param-4", parameter_type=api_pb2.DOUBLE, feasible_space=api_pb2.FeasibleSpace( max="5", min="1", list=[])) ]))) request = api_pb2.GetSuggestionsRequest( experiment=experiment, trials=trials, request_number=2, ) get_suggestion = self.test_server.invoke_unary_unary( method_descriptor=( api_pb2.DESCRIPTOR.services_by_name['Suggestion']. methods_by_name['GetSuggestions']), invocation_metadata={}, request=request, timeout=1) response, metadata, code, details = get_suggestion.termination() print(response.parameter_assignments) self.assertEqual(code, grpc.StatusCode.OK) self.assertEqual(2, len(response.parameter_assignments))
def test_get_suggestion(self): trials = [ api_pb2.Trial( name="first-trial", spec=api_pb2.TrialSpec( objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, objective_metric_name="Validation-Accuracy", goal=0.99), parameter_assignments=api_pb2.TrialSpec. ParameterAssignments(assignments=[ api_pb2.ParameterAssignment( name="architecture", value="[[3], [0, 1], [0, 0, 1], [2, 1, 0, 0]]", ), api_pb2.ParameterAssignment( name="nn_config", value="{'num_layers': 4}", ), ])), status=api_pb2.TrialStatus( observation=api_pb2.Observation(metrics=[ api_pb2.Metric(name="Validation-Accuracy", value="0.88"), ]), condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED, )), api_pb2.Trial( name="second-trial", spec=api_pb2.TrialSpec( objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, objective_metric_name="Validation-Accuracy", goal=0.99), parameter_assignments=api_pb2.TrialSpec. ParameterAssignments(assignments=[ api_pb2.ParameterAssignment( name="architecture", value="[[1], [0, 1], [2, 1, 1], [2, 1, 1, 0]]", ), api_pb2.ParameterAssignment( name="nn_config", value="{'num_layers': 4}", ), ], )), status=api_pb2.TrialStatus( observation=api_pb2.Observation(metrics=[ api_pb2.Metric(name="Validation-Accuracy", value="0.84"), ]), condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED, )) ] experiment = api_pb2.Experiment( name="enas-experiment", spec=api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec(algorithm_name="enas", ), objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, goal=0.9, objective_metric_name="Validation-Accuracy"), parallel_trial_count=2, max_trial_count=10, nas_config=api_pb2.NasConfig( graph_config=api_pb2.GraphConfig(num_layers=4, input_sizes=[32, 32, 8], output_sizes=[10]), operations=api_pb2.NasConfig.Operations(operation=[ api_pb2.Operation( operation_type="convolution", parameter_specs=api_pb2.Operation. ParameterSpecs(parameters=[ api_pb2.ParameterSpec( name="filter_size", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["5"])), api_pb2.ParameterSpec( name="num_filter", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["128"])), api_pb2.ParameterSpec( name="stride", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["1", "2"])), ])), api_pb2.Operation( operation_type="reduction", parameter_specs=api_pb2.Operation. ParameterSpecs(parameters=[ api_pb2.ParameterSpec( name="reduction_type", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["max_pooling"])), api_pb2.ParameterSpec( name="pool_size", parameter_type=api_pb2.INT, feasible_space=api_pb2.FeasibleSpace( min="2", max="3", step="1", list=[])), ])), ], )))) request = api_pb2.GetSuggestionsRequest( experiment=experiment, trials=trials, request_number=2, ) get_suggestion = self.test_server.invoke_unary_unary( method_descriptor=( api_pb2.DESCRIPTOR.services_by_name['Suggestion']. methods_by_name['GetSuggestions']), invocation_metadata={}, request=request, timeout=100) response, metadata, code, details = get_suggestion.termination() print(response.parameter_assignments) self.assertEqual(code, grpc.StatusCode.OK) self.assertEqual(2, len(response.parameter_assignments))
def test_get_suggestion(self, algorithm_name, algorithm_settings): experiment = api_pb2.Experiment( name="test", spec=api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec( algorithm_name=algorithm_name, algorithm_settings=[ api_pb2.AlgorithmSetting( name=name, value=value ) for name, value in algorithm_settings.items() ], ), objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, goal=0.9 ), parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs( parameters=[ api_pb2.ParameterSpec( name="param-1", parameter_type=api_pb2.INT, feasible_space=api_pb2.FeasibleSpace( max="5", min="1", list=[]), ), api_pb2.ParameterSpec( name="param-2", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["cat1", "cat2", "cat3"]) ), api_pb2.ParameterSpec( name="param-3", parameter_type=api_pb2.DISCRETE, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["3", "2", "6"]) ), api_pb2.ParameterSpec( name="param-4", parameter_type=api_pb2.DOUBLE, feasible_space=api_pb2.FeasibleSpace( max="5", min="1", list=[]) ) ] ) ) ) # Run the first suggestion with no previous trials in the request request = api_pb2.GetSuggestionsRequest( experiment=experiment, trials=[], current_request_number=2, ) get_suggestion = self.test_server.invoke_unary_unary( method_descriptor=(api_pb2.DESCRIPTOR .services_by_name['Suggestion'] .methods_by_name['GetSuggestions']), invocation_metadata={}, request=request, timeout=1) response, metadata, code, details = get_suggestion.termination() assert code == grpc.StatusCode.OK assert 2 == len(response.parameter_assignments) # Run the second suggestion with trials whose parameters are assigned in the first request trials = [ api_pb2.Trial( name="test-asfjh", spec=api_pb2.TrialSpec( objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, objective_metric_name="metric-2", goal=0.9 ), parameter_assignments=api_pb2.TrialSpec.ParameterAssignments( assignments=response.parameter_assignments[0].assignments ) ), status=api_pb2.TrialStatus( condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED, observation=api_pb2.Observation( metrics=[ api_pb2.Metric( name="metric-1", value="435" ), api_pb2.Metric( name="metric-2", value="5643" ), ] ) ) ), api_pb2.Trial( name="test-234hs", spec=api_pb2.TrialSpec( objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, objective_metric_name="metric-2", goal=0.9 ), parameter_assignments=api_pb2.TrialSpec.ParameterAssignments( assignments=response.parameter_assignments[1].assignments ) ), status=api_pb2.TrialStatus( condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED, observation=api_pb2.Observation( metrics=[ api_pb2.Metric( name="metric-1", value="123" ), api_pb2.Metric( name="metric-2", value="3028" ), ] ) ) ) ] request = api_pb2.GetSuggestionsRequest( experiment=experiment, trials=trials, current_request_number=2, ) get_suggestion = self.test_server.invoke_unary_unary( method_descriptor=(api_pb2.DESCRIPTOR .services_by_name['Suggestion'] .methods_by_name['GetSuggestions']), invocation_metadata={}, request=request, timeout=1) response, metadata, code, details = get_suggestion.termination() assert code == grpc.StatusCode.OK assert 2 == len(response.parameter_assignments)