def test_get_suggestion(self): experiment = api_pb2.Experiment( name="darts-experiment", spec=api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec( algorithm_name="darts", algorithm_settings=[ api_pb2.AlgorithmSetting(name="num_epoch", value="10") ], ), objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, objective_metric_name="Best-Genotype"), parallel_trial_count=1, max_trial_count=1, nas_config=api_pb2.NasConfig( graph_config=api_pb2.GraphConfig(num_layers=3, ), operations=api_pb2.NasConfig.Operations(operation=[ api_pb2.Operation( operation_type="separable_convolution", parameter_specs=api_pb2.Operation. ParameterSpecs(parameters=[ api_pb2.ParameterSpec( name="filter_size", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["3", "5"])), ])), ], )))) request = api_pb2.GetSuggestionsRequest( experiment=experiment, request_number=1, ) get_suggestion = self.test_server.invoke_unary_unary( method_descriptor=( api_pb2.DESCRIPTOR.services_by_name['Suggestion']. methods_by_name['GetSuggestions']), invocation_metadata={}, request=request, timeout=100) response, metadata, code, details = get_suggestion.termination() print(response.parameter_assignments) self.assertEqual(code, grpc.StatusCode.OK) self.assertEqual(1, len(response.parameter_assignments)) exp_algorithm_settings = {} for setting in experiment.spec.algorithm.algorithm_settings: exp_algorithm_settings[setting.name] = setting.value exp_num_layers = experiment.spec.nas_config.graph_config.num_layers exp_search_space = [ "separable_convolution_3x3", "separable_convolution_5x5" ] for pa in response.parameter_assignments[0].assignments: if (pa.name == "algorithm-settings"): algorithm_settings = pa.value.replace("\'", "\"") algorithm_settings = json.loads(algorithm_settings) self.assertDictContainsSubset(exp_algorithm_settings, algorithm_settings) elif (pa.name == "num-layers"): self.assertEqual(exp_num_layers, int(pa.value)) elif (pa.name == "search-space"): search_space = pa.value.replace("\'", "\"") search_space = json.loads(search_space) self.assertEqual(exp_search_space, search_space)
def test_get_suggestion(self): trials = [ api_pb2.Trial( name="first-trial", spec=api_pb2.TrialSpec( objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, objective_metric_name="Validation-Accuracy", goal=0.99), parameter_assignments=api_pb2.TrialSpec. ParameterAssignments(assignments=[ api_pb2.ParameterAssignment( name="architecture", value="[[3], [0, 1], [0, 0, 1], [2, 1, 0, 0]]", ), api_pb2.ParameterAssignment( name="nn_config", value="{'num_layers': 4}", ), ])), status=api_pb2.TrialStatus( observation=api_pb2.Observation(metrics=[ api_pb2.Metric(name="Validation-Accuracy", value="0.88"), ]), condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED, )), api_pb2.Trial( name="second-trial", spec=api_pb2.TrialSpec( objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, objective_metric_name="Validation-Accuracy", goal=0.99), parameter_assignments=api_pb2.TrialSpec. ParameterAssignments(assignments=[ api_pb2.ParameterAssignment( name="architecture", value="[[1], [0, 1], [2, 1, 1], [2, 1, 1, 0]]", ), api_pb2.ParameterAssignment( name="nn_config", value="{'num_layers': 4}", ), ], )), status=api_pb2.TrialStatus( observation=api_pb2.Observation(metrics=[ api_pb2.Metric(name="Validation-Accuracy", value="0.84"), ]), condition=api_pb2.TrialStatus.TrialConditionType.SUCCEEDED, )) ] experiment = api_pb2.Experiment( name="enas-experiment", spec=api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec(algorithm_name="enas", ), objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, goal=0.9, objective_metric_name="Validation-Accuracy"), parallel_trial_count=2, max_trial_count=10, nas_config=api_pb2.NasConfig( graph_config=api_pb2.GraphConfig(num_layers=4, input_sizes=[32, 32, 8], output_sizes=[10]), operations=api_pb2.NasConfig.Operations(operation=[ api_pb2.Operation( operation_type="convolution", parameter_specs=api_pb2.Operation. ParameterSpecs(parameters=[ api_pb2.ParameterSpec( name="filter_size", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["5"])), api_pb2.ParameterSpec( name="num_filter", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["128"])), api_pb2.ParameterSpec( name="stride", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["1", "2"])), ])), api_pb2.Operation( operation_type="reduction", parameter_specs=api_pb2.Operation. ParameterSpecs(parameters=[ api_pb2.ParameterSpec( name="reduction_type", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["max_pooling"])), api_pb2.ParameterSpec( name="pool_size", parameter_type=api_pb2.INT, feasible_space=api_pb2.FeasibleSpace( min="2", max="3", step="1", list=[])), ])), ], )))) request = api_pb2.GetSuggestionsRequest( experiment=experiment, trials=trials, request_number=2, ) get_suggestion = self.test_server.invoke_unary_unary( method_descriptor=( api_pb2.DESCRIPTOR.services_by_name['Suggestion']. methods_by_name['GetSuggestions']), invocation_metadata={}, request=request, timeout=100) response, metadata, code, details = get_suggestion.termination() print(response.parameter_assignments) self.assertEqual(code, grpc.StatusCode.OK) self.assertEqual(2, len(response.parameter_assignments))