def generate(param): algorithm_settings = [ api_pb2.AlgorithmSetting(name="eta", value=str(param.eta)), api_pb2.AlgorithmSetting(name="s_max", value=str(param.s_max)), api_pb2.AlgorithmSetting(name="r_l", value=str(param.r_l)), api_pb2.AlgorithmSetting(name="b_l", value=str(param.b_l)), api_pb2.AlgorithmSetting(name="r", value=str(param.r)), api_pb2.AlgorithmSetting(name="n", value=str(param.n)), api_pb2.AlgorithmSetting(name="current_s", value=str(param.current_s)), api_pb2.AlgorithmSetting(name="current_i", value=str(param.current_i)), api_pb2.AlgorithmSetting(name="resource_name", value=param.resource_name), api_pb2.AlgorithmSetting(name="evaluating_trials", value=str(param.evaluating_trials)) ] return api_pb2.AlgorithmSpec(algorithm_setting=algorithm_settings)
def _update_algorithm_extrasettings(self, experiment_name, sParams): as_list = [] for k, v in sParams.items(): as_list.append(api_pb2.AlgorithmSetting(name=k, value=str(v))) channel = grpc.beta.implementations.insecure_channel( self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: client.UpdateAlgorithmExtraSettings( api_pb2.UpdateAlgorithmExtraSettingsRequest( experiment_name=experiment_name, extra_algorithm_settings=as_list), 10)
def test_get_suggestion(self): trials = [ api_pb2.Trial( name="test-asfjh", spec=api_pb2.TrialSpec(objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, objective_metric_name="metric-2", goal=0.9), parameter_assignments=api_pb2.TrialSpec. ParameterAssignments(assignments=[ api_pb2.ParameterAssignment( name="param-1", value="2", ), api_pb2.ParameterAssignment( name="param-2", value="cat1", ), api_pb2.ParameterAssignment( name="param-3", value="2", ), api_pb2.ParameterAssignment( name="param-4", value="3.44", ) ])), status=api_pb2.TrialStatus(observation=api_pb2.Observation( metrics=[ api_pb2.Metric(name="metric=1", value="435"), api_pb2.Metric(name="metric=2", value="5643"), ]))), api_pb2.Trial( name="test-234hs", spec=api_pb2.TrialSpec(objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, objective_metric_name="metric-2", goal=0.9), parameter_assignments=api_pb2.TrialSpec. ParameterAssignments(assignments=[ api_pb2.ParameterAssignment( name="param-1", value="3", ), api_pb2.ParameterAssignment( name="param-2", value="cat2", ), api_pb2.ParameterAssignment( name="param-3", value="6", ), api_pb2.ParameterAssignment( name="param-4", value="4.44", ) ])), status=api_pb2.TrialStatus(observation=api_pb2.Observation( metrics=[ api_pb2.Metric(name="metric=1", value="123"), api_pb2.Metric(name="metric=2", value="3028"), ]))) ] experiment = api_pb2.Experiment( name="test", spec=api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec( algorithm_name="bayesianoptimization", algorithm_setting=[ api_pb2.AlgorithmSetting(name="random_state", value="10") ], ), objective=api_pb2.ObjectiveSpec(type=api_pb2.MAXIMIZE, goal=0.9), parameter_specs=api_pb2.ExperimentSpec. ParameterSpecs(parameters=[ api_pb2.ParameterSpec( name="param-1", parameter_type=api_pb2.INT, feasible_space=api_pb2.FeasibleSpace( max="5", min="1", list=[]), ), api_pb2.ParameterSpec(name="param-2", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["cat1", "cat2", "cat3"])), api_pb2.ParameterSpec( name="param-3", parameter_type=api_pb2.DISCRETE, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["3", "2", "6"])), api_pb2.ParameterSpec(name="param-4", parameter_type=api_pb2.DOUBLE, feasible_space=api_pb2.FeasibleSpace( max="5", min="1", list=[])) ]))) request = api_pb2.GetSuggestionsRequest( experiment=experiment, trials=trials, request_number=2, ) get_suggestion = self.test_server.invoke_unary_unary( method_descriptor=( api_pb2.DESCRIPTOR.services_by_name['Suggestion']. methods_by_name['GetSuggestions']), invocation_metadata={}, request=request, timeout=1) response, metadata, code, details = get_suggestion.termination() print(response.parameter_assignments) self.assertEqual(code, grpc.StatusCode.OK) self.assertEqual(2, len(response.parameter_assignments))
def test_validate_algorithm_settings(self): experiment_spec = [None] def call_validate(): experiment = api_pb2.Experiment(name="test", spec=experiment_spec[0]) request = api_pb2.ValidateAlgorithmSettingsRequest(experiment=experiment) validate_algorithm_settings = self.test_server.invoke_unary_unary( method_descriptor=(api_pb2.DESCRIPTOR .services_by_name['Suggestion'] .methods_by_name['ValidateAlgorithmSettings']), invocation_metadata={}, request=request, timeout=1) return validate_algorithm_settings.termination() # valid cases algorithm_spec = api_pb2.AlgorithmSpec( algorithm_name="tpe", algorithm_setting=[ api_pb2.AlgorithmSetting( name="random_state", value="10" ), api_pb2.AlgorithmSetting( name="gamma", value="0.25" ), api_pb2.AlgorithmSetting( name="prior_weight", value="1.0" ), api_pb2.AlgorithmSetting( name="n_EI_candidates", value="24" ), ], ) experiment_spec[0] = api_pb2.ExperimentSpec(algorithm=algorithm_spec) self.assertEqual(call_validate()[2], grpc.StatusCode.OK) # invalid cases experiment_spec[0] = api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown")) _, _, code, details = call_validate() self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) self.assertEqual(details, 'unknown algorithm name unknown') experiment_spec[0] = api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec( algorithm_name="random", algorithm_setting=[ api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")] )) _, _, code, details = call_validate() self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) self.assertEqual(details, 'unknown setting unknown_conf for algorithm random') experiment_spec[0] = api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec( algorithm_name="tpe", algorithm_setting=[ api_pb2.AlgorithmSetting(name="gamma", value="1.5")] )) _, _, code, details = call_validate() self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) self.assertEqual(details, 'gamma should be in the range of (0, 1)') experiment_spec[0] = api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec( algorithm_name="tpe", algorithm_setting=[ api_pb2.AlgorithmSetting(name="n_EI_candidates", value="0")] )) _, _, code, details = call_validate() self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) self.assertEqual(details, 'n_EI_candidates should be great than zero') experiment_spec[0] = api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec( algorithm_name="tpe", algorithm_setting=[ api_pb2.AlgorithmSetting(name="random_state", value="-1")] )) _, _, code, details = call_validate() self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) self.assertEqual(details, 'random_state should be great or equal than zero') experiment_spec[0] = api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec( algorithm_name="tpe", algorithm_setting=[ api_pb2.AlgorithmSetting(name="prior_weight", value="aaa")] )) _, _, code, details = call_validate() self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT) self.assertTrue(details.startswith('failed to validate prior_weight(aaa)'))
def test_get_suggestion(self): experiment = api_pb2.Experiment( name="darts-experiment", spec=api_pb2.ExperimentSpec( algorithm=api_pb2.AlgorithmSpec( algorithm_name="darts", algorithm_setting=[ api_pb2.AlgorithmSetting( name="num_epoch", value="10" ) ], ), objective=api_pb2.ObjectiveSpec( type=api_pb2.MAXIMIZE, objective_metric_name="Best-Genotype" ), parallel_trial_count=1, max_trial_count=1, nas_config=api_pb2.NasConfig( graph_config=api_pb2.GraphConfig( num_layers=3, ), operations=api_pb2.NasConfig.Operations( operation=[ api_pb2.Operation( operation_type="separable_convolution", parameter_specs=api_pb2.Operation.ParameterSpecs( parameters=[ api_pb2.ParameterSpec( name="filter_size", parameter_type=api_pb2.CATEGORICAL, feasible_space=api_pb2.FeasibleSpace( max=None, min=None, list=["3", "5"]) ), ] ) ), ], ) ) ) ) request = api_pb2.GetSuggestionsRequest( experiment=experiment, request_number=1, ) get_suggestion = self.test_server.invoke_unary_unary( method_descriptor=(api_pb2.DESCRIPTOR .services_by_name['Suggestion'] .methods_by_name['GetSuggestions']), invocation_metadata={}, request=request, timeout=100) response, metadata, code, details = get_suggestion.termination() print(response.parameter_assignments) self.assertEqual(code, grpc.StatusCode.OK) self.assertEqual(1, len(response.parameter_assignments)) exp_algorithm_settings = {} for setting in experiment.spec.algorithm.algorithm_setting: exp_algorithm_settings[setting.name] = setting.value exp_num_layers = experiment.spec.nas_config.graph_config.num_layers exp_search_space = ["separable_convolution_3x3", "separable_convolution_5x5"] for pa in response.parameter_assignments[0].assignments: if (pa.name == "algorithm-settings"): algorithm_settings = pa.value.replace("\'", "\"") algorithm_settings = json.loads(algorithm_settings) self.assertDictContainsSubset(exp_algorithm_settings, algorithm_settings) elif (pa.name == "num-layers"): self.assertEqual(exp_num_layers, int(pa.value)) elif (pa.name == "search-space"): search_space = pa.value.replace("\'", "\"") search_space = json.loads(search_space) self.assertEqual(exp_search_space, search_space)