def _get_experiment(self, name): channel = grpc.beta.implementations.insecure_channel( self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: exp = client.GetExperiment( api_pb2.GetExperimentRequest(experiment_name=name), 10) return exp.experiment
def GetEvaluationResult(self, experiment): channel = grpc.beta.implementations.insecure_channel(MANAGER_ADDRESS, MANAGER_PORT) with api_pb2.beta_create_Manager_stub(channel) as client: trials_resp = client.GetTrialList(api_pb2.GetTrialListRequest(experiment_name=experiment.experiment_name), 10) trials_list = trials_resp.trials completed_trials = dict() failed_trials = [] for t in trials_list: if t.status.condition == api_pb2.TrialStatus.TrialConditionType.SUCCEEDED: obslog_resp = client.GetObservationLog( api_pb2.GetObservationLogRequest( trial_name=t.name, metric_name=t.spec.objective.objective_metric_name ), 10 ) # Take only the latest metric value completed_trials[t.name] = float(obslog_resp.observation_log.metric_logs[-1].metric.value) if t.status.condition == api_pb2.TrialStatus.TrialConditionType.FAILED: failed_trials.append(t.name) n_completed = len(completed_trials) self.logger.info(">>> By now: {} Trials succeeded, {} Trials failed".format(n_completed, len(failed_trials))) for tname in completed_trials: self.logger.info("Trial: {}, Value: {}".format(tname, completed_trials[tname])) for tname in failed_trials: self.logger.info("Trial: {} was failed".format(tname)) if n_completed > 0: avg_metrics = sum(completed_trials.values()) / n_completed self.logger.info("The average is {}\n".format(avg_metrics)) return avg_metrics
def _get_trials(self, experiment_name): channel = grpc.beta.implementations.insecure_channel( self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: trials = client.GetTrialList( api_pb2.GetTrialListRequest(experiment_name=experiment_name), 10) return trials.trials
def _update_algorithm_extrasettings(self, experiment_name, sParams): as_list = [] for k, v in sParams.items(): as_list.append(api_pb2.AlgorithmSetting(name=k, value=str(v))) channel = grpc.beta.implementations.insecure_channel( self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: client.UpdateAlgorithmExtraSettings( api_pb2.UpdateAlgorithmExtraSettingsRequest( experiment_name=experiment_name, extra_algorithm_settings=as_list), 10)
def _get_algorithm_settings(self, experiment_name): channel = grpc.beta.implementations.insecure_channel( self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: alg = client.GetAlgorithmExtraSettings( api_pb2.GetAlgorithmExtraSettingsRequest( experiment_name=experiment_name), 10) params = alg.extra_algorithm_settings alg_settings = {} for param in params: alg_settings[param.name] = param.value return alg_settings
def getEvalHistory(self, experiment_name, obj_name, burn_in): worker_hist = [] x_train = [] y_train = [] channel = grpc.beta.implementations.insecure_channel( self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: trialsrep = client.GetTrialList( api_pb2.GetTrialListRequest(experiment_name=experiment_name), timeout) for t in trialsrep.trials: if t.status.condition == api_pb2.TrialStatus.TrialConditionType.SUCCEEDED: gwfrep = client.GetObservationLog( api_pb2.GetObservationLogRequest(trial_name=t.name, metric_name=obj_name), timeout) w = gwfrep.observation_log for ml in w.metric_logs: y_train.append(float(ml.metric.value)) x_train.append( t.spec.parameter_assignments.assignments) self.logger.info("%d completed trials are found.", len(x_train), extra={"Experiment": experiment_name}) if len(x_train) <= burn_in: x_train = [] y_train = [] self.logger.info( "Trials will be sampled until %d trials for burn-in are completed.", burn_in, extra={"experiment_name": experiment_name}) else: self.logger.debug("Completed trials: %r", x_train, extra={"experiment_name": experiment_name}) return x_train, y_train
def _get_experiment_param(self): # this function need to # 1) get the number of layers # 2) get the I/O size # 3) get the available operations # 4) get the optimization direction (i.e. minimize or maximize) # 5) get the objective name # 6) get the algorithm settings channel = grpc.beta.implementations.insecure_channel(MANAGER_ADDRESS, MANAGER_PORT) with api_pb2.beta_create_Manager_stub(channel) as client: api_experiment_param = client.GetExperiment(api_pb2.GetExperimentRequest(experiment_name=self.experiment_name), 10) # Get Search Space self.experiment_name = api_experiment_param.experiment.name self.opt_direction = api_experiment_param.experiment.spec.objective.type self.objective_name = api_experiment_param.experiment.spec.objective.objective_metric_name nas_config = api_experiment_param.experiment.spec.nas_config graph_config = nas_config.graph_config self.num_layers = int(graph_config.num_layers) self.input_sizes = list(map(int, graph_config.input_sizes)) self.output_sizes = list(map(int, graph_config.output_sizes)) search_space_raw = nas_config.operations search_space_object = SearchSpace(search_space_raw) self.search_space = search_space_object.search_space self.num_operations = search_space_object.num_operations self.print_search_space() # Get Experiment Parameters params_raw = api_experiment_param.experiment.spec.algorithm.algorithm_setting self.algorithm_settings = parseAlgorithmSettings(params_raw) self.print_algorithm_settings()
def parseParameters(self, experiment_name): channel = grpc.beta.implementations.insecure_channel( self.manager_addr, self.manager_port) params = [] with api_pb2.beta_create_Manager_stub(channel) as client: gsprep = client.GetAlgorithmExtraSettings( api_pb2.GetAlgorithmExtraSettingsRequest( experiment_name=experiment_name), timeout) params = gsprep.extra_algorithm_settings parsed_service_params = { "N": 100, "model_type": "gp", "max_features": "auto", "length_scale": 0.5, "noise": 0.0005, "nu": 1.5, "kernel_type": "matern", "n_estimators": 50, "mode": "pi", "trade_off": 0.01, "trial_hist": "", "burn_in": 10, } modes = ["pi", "ei"] model_types = ["gp", "rf"] kernel_types = ["matern", "rbf"] for param in params: if param.name in parsed_service_params.keys(): if param.name == "length_scale" or param.name == "noise" or param.name == "nu" or param.name == "trade_off": try: float(param.value) except ValueError: self.logger.warning( "Parameter must be float for %s: %s back to default value", param.name, param.value) else: parsed_service_params[param.name] = float(param.value) elif param.name == "N" or param.name == "n_estimators" or param.name == "burn_in": try: int(param.value) except ValueError: self.logger.warning( "Parameter must be int for %s: %s back to default value", param.name, param.value) else: parsed_service_params[param.name] = int(param.value) elif param.name == "kernel_type": if param.value != "rbf" and param.value != "matern": parsed_service_params[param.name] = param.value else: self.logger.warning( "Unknown Parameter for %s: %s back to default value", param.name, param.value) elif param.name == "mode" and param.value in modes: if param.value != "lcb" and param.value != "ei" and param.value != "pi": parsed_service_params[param.name] = param.value else: self.logger.warning( "Unknown Parameter for %s: %s back to default value", param.name, param.value) elif param.name == "model_type" and param.value in model_types: if param.value != "rf" and param.value != "gp": parsed_service_params[param.name] = param.value else: self.logger.warning( "Unknown Parameter for %s: %s back to default value", param.name, param.value) else: self.logger.warning("Unknown Parameter name: %s ", param.name) return parsed_service_params