def GetEvaluationResult(self, study): channel = grpc.beta.implementations.insecure_channel( MANAGER_ADDRESS, MANAGER_PORT) with api_pb2.beta_create_Manager_stub(channel) as client: gwfrep = client.GetWorkerFullInfo( api_pb2.GetWorkerFullInfoRequest(study_id=study.study_id, only_latest_log=True), 10) trials_list = gwfrep.worker_full_infos completed_trials = dict() for t in trials_list: if t.Worker.trial_id in study.prev_trial_ids and t.Worker.status == api_pb2.COMPLETED: for ml in t.metrics_logs: if ml.name == study.objective_name: completed_trials[t.Worker.trial_id] = float( ml.values[-1].value) n_complete = len(completed_trials) n_fail = study.num_trials - n_complete self.logger.info(">>> {} Trials succeeded, {} Trials failed:".format( n_complete, n_fail)) for tid in study.prev_trial_ids: if tid in completed_trials: self.logger.info("{}: {}".format(tid, completed_trials[tid])) else: self.logger.info("{}: Failed".format(tid)) if n_complete > 0: avg_metrics = sum(completed_trials.values()) / n_complete self.logger.info("The average is {}\n".format(avg_metrics)) return avg_metrics
def _get_study_param(self): # this function need to # 1) get the number of layers # 2) get the I/O size # 3) get the available operations # 4) get the optimization direction (i.e. minimize or maximize) # 5) get the objective name # 6) get the study name channel = grpc.beta.implementations.insecure_channel( MANAGER_ADDRESS, MANAGER_PORT) with api_pb2.beta_create_Manager_stub(channel) as client: api_study_param = client.GetStudy( api_pb2.GetStudyRequest(study_id=self.study_id), 10) self.study_name = api_study_param.study_config.name self.opt_direction = api_study_param.study_config.optimization_type self.objective_name = api_study_param.study_config.objective_value_name all_params = api_study_param.study_config.nas_config graph_config = all_params.graph_config self.num_layers = int(graph_config.num_layers) self.input_size = list(map(int, graph_config.input_size)) self.output_size = list(map(int, graph_config.output_size)) search_space_raw = all_params.operations search_space_object = SearchSpace(search_space_raw) self.search_space = search_space_object.search_space self.num_operations = search_space_object.num_operations self.print_search_space()
def getEvalHistory(self, studyID, obj_name, burn_in): worker_hist = [] x_train = [] y_train = [] channel = grpc.beta.implementations.insecure_channel( self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: gwfrep = client.GetWorkerFullInfo( api_pb2.GetWorkerFullInfoRequest(study_id=studyID, only_latest_log=True), 10) worker_hist = gwfrep.worker_full_infos #self.logger.debug("Eval Trials Log: %r", worker_hist, extra={"StudyID": studyID}) for w in worker_hist: if w.Worker.status == api_pb2.COMPLETED: for ml in w.metrics_logs: if ml.name == obj_name: y_train.append(float(ml.values[-1].value)) x_train.append(w.parameter_set) break self.logger.info("%d completed trials are found.", len(x_train), extra={"StudyID": studyID}) if len(x_train) <= burn_in: x_train = [] y_train = [] self.logger.info( "Trials will be sampled until %d trials for burn-in are completed.", burn_in, extra={"StudyID": studyID}) else: self.logger.debug("Completed trials: %r", x_train, extra={"StudyID": studyID}) return x_train, y_train
def getStudyConfig(self, studyID): channel = grpc.beta.implementations.insecure_channel( self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: gsrep = client.GetStudy(api_pb2.GetStudyRequest(study_id=studyID), 10) return gsrep.study_config
def registerTrials(self, trials): channel = grpc.beta.implementations.insecure_channel( self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: for i, t in enumerate(trials): ctrep = client.CreateTrial(api_pb2.CreateTrialRequest(trial=t), 10) trials[i].trial_id = ctrep.trial_id return trials
def GetEvaluationResult(self, studyID, trialID): worker_list = [] channel = grpc.beta.implementations.insecure_channel(self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: gwfrep = client.GetWorkerFullInfo(api_pb2.GetWorkerFullInfoRequest(study_id=studyID, trial_id=trialID, only_latest_log=False), 10) worker_list = gwfrep.worker_full_infos for w in worker_list: if w.Worker.status == api_pb2.COMPLETED: for ml in w.metrics_logs: if ml.name == self.objective_name: samples=self.get_featuremap_statistics(ml) return samples
def _get_suggestion_param(self, paramID): channel = grpc.beta.implementations.insecure_channel(self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: gsprep = client.GetSuggestionParameters(api_pb2.GetSuggestionParametersRequest(param_id=paramID), 10) params_raw = gsprep.suggestion_parameters suggestion_params = parseSuggestionParam(params_raw) self.suggestion_config = suggestion_params self.suggestion_config.update({"input_size":self.input_size[0]}) self.suggestion_config.update({"output_size":self.output_size[0]}) self.search_space.update({"max_layers_per_stage":self.suggestion_config["max_layers_per_stage"]}) self.logger.info("Suggestion Config: {}".format(self.suggestion_config))
def _get_suggestion_param(self): channel = grpc.beta.implementations.insecure_channel( MANAGER_ADDRESS, MANAGER_PORT) with api_pb2.beta_create_Manager_stub(channel) as client: api_suggestion_param = client.GetSuggestionParameters( api_pb2.GetSuggestionParametersRequest(param_id=self.param_id), 10) params_raw = api_suggestion_param.suggestion_parameters self.suggestion_config = parseSuggestionParam(params_raw) self.print_suggestion_params()
def GetSuggestions(self, request, context): if request.study_id != self.current_study_id: self.generate_arch(request) if self.current_itr==0: self.arch=self.generator.get_init_arch() elif self.current_itr<=self.restruct_itr: result = self.GetEvaluationResult(request.study_id, self.prev_trial_id) self.arch=self.generator.get_arch(self.arch, result) self.logger.info("Architecture at itr={}".format(self.current_itr)) self.logger.info(self.arch) arch_json=json.dumps(self.arch) config_json=json.dumps(self.suggestion_config) arch=str(arch_json).replace('\"', '\'') config=str(config_json).replace('\"', '\'') trials = [] trials.append(api_pb2.Trial( study_id=request.study_id, parameter_set=[ api_pb2.Parameter( name="architecture", value=arch, parameter_type= api_pb2.CATEGORICAL), api_pb2.Parameter( name="parameters", value=config, parameter_type= api_pb2.CATEGORICAL), api_pb2.Parameter( name="current_itr", value=str(self.current_itr), parameter_type= api_pb2.CATEGORICAL) ], ) ) channel = grpc.beta.implementations.insecure_channel(self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: for i, t in enumerate(trials): ctrep = client.CreateTrial(api_pb2.CreateTrialRequest(trial=t), 10) trials[i].trial_id = ctrep.trial_id self.prev_trial_id = ctrep.trial_id self.current_itr+=1 return api_pb2.GetSuggestionsReply(trials=trials)
def _get_search_space(self, studyID): channel = grpc.beta.implementations.insecure_channel(self.manager_addr, self.manager_port) with api_pb2.beta_create_Manager_stub(channel) as client: gsrep = client.GetStudy(api_pb2.GetStudyRequest(study_id=studyID), 10) self.objective_name = gsrep.study_config.objective_value_name all_params = gsrep.study_config.nas_config graph_config = all_params.graph_config search_space_raw = all_params.operations self.stages = int(graph_config.num_layers) self.input_size = list(map(int, graph_config.input_size)) self.output_size = list(map(int, graph_config.output_size)) search_space_object = SearchSpace(search_space_raw) self.search_space = search_space_object.search_space self.search_space.update({"stages":self.stages}) self.logger.info("Search Space: {}".format(self.search_space))
def SpawnTrials(self, study, trials): study.prev_trials = trials study.prev_trial_ids = list() self.logger.info("") channel = grpc.beta.implementations.insecure_channel( MANAGER_ADDRESS, MANAGER_PORT) with api_pb2.beta_create_Manager_stub(channel) as client: for i, t in enumerate(trials): ctrep = client.CreateTrial(api_pb2.CreateTrialRequest(trial=t), 10) trials[i].trial_id = ctrep.trial_id study.prev_trial_ids.append(ctrep.trial_id) self.logger.info(">>> {} Trials were created:".format( study.num_trials)) for t in study.prev_trial_ids: self.logger.info(t) self.logger.info("") study.ctrl_step += 1 return api_pb2.GetSuggestionsReply(trials=trials)
def parseParameters(self, paramID): channel = grpc.beta.implementations.insecure_channel( self.manager_addr, self.manager_port) params = [] with api_pb2.beta_create_Manager_stub(channel) as client: gsprep = client.GetSuggestionParameters( api_pb2.GetSuggestionParametersRequest(param_id=paramID), 10) params = gsprep.suggestion_parameters parsed_service_params = { "N": 100, "model_type": "gp", "max_features": "auto", "length_scale": 0.5, "noise": 0.0005, "nu": 1.5, "kernel_type": "matern", "n_estimators": 50, "mode": "pi", "trade_off": 0.01, "trial_hist": "", "burn_in": 10, } modes = ["pi", "ei"] model_types = ["gp", "rf"] kernel_types = ["matern", "rbf"] for param in params: if param.name in parsed_service_params.keys(): if param.name == "length_scale" or param.name == "noise" or param.name == "nu" or param.name == "trade_off": try: float(param.value) except ValueError: self.logger.warning( "Parameter must be float for %s: %s back to default value", param.name, param.value) else: parsed_service_params[param.name] = float(param.value) elif param.name == "N" or param.name == "n_estimators" or param.name == "burn_in": try: int(param.value) except ValueError: self.logger.warning( "Parameter must be int for %s: %s back to default value", param.name, param.value) else: parsed_service_params[param.name] = int(param.value) elif param.name == "kernel_type": if param.value != "rbf" and param.value != "matern": parsed_service_params[param.name] = param.value else: self.logger.warning( "Unknown Parameter for %s: %s back to default value", param.name, param.value) elif param.name == "mode" and param.value in modes: if param.value != "lcb" and param.value != "ei" and param.value != "pi": parsed_service_params[param.name] = param.value else: self.logger.warning( "Unknown Parameter for %s: %s back to default value", param.name, param.value) elif param.name == "model_type" and param.value in model_types: if param.value != "rf" and param.value != "gp": parsed_service_params[param.name] = param.value else: self.logger.warning( "Unknown Parameter for %s: %s back to default value", param.name, param.value) else: self.logger.warning("Unknown Parameter name: %s ", param.name) return parsed_service_params