def GetEvaluationResult(self, study):
channel = grpc.beta.implementations.insecure_channel(
MANAGER_ADDRESS, MANAGER_PORT)
with api_pb2.beta_create_Manager_stub(channel) as client:
gwfrep = client.GetWorkerFullInfo(
api_pb2.GetWorkerFullInfoRequest(study_id=study.study_id,
only_latest_log=True), 10)
trials_list = gwfrep.worker_full_infos
completed_trials = dict()
for t in trials_list:
if t.Worker.trial_id in study.prev_trial_ids and t.Worker.status == api_pb2.COMPLETED:
for ml in t.metrics_logs:
if ml.name == study.objective_name:
completed_trials[t.Worker.trial_id] = float(
ml.values[-1].value)
n_complete = len(completed_trials)
n_fail = study.num_trials - n_complete
self.logger.info(">>> {} Trials succeeded, {} Trials failed:".format(
n_complete, n_fail))
for tid in study.prev_trial_ids:
if tid in completed_trials:
self.logger.info("{}: {}".format(tid, completed_trials[tid]))
else:
self.logger.info("{}: Failed".format(tid))
if n_complete > 0:
avg_metrics = sum(completed_trials.values()) / n_complete
self.logger.info("The average is {}\n".format(avg_metrics))
return avg_metrics
def _get_study_param(self):
# this function need to
# 1) get the number of layers
# 2) get the I/O size
# 3) get the available operations
# 4) get the optimization direction (i.e. minimize or maximize)
# 5) get the objective name
# 6) get the study name
channel = grpc.beta.implementations.insecure_channel(
MANAGER_ADDRESS, MANAGER_PORT)
with api_pb2.beta_create_Manager_stub(channel) as client:
api_study_param = client.GetStudy(
api_pb2.GetStudyRequest(study_id=self.study_id), 10)
self.study_name = api_study_param.study_config.name
self.opt_direction = api_study_param.study_config.optimization_type
self.objective_name = api_study_param.study_config.objective_value_name
all_params = api_study_param.study_config.nas_config
graph_config = all_params.graph_config
self.num_layers = int(graph_config.num_layers)
self.input_size = list(map(int, graph_config.input_size))
self.output_size = list(map(int, graph_config.output_size))
search_space_raw = all_params.operations
search_space_object = SearchSpace(search_space_raw)
self.search_space = search_space_object.search_space
self.num_operations = search_space_object.num_operations
self.print_search_space()
def getEvalHistory(self, studyID, obj_name, burn_in):
worker_hist = []
x_train = []
y_train = []
channel = grpc.beta.implementations.insecure_channel(
self.manager_addr, self.manager_port)
with api_pb2.beta_create_Manager_stub(channel) as client:
gwfrep = client.GetWorkerFullInfo(
api_pb2.GetWorkerFullInfoRequest(study_id=studyID,
only_latest_log=True), 10)
worker_hist = gwfrep.worker_full_infos
#self.logger.debug("Eval Trials Log: %r", worker_hist, extra={"StudyID": studyID})
for w in worker_hist:
if w.Worker.status == api_pb2.COMPLETED:
for ml in w.metrics_logs:
if ml.name == obj_name:
y_train.append(float(ml.values[-1].value))
x_train.append(w.parameter_set)
break
self.logger.info("%d completed trials are found.",
len(x_train),
extra={"StudyID": studyID})
if len(x_train) <= burn_in:
x_train = []
y_train = []
self.logger.info(
"Trials will be sampled until %d trials for burn-in are completed.",
burn_in,
extra={"StudyID": studyID})
else:
self.logger.debug("Completed trials: %r",
x_train,
extra={"StudyID": studyID})
return x_train, y_train
def getStudyConfig(self, studyID):
channel = grpc.beta.implementations.insecure_channel(
self.manager_addr, self.manager_port)
with api_pb2.beta_create_Manager_stub(channel) as client:
gsrep = client.GetStudy(api_pb2.GetStudyRequest(study_id=studyID),
10)
return gsrep.study_config
def registerTrials(self, trials):
channel = grpc.beta.implementations.insecure_channel(
self.manager_addr, self.manager_port)
with api_pb2.beta_create_Manager_stub(channel) as client:
for i, t in enumerate(trials):
ctrep = client.CreateTrial(api_pb2.CreateTrialRequest(trial=t),
10)
trials[i].trial_id = ctrep.trial_id
return trials
def GetEvaluationResult(self, studyID, trialID):
worker_list = []
channel = grpc.beta.implementations.insecure_channel(self.manager_addr, self.manager_port)
with api_pb2.beta_create_Manager_stub(channel) as client:
gwfrep = client.GetWorkerFullInfo(api_pb2.GetWorkerFullInfoRequest(study_id=studyID, trial_id=trialID, only_latest_log=False), 10)
worker_list = gwfrep.worker_full_infos
for w in worker_list:
if w.Worker.status == api_pb2.COMPLETED:
for ml in w.metrics_logs:
if ml.name == self.objective_name:
samples=self.get_featuremap_statistics(ml)
return samples
def _get_suggestion_param(self, paramID):
channel = grpc.beta.implementations.insecure_channel(self.manager_addr, self.manager_port)
with api_pb2.beta_create_Manager_stub(channel) as client:
gsprep = client.GetSuggestionParameters(api_pb2.GetSuggestionParametersRequest(param_id=paramID), 10)
params_raw = gsprep.suggestion_parameters
suggestion_params = parseSuggestionParam(params_raw)
self.suggestion_config = suggestion_params
self.suggestion_config.update({"input_size":self.input_size[0]})
self.suggestion_config.update({"output_size":self.output_size[0]})
self.search_space.update({"max_layers_per_stage":self.suggestion_config["max_layers_per_stage"]})
self.logger.info("Suggestion Config: {}".format(self.suggestion_config))
def _get_suggestion_param(self):
channel = grpc.beta.implementations.insecure_channel(
MANAGER_ADDRESS, MANAGER_PORT)
with api_pb2.beta_create_Manager_stub(channel) as client:
api_suggestion_param = client.GetSuggestionParameters(
api_pb2.GetSuggestionParametersRequest(param_id=self.param_id),
10)
params_raw = api_suggestion_param.suggestion_parameters
self.suggestion_config = parseSuggestionParam(params_raw)
self.print_suggestion_params()
def GetSuggestions(self, request, context):
if request.study_id != self.current_study_id:
self.generate_arch(request)
if self.current_itr==0:
self.arch=self.generator.get_init_arch()
elif self.current_itr<=self.restruct_itr:
result = self.GetEvaluationResult(request.study_id, self.prev_trial_id)
self.arch=self.generator.get_arch(self.arch, result)
self.logger.info("Architecture at itr={}".format(self.current_itr))
self.logger.info(self.arch)
arch_json=json.dumps(self.arch)
config_json=json.dumps(self.suggestion_config)
arch=str(arch_json).replace('\"', '\'')
config=str(config_json).replace('\"', '\'')
trials = []
trials.append(api_pb2.Trial(
study_id=request.study_id,
parameter_set=[
api_pb2.Parameter(
name="architecture",
value=arch,
parameter_type= api_pb2.CATEGORICAL),
api_pb2.Parameter(
name="parameters",
value=config,
parameter_type= api_pb2.CATEGORICAL),
api_pb2.Parameter(
name="current_itr",
value=str(self.current_itr),
parameter_type= api_pb2.CATEGORICAL)
],
)
)
channel = grpc.beta.implementations.insecure_channel(self.manager_addr, self.manager_port)
with api_pb2.beta_create_Manager_stub(channel) as client:
for i, t in enumerate(trials):
ctrep = client.CreateTrial(api_pb2.CreateTrialRequest(trial=t), 10)
trials[i].trial_id = ctrep.trial_id
self.prev_trial_id = ctrep.trial_id
self.current_itr+=1
return api_pb2.GetSuggestionsReply(trials=trials)
def _get_search_space(self, studyID):
channel = grpc.beta.implementations.insecure_channel(self.manager_addr, self.manager_port)
with api_pb2.beta_create_Manager_stub(channel) as client:
gsrep = client.GetStudy(api_pb2.GetStudyRequest(study_id=studyID), 10)
self.objective_name = gsrep.study_config.objective_value_name
all_params = gsrep.study_config.nas_config
graph_config = all_params.graph_config
search_space_raw = all_params.operations
self.stages = int(graph_config.num_layers)
self.input_size = list(map(int, graph_config.input_size))
self.output_size = list(map(int, graph_config.output_size))
search_space_object = SearchSpace(search_space_raw)
self.search_space = search_space_object.search_space
self.search_space.update({"stages":self.stages})
self.logger.info("Search Space: {}".format(self.search_space))
def SpawnTrials(self, study, trials):
study.prev_trials = trials
study.prev_trial_ids = list()
self.logger.info("")
channel = grpc.beta.implementations.insecure_channel(
MANAGER_ADDRESS, MANAGER_PORT)
with api_pb2.beta_create_Manager_stub(channel) as client:
for i, t in enumerate(trials):
ctrep = client.CreateTrial(api_pb2.CreateTrialRequest(trial=t),
10)
trials[i].trial_id = ctrep.trial_id
study.prev_trial_ids.append(ctrep.trial_id)
self.logger.info(">>> {} Trials were created:".format(
study.num_trials))
for t in study.prev_trial_ids:
self.logger.info(t)
self.logger.info("")
study.ctrl_step += 1
return api_pb2.GetSuggestionsReply(trials=trials)
def parseParameters(self, paramID):
channel = grpc.beta.implementations.insecure_channel(
self.manager_addr, self.manager_port)
params = []
with api_pb2.beta_create_Manager_stub(channel) as client:
gsprep = client.GetSuggestionParameters(
api_pb2.GetSuggestionParametersRequest(param_id=paramID), 10)
params = gsprep.suggestion_parameters
parsed_service_params = {
"N": 100,
"model_type": "gp",
"max_features": "auto",
"length_scale": 0.5,
"noise": 0.0005,
"nu": 1.5,
"kernel_type": "matern",
"n_estimators": 50,
"mode": "pi",
"trade_off": 0.01,
"trial_hist": "",
"burn_in": 10,
}
modes = ["pi", "ei"]
model_types = ["gp", "rf"]
kernel_types = ["matern", "rbf"]
for param in params:
if param.name in parsed_service_params.keys():
if param.name == "length_scale" or param.name == "noise" or param.name == "nu" or param.name == "trade_off":
try:
float(param.value)
except ValueError:
self.logger.warning(
"Parameter must be float for %s: %s back to default value",
param.name, param.value)
else:
parsed_service_params[param.name] = float(param.value)
elif param.name == "N" or param.name == "n_estimators" or param.name == "burn_in":
try:
int(param.value)
except ValueError:
self.logger.warning(
"Parameter must be int for %s: %s back to default value",
param.name, param.value)
else:
parsed_service_params[param.name] = int(param.value)
elif param.name == "kernel_type":
if param.value != "rbf" and param.value != "matern":
parsed_service_params[param.name] = param.value
else:
self.logger.warning(
"Unknown Parameter for %s: %s back to default value",
param.name, param.value)
elif param.name == "mode" and param.value in modes:
if param.value != "lcb" and param.value != "ei" and param.value != "pi":
parsed_service_params[param.name] = param.value
else:
self.logger.warning(
"Unknown Parameter for %s: %s back to default value",
param.name, param.value)
elif param.name == "model_type" and param.value in model_types:
if param.value != "rf" and param.value != "gp":
parsed_service_params[param.name] = param.value
else:
self.logger.warning(
"Unknown Parameter for %s: %s back to default value",
param.name, param.value)
else:
self.logger.warning("Unknown Parameter name: %s ", param.name)
return parsed_service_params