def _get_experiment(self, name):
channel = grpc.beta.implementations.insecure_channel(
self.manager_addr, self.manager_port)
with api_pb2.beta_create_Manager_stub(channel) as client:
exp = client.GetExperiment(
api_pb2.GetExperimentRequest(experiment_name=name), 10)
return exp.experiment
def GetEvaluationResult(self, experiment):
channel = grpc.beta.implementations.insecure_channel(MANAGER_ADDRESS, MANAGER_PORT)
with api_pb2.beta_create_Manager_stub(channel) as client:
trials_resp = client.GetTrialList(api_pb2.GetTrialListRequest(experiment_name=experiment.experiment_name), 10)
trials_list = trials_resp.trials
completed_trials = dict()
failed_trials = []
for t in trials_list:
if t.status.condition == api_pb2.TrialStatus.TrialConditionType.SUCCEEDED:
obslog_resp = client.GetObservationLog(
api_pb2.GetObservationLogRequest(
trial_name=t.name,
metric_name=t.spec.objective.objective_metric_name
), 10
)
# Take only the latest metric value
completed_trials[t.name] = float(obslog_resp.observation_log.metric_logs[-1].metric.value)
if t.status.condition == api_pb2.TrialStatus.TrialConditionType.FAILED:
failed_trials.append(t.name)
n_completed = len(completed_trials)
self.logger.info(">>> By now: {} Trials succeeded, {} Trials failed".format(n_completed, len(failed_trials)))
for tname in completed_trials:
self.logger.info("Trial: {}, Value: {}".format(tname, completed_trials[tname]))
for tname in failed_trials:
self.logger.info("Trial: {} was failed".format(tname))
if n_completed > 0:
avg_metrics = sum(completed_trials.values()) / n_completed
self.logger.info("The average is {}\n".format(avg_metrics))
return avg_metrics
def _get_trials(self, experiment_name):
channel = grpc.beta.implementations.insecure_channel(
self.manager_addr, self.manager_port)
with api_pb2.beta_create_Manager_stub(channel) as client:
trials = client.GetTrialList(
api_pb2.GetTrialListRequest(experiment_name=experiment_name),
10)
return trials.trials
def _update_algorithm_extrasettings(self, experiment_name, sParams):
as_list = []
for k, v in sParams.items():
as_list.append(api_pb2.AlgorithmSetting(name=k, value=str(v)))
channel = grpc.beta.implementations.insecure_channel(
self.manager_addr, self.manager_port)
with api_pb2.beta_create_Manager_stub(channel) as client:
client.UpdateAlgorithmExtraSettings(
api_pb2.UpdateAlgorithmExtraSettingsRequest(
experiment_name=experiment_name,
extra_algorithm_settings=as_list), 10)
def _get_algorithm_settings(self, experiment_name):
channel = grpc.beta.implementations.insecure_channel(
self.manager_addr, self.manager_port)
with api_pb2.beta_create_Manager_stub(channel) as client:
alg = client.GetAlgorithmExtraSettings(
api_pb2.GetAlgorithmExtraSettingsRequest(
experiment_name=experiment_name), 10)
params = alg.extra_algorithm_settings
alg_settings = {}
for param in params:
alg_settings[param.name] = param.value
return alg_settings
def getEvalHistory(self, experiment_name, obj_name, burn_in):
worker_hist = []
x_train = []
y_train = []
channel = grpc.beta.implementations.insecure_channel(
self.manager_addr, self.manager_port)
with api_pb2.beta_create_Manager_stub(channel) as client:
trialsrep = client.GetTrialList(
api_pb2.GetTrialListRequest(experiment_name=experiment_name),
timeout)
for t in trialsrep.trials:
if t.status.condition == api_pb2.TrialStatus.TrialConditionType.SUCCEEDED:
gwfrep = client.GetObservationLog(
api_pb2.GetObservationLogRequest(trial_name=t.name,
metric_name=obj_name),
timeout)
w = gwfrep.observation_log
for ml in w.metric_logs:
y_train.append(float(ml.metric.value))
x_train.append(
t.spec.parameter_assignments.assignments)
self.logger.info("%d completed trials are found.",
len(x_train),
extra={"Experiment": experiment_name})
if len(x_train) <= burn_in:
x_train = []
y_train = []
self.logger.info(
"Trials will be sampled until %d trials for burn-in are completed.",
burn_in,
extra={"experiment_name": experiment_name})
else:
self.logger.debug("Completed trials: %r",
x_train,
extra={"experiment_name": experiment_name})
return x_train, y_train
def _get_experiment_param(self):
# this function need to
# 1) get the number of layers
# 2) get the I/O size
# 3) get the available operations
# 4) get the optimization direction (i.e. minimize or maximize)
# 5) get the objective name
# 6) get the algorithm settings
channel = grpc.beta.implementations.insecure_channel(MANAGER_ADDRESS, MANAGER_PORT)
with api_pb2.beta_create_Manager_stub(channel) as client:
api_experiment_param = client.GetExperiment(api_pb2.GetExperimentRequest(experiment_name=self.experiment_name), 10)
# Get Search Space
self.experiment_name = api_experiment_param.experiment.name
self.opt_direction = api_experiment_param.experiment.spec.objective.type
self.objective_name = api_experiment_param.experiment.spec.objective.objective_metric_name
nas_config = api_experiment_param.experiment.spec.nas_config
graph_config = nas_config.graph_config
self.num_layers = int(graph_config.num_layers)
self.input_sizes = list(map(int, graph_config.input_sizes))
self.output_sizes = list(map(int, graph_config.output_sizes))
search_space_raw = nas_config.operations
search_space_object = SearchSpace(search_space_raw)
self.search_space = search_space_object.search_space
self.num_operations = search_space_object.num_operations
self.print_search_space()
# Get Experiment Parameters
params_raw = api_experiment_param.experiment.spec.algorithm.algorithm_setting
self.algorithm_settings = parseAlgorithmSettings(params_raw)
self.print_algorithm_settings()
def parseParameters(self, experiment_name):
channel = grpc.beta.implementations.insecure_channel(
self.manager_addr, self.manager_port)
params = []
with api_pb2.beta_create_Manager_stub(channel) as client:
gsprep = client.GetAlgorithmExtraSettings(
api_pb2.GetAlgorithmExtraSettingsRequest(
experiment_name=experiment_name), timeout)
params = gsprep.extra_algorithm_settings
parsed_service_params = {
"N": 100,
"model_type": "gp",
"max_features": "auto",
"length_scale": 0.5,
"noise": 0.0005,
"nu": 1.5,
"kernel_type": "matern",
"n_estimators": 50,
"mode": "pi",
"trade_off": 0.01,
"trial_hist": "",
"burn_in": 10,
}
modes = ["pi", "ei"]
model_types = ["gp", "rf"]
kernel_types = ["matern", "rbf"]
for param in params:
if param.name in parsed_service_params.keys():
if param.name == "length_scale" or param.name == "noise" or param.name == "nu" or param.name == "trade_off":
try:
float(param.value)
except ValueError:
self.logger.warning(
"Parameter must be float for %s: %s back to default value",
param.name, param.value)
else:
parsed_service_params[param.name] = float(param.value)
elif param.name == "N" or param.name == "n_estimators" or param.name == "burn_in":
try:
int(param.value)
except ValueError:
self.logger.warning(
"Parameter must be int for %s: %s back to default value",
param.name, param.value)
else:
parsed_service_params[param.name] = int(param.value)
elif param.name == "kernel_type":
if param.value != "rbf" and param.value != "matern":
parsed_service_params[param.name] = param.value
else:
self.logger.warning(
"Unknown Parameter for %s: %s back to default value",
param.name, param.value)
elif param.name == "mode" and param.value in modes:
if param.value != "lcb" and param.value != "ei" and param.value != "pi":
parsed_service_params[param.name] = param.value
else:
self.logger.warning(
"Unknown Parameter for %s: %s back to default value",
param.name, param.value)
elif param.name == "model_type" and param.value in model_types:
if param.value != "rf" and param.value != "gp":
parsed_service_params[param.name] = param.value
else:
self.logger.warning(
"Unknown Parameter for %s: %s back to default value",
param.name, param.value)
else:
self.logger.warning("Unknown Parameter name: %s ", param.name)
return parsed_service_params