class TaskOptimizer:
def __init__(self, model_name, data_name, max_evals, cv_runs, logger, eval_by_rel):
self.model_name = model_name
self.data_name = data_name
self.max_evals = max_evals
self.cv_runs = cv_runs
self.logger = logger
self.eval_by_rel = eval_by_rel
self.model_param_space = ModelParamSpace(self.model_name)
def _obj(self, param_dict):
param_dict = self.model_param_space._convert_into_param(param_dict)
self.task = Task(
self.model_name, self.data_name, self.cv_runs,
param_dict, self.logger, self.eval_by_rel)
self.task.cv()
tf.reset_default_graph()
ret = {
"loss": -self.task.mrr,
"attachments": {
"raw_mrr": self.task.raw_mrr,
"raw_hits_at1": self.task.raw_hits_at1,
"raw_hits_at3": self.task.raw_hits_at3,
"raw_hits_at10": self.task.raw_hits_at10,
"hits_at1": self.task.hits_at1,
"hits_at3": self.task.hits_at3,
"hits_at10": self.task.hits_at10,
},
"status": STATUS_OK
}
return ret
def run(self):
trials = Trials()
best = fmin(
self._obj, self.model_param_space._build_space(),
tpe.suggest, self.max_evals, trials)
best_params = space_eval(self.model_param_space._build_space(), best)
best_params = self.model_param_space._convert_into_param(best_params)
trial_loss = np.asarray(trials.losses(), dtype=float)
best_ind = np.argmin(trial_loss)
mrr = -trial_loss[best_ind]
raw_mrr = trials.trial_attachments(trials.trials[best_ind])["raw_mrr"]
raw_hits_at1 = trials.trial_attachments(trials.trials[best_ind])["raw_hits_at1"]
raw_hits_at3 = trials.trial_attachments(trials.trials[best_ind])["raw_hits_at3"]
raw_hits_at10 = trials.trial_attachments(trials.trials[best_ind])["raw_hits_at10"]
hits_at1 = trials.trial_attachments(trials.trials[best_ind])["hits_at1"]
hits_at3 = trials.trial_attachments(trials.trials[best_ind])["hits_at3"]
hits_at10 = trials.trial_attachments(trials.trials[best_ind])["hits_at10"]
self.logger.info("-" * 50)
self.logger.info("Best CV Results:")
self.logger.info("Raw MRR: %.6f" % raw_mrr)
self.logger.info("Filtered MRR: %.6f" % mrr)
self.logger.info("Raw: Hits@1 %.3f Hits@3 %.3f Hits@10 %.3f" % (
raw_hits_at1, raw_hits_at3, raw_hits_at10))
self.logger.info("Filtered: Hits@1 %.3f Hits@3 %.3f Hits@10 %.3f" % (
hits_at1, hits_at3, hits_at10))
self.logger.info("Best Param:")
self.task._print_param_dict(best_params)
self.logger.info("-" * 50)
class TaskOptimizer:
def __init__(self,
model_name,
data_name,
cv_runs,
max_evals,
logger,
cvonfull,
savename='',
portion=100):
self.model_name = model_name
self.data_name = data_name
self.cv_runs = cv_runs
self.max_evals = max_evals
self.logger = logger
self.cvonfull = cvonfull
self.save_name = savename
self.model_param_space = ModelParamSpace(
self.model_name) # get the param dict via dict name
self.portion = portion
def _obj(self, param_dict):
param_dict = self.model_param_space._convert_into_param(param_dict)
self.task = Task(model_name=self.model_name,
data_name=self.data_name,
cv_runs=self.cv_runs,
params_dict=param_dict,
logger=self.logger,
save_name=self.save_name,
portion=self.portion)
self.task.cv(self.cvonfull)
tf.reset_default_graph()
ret = {
"loss": -self.task.eacc,
# "attachments": {
# "pacc": self.task.pacc,
# # "eacc": self.task.eacc,
# },
"status": STATUS_OK
}
return ret
def run(self):
trials = Trials()
best = fmin(self._obj, self.model_param_space._build_space(),
tpe.suggest, self.max_evals, trials)
best_params = space_eval(self.model_param_space._build_space(), best)
best_params = self.model_param_space._convert_into_param(best_params)
trial_loss = np.asarray(trials.losses(), dtype=float)
best_ind = np.argmin(trial_loss)
best_loss = -trial_loss[best_ind]
# best_pacc = trials.trial_attachments(trials.trials[best_ind])["pacc"]
# best_eacc = trials.trial_attachments(trials.trials[best_ind])["eacc"]
self.logger.info("-" * 50)
self.logger.info("Best Exact Accuracy %.3f " % (best_loss, ))
self.logger.info("Best Param:")
self.task._print_param_dict(best_params)
self.logger.info("-" * 50)
class TaskOptimizer:
def __init__(self, model_name, data_name, cv_runs, max_evals, logger):
self.model_name = model_name
self.data_name = data_name
self.cv_runs = cv_runs
self.max_evals = max_evals
self.logger = logger
self.model_param_space = ModelParamSpace(self.model_name)
def _obj(self, param_dict):
param_dict = self.model_param_space._convert_into_param(param_dict)
self.task = Task(self.model_name, self.data_name, self.cv_runs,
param_dict, self.logger)
self.task.cv()
#self.task.save()
tf.reset_default_graph()
ret = {
"loss": -self.task.eacc,
"attachments": {
"pacc": self.task.pacc,
# "eacc": self.task.eacc,
},
"status": STATUS_OK
}
return ret
def run(self):
trials = Trials()
best = fmin(self._obj,
space=self.model_param_space._build_space(),
algo=tpe.suggest,
max_evals=self.max_evals,
trials=trials)
best_params = space_eval(self.model_param_space._build_space(), best)
best_params = self.model_param_space._convert_into_param(best_params)
trial_loss = np.asarray(trials.losses(), dtype=float)
best_ind = np.argmin(trial_loss)
best_loss = -trial_loss[best_ind]
best_pacc = trials.trial_attachments(trials.trials[best_ind])["pacc"]
# best_eacc = trials.trial_attachments(trials.trials[best_ind])["eacc"]
self.logger.info("-" * 50)
self.logger.info(
"Best Exact Accuracy %.3f with Parital Accuracy %.3f" %
(best_loss, best_pacc))
self.logger.info("Best Param:")
self.task._print_param_dict(best_params)
self.logger.info("-" * 50)
class TaskOptimizer:
def __init__(self, model_name, max_evals, runs, logger):
self.model_name = model_name
self.max_evals = max_evals
self.runs = runs
self.logger = logger
self.model_param_space = ModelParamSpace(self.model_name)
def _obj(self, param_dict):
param_dict = self.model_param_space._convert_into_param(param_dict)
self.task = Task(self.model_name, self.runs, param_dict, self.logger)
self.task.cv()
tf.reset_default_graph()
ret = {
"loss": -self.task.ap,
"attachments": {
"loss": self.task.loss,
"acc": self.task.acc,
},
"status": STATUS_OK
}
return ret
def run(self):
trials = Trials()
best = fmin(self._obj, self.model_param_space._build_space(),
tpe.suggest, self.max_evals, trials)
best_params = space_eval(self.model_param_space._build_space(), best)
best_params = self.model_param_space._convert_into_param(best_params)
trial_loss = np.asarray(trials.losses(), dtype=float)
best_ind = np.argmin(trial_loss)
best_ap = trial_loss[best_ind]
best_loss = trials.trial_attachments(trials.trials[best_ind])["loss"]
best_acc = trials.trial_attachments(trials.trials[best_ind])["acc"]
self.logger.info("-" * 50)
self.logger.info("Best Average Precision: %.3f" % best_ap)
self.logger.info("with Loss %.3f, Accuracy %.3f" %
(best_loss, best_acc))
self.logger.info("Best Param:")
self.task._print_param_dict(best_params)
self.logger.info("-" * 50)
class TaskOptimizer:
def __init__(self, model_name, data_name, max_evals, cv_runs, logger):
self.model_name = model_name
self.data_name = data_name
self.max_evals = max_evals
self.cv_runs = cv_runs
self.logger = logger
self.model_param_space = ModelParamSpace(self.model_name)
def _obj(self, param_dict):
param_dict = self.model_param_space._convert_into_param(param_dict)
self.task = Task(self.model_name, self.data_name, self.cv_runs,
param_dict, self.logger)
self.task.cv()
tf.reset_default_graph()
ret = {
"loss": -self.task.acc,
"attachments": {
"loss": self.task.loss,
},
"status": STATUS_OK
}
return ret
def run(self):
trials = Trials()
best = fmin(self._obj, self.model_param_space._build_space(),
tpe.suggest, self.max_evals, trials)
best_params = space_eval(self.model_param_space._build_space(), best)
best_params = self.model_param_space._convert_into_param(best_params)
trial_loss = np.asarray(trials.losses(), dtype=float)
best_ind = np.argmin(trial_loss)
acc = -trial_loss[best_ind]
loss = trials.trial_attachments(trials.trials[best_ind])["loss"]
self.logger.info("-" * 50)
self.logger.info("Best CV Results:")
self.logger.info("Loss: %.3f" % loss)
self.logger.info("Accuracy: %.3f" % acc)
self.logger.info("Best Param:")
self.task._print_param_dict(best_params)
self.logger.info("-" * 50)
