class TaskOptimizer: def __init__(self, model_name, data_name, max_evals, cv_runs, logger, eval_by_rel): self.model_name = model_name self.data_name = data_name self.max_evals = max_evals self.cv_runs = cv_runs self.logger = logger self.eval_by_rel = eval_by_rel self.model_param_space = ModelParamSpace(self.model_name) def _obj(self, param_dict): param_dict = self.model_param_space._convert_into_param(param_dict) self.task = Task( self.model_name, self.data_name, self.cv_runs, param_dict, self.logger, self.eval_by_rel) self.task.cv() tf.reset_default_graph() ret = { "loss": -self.task.mrr, "attachments": { "raw_mrr": self.task.raw_mrr, "raw_hits_at1": self.task.raw_hits_at1, "raw_hits_at3": self.task.raw_hits_at3, "raw_hits_at10": self.task.raw_hits_at10, "hits_at1": self.task.hits_at1, "hits_at3": self.task.hits_at3, "hits_at10": self.task.hits_at10, }, "status": STATUS_OK } return ret def run(self): trials = Trials() best = fmin( self._obj, self.model_param_space._build_space(), tpe.suggest, self.max_evals, trials) best_params = space_eval(self.model_param_space._build_space(), best) best_params = self.model_param_space._convert_into_param(best_params) trial_loss = np.asarray(trials.losses(), dtype=float) best_ind = np.argmin(trial_loss) mrr = -trial_loss[best_ind] raw_mrr = trials.trial_attachments(trials.trials[best_ind])["raw_mrr"] raw_hits_at1 = trials.trial_attachments(trials.trials[best_ind])["raw_hits_at1"] raw_hits_at3 = trials.trial_attachments(trials.trials[best_ind])["raw_hits_at3"] raw_hits_at10 = trials.trial_attachments(trials.trials[best_ind])["raw_hits_at10"] hits_at1 = trials.trial_attachments(trials.trials[best_ind])["hits_at1"] hits_at3 = trials.trial_attachments(trials.trials[best_ind])["hits_at3"] hits_at10 = trials.trial_attachments(trials.trials[best_ind])["hits_at10"] self.logger.info("-" * 50) self.logger.info("Best CV Results:") self.logger.info("Raw MRR: %.6f" % raw_mrr) self.logger.info("Filtered MRR: %.6f" % mrr) self.logger.info("Raw: Hits@1 %.3f Hits@3 %.3f Hits@10 %.3f" % ( raw_hits_at1, raw_hits_at3, raw_hits_at10)) self.logger.info("Filtered: Hits@1 %.3f Hits@3 %.3f Hits@10 %.3f" % ( hits_at1, hits_at3, hits_at10)) self.logger.info("Best Param:") self.task._print_param_dict(best_params) self.logger.info("-" * 50)
class TaskOptimizer: def __init__(self, model_name, data_name, cv_runs, max_evals, logger, cvonfull, savename='', portion=100): self.model_name = model_name self.data_name = data_name self.cv_runs = cv_runs self.max_evals = max_evals self.logger = logger self.cvonfull = cvonfull self.save_name = savename self.model_param_space = ModelParamSpace( self.model_name) # get the param dict via dict name self.portion = portion def _obj(self, param_dict): param_dict = self.model_param_space._convert_into_param(param_dict) self.task = Task(model_name=self.model_name, data_name=self.data_name, cv_runs=self.cv_runs, params_dict=param_dict, logger=self.logger, save_name=self.save_name, portion=self.portion) self.task.cv(self.cvonfull) tf.reset_default_graph() ret = { "loss": -self.task.eacc, # "attachments": { # "pacc": self.task.pacc, # # "eacc": self.task.eacc, # }, "status": STATUS_OK } return ret def run(self): trials = Trials() best = fmin(self._obj, self.model_param_space._build_space(), tpe.suggest, self.max_evals, trials) best_params = space_eval(self.model_param_space._build_space(), best) best_params = self.model_param_space._convert_into_param(best_params) trial_loss = np.asarray(trials.losses(), dtype=float) best_ind = np.argmin(trial_loss) best_loss = -trial_loss[best_ind] # best_pacc = trials.trial_attachments(trials.trials[best_ind])["pacc"] # best_eacc = trials.trial_attachments(trials.trials[best_ind])["eacc"] self.logger.info("-" * 50) self.logger.info("Best Exact Accuracy %.3f " % (best_loss, )) self.logger.info("Best Param:") self.task._print_param_dict(best_params) self.logger.info("-" * 50)
class TaskOptimizer: def __init__(self, model_name, data_name, cv_runs, max_evals, logger): self.model_name = model_name self.data_name = data_name self.cv_runs = cv_runs self.max_evals = max_evals self.logger = logger self.model_param_space = ModelParamSpace(self.model_name) def _obj(self, param_dict): param_dict = self.model_param_space._convert_into_param(param_dict) self.task = Task(self.model_name, self.data_name, self.cv_runs, param_dict, self.logger) self.task.cv() #self.task.save() tf.reset_default_graph() ret = { "loss": -self.task.eacc, "attachments": { "pacc": self.task.pacc, # "eacc": self.task.eacc, }, "status": STATUS_OK } return ret def run(self): trials = Trials() best = fmin(self._obj, space=self.model_param_space._build_space(), algo=tpe.suggest, max_evals=self.max_evals, trials=trials) best_params = space_eval(self.model_param_space._build_space(), best) best_params = self.model_param_space._convert_into_param(best_params) trial_loss = np.asarray(trials.losses(), dtype=float) best_ind = np.argmin(trial_loss) best_loss = -trial_loss[best_ind] best_pacc = trials.trial_attachments(trials.trials[best_ind])["pacc"] # best_eacc = trials.trial_attachments(trials.trials[best_ind])["eacc"] self.logger.info("-" * 50) self.logger.info( "Best Exact Accuracy %.3f with Parital Accuracy %.3f" % (best_loss, best_pacc)) self.logger.info("Best Param:") self.task._print_param_dict(best_params) self.logger.info("-" * 50)
class TaskOptimizer: def __init__(self, model_name, max_evals, runs, logger): self.model_name = model_name self.max_evals = max_evals self.runs = runs self.logger = logger self.model_param_space = ModelParamSpace(self.model_name) def _obj(self, param_dict): param_dict = self.model_param_space._convert_into_param(param_dict) self.task = Task(self.model_name, self.runs, param_dict, self.logger) self.task.cv() tf.reset_default_graph() ret = { "loss": -self.task.ap, "attachments": { "loss": self.task.loss, "acc": self.task.acc, }, "status": STATUS_OK } return ret def run(self): trials = Trials() best = fmin(self._obj, self.model_param_space._build_space(), tpe.suggest, self.max_evals, trials) best_params = space_eval(self.model_param_space._build_space(), best) best_params = self.model_param_space._convert_into_param(best_params) trial_loss = np.asarray(trials.losses(), dtype=float) best_ind = np.argmin(trial_loss) best_ap = trial_loss[best_ind] best_loss = trials.trial_attachments(trials.trials[best_ind])["loss"] best_acc = trials.trial_attachments(trials.trials[best_ind])["acc"] self.logger.info("-" * 50) self.logger.info("Best Average Precision: %.3f" % best_ap) self.logger.info("with Loss %.3f, Accuracy %.3f" % (best_loss, best_acc)) self.logger.info("Best Param:") self.task._print_param_dict(best_params) self.logger.info("-" * 50)
class TaskOptimizer: def __init__(self, model_name, data_name, max_evals, cv_runs, logger): self.model_name = model_name self.data_name = data_name self.max_evals = max_evals self.cv_runs = cv_runs self.logger = logger self.model_param_space = ModelParamSpace(self.model_name) def _obj(self, param_dict): param_dict = self.model_param_space._convert_into_param(param_dict) self.task = Task(self.model_name, self.data_name, self.cv_runs, param_dict, self.logger) self.task.cv() tf.reset_default_graph() ret = { "loss": -self.task.acc, "attachments": { "loss": self.task.loss, }, "status": STATUS_OK } return ret def run(self): trials = Trials() best = fmin(self._obj, self.model_param_space._build_space(), tpe.suggest, self.max_evals, trials) best_params = space_eval(self.model_param_space._build_space(), best) best_params = self.model_param_space._convert_into_param(best_params) trial_loss = np.asarray(trials.losses(), dtype=float) best_ind = np.argmin(trial_loss) acc = -trial_loss[best_ind] loss = trials.trial_attachments(trials.trials[best_ind])["loss"] self.logger.info("-" * 50) self.logger.info("Best CV Results:") self.logger.info("Loss: %.3f" % loss) self.logger.info("Accuracy: %.3f" % acc) self.logger.info("Best Param:") self.task._print_param_dict(best_params) self.logger.info("-" * 50)