def objective(trial: Trial) -> float: trial.set_user_attr('method', 'hope') classifier = classifier_type if classifier is None: classifier = trial.suggest_categorical('classifier', ['SVM', 'EN', 'RF', 'LR']) else: trial.set_user_attr('classifier', classifier) dimensions = trial.suggest_int('dimensions', dimensions_range[0], dimensions_range[1]) # Set the inner trial seed _set_trial_seed(trial) model = embed_train.train_embed_hope( train_graph_filename=train_graph_filename, dimensions=dimensions, weighted=weighted, ) return predict_and_evaluate( model=model, graph=graph, graph_train=graph_train, testing_pos_edges=testing_pos_edges, seed=seed, trial=trial, labels=labels, node_list=node_list, classifier_type=classifier, prediction_task=prediction_task, )
def objective_func(trial: optuna.Trial) -> float: start = time.time() params = dict(**self._suggest(trial, param_prefix), **self.fixed_params) self.logger.info("Trial %s:", trial.number) self.logger.info("parameter = %s", params) arg, parameters = self.get_model_arguments(**params) recommender = self.recommender_class(self._data, *arg, **parameters) recommender.learn_with_optimizer(self.val_evaluator, trial) score = self.val_evaluator.get_score(recommender) end = time.time() time_spent = end - start self.logger.info( "Config %d obtained the following scores: %s within %f seconds.", trial.number, score, time_spent, ) val_score = score[self.val_evaluator.target_metric.name] # max_epoch will be stored in learnt_config for param_name, param_val in recommender.learnt_config.items(): trial.set_user_attr(param_name, param_val) add_score_to_trial(trial, score, self.val_evaluator.cutoff) return -val_score
def objective(trial: Trial) -> float: trial.set_user_attr('method', 'sdne') classifier = classifier_type if classifier is None: classifier = trial.suggest_categorical('classifier', ['SVM', 'EN', 'RF', 'LR']) else: trial.set_user_attr('classifier', classifier) alpha = trial.suggest_uniform('alpha', 0, 0.4) beta = trial.suggest_int('beta', 0, 30) epochs = trial.suggest_categorical('epochs', [5, 10, 15, 20, 25, 30]) # Set the inner trial seed _set_trial_seed(trial) model = embed_train.train_embed_sdne( train_graph_filename=train_graph_filename, alpha=alpha, beta=beta, epochs=epochs, weighted=weighted, ) return predict_and_evaluate( prediction_task=prediction_task, model=model, graph=graph, graph_train=graph_train, testing_pos_edges=testing_pos_edges, seed=study_seed, trial=trial, labels=labels, node_list=node_list, classifier_type=classifier, )
def _objective(self, trial: optuna.Trial): kwargs = self.get_parameters(trial) step = 0 result = None for df in self.data: step += 1 result, stats, cerebro, thestrats = self.backtest(df, **kwargs) if result is None: raise optuna.exceptions.TrialPruned("no result") trial.report(result, step) for k in stats.keys(): trial.set_user_attr(f'backtest_{k}', stats[k]) if trial.should_prune(): raise optuna.exceptions.TrialPruned("should_prune") # result = self.backtest(self.data[0], **kwargs) if result is None: raise optuna.exceptions.TrialPruned("no result") return result
def f(trial: Trial) -> float: x = trial.suggest_int("x", 1, 1) y = trial.suggest_categorical("y", (2.5, )) trial.set_user_attr("train_loss", 3) raise ValueError() return x + y # 3.5
def objective(trial: optuna.Trial) -> float: x = trial.suggest_float("x", -10, 10) y = trial.suggest_float("y", -10, 10) trial.report(x, 0) trial.report(y, 1) trial.set_user_attr("x", x) trial.set_system_attr("y", y) return f(x, y)
def f(trial: Trial) -> float: x = trial.suggest_int("x", 1, 1) y = trial.suggest_categorical("y", (2.5, )) assert isinstance(y, float) trial.set_user_attr("train_loss", 3) trial.set_system_attr("foo", "bar") value = x + y # 3.5 # Test reported intermediate values, although it in practice is not "intermediate". trial.report(value, step=0) return value
def evaluate_nav_goal(self, goal: MoveBaseActionGoal, trial: optuna.Trial): self.goal_publisher.publish(goal) start_time = self.sim.get_time() # wait till time for test is up or stopping condition has been reached while rclpy.ok(): passed_time = self.sim.get_time() - start_time # test timeout if passed_time > self.time_limit: goal_quat = goal.goal.target_pose.pose.orientation goal_rpy = transforms3d.euler.quat2euler( (goal_quat.w, goal_quat.x, goal_quat.y, goal_quat.z)) trial.set_user_attr( 'time_limit_reached', (goal.goal.target_pose.pose.position.x, goal.goal.target_pose.pose.position.y, goal_rpy[2])) return False, self.time_limit # test if the robot has fallen down pos, rpy = self.sim.get_robot_pose_rpy() if abs(rpy[0]) > math.radians(45) or abs(rpy[1]) > math.radians( 45) or pos[2] < self.trunk_height / 2: # add extra information to trial # todo include max vel parameter or something similar goal_quat = goal.goal.target_pose.pose.orientation goal_rpy = transforms3d.euler.quat2euler( (goal_quat.w, goal_quat.x, goal_quat.y, goal_quat.z)) trial.set_user_attr( 'fallen', (goal.goal.target_pose.pose.position.x, goal.goal.target_pose.pose.position.y, goal_rpy[2])) return True, self.time_limit # todo add test if the robot did not move for more than x seconds, to find oszillations more easily # test if we are successful if self.is_goal_reached(): return False, passed_time # give time to other algorithms to compute their responses # use wall time, as ros time is standing still time.sleep(0.01) self.sim.step_sim() # was stopped before finishing raise optuna.exceptions.OptunaError()
def objective(trial: Trial) -> float: trial.set_user_attr('method', 'node2vec') classifier = classifier_type if classifier is None: classifier = trial.suggest_categorical('classifier', ['SVM', 'EN', 'RF', 'LR']) else: trial.set_user_attr('classifier', classifier) dimensions = trial.suggest_int('dimensions', dimensions_range[0], dimensions_range[1]) walk_length = trial.suggest_categorical('walk_length', [8, 16, 32, 64, 128]) number_walks = trial.suggest_categorical('number_walks', [8, 16, 32, 64, 128, 256]) window_size = trial.suggest_int('window_size', 2, 6) p = trial.suggest_uniform('p', 0, 4.0) q = trial.suggest_uniform('q', 0, 4.0) # Set the inner trial seed _set_trial_seed(trial) model = embed_train.train_embed_node2vec( train_graph_filename=train_graph_filename, dimensions=dimensions, walk_length=walk_length, number_walks=number_walks, window_size=window_size, p=p, q=q, weighted=weighted, ) return predict_and_evaluate( prediction_task=prediction_task, model=model, graph=graph, graph_train=graph_train, testing_pos_edges=testing_pos_edges, seed=study_seed, trial=trial, labels=labels, node_list=node_list, classifier_type=classifier, )
def objective(trial: Trial): # x = trial.suggest_uniform('x', -10, 10) # return (x - 2) ** 2 clean_token_count_limit = int( trial.suggest_discrete_uniform('clean_token_count_limit', 20, 60000, 1)) trial.set_user_attr( 'run', { 'loss': 1.2, # -- store other results like this 'os_uname': os.uname(), 'clean_token_count_limit': clean_token_count_limit, 'attachments': { 'info': 'info', 'output': 'output' } }) if clean_token_count_limit < 10000: return None return float(clean_token_count_limit)**2
def objective_func(trial: optuna.Trial) -> float: start = time.time() params = dict(**self._suggest(trial), **self.fixed_params) self.logger.info("Trial %s:", trial.number) self.logger.info("parameter = %s", params) arg, parameters = self.get_model_arguments(**params) recommender = self.recommender_class(self._data, *arg, **parameters) recommender.learn_with_optimizer(self.val_evaluator, trial) score = self.val_evaluator.get_score(recommender) end = time.time() time_spent = end - start self.logger.info( "Config %d obtained the following scores: %s within %f seconds.", trial.number, score, time_spent, ) val_score = score[self.val_evaluator.target_metric.name] # max_epoch will be stored in learnt_config for param_name, param_val in recommender.learnt_config.items(): trial.set_user_attr(param_name, param_val) score_history: List[Tuple[int, Dict[ str, float]]] = trial.study.user_attrs.get("scores", []) score_history.append(( trial.number, { f"{score_name}@{self.val_evaluator.cutoff}": score_value for score_name, score_value in score.items() }, )) trial.study.set_user_attr("scores", score_history) return -val_score
def objective(trial: Trial) -> float: trial.set_user_attr('method', 'grarep') classifier = classifier_type if classifier is None: classifier = trial.suggest_categorical('classifier', ['SVM', 'EN', 'RF', 'LR']) else: trial.set_user_attr('classifier', classifier) # TODO: need to choose kstep in which it can divide the dimension dimensions = trial.suggest_int('dimensions', dimensions_range[0], dimensions_range[1]) kstep = trial.suggest_int('kstep', 1, 7) if dimensions % kstep != 0: raise optuna.structs.TrialPruned() # Set the inner trial seed _set_trial_seed(trial) model = embed_train.train_embed_grarep( train_graph_filename=train_graph_filename, dimensions=dimensions, kstep=kstep, weighted=weighted, ) return predict_and_evaluate( prediction_task=prediction_task, model=model, graph=graph, graph_train=graph_train, testing_pos_edges=testing_pos_edges, seed=study_seed, trial=trial, labels=labels, node_list=node_list, classifier_type=classifier, )
def objective(trial: Trial): # config dictionaries are modified during instantiation, so we need to deepcopy the originals to not lose them config = copy.deepcopy(base_config) if with_clustering: assert type(classifier) is not str config["classifier"] = copy.deepcopy(classifier) config["clustering"] = sample_clustering_config_with_optuna(trial) else: assert type(classifier) is str config["classifier"] = sample_classifier_config_with_optuna(trial, classifier) # store the config in the trial so that we can retrieve it later and use it to instantiate the best model - # don't ask me why it needs to be stored as a string, using the dict object did not work trial.set_user_attr("config", json.dumps(config)) # instantiate feature pipeline and classifier, transform the features pipeline, scoring = instantiate_pipeline(logger, config, with_clustering=with_clustering, use_caching=True, scorer_should_return_single_scalar=True, serialization_dir=serialization_dir / "pipeline" / f"trial_{trial.number:03}") cv = RepeatedKFold(n_splits=cv_num_splits, n_repeats=cv_num_repeats, random_state=optimization_random_seed) f1_scores_cv = cross_val_score(estimator=pipeline, X=train_X, y=train_y, n_jobs=cv_n_jobs, cv=cv, scoring=scoring, verbose=0) mean_f1 = f1_scores_cv.mean() return mean_f1
def objective(self, trial: Trial, return_model=False): X, y = self.make_xy(trial) # X, y = self.X, self.y loss_cv_train = [] cv_preds = np.zeros_like(y) cv_preds.fill(np.nan) models = [] for step, (train_idx, val_idx) in enumerate(self.split(X, y)): X_train = X[train_idx] X_val = X[val_idx] y_train = y[train_idx] y_val = y[val_idx] model = self.fit(trial, X_train, y_train, X_val, y_val, step) y_train_preds = model.predict(X_train) y_val_preds = model.predict(X_val) # from IPython.core.debugger import Pdb; # Pdb().set_trace() cv_preds[val_idx] = y_val_preds mask_done = ~np.isnan(cv_preds) intermediate_loss_train = self.loss_fn(y_train, y_train_preds, trial) intermediate_loss_val = self.loss_fn(y[mask_done], cv_preds[mask_done], trial) loss_cv_train.append(intermediate_loss_train) trial.report(intermediate_loss_val, step) if self.prune and trial.should_prune(step): raise optuna.structs.TrialPruned() models.append(model) if self.is_one_cv: break mask_done = ~np.isnan(cv_preds) loss_train = float(np.mean(loss_cv_train)) loss_val = float(self.loss_fn(y[mask_done], cv_preds[mask_done], trial)) trial.set_user_attr('train_loss', loss_train) trial.set_user_attr('val_loss', loss_val) trial.set_user_attr('is_one_cv', int(self.is_one_cv)) self.on_after_trial(trial, models, cv_preds, loss_val) if return_model: return loss_val, models, cv_preds return loss_val
def add_suggest(trial: optuna.Trial, user_attrs={}): """ Add hyperparam ranges to an optuna trial and typical user attrs. Usage: trial = optuna.trial.FixedTrial( params={ 'hidden_size': 128, } ) trial = add_suggest(trial) trainer = pl.Trainer() model = LSTM_PL(dict(**trial.params, **trial.user_attrs), dataset_train, dataset_test, cache_base_path, norm) trainer.fit(model) """ trial.suggest_loguniform("learning_rate", 1e-6, 1e-2) trial.suggest_uniform("attention_dropout", 0, 0.75) # we must have nhead<==hidden_size # so nhead_power.max()<==hidden_size_power.min() trial.suggest_discrete_uniform("hidden_size_power", 4, 10, 1) trial.suggest_discrete_uniform("hidden_out_size_power", 4, 9, 1) trial.suggest_discrete_uniform("nhead_power", 1, 4, 1) trial.suggest_int("nlayers", 1, 12) trial.suggest_categorical("use_lstm", [False, True]) trial.suggest_categorical("agg", ['last', 'max', 'mean', 'all']) user_attrs_default = { "batch_size": 16, "grad_clip": 40, "max_nb_epochs": 200, "num_workers": 4, "num_extra_target": 24 * 4, "vis_i": "670", "num_context": 24 * 4, "input_size": 18, "input_size_decoder": 17, "context_in_target": False, "output_size": 1, "patience": 3, 'min_std': 0.005, } [trial.set_user_attr(k, v) for k, v in user_attrs_default.items()] [trial.set_user_attr(k, v) for k, v in user_attrs.items()] return trial
def sample_a2c_params(trial: optuna.Trial): """Sampler for A2C hyperparameters.""" gamma = 1.0 - trial.suggest_float("gamma", 0.0001, 0.1, log=True) max_grad_norm = trial.suggest_float("max_grad_norm", 0.3, 5.0, log=True) gae_lambda = 1.0 - trial.suggest_float("gae_lambda", 0.001, 0.2, log=True) n_steps = 2**trial.suggest_int("exponent_n_steps", 3, 10) learning_rate = trial.suggest_float("lr", 1e-5, 1, log=True) ent_coef = trial.suggest_float("ent_coef", 0.00000001, 0.1, log=True) ortho_init = trial.suggest_categorical("ortho_init", [False, True]) net_arch = trial.suggest_categorical("net_arch", ["tiny", "small"]) activation_fn = trial.suggest_categorical("activation_fn", ["tanh", "relu"]) # Display true values trial.set_user_attr("gamma_", gamma) trial.set_user_attr("gae_lambda_", gae_lambda) trial.set_user_attr("n_steps", n_steps) net_arch = [{ "pi": [64], "vf": [64] } if net_arch == "tiny" else { "pi": [64, 64], "vf": [64, 64] }] activation_fn = { "tanh": nn.Tanh, "relu": nn.ReLU, }[activation_fn] return { "n_steps": n_steps, "gamma": gamma, "gae_lambda": gae_lambda, "learning_rate": learning_rate, "ent_coef": ent_coef, "max_grad_norm": max_grad_norm, "policy_kwargs": { "net_arch": net_arch, "activation_fn": activation_fn, "ortho_init": ortho_init, }, }
def __call__(self, trial: Trial) -> Optional[float]: """Suggest parameters then train the model.""" if self.model_kwargs is not None: problems = [ x for x in ("loss", "regularizer", "optimizer", "lr_scheduler", "training", "negative_sampler", "stopper") if x in self.model_kwargs ] if problems: raise ValueError( f"model_kwargs should not have: {problems}. {self}") # 2. Model _model_kwargs = _get_kwargs( trial=trial, prefix="model", default_kwargs_ranges=self.model.hpo_default, kwargs=self.model_kwargs, kwargs_ranges=self.model_kwargs_ranges, ) try: loss_default_kwargs_ranges = self.loss.hpo_default except AttributeError: logger.warning( "using a loss function with no hpo_default field: %s", self.loss) loss_default_kwargs_ranges = {} # 3. Loss _loss_kwargs = _get_kwargs( trial=trial, prefix="loss", default_kwargs_ranges=loss_default_kwargs_ranges, kwargs=self.loss_kwargs, kwargs_ranges=self.loss_kwargs_ranges, ) # 4. Regularizer _regularizer_kwargs: Optional[Mapping[str, Any]] if self.regularizer is None: _regularizer_kwargs = {} else: _regularizer_kwargs = _get_kwargs( trial=trial, prefix="regularizer", default_kwargs_ranges=self.regularizer.hpo_default, kwargs=self.regularizer_kwargs, kwargs_ranges=self.regularizer_kwargs_ranges, ) # 5. Optimizer _optimizer_kwargs = _get_kwargs( trial=trial, prefix="optimizer", default_kwargs_ranges=optimizers_hpo_defaults[self.optimizer], kwargs=self.optimizer_kwargs, kwargs_ranges=self.optimizer_kwargs_ranges, ) # 5.1 Learning Rate Scheduler _lr_scheduler_kwargs: Optional[Mapping[str, Any]] = None if self.lr_scheduler is not None: _lr_scheduler_kwargs = _get_kwargs( trial=trial, prefix="lr_scheduler", default_kwargs_ranges=lr_schedulers_hpo_defaults[ self.lr_scheduler], kwargs=self.lr_scheduler_kwargs, kwargs_ranges=self.lr_scheduler_kwargs_ranges, ) _negative_sampler_kwargs: Mapping[str, Any] if self.training_loop is not SLCWATrainingLoop: _negative_sampler_kwargs = {} elif self.negative_sampler is None: raise ValueError( "Negative sampler class must be made explicit when training under sLCWA" ) else: # TODO this fixes the issue for negative samplers, but does not generally address it. # For example, some of them obscure their arguments with **kwargs, so should we look # at the parent class? Sounds like something to put in class resolver by using the # inspect module. For now, this solution will rely on the fact that the sampler is a # direct descendent of a parent NegativeSampler direct_params = inspect.signature(self.negative_sampler).parameters parent_params = inspect.signature( self.negative_sampler.__bases__[0]).parameters valid_keys = set(direct_params).union(parent_params) - {"kwargs"} invalid_keys = set(self.negative_sampler_kwargs_ranges or []) - valid_keys if invalid_keys: raise ExtraKeysError(invalid_keys) _negative_sampler_kwargs = _get_kwargs( trial=trial, prefix="negative_sampler", default_kwargs_ranges=self.negative_sampler.hpo_default, kwargs=self.negative_sampler_kwargs, kwargs_ranges=self.negative_sampler_kwargs_ranges, ) _training_kwargs = _get_kwargs( trial=trial, prefix="training", default_kwargs_ranges=self.training_loop.hpo_default, kwargs=self.training_kwargs, kwargs_ranges=self.training_kwargs_ranges, ) # create result tracker to allow to gracefully close failed trials result_tracker = tracker_resolver.make( query=self.result_tracker, pos_kwargs=self.result_tracker_kwargs) _stopper_kwargs = dict(self.stopper_kwargs or {}) if self.stopper is not None and issubclass(self.stopper, EarlyStopper): self._update_stopper_callbacks(_stopper_kwargs, trial, metric=self.metric, result_tracker=result_tracker) try: result = pipeline( # 1. Dataset dataset=self.dataset, dataset_kwargs=self.dataset_kwargs, training=self.training, testing=self.testing, validation=self.validation, evaluation_entity_whitelist=self.evaluation_entity_whitelist, evaluation_relation_whitelist=self. evaluation_relation_whitelist, # 2. Model model=self.model, model_kwargs=_model_kwargs, # 3. Loss loss=self.loss, loss_kwargs=_loss_kwargs, # 4. Regularizer regularizer=self.regularizer, regularizer_kwargs=_regularizer_kwargs, # 5. Optimizer optimizer=self.optimizer, optimizer_kwargs=_optimizer_kwargs, clear_optimizer=True, # 5.1 Learning Rate Scheduler lr_scheduler=self.lr_scheduler, lr_scheduler_kwargs=_lr_scheduler_kwargs, # 6. Training Loop training_loop=self.training_loop, negative_sampler=self.negative_sampler, negative_sampler_kwargs=_negative_sampler_kwargs, # 7. Training training_loop_kwargs=self.training_loop_kwargs, training_kwargs=_training_kwargs, stopper=self.stopper, stopper_kwargs=_stopper_kwargs, # 8. Evaluation evaluator=self.evaluator, evaluator_kwargs=self.evaluator_kwargs, evaluation_kwargs=self.evaluation_kwargs, filter_validation_when_testing=self. filter_validation_when_testing, # 9. Tracker result_tracker=result_tracker, result_tracker_kwargs=None, # Misc. use_testing_data=False, # use validation set during HPO! device=self.device, ) except (MemoryError, RuntimeError) as e: # close run in result tracker result_tracker.end_run(success=False) # raise the error again (which will be catched in study.optimize) raise e else: if self.save_model_directory: model_directory = os.path.join(self.save_model_directory, str(trial.number)) os.makedirs(model_directory, exist_ok=True) result.save_to_directory(model_directory) trial.set_user_attr("random_seed", result.random_seed) for k, v in result.metric_results.to_flat_dict().items(): trial.set_user_attr(k, v) return result.metric_results.get_metric(self.metric)
def __call__(self, trial: Trial) -> Optional[float]: """Suggest parameters then train the model.""" if self.model_kwargs is not None: problems = [ x for x in ('loss', 'regularizer', 'optimizer', 'training', 'negative_sampler', 'stopper') if x in self.model_kwargs ] if problems: raise ValueError(f'model_kwargs should not have: {problems}. {self}') # 2. Model _model_kwargs = _get_kwargs( trial=trial, prefix='model', default_kwargs_ranges=self.model.hpo_default, kwargs=self.model_kwargs, kwargs_ranges=self.model_kwargs_ranges, ) try: loss_default_kwargs_ranges = self.loss.hpo_default except AttributeError: logger.warning('using a loss function with no hpo_default field: %s', self.loss) loss_default_kwargs_ranges = {} # 3. Loss _loss_kwargs = _get_kwargs( trial=trial, prefix='loss', default_kwargs_ranges=loss_default_kwargs_ranges, kwargs=self.loss_kwargs, kwargs_ranges=self.loss_kwargs_ranges, ) # 4. Regularizer _regularizer_kwargs: Optional[Mapping[str, Any]] if self.regularizer is None: _regularizer_kwargs = {} else: _regularizer_kwargs = _get_kwargs( trial=trial, prefix='regularizer', default_kwargs_ranges=self.regularizer.hpo_default, kwargs=self.regularizer_kwargs, kwargs_ranges=self.regularizer_kwargs_ranges, ) # 5. Optimizer _optimizer_kwargs = _get_kwargs( trial=trial, prefix='optimizer', default_kwargs_ranges=optimizers_hpo_defaults[self.optimizer], kwargs=self.optimizer_kwargs, kwargs_ranges=self.optimizer_kwargs_ranges, ) _negative_sampler_kwargs: Mapping[str, Any] if self.training_loop is not SLCWATrainingLoop: _negative_sampler_kwargs = {} else: _negative_sampler_kwargs = _get_kwargs( trial=trial, prefix='negative_sampler', default_kwargs_ranges={} if self.negative_sampler is None else self.negative_sampler.hpo_default, kwargs=self.negative_sampler_kwargs, kwargs_ranges=self.negative_sampler_kwargs_ranges, ) _training_kwargs = _get_kwargs( trial=trial, prefix='training', default_kwargs_ranges=self.training_loop.hpo_default, kwargs=self.training_kwargs, kwargs_ranges=self.training_kwargs_ranges, ) _stopper_kwargs = dict(self.stopper_kwargs or {}) if self.stopper is not None and issubclass(self.stopper, EarlyStopper): self._update_stopper_callbacks(_stopper_kwargs, trial) try: result = pipeline( # 1. Dataset dataset=self.dataset, dataset_kwargs=self.dataset_kwargs, training=self.training, testing=self.testing, validation=self.validation, evaluation_entity_whitelist=self.evaluation_entity_whitelist, evaluation_relation_whitelist=self.evaluation_relation_whitelist, # 2. Model model=self.model, model_kwargs=_model_kwargs, # 3. Loss loss=self.loss, loss_kwargs=_loss_kwargs, # 4. Regularizer regularizer=self.regularizer, regularizer_kwargs=_regularizer_kwargs, clear_optimizer=True, # 5. Optimizer optimizer=self.optimizer, optimizer_kwargs=_optimizer_kwargs, # 6. Training Loop training_loop=self.training_loop, negative_sampler=self.negative_sampler, negative_sampler_kwargs=_negative_sampler_kwargs, # 7. Training training_loop_kwargs=self.training_loop_kwargs, training_kwargs=_training_kwargs, stopper=self.stopper, stopper_kwargs=_stopper_kwargs, # 8. Evaluation evaluator=self.evaluator, evaluator_kwargs=self.evaluator_kwargs, evaluation_kwargs=self.evaluation_kwargs, filter_validation_when_testing=self.filter_validation_when_testing, # 9. Tracker result_tracker=self.result_tracker, result_tracker_kwargs=self.result_tracker_kwargs, # Misc. use_testing_data=False, # use validation set during HPO! device=self.device, ) except (MemoryError, RuntimeError) as e: trial.set_user_attr('failure', str(e)) # Will trigger Optuna to set the state of the trial as failed return None else: if self.save_model_directory: model_directory = os.path.join(self.save_model_directory, str(trial.number)) os.makedirs(model_directory, exist_ok=True) result.save_to_directory(model_directory) trial.set_user_attr('random_seed', result.random_seed) for k, v in result.metric_results.to_flat_dict().items(): trial.set_user_attr(k, v) return result.metric_results.get_metric(self.metric)
def f(trial: Trial) -> float: trial.set_user_attr("train_accuracy", 1) assert trial.user_attrs["train_accuracy"] == 1 return 0.0
def _set_trial_seed(trial: Trial) -> int: new_seed = random.randrange(sys.maxsize) np.random.seed(new_seed) random.seed(new_seed) trial.set_user_attr('inner_seed', new_seed) return new_seed
def __call__(self, trial: optuna.Trial): torch.manual_seed(self.seed) random.seed(self.seed) train_data_loader, val_data_loader = datasets2data_loaders( self.train_set, self.val_set, test_set=None, num_workers=1) epochs = self.hydra_cfg['parameters']['epochs'] # Calculate an objective value by using the extra arguments. model = SupervisedFastText(V=self.dictionary.size_total_vocab, num_classes=len( self.dictionary.label_vocab), embedding_dim=self.dim, pretrained_emb=self.pretrained_word_vectors, freeze=self.is_freeze, pooling=self.pooling).to(self.device) initial_lr = trial.suggest_loguniform( 'lr', self.hydra_cfg['optuna']['lr_min'], self.hydra_cfg['optuna']['lr_max']) optimizer = optim.SGD(model.parameters(), lr=initial_lr) # parameters for update learning rate num_tokens = self.dictionary.num_words learning_rate_schedule = self.hydra_cfg['parameters']['lr_update_rate'] total_num_processed_tokens_in_training = epochs * num_tokens num_processed_tokens = 0 local_processed_tokens = 0 N = len(train_data_loader.dataset) best_val_loss = np.finfo(0.).max best_val_acc = np.finfo(0.).min save_fname = os.getcwd() + '/' + '{}.pt'.format( trial.number) # file name to store best model's weights for epoch in range(epochs): # begin training phase sum_loss = 0. correct = 0 model.train() for sentence, label, n_tokens in train_data_loader: sentence, label = sentence.to(self.device), label.to( self.device) optimizer.zero_grad() output = model(sentence) loss = F.nll_loss(output, label) loss.backward() optimizer.step() pred = output.argmax(1, keepdim=False) correct += pred.eq(label).sum().item() sum_loss += loss.item() # update learning rate # ref: https://github.com/facebookresearch/fastText/blob/6d7c77cd33b23eec26198fdfe10419476b5364c7/src/fasttext.cc#L656 local_processed_tokens += n_tokens.item() if local_processed_tokens > learning_rate_schedule: num_processed_tokens += local_processed_tokens local_processed_tokens = 0 progress = num_processed_tokens / total_num_processed_tokens_in_training optimizer.param_groups[0]['lr'] = initial_lr * (1. - progress) train_loss = sum_loss / N train_acc = correct / N # end training phase val_loss, val_acc = evaluation(model, self.device, val_data_loader) progress = num_processed_tokens / total_num_processed_tokens_in_training # approximated progress self.logger.info( '\rProgress: {:.1f}% Avg. train loss: {:.4f}, train acc: {:.1f}%, ' 'Avg. val loss: {:.4f}, val acc: {:.1f}%'.format( progress * 100., train_loss, train_acc * 100, val_loss, val_acc * 100)) if self.metric == 'loss': trial.report(val_loss, epoch) else: trial.report(val_acc, epoch) if trial.should_prune(): raise optuna.exceptions.TrialPruned() # validation is_saved_model = False if self.metric == 'loss': if best_val_loss > val_loss: best_val_loss = val_loss best_val_acc = val_acc is_saved_model = True else: if best_val_acc < val_acc: best_val_loss = val_loss best_val_acc = val_acc is_saved_model = True if is_saved_model: torch.save(model.state_dict(), save_fname) trial.set_user_attr('val_loss', best_val_loss) trial.set_user_attr('val_acc', best_val_acc) trial.set_user_attr('model_path', save_fname) if self.metric == 'loss': return best_val_loss else: return best_val_acc