def _objective(self, trial: optuna.Trial): kwargs = self.get_parameters(trial) step = 0 result = None for df in self.data: step += 1 result, stats, cerebro, thestrats = self.backtest(df, **kwargs) if result is None: raise optuna.exceptions.TrialPruned("no result") trial.report(result, step) for k in stats.keys(): trial.set_user_attr(f'backtest_{k}', stats[k]) if trial.should_prune(): raise optuna.exceptions.TrialPruned("should_prune") # result = self.backtest(self.data[0], **kwargs) if result is None: raise optuna.exceptions.TrialPruned("no result") return result
def objective(trial: Trial, args): args.params = get_params(trial=trial, tunable_params=args.tunable_params, default_params=args.default_params) experiment, agent_ctor = dispatch_experiment( args, f'{args.logdir}/logs/{trial.number}') freeze_params(params=args.params, logdir=f'{args.logdir}/logs/{trial.number}') agent = experiment.configure_agent(agent_ctor) objective_value = -1e9 for epoch in range(1, args.epochs + 1): objective_value = experiment.run_trial(agent=agent, steps=args.steps) print(f'Trial-{trial.number}: {objective_value} in epoch {epoch}') if not np.isnan(objective_value): trial.report(objective_value, step=epoch) if trial.should_prune(): print(f'Trial {trial.number}: pruning.') raise optuna.exceptions.TrialPruned() if np.isnan(objective_value): return -1e9 else: return objective_value
def objective(trial: optuna.Trial) -> float: x = trial.suggest_float("x", -10, 10) y = trial.suggest_float("y", -10, 10) trial.report(x, 0) trial.report(y, 1) trial.set_user_attr("x", x) trial.set_system_attr("y", y) return f(x, y)
def _cross_validate(self, model, trial: optuna.Trial) \ -> Tuple[List[float], Dict[str, List[float]]]: """ Validate model using validation scheme defined in `__init__`. It handles pruning and some mlflow reporting. Parameters ---------- model: sklearn estimator Model returned by `create_model` specific for the subclass. trial: optuna.Trial Object that controls how parameters are drawn. Returns ------- List of scores and dict of additional scores. Raises ------ optuna.exceptions.TrialPruned when trial is not promising and should be pruned. The exception is further handled by optuna. """ x, y = self.dataset.x, self.dataset.y scores = [] additional_scores = { ans.name: [] for ans in self.additional_named_scorers } for i, (train_index, val_index) in enumerate(self.split_indices, start=1): x_train, x_val = x.iloc[train_index], x.iloc[val_index] y_train, y_val = y.iloc[train_index], y.iloc[val_index] model.fit(x_train, y_train, **self.fit_kwargs) score = self.scorer(model, x_val, y_val) scores.append(score) if self.use_mlflow: mlflow.log_metrics({self.scorer_name: score}, step=i) for fn, name in self.additional_named_scorers: additional_score = fn(model, x_val, y_val) additional_scores[name].append(additional_score) if self.use_mlflow: mlflow.log_metrics({name: additional_score}, step=i) if self.models_to_mlflow: mlflow_push_pickled(model, f'model_cv{i:02d}_') trial.report(score, i) if trial.should_prune(): if self.use_mlflow: mlflow.set_tag('PRUNED_FOLD', str(i + 1)) raise optuna.exceptions.TrialPruned() return scores, additional_scores
def objective(self, trial: Trial, return_model=False): X, y = self.make_xy(trial) # X, y = self.X, self.y loss_cv_train = [] cv_preds = np.zeros_like(y) cv_preds.fill(np.nan) models = [] for step, (train_idx, val_idx) in enumerate(self.split(X, y)): X_train = X[train_idx] X_val = X[val_idx] y_train = y[train_idx] y_val = y[val_idx] model = self.fit(trial, X_train, y_train, X_val, y_val, step) y_train_preds = model.predict(X_train) y_val_preds = model.predict(X_val) # from IPython.core.debugger import Pdb; # Pdb().set_trace() cv_preds[val_idx] = y_val_preds mask_done = ~np.isnan(cv_preds) intermediate_loss_train = self.loss_fn(y_train, y_train_preds, trial) intermediate_loss_val = self.loss_fn(y[mask_done], cv_preds[mask_done], trial) loss_cv_train.append(intermediate_loss_train) trial.report(intermediate_loss_val, step) if self.prune and trial.should_prune(step): raise optuna.structs.TrialPruned() models.append(model) if self.is_one_cv: break mask_done = ~np.isnan(cv_preds) loss_train = float(np.mean(loss_cv_train)) loss_val = float(self.loss_fn(y[mask_done], cv_preds[mask_done], trial)) trial.set_user_attr('train_loss', loss_train) trial.set_user_attr('val_loss', loss_val) trial.set_user_attr('is_one_cv', int(self.is_one_cv)) self.on_after_trial(trial, models, cv_preds, loss_val) if return_model: return loss_val, models, cv_preds return loss_val
def hp_search(trial: optuna.Trial): if torch.cuda.is_available(): logger.info("%s", torch.cuda.get_device_name(0)) global gopt opt = gopt # set config config = load_config(opt) config['opt'] = opt logger.info("%s", config) # set path set_path(config) # set search spaces lr = trial.suggest_loguniform('lr', 1e-6, 1e-3) # .suggest_float('lr', 1e-6, 1e-3, log=True) bsz = trial.suggest_categorical('batch_size', [32, 64, 128]) seed = trial.suggest_int('seed', 17, 42) epochs = trial.suggest_int('epochs', 1, opt.epoch) # prepare train, valid dataset train_loader, valid_loader = prepare_datasets(config, hp_search_bsz=bsz) with temp_seed(seed): # prepare model model = prepare_model(config) # create optimizer, scheduler, summary writer, scaler optimizer, scheduler, writer, scaler = prepare_osws(config, model, train_loader, hp_search_lr=lr) config['optimizer'] = optimizer config['scheduler'] = scheduler config['writer'] = writer config['scaler'] = scaler early_stopping = EarlyStopping(logger, patience=opt.patience, measure=opt.measure, verbose=1) best_eval_measure = float('inf') if opt.measure == 'loss' else -float('inf') for epoch in range(epochs): eval_loss, eval_acc = train_epoch(model, config, train_loader, valid_loader, epoch) if opt.measure == 'loss': eval_measure = eval_loss else: eval_measure = eval_acc # early stopping if early_stopping.validate(eval_measure, measure=opt.measure): break if opt.measure == 'loss': is_best = eval_measure < best_eval_measure else: is_best = eval_measure > best_eval_measure if is_best: best_eval_measure = eval_measure early_stopping.reset(best_eval_measure) early_stopping.status() trial.report(eval_acc, epoch) if trial.should_prune(): raise optuna.TrialPruned() return eval_acc
def f(trial: Trial) -> float: x = trial.suggest_int("x", 1, 1) y = trial.suggest_categorical("y", (2.5, )) assert isinstance(y, float) trial.set_user_attr("train_loss", 3) trial.set_system_attr("foo", "bar") value = x + y # 3.5 # Test reported intermediate values, although it in practice is not "intermediate". trial.report(value, step=0) return value
def hp_search(trial: optuna.Trial, model_name: str, dataset, label_nbr, metric_name, reference_class, device): """ objective function for optuna.study optimizes for epoch number, lr and batch_size :param trial: optuna.Trial, trial of optuna, which will optimize for hyperparameters :param model_name: name of the pretrained model :param dataset: huggingface/nlp dataset object :param label_nbr: number of label for the model to output :param metric_name: name of the metric to maximize :param reference_class: reference class to calculate metrics for :param device: device where the training will occur, cuda recommended :return: metric after training """ lr = trial.suggest_float("lr", 1e-7, 1e-4, log=True) batch_size = trial.suggest_categorical("batch_size", [2, 4, 6]) epochs = trial.suggest_int("epochs", 1, 5) model = MultilabeledSequenceModel(pretrained_model_name=model_name, label_nbr=label_nbr).to(device) optimizer = AdamW(params=model.parameters(), lr=lr) for epoch in range(epochs): train_epoch(model, optimizer, dataset, batch_size, device) labels, preds = evaluate(model, dataset, batch_size, device) metric = calculate_metric(metric_name, labels, preds, reference_class) trial.report(metric, epoch) if trial.should_prune(): raise optuna.TrialPruned() return metric
def fn_opt(trial: optuna.Trial) -> int: try: net_arch = trial.suggest_categorical('net_arch', ['CnnPolicy', 'LnCnnPolicy']) gamma = trial.suggest_categorical( 'gamma', [0.9, 0.95, 0.98, 0.99, 0.995, 0.999, 0.9999]) learning_rate = trial.suggest_loguniform('lr', 1e-5, 1) batch_size = trial.suggest_categorical('batch_size', [16, 32, 64, 128, 256, 512]) buffer_size = trial.suggest_categorical( 'buffer_size', [int(1e4), int(1e5), int(1e6)]) learning_starts = trial.suggest_categorical('learning_starts', [0, 25, 50, 75, 100]) gradient_steps = trial.suggest_categorical('gradient_steps', [5, 15, 20, 50, 100, 300]) ent_coef = trial.suggest_categorical( 'ent_coef', ['auto', 0.5, 0.1, 0.05, 0.01, 0.0001]) target_entropy = 'auto' if ent_coef == 'auto': target_entropy = trial.suggest_categorical( 'target_entropy', ['auto', -1, -10, -20, -50, -100]) env = fn_gym.FNGym(0.2) model = fn_sac.FNSAC(net_arch, env, gamma=gamma, learning_rate=learning_rate, batch_size=batch_size, buffer_size=buffer_size, learning_starts=learning_starts, gradient_steps=gradient_steps, ent_coef=ent_coef, target_entropy=target_entropy) for train_count in range(10): model.learn(total_timesteps=200) trial.report(env.get_running_reward(), (train_count + 1) * 200) if trial.should_prune(): raise optuna.TrialPruned() except KeyboardInterrupt: input('Keyboard Interrupt. Press any key to continue') raise ValueError("Exit Trial, Keyboard Interrupt") return env.get_running_reward()
def objective(self, trial: optuna.Trial) -> float: # set up the search space of the hyperparameters k = trial.suggest_int('num_topics', 1, self.max_num_topics) a = trial.suggest_categorical('alpha', list(np.arange(0.01, 1, 0.3)) + ['symmetric','asymmetric']) b = trial.suggest_categorical('eta', list(np.arange(0.01, 1, 0.3)) + ['symmetric']) chunksize = trial.suggest_int('chunksize', 100, 2000, step=100) passes = trial.suggest_int('passes', 1, 10, step=2) iterations = trial.suggest_int('iterations', 50, 500, step=50) # train the model using the hyperparamters suggested by Optuna finder = TopicsFinder(self.data, self.num_ngrams, self.addl_stop_words) model, cv = finder.fit_model( random_state=100, eval_every=None, chunksize=chunksize, passes=passes, iterations=iterations, num_topics = k, alpha = a, eta = b ) score = cv.get_coherence() # emit progress signal to the slot functions if (self.progress_signal is not None): self.progress_signal.emit({'study':self.studyname,'num_trial':trial.number}) # report an objective function value for a given step. The reported values are used by the pruners to determine whether this trial should be pruned. trial.report(score, 0) # handle pruning based on the intermediate value. if trial.should_prune(): raise optuna.TrialPruned() # save a trial info object to a file. trial_info = {'trial': trial, 'model': model, 'score': score} with open(f'{trial.number}.pickle', 'wb') as fout: pickle.dump(trial_info, fout) return score
def __call__(self, trial: optuna.Trial): torch.manual_seed(self.seed) random.seed(self.seed) train_data_loader, val_data_loader = datasets2data_loaders( self.train_set, self.val_set, test_set=None, num_workers=1) epochs = self.hydra_cfg['parameters']['epochs'] # Calculate an objective value by using the extra arguments. model = SupervisedFastText(V=self.dictionary.size_total_vocab, num_classes=len( self.dictionary.label_vocab), embedding_dim=self.dim, pretrained_emb=self.pretrained_word_vectors, freeze=self.is_freeze, pooling=self.pooling).to(self.device) initial_lr = trial.suggest_loguniform( 'lr', self.hydra_cfg['optuna']['lr_min'], self.hydra_cfg['optuna']['lr_max']) optimizer = optim.SGD(model.parameters(), lr=initial_lr) # parameters for update learning rate num_tokens = self.dictionary.num_words learning_rate_schedule = self.hydra_cfg['parameters']['lr_update_rate'] total_num_processed_tokens_in_training = epochs * num_tokens num_processed_tokens = 0 local_processed_tokens = 0 N = len(train_data_loader.dataset) best_val_loss = np.finfo(0.).max best_val_acc = np.finfo(0.).min save_fname = os.getcwd() + '/' + '{}.pt'.format( trial.number) # file name to store best model's weights for epoch in range(epochs): # begin training phase sum_loss = 0. correct = 0 model.train() for sentence, label, n_tokens in train_data_loader: sentence, label = sentence.to(self.device), label.to( self.device) optimizer.zero_grad() output = model(sentence) loss = F.nll_loss(output, label) loss.backward() optimizer.step() pred = output.argmax(1, keepdim=False) correct += pred.eq(label).sum().item() sum_loss += loss.item() # update learning rate # ref: https://github.com/facebookresearch/fastText/blob/6d7c77cd33b23eec26198fdfe10419476b5364c7/src/fasttext.cc#L656 local_processed_tokens += n_tokens.item() if local_processed_tokens > learning_rate_schedule: num_processed_tokens += local_processed_tokens local_processed_tokens = 0 progress = num_processed_tokens / total_num_processed_tokens_in_training optimizer.param_groups[0]['lr'] = initial_lr * (1. - progress) train_loss = sum_loss / N train_acc = correct / N # end training phase val_loss, val_acc = evaluation(model, self.device, val_data_loader) progress = num_processed_tokens / total_num_processed_tokens_in_training # approximated progress self.logger.info( '\rProgress: {:.1f}% Avg. train loss: {:.4f}, train acc: {:.1f}%, ' 'Avg. val loss: {:.4f}, val acc: {:.1f}%'.format( progress * 100., train_loss, train_acc * 100, val_loss, val_acc * 100)) if self.metric == 'loss': trial.report(val_loss, epoch) else: trial.report(val_acc, epoch) if trial.should_prune(): raise optuna.exceptions.TrialPruned() # validation is_saved_model = False if self.metric == 'loss': if best_val_loss > val_loss: best_val_loss = val_loss best_val_acc = val_acc is_saved_model = True else: if best_val_acc < val_acc: best_val_loss = val_loss best_val_acc = val_acc is_saved_model = True if is_saved_model: torch.save(model.state_dict(), save_fname) trial.set_user_attr('val_loss', best_val_loss) trial.set_user_attr('val_acc', best_val_acc) trial.set_user_attr('model_path', save_fname) if self.metric == 'loss': return best_val_loss else: return best_val_acc
class AllenNLPPruningCallback(EpochCallback): """AllenNLP callback to prune unpromising trials. See `the example <https://github.com/optuna/optuna/blob/master/ examples/allennlp/allennlp_simple.py>`__ if you want to add a proning callback which observes a metric. You can also see the tutorial of our AllenNLP integration on `AllenNLP Guide <https://guide.allennlp.org/hyperparameter-optimization>`_. .. note:: When :class:`~optuna.integration.AllenNLPPruningCallback` is instantiated in Python script, trial and monitor are mandatory. On the other hand, when :class:`~optuna.integration.AllenNLPPruningCallback` is used with :class:`~optuna.integration.AllenNLPExecutor`, ``trial`` and ``monitor`` would be ``None``. :class:`~optuna.integration.AllenNLPExecutor` sets environment variables for a study name, trial id, monitor, and storage. Then :class:`~optuna.integration.AllenNLPPruningCallback` loads them to restore ``trial`` and ``monitor``. Args: trial: A :class:`~optuna.trial.Trial` corresponding to the current evaluation of the objective function. monitor: An evaluation metric for pruning, e.g. ``validation_loss`` or ``validation_accuracy``. """ def __init__( self, trial: Optional[optuna.trial.Trial] = None, monitor: Optional[str] = None, ): _imports.check() if allennlp.__version__ < "1.0.0": raise Exception( "AllenNLPPruningCallback requires `allennlp`>=1.0.0.") # When `AllenNLPPruningCallback` is instantiated in Python script, # trial and monitor should not be `None`. if trial is not None and monitor is not None: self._trial = trial self._monitor = monitor # When `AllenNLPPruningCallback` is used with `AllenNLPExecutor`, # `trial` and `monitor` would be None. `AllenNLPExecutor` sets information # for a study name, trial id, monitor, and storage in environment variables. else: environment_variables = _get_environment_variables_for_trial() study_name = environment_variables["study_name"] trial_id = environment_variables["trial_id"] monitor = environment_variables["monitor"] storage = environment_variables["storage"] if study_name is None or trial_id is None or monitor is None or storage is None: message = ( "Fail to load study. Perhaps you attempt to use `AllenNLPPruningCallback`" " without `AllenNLPExecutor`. If you want to use a callback" " without an executor, you have to instantiate a callback with" "`trial` and `monitor. Please see the Optuna example: https://github.com/" "optuna/optuna/blob/master/examples/allennlp/allennlp_simple.py." ) raise RuntimeError(message) else: # If `stoage` is empty despite `study_name`, `trial_id`, # and `monitor` are not `None`, users attempt to use `AllenNLPPruningCallback` # with `AllenNLPExecutor` and in-memory storage. # `AllenNLPruningCallback` needs RDB or Redis storages to work. if storage == "": message = ( "If you want to use AllenNLPExecutor and AllenNLPPruningCallback," " you have to use RDB or Redis storage.") raise RuntimeError(message) study = load_study(study_name, storage, pruner=_create_pruner()) self._trial = Trial(study, int(trial_id)) self._monitor = monitor def __call__( self, trainer: "allennlp.training.GradientDescentTrainer", metrics: Dict[str, Any], epoch: int, is_master: bool, ) -> None: value = metrics.get(self._monitor) if value is None: return self._trial.report(float(value), epoch) if self._trial.should_prune(): raise optuna.TrialPruned()
def func_with_trial_pruned(trial: Trial) -> float: if report_value is not None: trial.report(report_value, 1) raise trial_pruned_class()
def func(trial: Trial) -> float: if intermediate is not None: trial.report(step=1, value=intermediate) raise TrialPruned
def objective(trial: opt.Trial): # only test dropping sozio economic facotrs drop_sozioeco = trial.suggest_categorical("drop_eco", [True, False]) # rest of preprocessing keeps default values # categrorial encoding, try identical encoders for all columns (for now) enc_name = trial.suggest_categorical("encoder", ["one-hot", "woe", "binary"]) enc = encoders[enc_name] clf_name = trial.suggest_categorical("classifier", ["rf", "xt", "gb"]) if clf_name == "rf": clf = RandomForestClassifier( n_estimators=trial.suggest_categorical( "rf_nest", [50, 100, 150, 200, 300, 500]), max_depth=trial.suggest_int("rf_max_depth", 3, 15), min_samples_split=trial.suggest_int("rf_min_split", 2, 5), min_samples_leaf=trial.suggest_int("rf_min_leaf", 1, 8), bootstrap=trial.suggest_categorical("rf_bootstrap", [True, False]), class_weight=trial.suggest_categorical( "rf_cl_weight", ["balanced", "balanced_subsample"]), ) elif clf_name == "xt": clf = ExtraTreesClassifier( n_estimators=trial.suggest_categorical( "xt_nest", [50, 100, 150, 200, 300, 500]), max_depth=trial.suggest_int("xt_max_depth", 3, 15), min_samples_split=trial.suggest_int("xt_min_split", 2, 5), min_samples_leaf=trial.suggest_int("xt_min_leaf", 1, 8), bootstrap=trial.suggest_categorical("xt_bootstrap", [True, False]), class_weight=trial.suggest_categorical( "xt_cl_weight", ["balanced", "balanced_subsample"]), ) elif clf_name == "gb": clf = GradientBoostingClassifier( learning_rate=trial.suggest_float("gb_lr", 0.01, 0.3), subsample=trial.suggest_float("gb_subsample", 0.5, 1.0), n_estimators=trial.suggest_categorical( "gb_nest", [50, 100, 150, 200, 300, 500]), max_depth=trial.suggest_int("gb_max_depth", 2, 7), min_samples_split=trial.suggest_int("gb_min_split", 2, 5), min_samples_leaf=trial.suggest_int("gb_min_leaf", 1, 8), n_iter_no_change=20, ) # k-fold over 5 split # on every split the average precision is calculated and optuna # can decide to prune this trial maes = [] i = 0 kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True) for train_idx, val_idx in kf.split(x, y): x_train = x.loc[train_idx] y_train = y.loc[train_idx] x_train = enc.fit_transform(x_train, y_train) clf.fit(x_train, y_train) x_val = x.loc[val_idx] x_val = enc.transform(x_val) y_val_prob = clf.predict_proba(x_val)[:, 1] y_val_true = y.loc[val_idx] avg_pre = me.average_precision_score(y_val_true, y_val_prob) maes.append(avg_pre) # after three folds, allow optuna to prune if i >= 3: trial.report(np.mean(maes), i) if trial.should_prune(): raise opt.exceptions.TrialPruned() i += 1 return np.mean(maes)
class AllenNLPPruningCallback(TrainerCallback): """AllenNLP callback to prune unpromising trials. See `the example <https://github.com/optuna/optuna-examples/tree/main/ allennlp/allennlp_simple.py>`__ if you want to add a pruning callback which observes a metric. You can also see the tutorial of our AllenNLP integration on `AllenNLP Guide <https://guide.allennlp.org/hyperparameter-optimization>`_. .. note:: When :class:`~optuna.integration.AllenNLPPruningCallback` is instantiated in Python script, trial and monitor are mandatory. On the other hand, when :class:`~optuna.integration.AllenNLPPruningCallback` is used with :class:`~optuna.integration.AllenNLPExecutor`, ``trial`` and ``monitor`` would be :obj:`None`. :class:`~optuna.integration.AllenNLPExecutor` sets environment variables for a study name, trial id, monitor, and storage. Then :class:`~optuna.integration.AllenNLPPruningCallback` loads them to restore ``trial`` and ``monitor``. .. note:: Currently, build-in pruners are supported except for :class:`~optuna.pruners.PatientPruner`. Args: trial: A :class:`~optuna.trial.Trial` corresponding to the current evaluation of the objective function. monitor: An evaluation metric for pruning, e.g. ``validation_loss`` or ``validation_accuracy``. """ def __init__( self, trial: Optional[optuna.trial.Trial] = None, monitor: Optional[str] = None, ): _imports.check() if version.parse(allennlp.__version__) < version.parse("2.0.0"): raise ImportError( "`AllenNLPPruningCallback` requires AllenNLP>=v2.0.0." "If you want to use a callback with an old version of AllenNLP, " "please install Optuna v2.5.0 by executing `pip install 'optuna==2.5.0'`." ) # When `AllenNLPPruningCallback` is instantiated in Python script, # trial and monitor should not be `None`. if trial is not None and monitor is not None: self._trial = trial self._monitor = monitor # When `AllenNLPPruningCallback` is used with `AllenNLPExecutor`, # `trial` and `monitor` would be None. `AllenNLPExecutor` sets information # for a study name, trial id, monitor, and storage in environment variables. else: environment_variables = _get_environment_variables_for_trial() study_name = environment_variables["study_name"] trial_id = environment_variables["trial_id"] monitor = environment_variables["monitor"] storage = environment_variables["storage"] if study_name is None or trial_id is None or monitor is None or storage is None: message = ( "Fail to load study. Perhaps you attempt to use `AllenNLPPruningCallback`" " without `AllenNLPExecutor`. If you want to use a callback" " without an executor, you have to instantiate a callback with" "`trial` and `monitor. Please see the Optuna example: https://github.com/" "optuna/optuna-examples/tree/main/allennlp/allennlp_simple.py." ) raise RuntimeError(message) else: # If `stoage` is empty despite `study_name`, `trial_id`, # and `monitor` are not `None`, users attempt to use `AllenNLPPruningCallback` # with `AllenNLPExecutor` and in-memory storage. # `AllenNLPruningCallback` needs RDB or Redis storages to work. if storage == "": message = ( "If you want to use AllenNLPExecutor and AllenNLPPruningCallback," " you have to use RDB or Redis storage." ) raise RuntimeError(message) study = load_study(study_name, storage, pruner=_create_pruner()) self._trial = Trial(study, int(trial_id)) self._monitor = monitor def on_epoch( self, trainer: "GradientDescentTrainer", metrics: Dict[str, Any], epoch: int, is_primary: bool = True, **kwargs: Any, ) -> None: """Check if a training reaches saturation. Args: trainer: AllenNLP's trainer metrics: Dictionary of metrics. epoch: Number of current epoch. is_primary: A flag for AllenNLP internal. """ if not is_primary: return None value = metrics.get(self._monitor) if value is None: return self._trial.report(float(value), epoch) if self._trial.should_prune(): raise optuna.TrialPruned()
def hp_search_optuna(trial: optuna.Trial): global gargs args = gargs # set config config = load_config(args) config['args'] = args logger.info("%s", config) # set path set_path(config) # create accelerator accelerator = Accelerator() config['accelerator'] = accelerator args.device = accelerator.device # set search spaces lr = trial.suggest_float('lr', 1e-5, 1e-3, log=True) bsz = trial.suggest_categorical('batch_size', [8, 16, 32, 64]) seed = trial.suggest_int('seed', 17, 42) epochs = trial.suggest_int('epochs', 1, args.epoch) # prepare train, valid dataset train_loader, valid_loader = prepare_datasets(config, hp_search_bsz=bsz) with temp_seed(seed): # prepare model model = prepare_model(config) # create optimizer, scheduler, summary writer model, optimizer, scheduler, writer = prepare_others(config, model, train_loader, lr=lr) # create secondary optimizer, scheduler _, optimizer_2nd, scheduler_2nd, _ = prepare_others( config, model, train_loader, lr=args.bert_lr_during_freezing) train_loader = accelerator.prepare(train_loader) valid_loader = accelerator.prepare(valid_loader) config['optimizer'] = optimizer config['scheduler'] = scheduler config['optimizer_2nd'] = optimizer_2nd config['scheduler_2nd'] = scheduler_2nd config['writer'] = writer total_batch_size = args.batch_size * accelerator.num_processes * args.gradient_accumulation_steps logger.info("***** Running training *****") logger.info(f" Num examples = {len(train_loader)}") logger.info(f" Num Epochs = {args.epoch}") logger.info( f" Instantaneous batch size per device = {args.batch_size}") logger.info( f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}" ) logger.info( f" Gradient Accumulation steps = {args.gradient_accumulation_steps}" ) logger.info(f" Total optimization steps = {args.max_train_steps}") early_stopping = EarlyStopping(logger, patience=args.patience, measure='f1', verbose=1) best_eval_f1 = -float('inf') for epoch in range(epochs): eval_loss, eval_f1, best_eval_f1 = train_epoch( model, config, train_loader, valid_loader, epoch, best_eval_f1) # early stopping if early_stopping.validate(eval_f1, measure='f1'): break if eval_f1 == best_eval_f1: early_stopping.reset(best_eval_f1) early_stopping.status() trial.report(eval_f1, epoch) if trial.should_prune(): raise optuna.TrialPruned() return eval_f1