def hp_search(trial: optuna.Trial):
if torch.cuda.is_available():
logger.info("%s", torch.cuda.get_device_name(0))
global gopt
opt = gopt
# set config
config = load_config(opt)
config['opt'] = opt
logger.info("%s", config)
# set path
set_path(config)
# set search spaces
lr = trial.suggest_loguniform('lr', 1e-6, 1e-3) # .suggest_float('lr', 1e-6, 1e-3, log=True)
bsz = trial.suggest_categorical('batch_size', [32, 64, 128])
seed = trial.suggest_int('seed', 17, 42)
epochs = trial.suggest_int('epochs', 1, opt.epoch)
# prepare train, valid dataset
train_loader, valid_loader = prepare_datasets(config, hp_search_bsz=bsz)
with temp_seed(seed):
# prepare model
model = prepare_model(config)
# create optimizer, scheduler, summary writer, scaler
optimizer, scheduler, writer, scaler = prepare_osws(config, model, train_loader, hp_search_lr=lr)
config['optimizer'] = optimizer
config['scheduler'] = scheduler
config['writer'] = writer
config['scaler'] = scaler
early_stopping = EarlyStopping(logger, patience=opt.patience, measure=opt.measure, verbose=1)
best_eval_measure = float('inf') if opt.measure == 'loss' else -float('inf')
for epoch in range(epochs):
eval_loss, eval_acc = train_epoch(model, config, train_loader, valid_loader, epoch)
if opt.measure == 'loss': eval_measure = eval_loss
else: eval_measure = eval_acc
# early stopping
if early_stopping.validate(eval_measure, measure=opt.measure): break
if opt.measure == 'loss': is_best = eval_measure < best_eval_measure
else: is_best = eval_measure > best_eval_measure
if is_best:
best_eval_measure = eval_measure
early_stopping.reset(best_eval_measure)
early_stopping.status()
trial.report(eval_acc, epoch)
if trial.should_prune():
raise optuna.TrialPruned()
return eval_acc
def hp_search(trial: optuna.Trial,
model_name: str,
dataset,
label_nbr,
metric_name,
reference_class,
device):
"""
objective function for optuna.study optimizes for epoch number, lr and batch_size
:param trial: optuna.Trial, trial of optuna, which will optimize for hyperparameters
:param model_name: name of the pretrained model
:param dataset: huggingface/nlp dataset object
:param label_nbr: number of label for the model to output
:param metric_name: name of the metric to maximize
:param reference_class: reference class to calculate metrics for
:param device: device where the training will occur, cuda recommended
:return: metric after training
"""
lr = trial.suggest_float("lr", 1e-7, 1e-4, log=True)
batch_size = trial.suggest_categorical("batch_size", [2, 4, 6])
epochs = trial.suggest_int("epochs", 1, 5)
model = MultilabeledSequenceModel(pretrained_model_name=model_name,
label_nbr=label_nbr).to(device)
optimizer = AdamW(params=model.parameters(), lr=lr)
for epoch in range(epochs):
train_epoch(model,
optimizer,
dataset,
batch_size,
device)
labels, preds = evaluate(model,
dataset,
batch_size,
device)
metric = calculate_metric(metric_name,
labels,
preds,
reference_class)
trial.report(metric, epoch)
if trial.should_prune():
raise optuna.TrialPruned()
return metric
def fn_opt(trial: optuna.Trial) -> int:
try:
net_arch = trial.suggest_categorical('net_arch',
['CnnPolicy', 'LnCnnPolicy'])
gamma = trial.suggest_categorical(
'gamma', [0.9, 0.95, 0.98, 0.99, 0.995, 0.999, 0.9999])
learning_rate = trial.suggest_loguniform('lr', 1e-5, 1)
batch_size = trial.suggest_categorical('batch_size',
[16, 32, 64, 128, 256, 512])
buffer_size = trial.suggest_categorical(
'buffer_size', [int(1e4), int(1e5), int(1e6)])
learning_starts = trial.suggest_categorical('learning_starts',
[0, 25, 50, 75, 100])
gradient_steps = trial.suggest_categorical('gradient_steps',
[5, 15, 20, 50, 100, 300])
ent_coef = trial.suggest_categorical(
'ent_coef', ['auto', 0.5, 0.1, 0.05, 0.01, 0.0001])
target_entropy = 'auto'
if ent_coef == 'auto':
target_entropy = trial.suggest_categorical(
'target_entropy', ['auto', -1, -10, -20, -50, -100])
env = fn_gym.FNGym(0.2)
model = fn_sac.FNSAC(net_arch,
env,
gamma=gamma,
learning_rate=learning_rate,
batch_size=batch_size,
buffer_size=buffer_size,
learning_starts=learning_starts,
gradient_steps=gradient_steps,
ent_coef=ent_coef,
target_entropy=target_entropy)
for train_count in range(10):
model.learn(total_timesteps=200)
trial.report(env.get_running_reward(), (train_count + 1) * 200)
if trial.should_prune():
raise optuna.TrialPruned()
except KeyboardInterrupt:
input('Keyboard Interrupt. Press any key to continue')
raise ValueError("Exit Trial, Keyboard Interrupt")
return env.get_running_reward()
def objective(self, trial: optuna.Trial) -> float:
# set up the search space of the hyperparameters
k = trial.suggest_int('num_topics', 1, self.max_num_topics)
a = trial.suggest_categorical('alpha', list(np.arange(0.01, 1, 0.3)) + ['symmetric','asymmetric'])
b = trial.suggest_categorical('eta', list(np.arange(0.01, 1, 0.3)) + ['symmetric'])
chunksize = trial.suggest_int('chunksize', 100, 2000, step=100)
passes = trial.suggest_int('passes', 1, 10, step=2)
iterations = trial.suggest_int('iterations', 50, 500, step=50)
# train the model using the hyperparamters suggested by Optuna
finder = TopicsFinder(self.data, self.num_ngrams, self.addl_stop_words)
model, cv = finder.fit_model(
random_state=100,
eval_every=None,
chunksize=chunksize,
passes=passes,
iterations=iterations,
num_topics = k,
alpha = a,
eta = b
)
score = cv.get_coherence()
# emit progress signal to the slot functions
if (self.progress_signal is not None):
self.progress_signal.emit({'study':self.studyname,'num_trial':trial.number})
# report an objective function value for a given step. The reported values are used by the pruners to determine whether this trial should be pruned.
trial.report(score, 0)
# handle pruning based on the intermediate value.
if trial.should_prune():
raise optuna.TrialPruned()
# save a trial info object to a file.
trial_info = {'trial': trial, 'model': model, 'score': score}
with open(f'{trial.number}.pickle', 'wb') as fout:
pickle.dump(trial_info, fout)
return score
class AllenNLPPruningCallback(EpochCallback):
"""AllenNLP callback to prune unpromising trials.
See `the example <https://github.com/optuna/optuna/blob/master/
examples/allennlp/allennlp_simple.py>`__
if you want to add a proning callback which observes a metric.
You can also see the tutorial of our AllenNLP integration on
`AllenNLP Guide <https://guide.allennlp.org/hyperparameter-optimization>`_.
.. note::
When :class:`~optuna.integration.AllenNLPPruningCallback` is instantiated in Python script,
trial and monitor are mandatory.
On the other hand, when :class:`~optuna.integration.AllenNLPPruningCallback` is used with
:class:`~optuna.integration.AllenNLPExecutor`, ``trial`` and ``monitor``
would be ``None``. :class:`~optuna.integration.AllenNLPExecutor` sets
environment variables for a study name, trial id, monitor, and storage.
Then :class:`~optuna.integration.AllenNLPPruningCallback`
loads them to restore ``trial`` and ``monitor``.
Args:
trial:
A :class:`~optuna.trial.Trial` corresponding to the current evaluation of the
objective function.
monitor:
An evaluation metric for pruning, e.g. ``validation_loss`` or
``validation_accuracy``.
"""
def __init__(
self,
trial: Optional[optuna.trial.Trial] = None,
monitor: Optional[str] = None,
):
_imports.check()
if allennlp.__version__ < "1.0.0":
raise Exception(
"AllenNLPPruningCallback requires `allennlp`>=1.0.0.")
# When `AllenNLPPruningCallback` is instantiated in Python script,
# trial and monitor should not be `None`.
if trial is not None and monitor is not None:
self._trial = trial
self._monitor = monitor
# When `AllenNLPPruningCallback` is used with `AllenNLPExecutor`,
# `trial` and `monitor` would be None. `AllenNLPExecutor` sets information
# for a study name, trial id, monitor, and storage in environment variables.
else:
environment_variables = _get_environment_variables_for_trial()
study_name = environment_variables["study_name"]
trial_id = environment_variables["trial_id"]
monitor = environment_variables["monitor"]
storage = environment_variables["storage"]
if study_name is None or trial_id is None or monitor is None or storage is None:
message = (
"Fail to load study. Perhaps you attempt to use `AllenNLPPruningCallback`"
" without `AllenNLPExecutor`. If you want to use a callback"
" without an executor, you have to instantiate a callback with"
"`trial` and `monitor. Please see the Optuna example: https://github.com/"
"optuna/optuna/blob/master/examples/allennlp/allennlp_simple.py."
)
raise RuntimeError(message)
else:
# If `stoage` is empty despite `study_name`, `trial_id`,
# and `monitor` are not `None`, users attempt to use `AllenNLPPruningCallback`
# with `AllenNLPExecutor` and in-memory storage.
# `AllenNLPruningCallback` needs RDB or Redis storages to work.
if storage == "":
message = (
"If you want to use AllenNLPExecutor and AllenNLPPruningCallback,"
" you have to use RDB or Redis storage.")
raise RuntimeError(message)
study = load_study(study_name,
storage,
pruner=_create_pruner())
self._trial = Trial(study, int(trial_id))
self._monitor = monitor
def __call__(
self,
trainer: "allennlp.training.GradientDescentTrainer",
metrics: Dict[str, Any],
epoch: int,
is_master: bool,
) -> None:
value = metrics.get(self._monitor)
if value is None:
return
self._trial.report(float(value), epoch)
if self._trial.should_prune():
raise optuna.TrialPruned()
def objective(trial: opt.Trial):
# only test dropping sozio economic facotrs
drop_sozioeco = trial.suggest_categorical("drop_eco", [True, False])
# rest of preprocessing keeps default values
# categrorial encoding, try identical encoders for all columns (for now)
enc_name = trial.suggest_categorical("encoder",
["one-hot", "woe", "binary"])
enc = encoders[enc_name]
clf_name = trial.suggest_categorical("classifier", ["rf", "xt", "gb"])
if clf_name == "rf":
clf = RandomForestClassifier(
n_estimators=trial.suggest_categorical(
"rf_nest", [50, 100, 150, 200, 300, 500]),
max_depth=trial.suggest_int("rf_max_depth", 3, 15),
min_samples_split=trial.suggest_int("rf_min_split", 2, 5),
min_samples_leaf=trial.suggest_int("rf_min_leaf", 1, 8),
bootstrap=trial.suggest_categorical("rf_bootstrap", [True, False]),
class_weight=trial.suggest_categorical(
"rf_cl_weight", ["balanced", "balanced_subsample"]),
)
elif clf_name == "xt":
clf = ExtraTreesClassifier(
n_estimators=trial.suggest_categorical(
"xt_nest", [50, 100, 150, 200, 300, 500]),
max_depth=trial.suggest_int("xt_max_depth", 3, 15),
min_samples_split=trial.suggest_int("xt_min_split", 2, 5),
min_samples_leaf=trial.suggest_int("xt_min_leaf", 1, 8),
bootstrap=trial.suggest_categorical("xt_bootstrap", [True, False]),
class_weight=trial.suggest_categorical(
"xt_cl_weight", ["balanced", "balanced_subsample"]),
)
elif clf_name == "gb":
clf = GradientBoostingClassifier(
learning_rate=trial.suggest_float("gb_lr", 0.01, 0.3),
subsample=trial.suggest_float("gb_subsample", 0.5, 1.0),
n_estimators=trial.suggest_categorical(
"gb_nest", [50, 100, 150, 200, 300, 500]),
max_depth=trial.suggest_int("gb_max_depth", 2, 7),
min_samples_split=trial.suggest_int("gb_min_split", 2, 5),
min_samples_leaf=trial.suggest_int("gb_min_leaf", 1, 8),
n_iter_no_change=20,
)
# k-fold over 5 split
# on every split the average precision is calculated and optuna
# can decide to prune this trial
maes = []
i = 0
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
for train_idx, val_idx in kf.split(x, y):
x_train = x.loc[train_idx]
y_train = y.loc[train_idx]
x_train = enc.fit_transform(x_train, y_train)
clf.fit(x_train, y_train)
x_val = x.loc[val_idx]
x_val = enc.transform(x_val)
y_val_prob = clf.predict_proba(x_val)[:, 1]
y_val_true = y.loc[val_idx]
avg_pre = me.average_precision_score(y_val_true, y_val_prob)
maes.append(avg_pre)
# after three folds, allow optuna to prune
if i >= 3:
trial.report(np.mean(maes), i)
if trial.should_prune():
raise opt.exceptions.TrialPruned()
i += 1
return np.mean(maes)
class AllenNLPPruningCallback(TrainerCallback):
"""AllenNLP callback to prune unpromising trials.
See `the example <https://github.com/optuna/optuna-examples/tree/main/
allennlp/allennlp_simple.py>`__
if you want to add a pruning callback which observes a metric.
You can also see the tutorial of our AllenNLP integration on
`AllenNLP Guide <https://guide.allennlp.org/hyperparameter-optimization>`_.
.. note::
When :class:`~optuna.integration.AllenNLPPruningCallback` is instantiated in Python script,
trial and monitor are mandatory.
On the other hand, when :class:`~optuna.integration.AllenNLPPruningCallback` is used with
:class:`~optuna.integration.AllenNLPExecutor`, ``trial`` and ``monitor``
would be :obj:`None`. :class:`~optuna.integration.AllenNLPExecutor` sets
environment variables for a study name, trial id, monitor, and storage.
Then :class:`~optuna.integration.AllenNLPPruningCallback`
loads them to restore ``trial`` and ``monitor``.
.. note::
Currently, build-in pruners are supported except for
:class:`~optuna.pruners.PatientPruner`.
Args:
trial:
A :class:`~optuna.trial.Trial` corresponding to the current evaluation of the
objective function.
monitor:
An evaluation metric for pruning, e.g. ``validation_loss`` or
``validation_accuracy``.
"""
def __init__(
self,
trial: Optional[optuna.trial.Trial] = None,
monitor: Optional[str] = None,
):
_imports.check()
if version.parse(allennlp.__version__) < version.parse("2.0.0"):
raise ImportError(
"`AllenNLPPruningCallback` requires AllenNLP>=v2.0.0."
"If you want to use a callback with an old version of AllenNLP, "
"please install Optuna v2.5.0 by executing `pip install 'optuna==2.5.0'`."
)
# When `AllenNLPPruningCallback` is instantiated in Python script,
# trial and monitor should not be `None`.
if trial is not None and monitor is not None:
self._trial = trial
self._monitor = monitor
# When `AllenNLPPruningCallback` is used with `AllenNLPExecutor`,
# `trial` and `monitor` would be None. `AllenNLPExecutor` sets information
# for a study name, trial id, monitor, and storage in environment variables.
else:
environment_variables = _get_environment_variables_for_trial()
study_name = environment_variables["study_name"]
trial_id = environment_variables["trial_id"]
monitor = environment_variables["monitor"]
storage = environment_variables["storage"]
if study_name is None or trial_id is None or monitor is None or storage is None:
message = (
"Fail to load study. Perhaps you attempt to use `AllenNLPPruningCallback`"
" without `AllenNLPExecutor`. If you want to use a callback"
" without an executor, you have to instantiate a callback with"
"`trial` and `monitor. Please see the Optuna example: https://github.com/"
"optuna/optuna-examples/tree/main/allennlp/allennlp_simple.py."
)
raise RuntimeError(message)
else:
# If `stoage` is empty despite `study_name`, `trial_id`,
# and `monitor` are not `None`, users attempt to use `AllenNLPPruningCallback`
# with `AllenNLPExecutor` and in-memory storage.
# `AllenNLPruningCallback` needs RDB or Redis storages to work.
if storage == "":
message = (
"If you want to use AllenNLPExecutor and AllenNLPPruningCallback,"
" you have to use RDB or Redis storage."
)
raise RuntimeError(message)
study = load_study(study_name, storage, pruner=_create_pruner())
self._trial = Trial(study, int(trial_id))
self._monitor = monitor
def on_epoch(
self,
trainer: "GradientDescentTrainer",
metrics: Dict[str, Any],
epoch: int,
is_primary: bool = True,
**kwargs: Any,
) -> None:
"""Check if a training reaches saturation.
Args:
trainer:
AllenNLP's trainer
metrics:
Dictionary of metrics.
epoch:
Number of current epoch.
is_primary:
A flag for AllenNLP internal.
"""
if not is_primary:
return None
value = metrics.get(self._monitor)
if value is None:
return
self._trial.report(float(value), epoch)
if self._trial.should_prune():
raise optuna.TrialPruned()
def hp_search_optuna(trial: optuna.Trial):
global gargs
args = gargs
# set config
config = load_config(args)
config['args'] = args
logger.info("%s", config)
# set path
set_path(config)
# create accelerator
accelerator = Accelerator()
config['accelerator'] = accelerator
args.device = accelerator.device
# set search spaces
lr = trial.suggest_float('lr', 1e-5, 1e-3, log=True)
bsz = trial.suggest_categorical('batch_size', [8, 16, 32, 64])
seed = trial.suggest_int('seed', 17, 42)
epochs = trial.suggest_int('epochs', 1, args.epoch)
# prepare train, valid dataset
train_loader, valid_loader = prepare_datasets(config, hp_search_bsz=bsz)
with temp_seed(seed):
# prepare model
model = prepare_model(config)
# create optimizer, scheduler, summary writer
model, optimizer, scheduler, writer = prepare_others(config,
model,
train_loader,
lr=lr)
# create secondary optimizer, scheduler
_, optimizer_2nd, scheduler_2nd, _ = prepare_others(
config, model, train_loader, lr=args.bert_lr_during_freezing)
train_loader = accelerator.prepare(train_loader)
valid_loader = accelerator.prepare(valid_loader)
config['optimizer'] = optimizer
config['scheduler'] = scheduler
config['optimizer_2nd'] = optimizer_2nd
config['scheduler_2nd'] = scheduler_2nd
config['writer'] = writer
total_batch_size = args.batch_size * accelerator.num_processes * args.gradient_accumulation_steps
logger.info("***** Running training *****")
logger.info(f" Num examples = {len(train_loader)}")
logger.info(f" Num Epochs = {args.epoch}")
logger.info(
f" Instantaneous batch size per device = {args.batch_size}")
logger.info(
f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}"
)
logger.info(
f" Gradient Accumulation steps = {args.gradient_accumulation_steps}"
)
logger.info(f" Total optimization steps = {args.max_train_steps}")
early_stopping = EarlyStopping(logger,
patience=args.patience,
measure='f1',
verbose=1)
best_eval_f1 = -float('inf')
for epoch in range(epochs):
eval_loss, eval_f1, best_eval_f1 = train_epoch(
model, config, train_loader, valid_loader, epoch, best_eval_f1)
# early stopping
if early_stopping.validate(eval_f1, measure='f1'): break
if eval_f1 == best_eval_f1:
early_stopping.reset(best_eval_f1)
early_stopping.status()
trial.report(eval_f1, epoch)
if trial.should_prune():
raise optuna.TrialPruned()
return eval_f1
def __call__(self, trial: optuna.Trial):
torch.manual_seed(self.seed)
random.seed(self.seed)
train_data_loader, val_data_loader = datasets2data_loaders(
self.train_set, self.val_set, test_set=None, num_workers=1)
epochs = self.hydra_cfg['parameters']['epochs']
# Calculate an objective value by using the extra arguments.
model = SupervisedFastText(V=self.dictionary.size_total_vocab,
num_classes=len(
self.dictionary.label_vocab),
embedding_dim=self.dim,
pretrained_emb=self.pretrained_word_vectors,
freeze=self.is_freeze,
pooling=self.pooling).to(self.device)
initial_lr = trial.suggest_loguniform(
'lr', self.hydra_cfg['optuna']['lr_min'],
self.hydra_cfg['optuna']['lr_max'])
optimizer = optim.SGD(model.parameters(), lr=initial_lr)
# parameters for update learning rate
num_tokens = self.dictionary.num_words
learning_rate_schedule = self.hydra_cfg['parameters']['lr_update_rate']
total_num_processed_tokens_in_training = epochs * num_tokens
num_processed_tokens = 0
local_processed_tokens = 0
N = len(train_data_loader.dataset)
best_val_loss = np.finfo(0.).max
best_val_acc = np.finfo(0.).min
save_fname = os.getcwd() + '/' + '{}.pt'.format(
trial.number) # file name to store best model's weights
for epoch in range(epochs):
# begin training phase
sum_loss = 0.
correct = 0
model.train()
for sentence, label, n_tokens in train_data_loader:
sentence, label = sentence.to(self.device), label.to(
self.device)
optimizer.zero_grad()
output = model(sentence)
loss = F.nll_loss(output, label)
loss.backward()
optimizer.step()
pred = output.argmax(1, keepdim=False)
correct += pred.eq(label).sum().item()
sum_loss += loss.item()
# update learning rate
# ref: https://github.com/facebookresearch/fastText/blob/6d7c77cd33b23eec26198fdfe10419476b5364c7/src/fasttext.cc#L656
local_processed_tokens += n_tokens.item()
if local_processed_tokens > learning_rate_schedule:
num_processed_tokens += local_processed_tokens
local_processed_tokens = 0
progress = num_processed_tokens / total_num_processed_tokens_in_training
optimizer.param_groups[0]['lr'] = initial_lr * (1. -
progress)
train_loss = sum_loss / N
train_acc = correct / N
# end training phase
val_loss, val_acc = evaluation(model, self.device, val_data_loader)
progress = num_processed_tokens / total_num_processed_tokens_in_training # approximated progress
self.logger.info(
'\rProgress: {:.1f}% Avg. train loss: {:.4f}, train acc: {:.1f}%, '
'Avg. val loss: {:.4f}, val acc: {:.1f}%'.format(
progress * 100., train_loss, train_acc * 100, val_loss,
val_acc * 100))
if self.metric == 'loss':
trial.report(val_loss, epoch)
else:
trial.report(val_acc, epoch)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
# validation
is_saved_model = False
if self.metric == 'loss':
if best_val_loss > val_loss:
best_val_loss = val_loss
best_val_acc = val_acc
is_saved_model = True
else:
if best_val_acc < val_acc:
best_val_loss = val_loss
best_val_acc = val_acc
is_saved_model = True
if is_saved_model:
torch.save(model.state_dict(), save_fname)
trial.set_user_attr('val_loss', best_val_loss)
trial.set_user_attr('val_acc', best_val_acc)
trial.set_user_attr('model_path', save_fname)
if self.metric == 'loss':
return best_val_loss
else:
return best_val_acc
