def __init__(
self,
study_name, # type: str
storage, # type: Union[str, storages.BaseStorage]
sampler=None, # type: samplers.BaseSampler
pruner=None, # type: pruners.BasePruner
direction='minimize', # type: str
):
# type: (...) -> None
self.study_name = study_name
self.storage = storages.get_storage(storage)
self.sampler = sampler or samplers.TPESampler()
self.pruner = pruner or pruners.MedianPruner()
self.study_id = self.storage.get_study_id_from_name(study_name)
self.logger = logging.get_logger(__name__)
if direction == 'minimize':
_direction = structs.StudyDirection.MINIMIZE
elif direction == 'maximize':
_direction = structs.StudyDirection.MAXIMIZE
else:
raise ValueError(
'Please set either \'minimize\' or \'maximize\' to direction.')
# TODO(Yanase): Implement maximization.
if _direction == structs.StudyDirection.MAXIMIZE:
raise ValueError(
'Optimization direction of study {} is set to `MAXIMIZE`. '
'Currently, Optuna supports `MINIMIZE` only.'.format(
study_name))
self.storage.set_study_direction(self.study_id, _direction)
def __init__(
self,
study_name, # type: str
storage, # type: Union[str, storages.BaseStorage]
sampler=None, # type: samplers.BaseSampler
pruner=None # type: pruners.BasePruner
):
# type: (...) -> None
self.study_name = study_name
storage = storages.get_storage(storage)
study_id = storage.get_study_id_from_name(study_name)
super(Study, self).__init__(study_id, storage)
self.sampler = sampler or samplers.TPESampler()
self.pruner = pruner or pruners.MedianPruner()
if (isinstance(self.sampler, samplers.TPESampler)
and isinstance(self.pruner, pruners.HyperbandPruner)):
msg = (
"The algorithm of TPESampler and HyperbandPruner might behave in a different way "
"from the paper of Hyperband because the sampler uses all the trials including "
"ones of brackets other than that of currently running trial")
warnings.warn(msg, UserWarning)
_logger.warning(msg)
self._optimize_lock = threading.Lock()
def __init__(
self,
study_name, # type: str
storage, # type: Union[str, storages.BaseStorage]
sampler=None, # type: samplers.BaseSampler
pruner=None, # type: pruners.BasePruner
):
# type: (...) -> None
self.study_name = study_name
self.storage = storages.get_storage(storage)
self.sampler = sampler or samplers.TPESampler()
self.pruner = pruner or pruners.MedianPruner()
self.study_id = self.storage.get_study_id_from_name(study_name)
self.logger = logging.get_logger(__name__)
def __init__(
self,
study_name: str,
storage: Union[str, storages.BaseStorage],
sampler: Optional["samplers.BaseSampler"] = None,
pruner: Optional[pruners.BasePruner] = None,
) -> None:
self.study_name = study_name
storage = storages.get_storage(storage)
study_id = storage.get_study_id_from_name(study_name)
super(Study, self).__init__(study_id, storage)
self.sampler = sampler or samplers.TPESampler()
self.pruner = pruner or pruners.MedianPruner()
self._optimize_lock = threading.Lock()
self._stop_flag = False
def __init__(
self,
study_name, # type: str
storage, # type: Union[str, storages.BaseStorage]
sampler=None, # type: samplers.BaseSampler
pruner=None, # type: pruners.BasePruner
):
# type: (...) -> None
self.study_name = study_name
storage = storages.get_storage(storage)
study_id = storage.get_study_id_from_name(study_name)
super(Study, self).__init__(study_id, storage)
self.sampler = sampler or samplers.TPESampler()
self.pruner = pruner or pruners.MedianPruner()
self._optimize_lock = threading.Lock()
def __init__(
self,
study_name, # type: str
storage, # type: Union[str, storages.BaseStorage]
sampler=None, # type: samplers.BaseSampler
pruner=None, # type: pruners.BasePruner
force_garbage_collection=True, # type: bool
):
# type: (...) -> None
self.study_name = study_name
storage = storages.get_storage(storage)
study_id = storage.get_study_id_from_name(study_name)
super(Study, self).__init__(study_id, storage)
self.sampler = sampler or samplers.TPESampler()
self.pruner = pruner or pruners.MedianPruner()
self.logger = logging.get_logger(__name__)
self._optimize_lock = threading.Lock()
self.force_garbage_collection = force_garbage_collection
from optuna.testing.sampler import DeterministicRelativeSampler
from optuna.testing.storage import StorageSupplier
from optuna.trial import Trial
from optuna.trial import TrialState
try:
import chainermn
from chainermn.communicators.communicator_base import CommunicatorBase # NOQA
_available = True
except ImportError:
_available = False
STORAGE_MODES = ["sqlite"]
PRUNER_INIT_FUNCS = [
lambda: pruners.MedianPruner(), lambda: pruners.SuccessiveHalvingPruner()
]
class Func(object):
def __init__(self) -> None:
self.suggested_values: Dict[int, Dict[str, Any]] = {}
def __call__(self, trial: ChainerMNTrial, comm: CommunicatorBase) -> float:
x = trial.suggest_uniform("x", -10, 10)
y = trial.suggest_loguniform("y", 20, 30)
z = trial.suggest_categorical("z", (-1.0, 1.0))
self.suggested_values[trial.number] = {}
from types import TracebackType # NOQA
from typing import Any # NOQA
from typing import Callable # NOQA
from typing import Dict # NOQA
from typing import Optional # NOQA
from typing import Type # NOQA
try:
import chainermn
from chainermn.communicators.communicator_base import CommunicatorBase # NOQA
_available = True
except ImportError:
_available = False
STORAGE_MODES = ['new', 'common']
PRUNER_INIT_FUNCS = [lambda: pruners.MedianPruner(), lambda: pruners.SuccessiveHalvingPruner()]
CACHE_MODES = [True, False]
def setup_module():
# type: () -> None
StorageSupplier.setup_common_tempfile()
def teardown_module():
# type: () -> None
StorageSupplier.teardown_common_tempfile()
def train(
self,
texts: List,
tags: List,
model_name: str = "bert_email_router",
last_checkpoint: Union[str, bool] = None,
experiment_name: str = None,
hp_space: Optional[Callable[["optuna.Trial"], Dict[str,
float]]] = None,
):
"""[train the model]
Args:
texts (List): [list of text sequences]
tags (List): [list of list of tags to train on]
model_name (str, optional): [name of the model, used for outputdir and mlflow name]. Defaults to "bert_ner_test".
last_checkpoint (Union[str, bool], optional): [if boolean true then keep modelling from last checkpoint or if modelname string if we want to continue training on that checkpoint]. Defaults to None.
experiment_name (str, optional): [name of mlflow experiment to start]
"""
train_texts, val_texts, train_tags, val_tags = model_utils.train_test_split(
texts, tags, test_size=0.1)
train_encodings = self.tokenizer(
train_texts,
is_split_into_words=False,
return_offsets_mapping=False,
padding="max_length",
max_length=self.params.max_len,
truncation=True,
)
train_labels = [self.params.tag2id[tag] for tag in train_tags]
train_data = self.NerDataset(train_encodings, train_labels)
val_encodings = self.tokenizer(
val_texts,
is_split_into_words=False,
return_offsets_mapping=False,
padding="max_length",
max_length=self.params.max_len,
truncation=True,
)
val_labels = [self.params.tag2id[tag] for tag in val_tags]
val_data = self.NerDataset(val_encodings, val_labels)
num_train_steps = int(
len(train_data) / self.params.batch_size * self.params.epochs)
num_warmup_steps = int(num_train_steps * self.params.warmup_proportion)
training_args = self.set_trainings_args(
model_name=model_name, num_warmup_steps=num_warmup_steps)
if experiment_name is not None:
mlflow.set_experiment(experiment_name)
if hp_space:
self.trainer = Trainer(
model_init=self.load_model,
train_dataset=train_data,
eval_dataset=val_data,
args=training_args,
tokenizer=self.tokenizer,
compute_metrics=model_utils.compute_metrics,
callbacks=[
model_utils.TensorBoardCallbackWrap,
model_utils.MLflowCallbackWrap
],
)
self.trainer.remove_callback(TensorBoardCallback)
self.trainer.remove_callback(MLflowCallback)
best_trial = self.trainer.hyperparameter_search(
hp_space=hp_space,
direction=
"minimize", # probably should be minimize (now it's maximize)
backend="optuna",
n_trials=settings.N_TRIALS, # number of trials
pruner=pruners.MedianPruner(n_startup_trials=5,
n_warmup_steps=30,
interval_steps=10),
hp_name=model_utils.hp_name_optuna,
)
metrics = {}
metrics["loss"] = best_trial.objective
logger.info(
f"Hyperparameter optimization finished: Best hyperparameters are {best_trial.hyperparameters}"
)
else:
if last_checkpoint is not None:
checkpoint = last_checkpoint
else:
checkpoint = None
self.trainer = Trainer(
model=self.model,
train_dataset=train_data,
eval_dataset=val_data,
args=training_args,
tokenizer=self.tokenizer,
compute_metrics=model_utils.compute_metrics,
callbacks=[
model_utils.TensorBoardCallbackWrap,
model_utils.MLflowCallbackWrap
],
)
self.trainer.remove_callback(TensorBoardCallback)
self.trainer.remove_callback(MLflowCallback)
train_result = self.trainer.train(
resume_from_checkpoint=checkpoint)
metrics = train_result.metrics
self.model.save_pretrained(f"{model_name}/model")
self.tokenizer.save_pretrained(f"{model_name}/tokenizer")
self.save_model_mlflow(
model_name=model_name,
model_out_dir=f"{model_name}/model",
tokenizer_out_dir=f"{model_name}/tokenizer",
)
max_train_samples = len(train_data)
metrics["train_samples"] = min(max_train_samples, len(train_data))
self.trainer.log_metrics("train", metrics)
self.trainer.save_metrics("train", metrics)
self.trainer.save_state()
if args.fine_tune:
output_fine_tune = fine_tune.fine_tune(
models, dataset, train_args['fine_tune_args'])
print(f"Train:\t{output['acc']['train']:.5f}->{output_fine_tune['acc']['train']:.5f}")
print(f"Val:\t{output['acc']['val']:.5f}->{output_fine_tune['acc']['val']:.5f}")
print(f"Test:\t{output['acc']['test']:.5f}->{output_fine_tune['acc']['test']:.5f}")
output = output_fine_tune
_attach(trial, output)
return output['acc']['val']
# Optimize
study = optuna.create_study(
direction='maximize', pruner=pruners.MedianPruner())
study.optimize(lambda trial: objective(trial, args), n_trials=args.n_trials)
def _save(out_dir, study):
os.makedirs(out_dir, exist_ok=True)
out_dir = Path(out_dir)
best_params = study.best_trial.params
print('Best Trial: ', best_params)
with open(out_dir / 'best_params.json', 'w') as f:
json.dump(best_params, f)
attrs = study.best_trial.user_attrs
accs = {
'train': attrs['acc_train'],
