def __init__( self, study_name, # type: str storage, # type: Union[str, storages.BaseStorage] sampler=None, # type: samplers.BaseSampler pruner=None, # type: pruners.BasePruner direction='minimize', # type: str ): # type: (...) -> None self.study_name = study_name self.storage = storages.get_storage(storage) self.sampler = sampler or samplers.TPESampler() self.pruner = pruner or pruners.MedianPruner() self.study_id = self.storage.get_study_id_from_name(study_name) self.logger = logging.get_logger(__name__) if direction == 'minimize': _direction = structs.StudyDirection.MINIMIZE elif direction == 'maximize': _direction = structs.StudyDirection.MAXIMIZE else: raise ValueError( 'Please set either \'minimize\' or \'maximize\' to direction.') # TODO(Yanase): Implement maximization. if _direction == structs.StudyDirection.MAXIMIZE: raise ValueError( 'Optimization direction of study {} is set to `MAXIMIZE`. ' 'Currently, Optuna supports `MINIMIZE` only.'.format( study_name)) self.storage.set_study_direction(self.study_id, _direction)
def __init__( self, study_name, # type: str storage, # type: Union[str, storages.BaseStorage] sampler=None, # type: samplers.BaseSampler pruner=None # type: pruners.BasePruner ): # type: (...) -> None self.study_name = study_name storage = storages.get_storage(storage) study_id = storage.get_study_id_from_name(study_name) super(Study, self).__init__(study_id, storage) self.sampler = sampler or samplers.TPESampler() self.pruner = pruner or pruners.MedianPruner() if (isinstance(self.sampler, samplers.TPESampler) and isinstance(self.pruner, pruners.HyperbandPruner)): msg = ( "The algorithm of TPESampler and HyperbandPruner might behave in a different way " "from the paper of Hyperband because the sampler uses all the trials including " "ones of brackets other than that of currently running trial") warnings.warn(msg, UserWarning) _logger.warning(msg) self._optimize_lock = threading.Lock()
def __init__( self, study_name, # type: str storage, # type: Union[str, storages.BaseStorage] sampler=None, # type: samplers.BaseSampler pruner=None, # type: pruners.BasePruner ): # type: (...) -> None self.study_name = study_name self.storage = storages.get_storage(storage) self.sampler = sampler or samplers.TPESampler() self.pruner = pruner or pruners.MedianPruner() self.study_id = self.storage.get_study_id_from_name(study_name) self.logger = logging.get_logger(__name__)
def __init__( self, study_name: str, storage: Union[str, storages.BaseStorage], sampler: Optional["samplers.BaseSampler"] = None, pruner: Optional[pruners.BasePruner] = None, ) -> None: self.study_name = study_name storage = storages.get_storage(storage) study_id = storage.get_study_id_from_name(study_name) super(Study, self).__init__(study_id, storage) self.sampler = sampler or samplers.TPESampler() self.pruner = pruner or pruners.MedianPruner() self._optimize_lock = threading.Lock() self._stop_flag = False
def __init__( self, study_name, # type: str storage, # type: Union[str, storages.BaseStorage] sampler=None, # type: samplers.BaseSampler pruner=None, # type: pruners.BasePruner ): # type: (...) -> None self.study_name = study_name storage = storages.get_storage(storage) study_id = storage.get_study_id_from_name(study_name) super(Study, self).__init__(study_id, storage) self.sampler = sampler or samplers.TPESampler() self.pruner = pruner or pruners.MedianPruner() self._optimize_lock = threading.Lock()
def __init__( self, study_name, # type: str storage, # type: Union[str, storages.BaseStorage] sampler=None, # type: samplers.BaseSampler pruner=None, # type: pruners.BasePruner force_garbage_collection=True, # type: bool ): # type: (...) -> None self.study_name = study_name storage = storages.get_storage(storage) study_id = storage.get_study_id_from_name(study_name) super(Study, self).__init__(study_id, storage) self.sampler = sampler or samplers.TPESampler() self.pruner = pruner or pruners.MedianPruner() self.logger = logging.get_logger(__name__) self._optimize_lock = threading.Lock() self.force_garbage_collection = force_garbage_collection
from optuna.testing.sampler import DeterministicRelativeSampler from optuna.testing.storage import StorageSupplier from optuna.trial import Trial from optuna.trial import TrialState try: import chainermn from chainermn.communicators.communicator_base import CommunicatorBase # NOQA _available = True except ImportError: _available = False STORAGE_MODES = ["sqlite"] PRUNER_INIT_FUNCS = [ lambda: pruners.MedianPruner(), lambda: pruners.SuccessiveHalvingPruner() ] class Func(object): def __init__(self) -> None: self.suggested_values: Dict[int, Dict[str, Any]] = {} def __call__(self, trial: ChainerMNTrial, comm: CommunicatorBase) -> float: x = trial.suggest_uniform("x", -10, 10) y = trial.suggest_loguniform("y", 20, 30) z = trial.suggest_categorical("z", (-1.0, 1.0)) self.suggested_values[trial.number] = {}
from types import TracebackType # NOQA from typing import Any # NOQA from typing import Callable # NOQA from typing import Dict # NOQA from typing import Optional # NOQA from typing import Type # NOQA try: import chainermn from chainermn.communicators.communicator_base import CommunicatorBase # NOQA _available = True except ImportError: _available = False STORAGE_MODES = ['new', 'common'] PRUNER_INIT_FUNCS = [lambda: pruners.MedianPruner(), lambda: pruners.SuccessiveHalvingPruner()] CACHE_MODES = [True, False] def setup_module(): # type: () -> None StorageSupplier.setup_common_tempfile() def teardown_module(): # type: () -> None StorageSupplier.teardown_common_tempfile()
def train( self, texts: List, tags: List, model_name: str = "bert_email_router", last_checkpoint: Union[str, bool] = None, experiment_name: str = None, hp_space: Optional[Callable[["optuna.Trial"], Dict[str, float]]] = None, ): """[train the model] Args: texts (List): [list of text sequences] tags (List): [list of list of tags to train on] model_name (str, optional): [name of the model, used for outputdir and mlflow name]. Defaults to "bert_ner_test". last_checkpoint (Union[str, bool], optional): [if boolean true then keep modelling from last checkpoint or if modelname string if we want to continue training on that checkpoint]. Defaults to None. experiment_name (str, optional): [name of mlflow experiment to start] """ train_texts, val_texts, train_tags, val_tags = model_utils.train_test_split( texts, tags, test_size=0.1) train_encodings = self.tokenizer( train_texts, is_split_into_words=False, return_offsets_mapping=False, padding="max_length", max_length=self.params.max_len, truncation=True, ) train_labels = [self.params.tag2id[tag] for tag in train_tags] train_data = self.NerDataset(train_encodings, train_labels) val_encodings = self.tokenizer( val_texts, is_split_into_words=False, return_offsets_mapping=False, padding="max_length", max_length=self.params.max_len, truncation=True, ) val_labels = [self.params.tag2id[tag] for tag in val_tags] val_data = self.NerDataset(val_encodings, val_labels) num_train_steps = int( len(train_data) / self.params.batch_size * self.params.epochs) num_warmup_steps = int(num_train_steps * self.params.warmup_proportion) training_args = self.set_trainings_args( model_name=model_name, num_warmup_steps=num_warmup_steps) if experiment_name is not None: mlflow.set_experiment(experiment_name) if hp_space: self.trainer = Trainer( model_init=self.load_model, train_dataset=train_data, eval_dataset=val_data, args=training_args, tokenizer=self.tokenizer, compute_metrics=model_utils.compute_metrics, callbacks=[ model_utils.TensorBoardCallbackWrap, model_utils.MLflowCallbackWrap ], ) self.trainer.remove_callback(TensorBoardCallback) self.trainer.remove_callback(MLflowCallback) best_trial = self.trainer.hyperparameter_search( hp_space=hp_space, direction= "minimize", # probably should be minimize (now it's maximize) backend="optuna", n_trials=settings.N_TRIALS, # number of trials pruner=pruners.MedianPruner(n_startup_trials=5, n_warmup_steps=30, interval_steps=10), hp_name=model_utils.hp_name_optuna, ) metrics = {} metrics["loss"] = best_trial.objective logger.info( f"Hyperparameter optimization finished: Best hyperparameters are {best_trial.hyperparameters}" ) else: if last_checkpoint is not None: checkpoint = last_checkpoint else: checkpoint = None self.trainer = Trainer( model=self.model, train_dataset=train_data, eval_dataset=val_data, args=training_args, tokenizer=self.tokenizer, compute_metrics=model_utils.compute_metrics, callbacks=[ model_utils.TensorBoardCallbackWrap, model_utils.MLflowCallbackWrap ], ) self.trainer.remove_callback(TensorBoardCallback) self.trainer.remove_callback(MLflowCallback) train_result = self.trainer.train( resume_from_checkpoint=checkpoint) metrics = train_result.metrics self.model.save_pretrained(f"{model_name}/model") self.tokenizer.save_pretrained(f"{model_name}/tokenizer") self.save_model_mlflow( model_name=model_name, model_out_dir=f"{model_name}/model", tokenizer_out_dir=f"{model_name}/tokenizer", ) max_train_samples = len(train_data) metrics["train_samples"] = min(max_train_samples, len(train_data)) self.trainer.log_metrics("train", metrics) self.trainer.save_metrics("train", metrics) self.trainer.save_state()
if args.fine_tune: output_fine_tune = fine_tune.fine_tune( models, dataset, train_args['fine_tune_args']) print(f"Train:\t{output['acc']['train']:.5f}->{output_fine_tune['acc']['train']:.5f}") print(f"Val:\t{output['acc']['val']:.5f}->{output_fine_tune['acc']['val']:.5f}") print(f"Test:\t{output['acc']['test']:.5f}->{output_fine_tune['acc']['test']:.5f}") output = output_fine_tune _attach(trial, output) return output['acc']['val'] # Optimize study = optuna.create_study( direction='maximize', pruner=pruners.MedianPruner()) study.optimize(lambda trial: objective(trial, args), n_trials=args.n_trials) def _save(out_dir, study): os.makedirs(out_dir, exist_ok=True) out_dir = Path(out_dir) best_params = study.best_trial.params print('Best Trial: ', best_params) with open(out_dir / 'best_params.json', 'w') as f: json.dump(best_params, f) attrs = study.best_trial.user_attrs accs = { 'train': attrs['acc_train'],