def objective(trial: Trial) -> float:
trial.set_user_attr('method', 'hope')
classifier = classifier_type
if classifier is None:
classifier = trial.suggest_categorical('classifier',
['SVM', 'EN', 'RF', 'LR'])
else:
trial.set_user_attr('classifier', classifier)
dimensions = trial.suggest_int('dimensions', dimensions_range[0],
dimensions_range[1])
# Set the inner trial seed
_set_trial_seed(trial)
model = embed_train.train_embed_hope(
train_graph_filename=train_graph_filename,
dimensions=dimensions,
weighted=weighted,
)
return predict_and_evaluate(
model=model,
graph=graph,
graph_train=graph_train,
testing_pos_edges=testing_pos_edges,
seed=seed,
trial=trial,
labels=labels,
node_list=node_list,
classifier_type=classifier,
prediction_task=prediction_task,
)
def objective_func(trial: optuna.Trial) -> float:
start = time.time()
params = dict(**self._suggest(trial, param_prefix),
**self.fixed_params)
self.logger.info("Trial %s:", trial.number)
self.logger.info("parameter = %s", params)
arg, parameters = self.get_model_arguments(**params)
recommender = self.recommender_class(self._data, *arg,
**parameters)
recommender.learn_with_optimizer(self.val_evaluator, trial)
score = self.val_evaluator.get_score(recommender)
end = time.time()
time_spent = end - start
self.logger.info(
"Config %d obtained the following scores: %s within %f seconds.",
trial.number,
score,
time_spent,
)
val_score = score[self.val_evaluator.target_metric.name]
# max_epoch will be stored in learnt_config
for param_name, param_val in recommender.learnt_config.items():
trial.set_user_attr(param_name, param_val)
add_score_to_trial(trial, score, self.val_evaluator.cutoff)
return -val_score
def objective(trial: Trial) -> float:
trial.set_user_attr('method', 'sdne')
classifier = classifier_type
if classifier is None:
classifier = trial.suggest_categorical('classifier',
['SVM', 'EN', 'RF', 'LR'])
else:
trial.set_user_attr('classifier', classifier)
alpha = trial.suggest_uniform('alpha', 0, 0.4)
beta = trial.suggest_int('beta', 0, 30)
epochs = trial.suggest_categorical('epochs', [5, 10, 15, 20, 25, 30])
# Set the inner trial seed
_set_trial_seed(trial)
model = embed_train.train_embed_sdne(
train_graph_filename=train_graph_filename,
alpha=alpha,
beta=beta,
epochs=epochs,
weighted=weighted,
)
return predict_and_evaluate(
prediction_task=prediction_task,
model=model,
graph=graph,
graph_train=graph_train,
testing_pos_edges=testing_pos_edges,
seed=study_seed,
trial=trial,
labels=labels,
node_list=node_list,
classifier_type=classifier,
)
def _objective(self, trial: optuna.Trial):
kwargs = self.get_parameters(trial)
step = 0
result = None
for df in self.data:
step += 1
result, stats, cerebro, thestrats = self.backtest(df, **kwargs)
if result is None:
raise optuna.exceptions.TrialPruned("no result")
trial.report(result, step)
for k in stats.keys():
trial.set_user_attr(f'backtest_{k}', stats[k])
if trial.should_prune():
raise optuna.exceptions.TrialPruned("should_prune")
# result = self.backtest(self.data[0], **kwargs)
if result is None:
raise optuna.exceptions.TrialPruned("no result")
return result
def f(trial: Trial) -> float:
x = trial.suggest_int("x", 1, 1)
y = trial.suggest_categorical("y", (2.5, ))
trial.set_user_attr("train_loss", 3)
raise ValueError()
return x + y # 3.5
def objective(trial: optuna.Trial) -> float:
x = trial.suggest_float("x", -10, 10)
y = trial.suggest_float("y", -10, 10)
trial.report(x, 0)
trial.report(y, 1)
trial.set_user_attr("x", x)
trial.set_system_attr("y", y)
return f(x, y)
def f(trial: Trial) -> float:
x = trial.suggest_int("x", 1, 1)
y = trial.suggest_categorical("y", (2.5, ))
assert isinstance(y, float)
trial.set_user_attr("train_loss", 3)
trial.set_system_attr("foo", "bar")
value = x + y # 3.5
# Test reported intermediate values, although it in practice is not "intermediate".
trial.report(value, step=0)
return value
def evaluate_nav_goal(self, goal: MoveBaseActionGoal, trial: optuna.Trial):
self.goal_publisher.publish(goal)
start_time = self.sim.get_time()
# wait till time for test is up or stopping condition has been reached
while rclpy.ok():
passed_time = self.sim.get_time() - start_time
# test timeout
if passed_time > self.time_limit:
goal_quat = goal.goal.target_pose.pose.orientation
goal_rpy = transforms3d.euler.quat2euler(
(goal_quat.w, goal_quat.x, goal_quat.y, goal_quat.z))
trial.set_user_attr(
'time_limit_reached',
(goal.goal.target_pose.pose.position.x,
goal.goal.target_pose.pose.position.y, goal_rpy[2]))
return False, self.time_limit
# test if the robot has fallen down
pos, rpy = self.sim.get_robot_pose_rpy()
if abs(rpy[0]) > math.radians(45) or abs(rpy[1]) > math.radians(
45) or pos[2] < self.trunk_height / 2:
# add extra information to trial
# todo include max vel parameter or something similar
goal_quat = goal.goal.target_pose.pose.orientation
goal_rpy = transforms3d.euler.quat2euler(
(goal_quat.w, goal_quat.x, goal_quat.y, goal_quat.z))
trial.set_user_attr(
'fallen',
(goal.goal.target_pose.pose.position.x,
goal.goal.target_pose.pose.position.y, goal_rpy[2]))
return True, self.time_limit
# todo add test if the robot did not move for more than x seconds, to find oszillations more easily
# test if we are successful
if self.is_goal_reached():
return False, passed_time
# give time to other algorithms to compute their responses
# use wall time, as ros time is standing still
time.sleep(0.01)
self.sim.step_sim()
# was stopped before finishing
raise optuna.exceptions.OptunaError()
def objective(trial: Trial) -> float:
trial.set_user_attr('method', 'node2vec')
classifier = classifier_type
if classifier is None:
classifier = trial.suggest_categorical('classifier',
['SVM', 'EN', 'RF', 'LR'])
else:
trial.set_user_attr('classifier', classifier)
dimensions = trial.suggest_int('dimensions', dimensions_range[0],
dimensions_range[1])
walk_length = trial.suggest_categorical('walk_length',
[8, 16, 32, 64, 128])
number_walks = trial.suggest_categorical('number_walks',
[8, 16, 32, 64, 128, 256])
window_size = trial.suggest_int('window_size', 2, 6)
p = trial.suggest_uniform('p', 0, 4.0)
q = trial.suggest_uniform('q', 0, 4.0)
# Set the inner trial seed
_set_trial_seed(trial)
model = embed_train.train_embed_node2vec(
train_graph_filename=train_graph_filename,
dimensions=dimensions,
walk_length=walk_length,
number_walks=number_walks,
window_size=window_size,
p=p,
q=q,
weighted=weighted,
)
return predict_and_evaluate(
prediction_task=prediction_task,
model=model,
graph=graph,
graph_train=graph_train,
testing_pos_edges=testing_pos_edges,
seed=study_seed,
trial=trial,
labels=labels,
node_list=node_list,
classifier_type=classifier,
)
def objective(trial: Trial):
# x = trial.suggest_uniform('x', -10, 10)
# return (x - 2) ** 2
clean_token_count_limit = int(
trial.suggest_discrete_uniform('clean_token_count_limit', 20, 60000,
1))
trial.set_user_attr(
'run',
{
'loss': 1.2,
# -- store other results like this
'os_uname': os.uname(),
'clean_token_count_limit': clean_token_count_limit,
'attachments': {
'info': 'info',
'output': 'output'
}
})
if clean_token_count_limit < 10000:
return None
return float(clean_token_count_limit)**2
def objective_func(trial: optuna.Trial) -> float:
start = time.time()
params = dict(**self._suggest(trial), **self.fixed_params)
self.logger.info("Trial %s:", trial.number)
self.logger.info("parameter = %s", params)
arg, parameters = self.get_model_arguments(**params)
recommender = self.recommender_class(self._data, *arg,
**parameters)
recommender.learn_with_optimizer(self.val_evaluator, trial)
score = self.val_evaluator.get_score(recommender)
end = time.time()
time_spent = end - start
self.logger.info(
"Config %d obtained the following scores: %s within %f seconds.",
trial.number,
score,
time_spent,
)
val_score = score[self.val_evaluator.target_metric.name]
# max_epoch will be stored in learnt_config
for param_name, param_val in recommender.learnt_config.items():
trial.set_user_attr(param_name, param_val)
score_history: List[Tuple[int, Dict[
str, float]]] = trial.study.user_attrs.get("scores", [])
score_history.append((
trial.number,
{
f"{score_name}@{self.val_evaluator.cutoff}": score_value
for score_name, score_value in score.items()
},
))
trial.study.set_user_attr("scores", score_history)
return -val_score
def objective(trial: Trial) -> float:
trial.set_user_attr('method', 'grarep')
classifier = classifier_type
if classifier is None:
classifier = trial.suggest_categorical('classifier',
['SVM', 'EN', 'RF', 'LR'])
else:
trial.set_user_attr('classifier', classifier)
# TODO: need to choose kstep in which it can divide the dimension
dimensions = trial.suggest_int('dimensions', dimensions_range[0],
dimensions_range[1])
kstep = trial.suggest_int('kstep', 1, 7)
if dimensions % kstep != 0:
raise optuna.structs.TrialPruned()
# Set the inner trial seed
_set_trial_seed(trial)
model = embed_train.train_embed_grarep(
train_graph_filename=train_graph_filename,
dimensions=dimensions,
kstep=kstep,
weighted=weighted,
)
return predict_and_evaluate(
prediction_task=prediction_task,
model=model,
graph=graph,
graph_train=graph_train,
testing_pos_edges=testing_pos_edges,
seed=study_seed,
trial=trial,
labels=labels,
node_list=node_list,
classifier_type=classifier,
)
def objective(trial: Trial):
# config dictionaries are modified during instantiation, so we need to deepcopy the originals to not lose them
config = copy.deepcopy(base_config)
if with_clustering:
assert type(classifier) is not str
config["classifier"] = copy.deepcopy(classifier)
config["clustering"] = sample_clustering_config_with_optuna(trial)
else:
assert type(classifier) is str
config["classifier"] = sample_classifier_config_with_optuna(trial, classifier)
# store the config in the trial so that we can retrieve it later and use it to instantiate the best model -
# don't ask me why it needs to be stored as a string, using the dict object did not work
trial.set_user_attr("config", json.dumps(config))
# instantiate feature pipeline and classifier, transform the features
pipeline, scoring = instantiate_pipeline(logger,
config,
with_clustering=with_clustering,
use_caching=True,
scorer_should_return_single_scalar=True,
serialization_dir=serialization_dir / "pipeline" / f"trial_{trial.number:03}")
cv = RepeatedKFold(n_splits=cv_num_splits,
n_repeats=cv_num_repeats,
random_state=optimization_random_seed)
f1_scores_cv = cross_val_score(estimator=pipeline,
X=train_X,
y=train_y,
n_jobs=cv_n_jobs,
cv=cv,
scoring=scoring,
verbose=0)
mean_f1 = f1_scores_cv.mean()
return mean_f1
def objective(self, trial: Trial, return_model=False):
X, y = self.make_xy(trial)
# X, y = self.X, self.y
loss_cv_train = []
cv_preds = np.zeros_like(y)
cv_preds.fill(np.nan)
models = []
for step, (train_idx, val_idx) in enumerate(self.split(X, y)):
X_train = X[train_idx]
X_val = X[val_idx]
y_train = y[train_idx]
y_val = y[val_idx]
model = self.fit(trial, X_train, y_train, X_val, y_val, step)
y_train_preds = model.predict(X_train)
y_val_preds = model.predict(X_val)
# from IPython.core.debugger import Pdb;
# Pdb().set_trace()
cv_preds[val_idx] = y_val_preds
mask_done = ~np.isnan(cv_preds)
intermediate_loss_train = self.loss_fn(y_train, y_train_preds, trial)
intermediate_loss_val = self.loss_fn(y[mask_done], cv_preds[mask_done], trial)
loss_cv_train.append(intermediate_loss_train)
trial.report(intermediate_loss_val, step)
if self.prune and trial.should_prune(step):
raise optuna.structs.TrialPruned()
models.append(model)
if self.is_one_cv:
break
mask_done = ~np.isnan(cv_preds)
loss_train = float(np.mean(loss_cv_train))
loss_val = float(self.loss_fn(y[mask_done], cv_preds[mask_done], trial))
trial.set_user_attr('train_loss', loss_train)
trial.set_user_attr('val_loss', loss_val)
trial.set_user_attr('is_one_cv', int(self.is_one_cv))
self.on_after_trial(trial, models, cv_preds, loss_val)
if return_model:
return loss_val, models, cv_preds
return loss_val
def add_suggest(trial: optuna.Trial, user_attrs={}):
"""
Add hyperparam ranges to an optuna trial and typical user attrs.
Usage:
trial = optuna.trial.FixedTrial(
params={
'hidden_size': 128,
}
)
trial = add_suggest(trial)
trainer = pl.Trainer()
model = LSTM_PL(dict(**trial.params, **trial.user_attrs), dataset_train,
dataset_test, cache_base_path, norm)
trainer.fit(model)
"""
trial.suggest_loguniform("learning_rate", 1e-6, 1e-2)
trial.suggest_uniform("attention_dropout", 0, 0.75)
# we must have nhead<==hidden_size
# so nhead_power.max()<==hidden_size_power.min()
trial.suggest_discrete_uniform("hidden_size_power", 4, 10, 1)
trial.suggest_discrete_uniform("hidden_out_size_power", 4, 9, 1)
trial.suggest_discrete_uniform("nhead_power", 1, 4, 1)
trial.suggest_int("nlayers", 1, 12)
trial.suggest_categorical("use_lstm", [False, True])
trial.suggest_categorical("agg", ['last', 'max', 'mean', 'all'])
user_attrs_default = {
"batch_size": 16,
"grad_clip": 40,
"max_nb_epochs": 200,
"num_workers": 4,
"num_extra_target": 24 * 4,
"vis_i": "670",
"num_context": 24 * 4,
"input_size": 18,
"input_size_decoder": 17,
"context_in_target": False,
"output_size": 1,
"patience": 3,
'min_std': 0.005,
}
[trial.set_user_attr(k, v) for k, v in user_attrs_default.items()]
[trial.set_user_attr(k, v) for k, v in user_attrs.items()]
return trial
def sample_a2c_params(trial: optuna.Trial):
"""Sampler for A2C hyperparameters."""
gamma = 1.0 - trial.suggest_float("gamma", 0.0001, 0.1, log=True)
max_grad_norm = trial.suggest_float("max_grad_norm", 0.3, 5.0, log=True)
gae_lambda = 1.0 - trial.suggest_float("gae_lambda", 0.001, 0.2, log=True)
n_steps = 2**trial.suggest_int("exponent_n_steps", 3, 10)
learning_rate = trial.suggest_float("lr", 1e-5, 1, log=True)
ent_coef = trial.suggest_float("ent_coef", 0.00000001, 0.1, log=True)
ortho_init = trial.suggest_categorical("ortho_init", [False, True])
net_arch = trial.suggest_categorical("net_arch", ["tiny", "small"])
activation_fn = trial.suggest_categorical("activation_fn",
["tanh", "relu"])
# Display true values
trial.set_user_attr("gamma_", gamma)
trial.set_user_attr("gae_lambda_", gae_lambda)
trial.set_user_attr("n_steps", n_steps)
net_arch = [{
"pi": [64],
"vf": [64]
} if net_arch == "tiny" else {
"pi": [64, 64],
"vf": [64, 64]
}]
activation_fn = {
"tanh": nn.Tanh,
"relu": nn.ReLU,
}[activation_fn]
return {
"n_steps": n_steps,
"gamma": gamma,
"gae_lambda": gae_lambda,
"learning_rate": learning_rate,
"ent_coef": ent_coef,
"max_grad_norm": max_grad_norm,
"policy_kwargs": {
"net_arch": net_arch,
"activation_fn": activation_fn,
"ortho_init": ortho_init,
},
}
def __call__(self, trial: Trial) -> Optional[float]:
"""Suggest parameters then train the model."""
if self.model_kwargs is not None:
problems = [
x for x in ("loss", "regularizer", "optimizer", "lr_scheduler",
"training", "negative_sampler", "stopper")
if x in self.model_kwargs
]
if problems:
raise ValueError(
f"model_kwargs should not have: {problems}. {self}")
# 2. Model
_model_kwargs = _get_kwargs(
trial=trial,
prefix="model",
default_kwargs_ranges=self.model.hpo_default,
kwargs=self.model_kwargs,
kwargs_ranges=self.model_kwargs_ranges,
)
try:
loss_default_kwargs_ranges = self.loss.hpo_default
except AttributeError:
logger.warning(
"using a loss function with no hpo_default field: %s",
self.loss)
loss_default_kwargs_ranges = {}
# 3. Loss
_loss_kwargs = _get_kwargs(
trial=trial,
prefix="loss",
default_kwargs_ranges=loss_default_kwargs_ranges,
kwargs=self.loss_kwargs,
kwargs_ranges=self.loss_kwargs_ranges,
)
# 4. Regularizer
_regularizer_kwargs: Optional[Mapping[str, Any]]
if self.regularizer is None:
_regularizer_kwargs = {}
else:
_regularizer_kwargs = _get_kwargs(
trial=trial,
prefix="regularizer",
default_kwargs_ranges=self.regularizer.hpo_default,
kwargs=self.regularizer_kwargs,
kwargs_ranges=self.regularizer_kwargs_ranges,
)
# 5. Optimizer
_optimizer_kwargs = _get_kwargs(
trial=trial,
prefix="optimizer",
default_kwargs_ranges=optimizers_hpo_defaults[self.optimizer],
kwargs=self.optimizer_kwargs,
kwargs_ranges=self.optimizer_kwargs_ranges,
)
# 5.1 Learning Rate Scheduler
_lr_scheduler_kwargs: Optional[Mapping[str, Any]] = None
if self.lr_scheduler is not None:
_lr_scheduler_kwargs = _get_kwargs(
trial=trial,
prefix="lr_scheduler",
default_kwargs_ranges=lr_schedulers_hpo_defaults[
self.lr_scheduler],
kwargs=self.lr_scheduler_kwargs,
kwargs_ranges=self.lr_scheduler_kwargs_ranges,
)
_negative_sampler_kwargs: Mapping[str, Any]
if self.training_loop is not SLCWATrainingLoop:
_negative_sampler_kwargs = {}
elif self.negative_sampler is None:
raise ValueError(
"Negative sampler class must be made explicit when training under sLCWA"
)
else:
# TODO this fixes the issue for negative samplers, but does not generally address it.
# For example, some of them obscure their arguments with **kwargs, so should we look
# at the parent class? Sounds like something to put in class resolver by using the
# inspect module. For now, this solution will rely on the fact that the sampler is a
# direct descendent of a parent NegativeSampler
direct_params = inspect.signature(self.negative_sampler).parameters
parent_params = inspect.signature(
self.negative_sampler.__bases__[0]).parameters
valid_keys = set(direct_params).union(parent_params) - {"kwargs"}
invalid_keys = set(self.negative_sampler_kwargs_ranges
or []) - valid_keys
if invalid_keys:
raise ExtraKeysError(invalid_keys)
_negative_sampler_kwargs = _get_kwargs(
trial=trial,
prefix="negative_sampler",
default_kwargs_ranges=self.negative_sampler.hpo_default,
kwargs=self.negative_sampler_kwargs,
kwargs_ranges=self.negative_sampler_kwargs_ranges,
)
_training_kwargs = _get_kwargs(
trial=trial,
prefix="training",
default_kwargs_ranges=self.training_loop.hpo_default,
kwargs=self.training_kwargs,
kwargs_ranges=self.training_kwargs_ranges,
)
# create result tracker to allow to gracefully close failed trials
result_tracker = tracker_resolver.make(
query=self.result_tracker, pos_kwargs=self.result_tracker_kwargs)
_stopper_kwargs = dict(self.stopper_kwargs or {})
if self.stopper is not None and issubclass(self.stopper, EarlyStopper):
self._update_stopper_callbacks(_stopper_kwargs,
trial,
metric=self.metric,
result_tracker=result_tracker)
try:
result = pipeline(
# 1. Dataset
dataset=self.dataset,
dataset_kwargs=self.dataset_kwargs,
training=self.training,
testing=self.testing,
validation=self.validation,
evaluation_entity_whitelist=self.evaluation_entity_whitelist,
evaluation_relation_whitelist=self.
evaluation_relation_whitelist,
# 2. Model
model=self.model,
model_kwargs=_model_kwargs,
# 3. Loss
loss=self.loss,
loss_kwargs=_loss_kwargs,
# 4. Regularizer
regularizer=self.regularizer,
regularizer_kwargs=_regularizer_kwargs,
# 5. Optimizer
optimizer=self.optimizer,
optimizer_kwargs=_optimizer_kwargs,
clear_optimizer=True,
# 5.1 Learning Rate Scheduler
lr_scheduler=self.lr_scheduler,
lr_scheduler_kwargs=_lr_scheduler_kwargs,
# 6. Training Loop
training_loop=self.training_loop,
negative_sampler=self.negative_sampler,
negative_sampler_kwargs=_negative_sampler_kwargs,
# 7. Training
training_loop_kwargs=self.training_loop_kwargs,
training_kwargs=_training_kwargs,
stopper=self.stopper,
stopper_kwargs=_stopper_kwargs,
# 8. Evaluation
evaluator=self.evaluator,
evaluator_kwargs=self.evaluator_kwargs,
evaluation_kwargs=self.evaluation_kwargs,
filter_validation_when_testing=self.
filter_validation_when_testing,
# 9. Tracker
result_tracker=result_tracker,
result_tracker_kwargs=None,
# Misc.
use_testing_data=False, # use validation set during HPO!
device=self.device,
)
except (MemoryError, RuntimeError) as e:
# close run in result tracker
result_tracker.end_run(success=False)
# raise the error again (which will be catched in study.optimize)
raise e
else:
if self.save_model_directory:
model_directory = os.path.join(self.save_model_directory,
str(trial.number))
os.makedirs(model_directory, exist_ok=True)
result.save_to_directory(model_directory)
trial.set_user_attr("random_seed", result.random_seed)
for k, v in result.metric_results.to_flat_dict().items():
trial.set_user_attr(k, v)
return result.metric_results.get_metric(self.metric)
def __call__(self, trial: Trial) -> Optional[float]:
"""Suggest parameters then train the model."""
if self.model_kwargs is not None:
problems = [
x
for x in ('loss', 'regularizer', 'optimizer', 'training', 'negative_sampler', 'stopper')
if x in self.model_kwargs
]
if problems:
raise ValueError(f'model_kwargs should not have: {problems}. {self}')
# 2. Model
_model_kwargs = _get_kwargs(
trial=trial,
prefix='model',
default_kwargs_ranges=self.model.hpo_default,
kwargs=self.model_kwargs,
kwargs_ranges=self.model_kwargs_ranges,
)
try:
loss_default_kwargs_ranges = self.loss.hpo_default
except AttributeError:
logger.warning('using a loss function with no hpo_default field: %s', self.loss)
loss_default_kwargs_ranges = {}
# 3. Loss
_loss_kwargs = _get_kwargs(
trial=trial,
prefix='loss',
default_kwargs_ranges=loss_default_kwargs_ranges,
kwargs=self.loss_kwargs,
kwargs_ranges=self.loss_kwargs_ranges,
)
# 4. Regularizer
_regularizer_kwargs: Optional[Mapping[str, Any]]
if self.regularizer is None:
_regularizer_kwargs = {}
else:
_regularizer_kwargs = _get_kwargs(
trial=trial,
prefix='regularizer',
default_kwargs_ranges=self.regularizer.hpo_default,
kwargs=self.regularizer_kwargs,
kwargs_ranges=self.regularizer_kwargs_ranges,
)
# 5. Optimizer
_optimizer_kwargs = _get_kwargs(
trial=trial,
prefix='optimizer',
default_kwargs_ranges=optimizers_hpo_defaults[self.optimizer],
kwargs=self.optimizer_kwargs,
kwargs_ranges=self.optimizer_kwargs_ranges,
)
_negative_sampler_kwargs: Mapping[str, Any]
if self.training_loop is not SLCWATrainingLoop:
_negative_sampler_kwargs = {}
else:
_negative_sampler_kwargs = _get_kwargs(
trial=trial,
prefix='negative_sampler',
default_kwargs_ranges={} if self.negative_sampler is None else self.negative_sampler.hpo_default,
kwargs=self.negative_sampler_kwargs,
kwargs_ranges=self.negative_sampler_kwargs_ranges,
)
_training_kwargs = _get_kwargs(
trial=trial,
prefix='training',
default_kwargs_ranges=self.training_loop.hpo_default,
kwargs=self.training_kwargs,
kwargs_ranges=self.training_kwargs_ranges,
)
_stopper_kwargs = dict(self.stopper_kwargs or {})
if self.stopper is not None and issubclass(self.stopper, EarlyStopper):
self._update_stopper_callbacks(_stopper_kwargs, trial)
try:
result = pipeline(
# 1. Dataset
dataset=self.dataset,
dataset_kwargs=self.dataset_kwargs,
training=self.training,
testing=self.testing,
validation=self.validation,
evaluation_entity_whitelist=self.evaluation_entity_whitelist,
evaluation_relation_whitelist=self.evaluation_relation_whitelist,
# 2. Model
model=self.model,
model_kwargs=_model_kwargs,
# 3. Loss
loss=self.loss,
loss_kwargs=_loss_kwargs,
# 4. Regularizer
regularizer=self.regularizer,
regularizer_kwargs=_regularizer_kwargs,
clear_optimizer=True,
# 5. Optimizer
optimizer=self.optimizer,
optimizer_kwargs=_optimizer_kwargs,
# 6. Training Loop
training_loop=self.training_loop,
negative_sampler=self.negative_sampler,
negative_sampler_kwargs=_negative_sampler_kwargs,
# 7. Training
training_loop_kwargs=self.training_loop_kwargs,
training_kwargs=_training_kwargs,
stopper=self.stopper,
stopper_kwargs=_stopper_kwargs,
# 8. Evaluation
evaluator=self.evaluator,
evaluator_kwargs=self.evaluator_kwargs,
evaluation_kwargs=self.evaluation_kwargs,
filter_validation_when_testing=self.filter_validation_when_testing,
# 9. Tracker
result_tracker=self.result_tracker,
result_tracker_kwargs=self.result_tracker_kwargs,
# Misc.
use_testing_data=False, # use validation set during HPO!
device=self.device,
)
except (MemoryError, RuntimeError) as e:
trial.set_user_attr('failure', str(e))
# Will trigger Optuna to set the state of the trial as failed
return None
else:
if self.save_model_directory:
model_directory = os.path.join(self.save_model_directory, str(trial.number))
os.makedirs(model_directory, exist_ok=True)
result.save_to_directory(model_directory)
trial.set_user_attr('random_seed', result.random_seed)
for k, v in result.metric_results.to_flat_dict().items():
trial.set_user_attr(k, v)
return result.metric_results.get_metric(self.metric)
def f(trial: Trial) -> float:
trial.set_user_attr("train_accuracy", 1)
assert trial.user_attrs["train_accuracy"] == 1
return 0.0
def _set_trial_seed(trial: Trial) -> int:
new_seed = random.randrange(sys.maxsize)
np.random.seed(new_seed)
random.seed(new_seed)
trial.set_user_attr('inner_seed', new_seed)
return new_seed
def __call__(self, trial: optuna.Trial):
torch.manual_seed(self.seed)
random.seed(self.seed)
train_data_loader, val_data_loader = datasets2data_loaders(
self.train_set, self.val_set, test_set=None, num_workers=1)
epochs = self.hydra_cfg['parameters']['epochs']
# Calculate an objective value by using the extra arguments.
model = SupervisedFastText(V=self.dictionary.size_total_vocab,
num_classes=len(
self.dictionary.label_vocab),
embedding_dim=self.dim,
pretrained_emb=self.pretrained_word_vectors,
freeze=self.is_freeze,
pooling=self.pooling).to(self.device)
initial_lr = trial.suggest_loguniform(
'lr', self.hydra_cfg['optuna']['lr_min'],
self.hydra_cfg['optuna']['lr_max'])
optimizer = optim.SGD(model.parameters(), lr=initial_lr)
# parameters for update learning rate
num_tokens = self.dictionary.num_words
learning_rate_schedule = self.hydra_cfg['parameters']['lr_update_rate']
total_num_processed_tokens_in_training = epochs * num_tokens
num_processed_tokens = 0
local_processed_tokens = 0
N = len(train_data_loader.dataset)
best_val_loss = np.finfo(0.).max
best_val_acc = np.finfo(0.).min
save_fname = os.getcwd() + '/' + '{}.pt'.format(
trial.number) # file name to store best model's weights
for epoch in range(epochs):
# begin training phase
sum_loss = 0.
correct = 0
model.train()
for sentence, label, n_tokens in train_data_loader:
sentence, label = sentence.to(self.device), label.to(
self.device)
optimizer.zero_grad()
output = model(sentence)
loss = F.nll_loss(output, label)
loss.backward()
optimizer.step()
pred = output.argmax(1, keepdim=False)
correct += pred.eq(label).sum().item()
sum_loss += loss.item()
# update learning rate
# ref: https://github.com/facebookresearch/fastText/blob/6d7c77cd33b23eec26198fdfe10419476b5364c7/src/fasttext.cc#L656
local_processed_tokens += n_tokens.item()
if local_processed_tokens > learning_rate_schedule:
num_processed_tokens += local_processed_tokens
local_processed_tokens = 0
progress = num_processed_tokens / total_num_processed_tokens_in_training
optimizer.param_groups[0]['lr'] = initial_lr * (1. -
progress)
train_loss = sum_loss / N
train_acc = correct / N
# end training phase
val_loss, val_acc = evaluation(model, self.device, val_data_loader)
progress = num_processed_tokens / total_num_processed_tokens_in_training # approximated progress
self.logger.info(
'\rProgress: {:.1f}% Avg. train loss: {:.4f}, train acc: {:.1f}%, '
'Avg. val loss: {:.4f}, val acc: {:.1f}%'.format(
progress * 100., train_loss, train_acc * 100, val_loss,
val_acc * 100))
if self.metric == 'loss':
trial.report(val_loss, epoch)
else:
trial.report(val_acc, epoch)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
# validation
is_saved_model = False
if self.metric == 'loss':
if best_val_loss > val_loss:
best_val_loss = val_loss
best_val_acc = val_acc
is_saved_model = True
else:
if best_val_acc < val_acc:
best_val_loss = val_loss
best_val_acc = val_acc
is_saved_model = True
if is_saved_model:
torch.save(model.state_dict(), save_fname)
trial.set_user_attr('val_loss', best_val_loss)
trial.set_user_attr('val_acc', best_val_acc)
trial.set_user_attr('model_path', save_fname)
if self.metric == 'loss':
return best_val_loss
else:
return best_val_acc
