def create_sampler(sampler_mode: str) -> BaseSampler:
if sampler_mode == "random":
return RandomSampler()
elif sampler_mode == "tpe":
return TPESampler()
elif sampler_mode == "cmaes":
return CmaEsSampler()
else:
assert False
def optuna_search_evo():
try:
if path.exists("study_evo.pkl"):
study = joblib.load('study_evo.pkl')
else:
study = optuna.create_study(direction='maximize', sampler=CmaEsSampler())
study.optimize(objective_evo, n_trials=300)
finally:
joblib.dump(study, 'study_evo.pkl')
print('Optuna search done...')
print('Best trial:')
print(study.best_trial)
print('Best params:')
print(study.best_params)
print('Best value:')
print(study.best_value)
def optimize(self) -> TuningResult:
"""
Method performs a hyperparameter optimization run according to the selected HPO-method.
:return: result: TuningResult
TuningResult-object that contains the results of this optimization run.
"""
# Select the specified HPO-tuning method
if self.hpo_method == 'CMA-ES':
this_optimizer = CmaEsSampler(seed=self.random_seed)
elif self.hpo_method == 'TPE':
this_optimizer = TPESampler(seed=self.random_seed)
elif self.hpo_method == 'RandomSearch':
this_optimizer = RandomSampler(seed=self.random_seed)
else:
raise Exception('Unknown HPO-method!')
# Create a study object and specify the optimization direction
study_name = 'hpo_study'
study_storage = 'sqlite:///hpo.db'
# Optimize on the predefined n_func_evals and measure the wall clock times
# start_time = time.time()
self.times = [] # Initialize a list for saving the wall clock times
# Delete old study objects ('fresh start') >> otherwise the old results will be included
try:
optuna.delete_study(study_name, study_storage)
except:
print('No old optuna study objects found!')
# Use a warmstart configuration?
if self.do_warmstart == 'Yes':
try:
# Create a new study
warmstart_study = optuna.create_study(direction='minimize',
storage=study_storage,
study_name=study_name,
load_if_exists=False)
# Retrieve the warmstart hyperparameters for the ML-algorithm
warmstart_params = self.get_warmstart_configuration()
# Initialize a dictionary for the warmstart HP-configuration
warmstart_dict = {}
# Iterate over all hyperparameters of this ML-algorithm's tuned HP-space and append the default values
# to the dictionary
for i in range(len(self.hp_space)):
this_param = self.hp_space[i].name
this_warmstart_value = warmstart_params[this_param]
# For some HPs (e.g. max_depth of RF) the default value is None, although their typical dtype is
# different (e.g. int)
if this_warmstart_value is None and type(
self.hp_space[i]) == skopt.space.space.Integer:
# Try to impute these values by the mean value
this_warmstart_value = int(
0.5 *
(self.hp_space[i].low + self.hp_space[i].high))
# Add the warm start HP-value to the dictionary
warmstart_dict[this_param] = this_warmstart_value
# Enqueue a trial with the warm start HP-values
warmstart_study.enqueue_trial(params=warmstart_dict)
# Optimize to ensure that the warm start configuration is evaluated first (e.g. for parallel processes)
warmstart_study.optimize(func=self.objective, n_trials=1)
# Set flag to indicate that a warmstart took place
did_warmstart = True
except:
print('Warmstarting optuna failed!')
# Set flag to indicate that NO warmstart took place
did_warmstart = False
# No warmstart requested
else:
# Set flag to indicate that NO warmstart took place
did_warmstart = False
# Create a new study or reload the warmstart study (if available and requested)
study = optuna.create_study(sampler=this_optimizer,
direction='minimize',
study_name=study_name,
storage=study_storage,
load_if_exists=True)
# If a warm start took place, reduce the number of remaining function evaluations to ensure comparability
# (equal budgets)
if did_warmstart:
n_func_evals = self.n_func_evals - 1
else:
n_func_evals = self.n_func_evals
# Start the optimization
try:
study.optimize(func=self.objective,
n_trials=n_func_evals,
n_jobs=self.n_workers)
run_successful = True
# Algorithm crashed
except:
# Add a warning here
run_successful = False
# If the optimization run was successful, determine the optimization results
if run_successful:
# Create a TuningResult-object to store the optimization results
# Transformation of the results into a TuningResult-Object
all_trials = study.get_trials()
best_configuration = study.best_params
best_val_loss = study.best_value
start_times = [] # Start time of each trial
finish_times = [] # Finish time of each trial
# evaluation_ids = [] # Number the evaluations / iterations of this run
unsorted_losses = [] # Loss of each iteration
unsorted_configurations = () # HP-configuration of each iteration
# Number the evaluations / iterations of this run
evaluation_ids = list(range(1, len(all_trials) + 1))
for i in range(len(all_trials)):
start_times.append(all_trials[i].datetime_start)
finish_times.append(all_trials[i].datetime_complete)
# evaluation_ids.append(all_trials[i].number)
unsorted_losses.append(all_trials[i].value)
unsorted_configurations = unsorted_configurations + (
all_trials[i].params, )
abs_start_time = min(start_times) # start time of the first trial
unsorted_timestamps = []
for i in range(len(start_times)):
this_time = finish_times[
i] - abs_start_time # time difference to the start of the first trial
this_timestamp = this_time.total_seconds(
) # conversion into float value
unsorted_timestamps.append(this_timestamp)
wall_clock_time = max(unsorted_timestamps)
ids = list(range(1, len(all_trials) + 1))
temp_dict = {
'ids': ids,
'timestamps [finished]': unsorted_timestamps,
'losses': unsorted_losses,
'configurations': unsorted_configurations,
}
unsorted_df = pd.DataFrame.from_dict(data=temp_dict)
unsorted_df.set_index('ids', inplace=True)
# Sort DataFrame according to timestamps (necessary for multiprocessing)
sorted_df = unsorted_df.sort_values(by=['timestamps [finished]'],
ascending=True,
inplace=False)
timestamps = list(sorted_df['timestamps [finished]'])
losses = list(sorted_df['losses'])
configurations = tuple(sorted_df['configurations'])
# Optuna uses full budgets for its HPO methods
budget = [100.0] * len(losses)
# Compute the loss on the test set for the best found configuration
test_loss = self.train_evaluate_ml_model(params=best_configuration,
cv_mode=False,
test_mode=True)
# Run not successful (algorithm crashed)
else:
evaluation_ids, timestamps, losses, configurations, best_val_loss, best_configuration, wall_clock_time, \
test_loss, budget = self.impute_results_for_crash()
# Pass the results to a TuningResult-object
result = TuningResult(evaluation_ids=evaluation_ids,
timestamps=timestamps,
losses=losses,
configurations=configurations,
best_val_loss=best_val_loss,
best_configuration=best_configuration,
wall_clock_time=wall_clock_time,
test_loss=test_loss,
successful=run_successful,
did_warmstart=did_warmstart,
budget=budget)
return result
action='store_true')
parser.add_argument('--json',
help="Print best params in json",
action='store_true')
args = parser.parse_args()
seed = np.random.randint(2**32 - 1)
n_startup_trials = args.startup
if args.sampler == "TPE":
sampler = TPESampler(n_startup_trials=n_startup_trials,
seed=seed,
multivariate=False)
elif args.sampler == "CMAES":
sampler = CmaEsSampler(n_startup_trials=n_startup_trials, seed=seed)
else:
print("sampler not correctly specified")
if args.results_only:
study = optuna.load_study(study_name=args.name,
storage=args.storage,
sampler=sampler)
else:
study = optuna.create_study(study_name=args.name,
storage=args.storage,
direction='minimize',
sampler=sampler,
load_if_exists=True)
study.set_user_attr("sampler", args.sampler)
if args.robot == "wolfgang":
optimizer = keras.optimizers.RMSprop(learning_rate=lr)
model.compile(optimizer=optimizer, loss='mse')
model.fit(X_train, y_train, epochs=10, batch_size=128, validation_data=(X_val, y_val), verbose=1)
y_pred = model.predict(X_val)
val_loss = sqrt(mean_squared_error(y_val, y_pred))
return val_loss
# 3. Create a study object an optimize the objective function
if SAMPLER == 'TPE':
thisSampler = TPESampler()
elif SAMPLER == 'CMA-ES':
thisSampler = CmaEsSampler()
else:
thisSampler = RandomSampler()
if ALGORITHM == 'RandomForestRegressor':
thisObjective = rf_objective
elif ALGORITHM == 'Keras':
thisObjective = keras_objective
study = optuna.create_study(sampler=thisSampler, direction='minimize')
study.optimize(func=thisObjective, n_trials=100)
# >> test how to access the trials data
all_trials = study.get_trials() # List of Froze-Trial objects
df_trials = study.trials_dataframe() # Pandas DataFrame