def hyperparameter_search(n_jobs: int, params: dict, name: str = 'hyperparameter_search'): _, _, samples = data.pipeline(params['data']['n_input_steps'], params['data']['n_output_steps'], params['paths']['data']) datasets = data.get_datasets(samples, params['data']['n_input_steps']) # set up ax from ax.service.ax_client import AxClient ax_client = AxClient(enforce_sequential_optimization=False) # define hyperparameter bounds ax_client.create_experiment(name=name, parameters=[{ "name": "num_epochs", "type": "range", "bounds": [150, 200] }, { "name": "learning_rate", "type": "range", "bounds": [5e-4, 1e-3], "log_scale": True }, { "name": "batch_size", "type": "range", "bounds": [64, 1024] }, { "name": "variational_dropout_p", "type": "range", "bounds": [0.2, 0.5] }], objective_name='loss', minimize=True) for job in range(n_jobs): parameters, trial_index = ax_client.get_next_trial() ax_client.complete_trial(trial_index=trial_index, raw_data=train_evaluate( parameters, datasets=datasets)['loss']) print(f'Best parameters found after {n_jobs}:') print(ax_client.get_best_parameters()) ax_client.save_to_json_file()
opts.max_conv_size = parameters['max_conv_size'] opts.dense_kernel_size = parameters['dense_kernel_size'] opts.batch_size = 64 # parameters['batch_size'] opts.learning_rate = parameters['learning_rate'] opts.epochs = cmd_line_opts.epochs # max to run, we also use early stopping # run start_time = time.time() # final_loss = train.train_in_subprocess(opts) final_loss = train.train(opts) log_record.append(time.time() - start_time) log_record.append(final_loss) # complete trial if final_loss is None: print("ax trial", trial_index, "failed?") ax.log_trial_failure(trial_index=trial_index) else: ax.complete_trial(trial_index=trial_index, raw_data={'final_loss': (final_loss, 0)}) print("CURRENT_BEST", ax.get_best_parameters()) # flush log log_msg = "\t".join(map(str, log_record)) print(log_msg, file=log) print(log_msg) log.flush() # save ax state ax.save_to_json_file()
y_train_partial[train_index], X_train_partial[val_index], y_train_partial[val_index]) } for i in range(25): skf = StratifiedKFold(n_splits=10, random_state=7, shuffle=True) for train_index, val_index in skf.split(np.zeros(X_train_partial.shape[0]), y_train_partial): print("Train index: ", train_index, " shape: ", train_index.shape, "Validation index: ", val_index, "shape: ", val_index.shape) parameters, trial_index = ax_client.get_next_trial() ax_client.complete_trial(trial_index=trial_index, raw_data=evaluate(parameters)) ax_client.save_to_json_file() ax_client.get_trials_data_frame().sort_values('keras_cv') best_parameters, values = ax_client.get_best_parameters() # the best set of parameters. for k in best_parameters.items(): print(k) print() # the best score achieved. means, covariances = values print(means)
# for i in range(25): # parameters, trial_index = ax_client.get_next_trial() # # Local evaluation here can be replaced with deployment to external system. # ax_client.complete_trial(trial_index=trial_index, raw_data=evaluate(parameters)) # # _, trial_index = ax_client.get_next_trial() # ax_client.log_trial_failure(trial_index=trial_index) # # ax_client.get_trials_data_frame().sort_values('trial_index') # best_parameters, values = ax_client.get_best_parameters() from ax.utils.notebook.plotting import render, init_notebook_plotting from ax.plot.contour import plot_contour plot = plot_contour( model=gpei, param_x=opt_list[0], param_y=opt_list[1], metric_name="base", ) render(plot) ax_client.generation_strategy.model = gpei init_notebook_plotting(offline=True) # render(ax_client.get_contour_plot()) render(ax_client.get_contour_plot(param_x=opt_list[0], param_y=opt_list[0])) #, metric_name=base)) # render(ax_client.get_optimization_trace(objective_optimum=hartmann6.fmin)) # Objective_optimum is optional. ax_client.save_to_json_file() # For custom filepath, pass `filepath` argument. restored_ax_client = AxClient.load_from_json_file( ) # For custom filepath, pass `filepath` argument.
#batch_params = {} #for exp_param in exp_params: # parameters, trial_index = ax_client.attach_trial(parameters={"alpha": exp_param.alpha}) # batch_params[trial_index] = parameters.get('alpha') # run #trial_index_to_results =runner.run_exp(batch_params) #print(trial_index_to_results) #for trial_index in batch_params.keys(): # results = trial_index_to_results[trial_index] # ax_client.complete_trial(trial_index=trial_index, raw_data=results) # run the opti loop for i in range(epocs): batch_params = {} for j in range(batch_size): parameters, trial_index = ax_client.get_next_trial() batch_params[trial_index] = parameters.get('alpha') # run trial_index_to_results =runner.run_exp(batch_params) for trial_index in batch_params.keys(): results = trial_index_to_results[trial_index] ax_client.complete_trial(trial_index=trial_index, raw_data=results) # save the results opti_dir = Path('.') filePaht = opti_dir/f'{workload}_{awareness}_bo.json' ax_client.save_to_json_file(filepath=filePaht)