def tune_SVM(dataset, svm_parameters=svm_parameters, is_marker=False, seed=0, total_trials=15): """ function to get baseline for standalone SVM """ #set a seed so that splits are the same across different model tests np.random.seed(seed) #make data splits train_df, test_df = make_split(dataset) #for rf, go straight to k-fold train_dataset = MicroDataset(train_df, is_marker=is_marker) X = train_dataset.matrix.detach().numpy() y = train_dataset.y.detach().numpy() def evaluation_func(p): #the function used for the hyperparameter tuning model = SVC(probability=True) [setattr(model, a, np.power(2., q)) for a, q in p.items()] roc, model = run_training(X, y, model=model) auc_val = auc(roc[0], roc[1]) if math.isnan(auc_val): auc_val = 0 return (auc_val) best_parameters, best_values, experiment, model = optimize( parameters=svm_parameters, evaluation_function=evaluation_func, total_trials=total_trials, minimize=False) #make a model with the best parameters best_classifier = SVC(probability=True) [ setattr(best_classifier, a, np.power(2., q)) for a, q in best_parameters.items() ] # fit it on the full train, valid data best_classifier.fit(X, y) # Get AUC for the full setup on the test set test_dataset = MicroDataset(test_df, is_marker=is_marker) test_preds = best_classifier.predict_proba( test_dataset.matrix.detach().numpy()) test_roc = roc_curve(test_dataset.y.detach().numpy(), test_preds[:, 1]) best_val = auc(test_roc[0], test_roc[1]) data_name = dataset.index[0] if is_marker == False: data_type = 'Abundance' else: data_type = 'Marker' return (['SVM', data_type, data_name, best_parameters, best_val])
def ax_cube(objective, n_trials, n_dim, with_count=False, method=None): global feval_count feval_count = 0 rt = get_logger('ax') rt.setLevel(CRITICAL) def evaluation_func(prms): global feval_count feval_count += 1 return objective([prms["u" + str(i)] for i in range(n_dim)]) parameters = [{ "name": "u" + str(i), "type": "range", "bounds": [0.0, 1.0], } for i in range(n_dim)] best_parameters, best_values, experiment, model = optimize( parameters=parameters, evaluation_function=evaluation_func, minimize=True, total_trials=n_trials) best_x = [best_parameters['u' + str(i)] for i in range(n_dim)] best_val = best_values[0]['objective'] return (best_val, best_x, feval_count) if with_count else (best_val, best_x)
def ax_cube(objective, scale, n_trials): def evaluation_func(prms): return objective([prms["u1"], prms["u2"], prms["u3"]])[0] best_parameters, best_values, experiment, model = optimize( parameters=[ { "name": "u1", "type": "range", "bounds": [-scale, scale], }, { "name": "u2", "type": "range", "bounds": [-scale, scale], }, { "name": "u3", "type": "range", "bounds": [-scale, scale], }, ], # Booth function evaluation_function=evaluation_func, minimize=True, total_trials=n_trials) return best_values[0]['objective']
def evaluate(self, error_yielding_function): random_seed = self.total_trials best_parameters, values, experiment, model = ax.optimize( parameters=[ { 'name': 'beta1', 'type': 'choice', 'values': [0.90, 0.95] }, { 'name': 'lr', 'type': 'range', 'bounds': [0.0008, 0.003] }, { 'name': 'N_sma_threshold', 'type': 'choice', 'values': [4, 5] }, { 'name': 'k', 'type': 'choice', 'values': [6, 8] }, ], evaluation_function=error_yielding_function, objective_name='accuracy', total_trials=self.total_trials, random_seed=random_seed) return best_parameters
def ax_train(rcml, model_params: dict, config_params: dict): # ---------------------------------------------------------------------------------------------------- # cross-validation folds # ---------------------------------------------------------------------------------------------------- global_best_model = None global_best_test_accuracy = 0 parameters = [ { "name": "max_depth", "type": "range", "bounds": [9, 17], "parameter_type": ParameterType.INT }, { "name": "max_features", "type": "range", "bounds": [0.20, 0.6], "parameter_type": ParameterType.FLOAT }, { "name": "n_estimators", "type": "range", "bounds": [100, 200], "parameter_type": ParameterType.INT } ] best_parameters, best_values, experiment, model = optimize( parameters=parameters, evaluation_function=lambda params: ax_train_proxy(model_params=model_params, config_params=config_params, ax_params=params), minimize=False, total_trials=10, objective_name='accuracy', ) print("Ax Optimization Results:") print(best_parameters) print(best_values) # save model # rcml.save_best_model(global_best_model) # ---------------------------------------------------------------------------------------------------- # TODO: baseline [ single-node - multi-GPU w/ dask ] # ---------------------------------------------------------------------------------------------------- return global_best_model, global_best_test_accuracy
def veritize(self): total_trials = (DashVerum.v_resp['metric']['num_trials'] if DashVerum.v_resp['metric']['num_trials'] is not None else 2) best_parameters, best_values, experiment, model = optimize( parameters=self.init_parameters(), evaluation_function=self.evaluation_fn, minimize=DashVerum.v_resp['metric']['minimize'], total_trials=total_trials) if DashVerum.v_resp['return']: try: DashVerum.learn.model.ps = best_parameters['dropout_ps'] except: pass # try: DashVerum.learn.model.wd = best_parameters['weight_decay'] # except: pass try: DashVerum.learn.model.use_bn = best_parameters['use_bn'] except: pass return_list = [DashVerum.learn] try: return_list.append(best_parameters['learning_rate']) except: return_list.append( DashVerum.v_resp['learning_rate']['default']) try: return_list.append(best_parameters['num_epochs']) except: return_list.append(DashVerum.v_resp['num_epochs']['default']) momentum_list = [] try: momentum_list.append(best_parameters['momentum0']) except: momentum_list.append(DashVerum.v_resp['momentum0']['default']) try: momentum_list.append(best_parameters['momentum1']) except: momentum_list.append(DashVerum.v_resp['momentum1']['default']) return_list.append(tuple(momentum_list)) return tuple(return_list) else: print(json.dumps(best_parameters, indent=4))
def optimize(self, **kwargs): """ :param kwargs: kwargs for ax.optimize """ def evaluate(kwargs: Dict[str, float]): run_offseason = self._offseason_runner_builder(**kwargs) run_league(self._league, run_offseason) return calc_brier_score(self._league) best_parameters, best_values, experiment, model = optimize( parameters=[p.to_ax() for p in self._params.values()], evaluation_function=evaluate, minimize=True, **kwargs, ) for name, best_value in best_parameters.items(): self._params[name] = self._params[name].update_value(best_value) return best_parameters, best_values, experiment, model
def tune_MAML_FFNN(datasets, is_marker=False, seed=0, total_trials=15): #set a seed so that splits are the same across different model tests np.random.seed(seed) #make data splits splits = [make_split(df) for df in datasets] #do a train/val/test split for the first dataset train, test = make_split(datasets[0]) train, valid = make_split(train) splits[0] = (train, valid) trains = [s[0] for s in splits] vals = [s[1] for s in splits] #build evaluation function eval_func = build_MAML_FFNN_eval_func(train, valid, test, all_trains=trains, all_valids=vals, is_marker=is_marker) #selectthe parameters based on the model_name parameters = MAML_FFNN_parameters #optimize hyperparapeter with Facebook's ax tuning best_parameters, best_values, experiment, model = optimize( parameters=parameters, evaluation_function=eval_func, total_trials=total_trials, minimize=False) data_name = train.dataset.values[0] if is_marker == False: data_type = 'Abundance' else: data_type = 'Marker' model_name = 'MAML_FFNN' return ([model_name, data_type, data_name, best_parameters, best_values])
def svm_eval(X, y, svm_hyperparams, n_folds=5, total_trials=10): # find the best hyperamas vis k-fold cross-validation def evaluation_func(p): #the function used for the hyperparameter tuning model = SVC(probability=True) [setattr(model, a, np.power(2., q)) for a, q in p.items()] roc, model = run_training(X, y, n_folds=n_folds, model=model) auc_val = auc(roc[0], roc[1]) if math.isnan(auc_val): auc_val = 0 return (auc_val) best_parameters, best_values, experiment, model = optimize( parameters=svm_hyperparams, evaluation_function=evaluation_func, total_trials=total_trials, minimize=False) return (best_parameters)
def rf_eval(X, y, rf_hyperparams, n_folds=5, total_trials=10): # find the best hyperamas vis k-fold cross-validation def evaluation_func(p): #the function used for the hyperparameter tuning model = RandomForestClassifier() [setattr(model, a, q) for a, q in p.items()] roc, model = run_training(X, y, n_folds=n_folds, model=model) auc_val = auc(roc[0], roc[1]) if math.isnan(auc_val): auc_val = 0 return (auc_val) best_parameters, best_values, experiment, model = optimize( parameters=rf_hyperparams, evaluation_function=evaluation_func, total_trials=total_trials, minimize=False) return (best_parameters)
def dont_test_intro_example(): """ https://ax.dev/ """ best_parameters, best_values, experiment, model = optimize( parameters=[ { "name": "x1", "type": "range", "bounds": [-10.0, 10.0], }, { "name": "x2", "type": "range", "bounds": [-10.0, 10.0], }, ], # Booth function evaluation_function=lambda p: (p["x1"] + 2 * p["x2"] - 7) ** 2 + (2 * p["x1"] + p["x2"] - 5) ** 2, minimize=True, ) return best_values
def ax_cube(objective,scale, n_trials, n_dim, with_count=False): global feval_count feval_count = 0 def evaluation_func(prms): global feval_count feval_count += 1 return objective([prms["u"]+str(i) for i in range(n_dim)])[0] parameters = [ { "name": "u"+str(i), "type": "range", "bounds": [-scale, scale], } for i in range(n_dim)] best_parameters, best_values, experiment, model = optimize( parameters=parameters, # Booth function evaluation_function=evaluation_func, minimize=True, total_trials=n_trials) return (best_values[0]['objective'], feval_count) if with_count else best_values[0]['objective']
def optimize(self): # ax.optimizeをコールする # ax.optimizeは最適化が終了するまで復帰して来ない # ax.optimize内でevaluation_functionが試行回数分コールされる best_parameters, best_values, experiment, model = optimize( parameters=[ # 最適化対象パラメータの定義 { "name": "x1", "type": "range", "bounds": [-10.0, 10.0], }, { "name": "x2", "type": "range", "bounds": [-10.0, 10.0], }, ], # 評価関数(オリジナルコード) # evaluation_function=lambda p: (p["x1"] + 2*p["x2"] - 7)**2 + (2*p["x1"] + p["x2"] - 5)**2, evaluation_function=self.evaluation_function, # 評価関数 objective_name="score", # 評価指標の名前 minimize=True, # 最適化=最小化 ) # best_parameters contains {'x1': 1.02, 'x2': 2.97}; # the global min is (1, 3) # ベストパラメータ=最適解に最も近づいた値 print(f"best_parameters = {best_parameters}") # その他指標 means, covariances = best_values print(f"means = {means}") if self.cb_end: # 終了コールバック:Unityアプリへベストパラメータを通知する x1 = best_parameters['x1'] x2 = best_parameters['x2'] x = np.array([x1, x2]) self.cb_end(x)
def optimize(self, func, bounds, args=(), x0=None, **kwargs): def optimizing_func(x): return func(list(x.values()), *args) parameters = [{ "name": f"x{i}", "type": "range", "bounds": [float(b[0]), float(b[1])], "value_type": "float", "log_scale": self._log_scale } for i, b in enumerate(bounds)] best_parameters, best_values, experiment, model = optimize( experiment_name="optimization", objective_name="seismic_metrics", evaluation_function=optimizing_func, parameters=parameters, total_trials=self._num_evals, minimize=True ) return list(best_parameters.values()), {}
def ax_optimize(scoring_function, tunable_hyperparameters, iterations): """Construct and run a full optimization loop. Convert the hyperparameters in to the accepted format by ``Ax`` and adapt the scoring function to pass the configuration as kwargs. Args: scoring_function (function): A function that performs scoring over params. tunable_hyperparameters (dict): A python dict with hyperparameters. iterations (int): Number of tuning iterations to perform. """ parameters = convert_hyperparameters(tunable_hyperparameters) evaluation_function = adapt_scoring_function(scoring_function) best_params = optimize(parameters=parameters, evaluation_function=evaluation_function, total_trials=iterations, minimize=False)[0] return evaluation_function(best_params)
best_parameters, best_values, experiment, model = optimize( parameters=[ { "name": "decay_start", "type": "range", "bounds": [100, 40000], "value_type": "int", "log_scale": True }, { "name": "lr_A", "type": "range", "bounds": [1e-5, 500e-5], "value_type": "float", "log_scale": True }, { "name": "lr_B", "type": "range", "bounds": [100.0, 40000.0], "value_type": "float", "log_scale": True }, { "name": "epochs_between_updates", "type": "range", "bounds": [1, 6], "value_type": "int", "log_scale": False }, { "name": "decoder_rnn_dim", "type": "range", "bounds": [768, 1280], "value_type": "int", "log_scale": True }, { "name": "p_attention_dropout", "type": "range", "bounds": [0, 0.5], "value_type": "float", "log_scale": False }, { "name": "p_decoder_dropout", "type": "range", "bounds": [0, 0.5], "value_type": "float", "log_scale": False }, { "name": "attention_rnn_dim", "type": "range", "bounds": [1024, 4096], "value_type": "int", "log_scale": True }, { "name": "attention_dim", "type": "range", "bounds": [64, 512], "value_type": "int", "log_scale": True }, { "name": "attention_location_n_filters", "type": "range", "bounds": [16, 128], "value_type": "int", "log_scale": True }, { "name": "speaker_embedding_dim", "type": "range", "bounds": [64, 2048], "value_type": "int", "log_scale": True }, ], # Booth function total_trials=15, evaluation_function=train_experiment, minimize=False, objective_name='validation_average_max_attention_weight', )
def tune_shared_encoder(datasets, model_name='DAE', is_marker=False, seed=0, total_trials=15): if model_name == 'SAE': # we only have 10 possible hyperparam combos for SAE -- this is just gridsearch total_trials = 10 #set a seed so that splits are the same across different model tests np.random.seed(seed) #make data splits #doing train/valid/test splits across all datasets # ==> note - we don't really need test split for t splits = [make_split(df) for df in datasets] #splits = [(make_split(a[0]), a[1]) for a in splits ] #redo the split for the dataset of interest (the one at idx 0) # so we include a test set train, test = make_split(datasets[0]) train, valid = make_split(train) splits[0] = (train, valid) trains = pd.concat([s[0] for s in splits], axis=0).reset_index(drop=True) vals = pd.concat([s[1] for s in splits], axis=0).reset_index(drop=True) #tests = pd.concat( [s[1] for s in splits], axis=0 ).reset_index(drop=True) # for i in range(1, len(trains)): # # we don't need the test set for the dataset's we aren't testing on # trains[i] = pd.concat( [trains[i],tests[i]], axis = 0 ) eval_func = build_sharing_encoder_eval_func(train, valid, test, all_trains=trains, all_valids=vals, model_name=model_name, is_marker=is_marker) #selectthe parameters based on the model_name if model_name == 'DAE': parameters = DAE_parameters if model_name == 'SAE': parameters = SAE_parameters if model_name == 'VAE': parameters = VAE_parameters #optimize hyperparapeter with Facebook's ax tuning best_parameters, best_values, experiment, model = optimize( parameters=parameters, evaluation_function=eval_func, total_trials=total_trials, minimize=False) data_name = train.dataset.values[0] if is_marker == False: data_type = 'Abundance' else: data_type = 'Marker' return ([model_name, data_type, data_name, best_parameters, best_values])
best_parameters, best_values, experiment, model = optimize( parameters=[ { "name": "number_of_twisted_points", "type": "range", "value_type": "int", "bounds": [2, 6], }, { "name": "from_last_twisted_point_to_forward", "type": "range", "value_type": "int", "bounds": [2, 5], }, { "name": "direction_error", "type": "fixed", "value_type": "float", "value": 10 / 100 }, { "name": "consecutive_error", "type": "range", "value_type": "float", "bounds": [0.5 / 100, 25 / 100], }, { "name": "ema_fast_v", "type": "choice", "value_type": "int", "values": [9, 14, 12, 15, 17], }, { "name": "ema_slow_v", "type": "choice", "value_type": "int", "values": [21, 28, 32, 45, 50], }, { "name": "prediction_index", "type": "choice", "value_type": "int", "values": [4, 5, 6, 7, 8, 9, 10], }, ], # Booth function evaluation_function=lambda v: evaluate_moving_avg( number_of_twisted_points=int(v["number_of_twisted_points"]), from_last_twisted_point_to_forward=int(v[ "from_last_twisted_point_to_forward"]), direction_error=float(v["direction_error"]), consecutive_error=float(v["consecutive_error"]), ema_fast_v=int(v["ema_fast_v"]), ema_slow_v=int(v["ema_slow_v"]), prediction_index=int(v["prediction_index"]), ), minimize=False, )
best_parameters, best_values, experiment, model = optimize( parameters=[ { "name": "factory_s", "type": "range", "bounds": [0.0, 30.0], }, { "name": "factory_Q", "type": "range", "bounds": [0.0, 30.0], }, { "name": "w1_s", "type": "range", "bounds": [0.0, 20.0], }, { "name": "w1_Q", "type": "range", "bounds": [0.0, 20.0], }, { "name": "w2_s", "type": "range", "bounds": [0.0, 20.0], }, { "name": "w2_Q", "type": "range", "bounds": [0.0, 20.0], }, { "name": "w3_s", "type": "range", "bounds": [0.0, 20.0], }, { "name": "w3_Q", "type": "range", "bounds": [0.0, 20.0], }, { "name": "w4_s", "type": "range", "bounds": [0.0, 20.0], }, { "name": "w4_Q", "type": "range", "bounds": [0.0, 20.0], }, ], evaluation_function=f_policy, minimize=False, total_trials=200, )
def main(): # TODO model 1 numexpochs 30, model 2 - maxxxx parser = ArgumentParser() parser.add_argument('--skip_train', help='if skip train', action='store_true', default=False) parser.add_argument('--model_path', help='if skip train - path model to load', type=str, default="models/model1.pth") parser.add_argument('--num_epochs', type=int, default=30) parser.add_argument('--msg', help='msg to write in log file', type=str, default='') parser.add_argument('--n_epochs_stop', help='early stopping in training', type=int, default=10) parser.add_argument('--comp', action='store_true', default=False) parser.add_argument('--total_trails', type=int, default=70) parser.add_argument('--debug', action='store_true', default=False) parser.add_argument('--search_hyperparams', action='store_true', default=False) args = parser.parse_args() # Gets or creates a logger logging.config.fileConfig('logging.conf') logger = logging.getLogger(__name__) # Data paths data_dir = "Data/" if args.debug: path_train = data_dir + "small_train.labeled" path_test = data_dir + "small_test.labeled" else: path_train = data_dir + "train.labeled" path_test = data_dir + "test.labeled" logger.debug(f"Starting train: {path_train} test:{path_test}") if args.search_hyperparams: best_parameters, best_values, experiment, model = optimize( parameters=[ { "name": "accumulate_grad_step", "type": "choice", "is_numeric": False, "values": [2, 5, 10, 20], }, { "name": "optimizer_method", "type": "choice", "is_numeric": False, "values": ["{'optim':optim.Adam, 'lr':0.001, 'betas':(0.9, 0.999)}", "{'optim':optim.Adam, 'lr':0.01, 'betas':(0.9, 0.9)}", "{'optim':optim.Adam, 'lr':0.001, 'betas':(0.9, 0.9)}", "{'optim':optim.Adam, 'lr':0.0005, 'betas':(0.9, 0.9)}", "{'optim':optim.SGD, 'lr':0.01}", "{'optim':optim.SGD, 'lr':0.1}", "{'optim':optim.SGD, 'lr':0.001}", "{'optim':optim.SGD, 'lr':1.0}", "{'optim':optim.AdamW, 'lr':0.001, 'betas':(0.9, 0.999)}", "{'optim':optim.AdamW, 'lr':0.002, 'betas':(0.9, 0.9)}", "{'optim':optim.AdamW, 'lr':0.0005, 'betas':(0.9, 0.9)}", "{'optim':optim.AdamW, 'lr':0.001, 'betas':(0.9, 0.9)}", "{'optim':optim.Adadelta, 'lr':1}", "{'optim':optim.Adadelta, 'lr':2}"], }, { "name": "lstm_hidden_dim", "type": "choice", "is_numeric": False, "values": [125, 200, 300, 400, 0], }, { "name": "word_embedding_name_or_size_and_freeze_flag", "type": "choice", "is_numeric": False, "values": ["('glove.6B.50d', True)", "('glove.6B.50d', False)", "('glove.6B.100d', True)", "('glove.6B.100d', False)", "('glove.6B.200d', True)", "('glove.6B.200d', False)", "('glove.6B.300d', True)", "('glove.6B.300d', False)", "('glove.840B.300d', True)", "('glove.840B.300d', False)", "('fasttext.en.300d', True)", "('fasttext.en.300d', False)", "('25', False)", "('50', False)", "('100', False)", "('200', False)", "('300', False)"] }, { "name": "tag_embedding_dim", "type": "choice", "is_numeric": False, "values": [5, 10, 25, 50, 150], }, { "name": "mlp_hidden_dim", "type": "choice", "is_numeric": False, "values": [50, 100, 200, 300, 400, 500], }, { "name": "bilstm_layers", "type": "choice", "is_numeric": False, "values": [1, 2, 3, 4] }, { "name": "dropout_alpha", "type": "choice", "is_numeric": False, "values": [0.0, 0.05, 0.1, 0.25] }, { "name": "lstm_dropout", "type": "choice", "is_numeric": False, "values": [0.0, 0.15, 0.3] }, { "name": "activation", "type": "choice", "is_numeric": False, "values": ["nn.Tanh", "nn.ReLU", "nn.Sigmoid", "nn.LeakyReLU"] }, { "name": "min_freq", "type": "choice", "is_numeric": False, "values": [1, 2, 3, 5] }, { "name": "mlp_dropout", "type": "choice", "is_numeric": False, "values": [0.0, 0.15, 0.3] }, ], evaluation_function=lambda p: optimization_wrapper(args, logger, path_train, path_test, p), minimize=False, total_trials=args.total_trails, objective_name="UAS accuracy", ) logger.debug(f"{best_parameters},{best_values[0]}") else: optimization_wrapper(args, logger, path_train, path_test, parameters_advanced_model)
best_parameters, best_values, _, _ = optimize( # https://ax.dev/docs/core.html#search-space-and-parameters parameters=[ {"name": "INITIAL_LR", "type": "range", "bounds": [5e-4, 1e-3] }, {"name": "GAMMA", "type": "range", "bounds": [0.8, 0.999] }, {"name": "FILTERS", "type": "choice", "values": [ '(128, 128, 256)', '(64, 128, 256)', '(128, 256, 256)', ] }, {"name": "OUTPUT_FC_HIDDEN_SIZES", "type": "choice", "values": ['(256, 256)', '(128, 256)', '(512, 256)', '(1024, 256)', '(2048, 256)', '(2048,)', ] }, { "name": "KERNEL_SIZES", "type": "choice", "values": [ "('(1, 8, 8)', '(1, 4, 4)', '(4, 3, 3)')", "('(1, 8, 8)', '(1, 4, 4)', '(4, 1, 1)')", "('(1, 8, 8)', '(1, 3, 3)', '(4, 3, 3)')", "('(1, 8, 8)', '(1, 5, 5)', '(4, 3, 3)')", ], }, { "name": "STRIDE_SIZES", "type": "choice", "values": [ "('(1, 3, 3)', '(1, 3, 3)', '(1, 3, 3)')", "('(1, 4, 4)', '(1, 2, 2)', '(1, 1, 1)')", "('(1, 5, 5)', '(1, 3, 3)', '(1, 3, 3)')", "('(1, 4, 4)', '(1, 2, 2)', '(1, 3, 3)')", ], }, {"name": "OUTPUT_HIDDEN_DROPOUT", "type": "choice", "values": [0, 0.1, 0.2], "value_type": 'float', }, {"name": "CATEGORICAL", "type": "choice", "values": [True, False] }, {"name": "SUPPORT_RANGE", "type": "choice", "values": ['(-10, 20)', '(-10, 10)'] }, {"name": "NOISY", "type": "choice", "values": [True, False] }, {"name": "DUELING", "type": "choice", "values": [True, False] }, # {"name": "GRAYSCALE", # "type": "choice", # "values": [True, False] # }, ], evaluation_function=visual_banana_tuning, minimize=False, arms_per_trial=1, total_trials=NUM_TRIALS )
def main(): parser = argparse.ArgumentParser() parser.add_argument("-config", type=str, required=True) parser.add_argument("-mode", type=str, default="train") parser.add_argument("-hyper", action='store_true') parser.add_argument("-es", action='store_true') args = parser.parse_args() global params params = runpy.run_path(args.config).get('model_params', None) pl.seed_everything(params["random_seed"]) if params["file_type"] == "sdf": loader = SDFDataLoader(params["train_data_file"], label_props=params["label_cols"]) else: loader = SMILESDataLoader(params["train_data_file"], smiles_col=params["smiles_col"], label_props=params["label_cols"]) from sklearn.model_selection import train_test_split X_train, X_test, y_train, y_val = train_test_split( loader.mols, loader.labels_list, shuffle=True, train_size=0.8, random_state=params["random_seed"]) molecule_dataset_train = MoleculeDataset(X_train, y_train) molecule_dataset_test = MoleculeDataset(X_test, y_val) if params["use_gpu"] and torch.cuda.is_available(): device = torch.device('cuda:0') gpu = 1 else: device = torch.device('cpu') gpu = 0 callbacks = [EarlyStopping(monitor='val_loss')] if args.hyper: params["molecule_dataset_train"] = molecule_dataset_train params["molecule_dataset_test"] = molecule_dataset_test params["device"] = device params["gpu"] = gpu params["es"] = True if args.es else False results = ax.optimize(params["hyper"]["parameters"], evaluation_function, random_seed=params["random_seed"], minimize=params["minimize"], total_trials=params["hyper"]["trials"]) print(results) else: data_loader_train = data.DataLoader(molecule_dataset_train, batch_size=params["batch_size"], shuffle=False, collate_fn=gcn_collate_fn) data_loader_val = data.DataLoader(molecule_dataset_test, batch_size=params["batch_size"], shuffle=False, collate_fn=gcn_collate_fn) model = GraphConvModel(device_ext=device, task=params["task"], conv_layer_sizes=params["conv_layer_sizes"], fingerprints_size=params["fingerprints_size"], mlp_layer_sizes=params["mlp_layer_sizes"], lr=params["lr"]) callbacks = [] checkpoint_callback = ModelCheckpoint( monitor='val_' + params["metrics"], dirpath=params["check_point"]["dirpath"], filename="model-{epoch:02d}-{val_" + params["metrics"] + ":.5f}", save_top_k=params["check_point"]["save_top_k"], mode='min' if params["minimize"] else 'max') callbacks.append(checkpoint_callback) if args.es: early_stop_callback = EarlyStopping( min_delta=params["early_stopping"]["min_delta"], patience=params["early_stopping"]["patience"], verbose=params["early_stopping"]["verbose"], monitor="val_" + params["metrics"], mode='min' if params["minimize"] else 'max') callbacks.append(early_stop_callback) trainer = pl.Trainer(max_epochs=params["num_epochs"], gpus=gpu, callbacks=callbacks) trainer.fit(model, data_loader_train, data_loader_val) print_result(trainer)
def _opt_svaal(self, objective_name="val_f1_macro", minimise=None): """ Optimisation routine for SVAAL model Notes ----- This model differs from SVAE and FDP as SVAAL is not necessarily trained on full data. """ start_time = datetime.now() best_parameters, best_values, experiment, model = optimize( parameters=[ { "name": "epochs", "type": "range", "value_type": "int", "bounds": [10, 100], }, { "name": "batch_size", "type": "range", "value_type": "int", "bounds": [16, 64], }, # { # "name": "tl_embedding_dim", # "type": "fixed", # "value_type": "int", # "value": 64, # }, # { # "name": "tl_hidden_dim", # "type": "fixed", # "value_type": "int", # "value": 64, # }, # { # "name": "tl_rnn_type", # "type": "fixed", # "value_type": "str", # "value": "gru", # }, # { # "name": "tl_learning_rate", # "type": "fixed", # "value_type": "float", # "value": 0.01, # }, { "name": "latent_size", "type": "range", "value_type": "int", "bounds": [16, 128], }, { "name": "disc_fc_dim", "type": "range", "value_type": "int", "bounds": [64, 256] }, { "name": "disc_learning_rate", "type": "range", "value_type": "float", "bounds": [0.0001, 0.1] }, # { # "name": "svae_rnn_type", # "type": "fixed", # "value_type": "str", # "value": "gru", # }, { "name": "svae_embedding_dim", "type": "range", "value_type": "int", "bounds": [128, 1024], }, { "name": "svae_hidden_dim", "type": "range", "value_type": "int", "bounds": [128, 1024], }, # { # "name": "svae_num_layers", # "type": "fixed", # "value_type": "int", # "value": 1, # }, # { # "name": "svae_bidirectional", # "type": "fixed", # "value_type": "bool", # "value": False, # }, { "name": "svae_word_dropout", "type": "range", "value_type": "float", "bounds": [0, 1], }, { "name": "svae_embedding_dropout", "type": "range", "value_type": "float", "bounds": [0, 1], }, { "name": "svae_k", "type": "range", "value_type": "float", "bounds": [0, 1], }, { "name": "svae_x0", "type": "range", "value_type": "int", "bounds": [250, 5000], }, # { # "name": "svae_adv_hyperparameter", # "type": "fixed", # "value_type": "int", # "value": 1, # }, { "name": "svae_learning_rate", "type": "range", "value_type": "float", "bounds": [0.0001, 0.1] }, ], evaluation_function=self.exp.train_single_cycle, minimize=minimise, objective_name=objective_name, total_trials=self.trials) finish_time = datetime.now() run_time = (finish_time - start_time).total_seconds() / 60 data = { "name": "SEQ-SVAE-BBN-GRU", "info": { "start timestamp": start_time, "finish timestamp": finish_time, "run time": run_time }, "settings": { "trials": self.trials, "object name": objective_name, "minimise": minimise }, "results": { "best parameters": best_parameters, "best value": best_values[0][objective_name] } } print(data) # Post results to mongodb self.mongo_coll_conn.post(data)
def _opt_full_data_performance(self, objective_name="f1_macro", minimise=None): """ Optimisation routine for full data performance of task learner """ start_time = datetime.now() best_parameters, best_values, _, _ = optimize( parameters=[ { "name": "epochs", "type": "range", "value_type": "int", "bounds": [16, 128] }, { "name": "batch_size", "type": "range", "value_type": "int", "bounds": [16, 64], }, { "name": "tl_rnn_type", "type": "fixed", "value_type": "str", "value": "gru", }, { "name": "tl_embedding_dim", "type": "range", "value_type": "int", "bounds": [128, 1024], }, { "name": "tl_hidden_dim", "type": "range", "value_type": "int", "bounds": [128, 1024], }, { "name": "learning_rate", # TODO: Will need to update to tl_ in the future "type": "range", "value_type": "float", "bounds": [0.00001, 0.1] }, ], evaluation_function=self.exp._full_data_performance, minimize=minimise, objective_name=objective_name, total_trials=self.trials) finish_time = datetime.now() run_time = (finish_time - start_time).total_seconds() / 60 # TODO: Will put into a decorated in the future... data = { "name": f"FDP-{self.task_type}-{self.data_name}", "info": { "start timestamp": start_time, "finish timestamp": finish_time, "run time": run_time }, "config": self.config, "settings": { "trials": self.trials, "object name": objective_name, "minimise": minimise }, "results": { "best parameters": best_parameters, "best value": best_values[0][objective_name] } } # # Post results to mongodb self.mongo_coll_conn.post(data)
def main(): """ This program's entrypoint. """ # Parse command line arguments. psr = argparse.ArgumentParser( description="Hyper-parameter optimizer for train.py.") psr, psr_verify = cl_args.add_training(psr) psr.add_argument("--opt-trials", default=DEFAULT_TLS_OPT, help="The number of optimization trials to run.", type=int) psr.add_argument( "--exhaustive", action="store_true", help=("Try all combinations of parameters. Incompatible with " "parameters of type \"range\".")) args = psr_verify(psr.parse_args()) tls_opt = args.opt_trials tls_cnf = args.conf_trials no_rand = args.no_rand # Define the optimization parameters. params = [ { "name": "data_dir", "type": "fixed", "value": args.data_dir }, { "name": "model", "type": "choice", "values": ["SvmSklearn", "LrSklearn"] }, { "name": "conf_trials", "type": "fixed", "value": tls_cnf }, { "name": "max_attempts", "type": "fixed", "value": args.max_attempts }, { "name": "num_gpus", "type": "fixed", "value": 0 }, { "name": "warmup_percent", "type": "fixed", "value": 10 }, { "name": "num_sims", "type": "fixed", "value": args.num_sims }, # { # "name": "train_batch", # "type": "fixed", # "value": 10 # }, # { # "name": "learning_rate", # "type": "range", # "bounds": [0.0001, 0.01] # }, # { # "name": "momentum", # "type": "range", # "bounds": [0.09, 0.99] # }, { "name": "kernel", "type": "choice", "values": ["linear", "poly", "rbf", "sigmoid"] }, # { # "name": "degree", # "type": "range", # "bounds": [0, 20] # }, # Represent degree as a choice parameter so that it is # compatible with exhaustive mode. { "name": "degree", "type": "choice", "values": list(range(0, 21)) }, { "name": "penalty", "type": "choice", "values": ["l1", "l2"] }, { "name": "no_rand", "type": "fixed", "value": no_rand }, { "name": "timeout_s", "type": "fixed", "value": args.timeout_s }, { "name": "out_dir", "type": "fixed", "value": args.out_dir } ] # If we are using early stopping, then "epochs" is unnecessary but # "val_patience" and "val_improvement_thresh" are also candidates for # optimization. if args.early_stop: params.extend([{ "name": "early_stop", "type": "fixed", "value": True }, { "name": "val_patience", "type": "range", "bounds": [5, 20] }, { "name": "val_improvement_thresh", "type": "range", "bounds": [0.01, 0.1] }]) else: params.extend([ # { # "name": "epochs", # "type": "range", # "bounds": [1, 100] # }, ]) tim_srt_s = time.time() if args.exhaustive: for param in params: assert param["type"] != "range", \ f"Exhaustive mode does not support range parameters: {param}" fixed = { param["name"]: param["value"] for param in params if param["type"] == "fixed" } to_vary = [[(param["name"], value) for value in param["values"]] for param in params if param["type"] == "choice"] print( f"Varying these parameters, with {tls_cnf} sub-trials(s) for each " f"configuration: {[pairs[0][0] for pairs in to_vary]}") cnfs = [{ **fixed, **dict(params) } for params in itertools.product(*to_vary)] print(f"Total trials: {len(cnfs) * tls_cnf}") if defaults.SYNC: res = [train.run_trials(cnf)[0] for cnf in cnfs] else: with multiprocessing.Pool() as pol: res = pol.map(lambda cnf: train.run_trials(cnf)[0], cnfs) best_idx = np.argmin(np.array(res)) best_params = cnfs[best_idx] best_err = res[best_idx] else: print((f"Running {tls_opt} optimization trial(s), with {tls_cnf} " "sub-trial(s) for each configuration.")) best_params, best_vals, _, _ = ax.optimize( parameters=params, evaluation_function=lambda cnf: train.run_trials(cnf)[0], minimize=True, total_trials=args.opt_trials, random_seed=utils.SEED if no_rand else None) best_err = best_vals[0]["objective"] print((f"Done with hyper-parameter optimization - " f"{time.time() - tim_srt_s:.2f} seconds")) print(f"\nBest params: {best_params}") print(f"Best error: {best_err:.4f}%")
def _opt_svae(self, objective_name="train_loss", minimise=None): """ Optimisation routine for full data performance of SVAE """ start_time = datetime.now() best_parameters, best_values, _, _ = optimize( parameters=[ { "name": "epochs", "type": "range", "value_type": "int", "bounds": [25, 100], }, { "name": "batch_size", "type": "range", "value_type": "int", "bounds": [16, 64], }, { "name": "svae_embedding_dim", "type": "range", "value_type": "int", "bounds": [128, 1024], }, { "name": "svae_hidden_dim", "type": "range", "value_type": "int", "bounds": [128, 1024], }, { "name": "svae_rnn_type", "type": "fixed", "value_type": "str", "value": "gru", }, { "name": "svae_num_layers", "type": "fixed", "value_type": "int", "value": 1, }, { "name": "svae_bidirectional", "type": "fixed", "value_type": "bool", "value": True, }, { "name": "svae_latent_size", "type": "range", "value_type": "int", "bounds": [16, 256], }, { "name": "svae_word_dropout", "type": "range", "value_type": "float", "bounds": [0, 1], }, { "name": "svae_embedding_dropout", "type": "range", "value_type": "float", "bounds": [0, 1], }, { "name": "svae_k", "type": "range", "value_type": "float", "bounds": [0, 1], }, { "name": "svae_x0", "type": "range", "value_type": "int", "bounds": [250, 5000], }, ], evaluation_function=self.exp._svae, minimize=minimise, objective_name=objective_name, total_trials=self.trials) finish_time = datetime.now() run_time = (finish_time - start_time).total_seconds() / 60 # TODO: Will put into a decorated in the future... data = { "name": "SVAE", "info": { "start timestamp": start_time, "finish timestamp": finish_time, "run time": run_time }, "settings": { "trials": self.trials, "object name": objective_name, "minimise": minimise }, "results": { "best parameters": best_parameters, "best value": best_values[0][objective_name] } } # # Post results to mongodb self.mongo_coll_conn.post(data)
#Loss export stuff: model_loss_dict = {} best_parameters, values, experiment, model = ax.optimize( parameters=[ { "name": "fc1s", "type": "range", "bounds": [8, 128] }, # [8, 64] { "name": "fc2s", "type": "range", "bounds": [4, 64] }, # [4, 32] { "name": "fc3s", "type": "range", "bounds": [4, 64] } # [4, 32] ], evaluation_function=create_train_eval, minimize=True, # minimize since objective is loss total_trials=options.trials, # default 20 arms_per_trial=1, # default 1 objective_name='BCE_loss', ) filename = 'BO_model_losses_{}b.json'.format(options.bits) with open(os.path.join(options.outputDir, filename), 'w') as fp: json.dump(model_loss_dict, fp)
best_parameters, best_values, _, _ = optimize( # https://ax.dev/docs/core.html#search-space-and-parameters parameters=[ { "name": "INITIAL_LR", "type": "range", "bounds": [5e-6, 5e-3] }, { "name": "GAMMA", "type": "range", "bounds": [0.8, 0.999] }, { "name": "ACTION_REPEATS", "type": "choice", "values": [1, 2, 3, 4] }, { "name": "UPDATE_FREQUENCY", "type": "choice", "values": [4, 16, 64] }, { "name": "MLP_FEATURES_HIDDEN", "type": "choice", "values": ['(64, 64)', '(128, 128)', '(512, 512)', '(512, )'] }, { "name": "OUTPUT_FC_HIDDEN_SIZES", "type": "choice", "values": ['(64, 64)', '(128, 64)', '(64, 32)', '(512, )'] }, { "name": "OUTPUT_HIDDEN_DROPOUT", "type": "choice", "values": [0, 0.1, 0.2, 0.3, 0.4, 0.5], "value_type": 'float', }, { "name": "CATEGORICAL", "type": "choice", "values": [True, False] }, { "name": "SUPPORT_RANGE", "type": "choice", "values": ['(-10, 10)', '(-50, 50)', '(-100, 100)'] }, { "name": "NOISY", "type": "choice", "values": [True, False] }, { "name": "DUELING", "type": "choice", "values": [True, False] }, ], evaluation_function=banana_tuning, minimize=False, arms_per_trial=1, total_trials=NUM_TRIALS)
ret = evaluate(params, x_train, y_train, x_valid, y_valid) return {"validation_loss": ret["validation_loss"]} best_parameters, best_values, _, _ = optimize( parameters=[{ "name": "batch_size", "type": "range", "bounds": [16, 512] }, { "name": "epochs", "type": "range", "bounds": [1, 400] }, { "name": "learning_rate", "type": "range", "bounds": [1e-6, 0.01], "log_scale": True }, { "name": "hidden_dim", "type": "range", "bounds": [10, 200] }, { "name": "layer_dim", "type": "range", "bounds": [1, 8] }], evaluation_function=eval_with_data, minimize=True, objective_name="validation_loss") print("Best parameters:", best_parameters) print("Best values:", best_values)
# Setup dataloaders with our dataset train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=10, pin_memory=True) # FFS, have to use numworkers = 0 because apparently h5 objects can't be pickled, https://github.com/WuJie1010/Facial-Expression-Recognition.Pytorch/issues/69 validate_loader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=True, num_workers=10, pin_memory=True) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=test_size, shuffle=False, num_workers=10, pin_memory=True) best_parameters, values, experiment, model = ax.optimize( parameters=[ {"name": "fc1s", "type": "range", "bounds": [8, 64]}, {"name": "fc2s", "type": "range", "bounds": [4, 32]}, {"name": "fc3s", "type": "range", "bounds": [4, 32]} ], evaluation_function=create_train_eval, objective_name='accuracy', ) print("~*~*~*~*~*~*~*~RESULTS~*~*~*~*~*~*~*~") print("Best Params: {}".format(best_parameters)) means, covariances = values print("Means: {}".format(means)) print("Covariances: {}".format(covariances)) # Time for filenames now = datetime.now() time = now.strftime("%d-%m-%Y_%H-%M-%S") results_file = open("{}BO_Results_Summary-{}b_{}".format(options.outputDir,options.bits,time),"a")