def test_study_optimize_command(options): # type: (List[str]) -> None with StorageConfigSupplier(TEST_CONFIG_TEMPLATE) as (storage_url, config_path): storage = RDBStorage(storage_url) study_name = storage.get_study_name_from_id( storage.create_new_study_id()) command = [ 'optuna', 'study', 'optimize', '--study', study_name, '--n-trials', '10', __file__, 'objective_func' ] command = _add_option(command, '--storage', storage_url, 'storage' in options) command = _add_option(command, '--config', config_path, 'config' in options) subprocess.check_call(command) study = optuna.Study(storage=storage_url, study_name=study_name) assert len(study.trials) == 10 assert 'x' in study.best_params # Check if a default value of study_name is stored in the storage. assert storage.get_study_name_from_id( study.study_id).startswith(DEFAULT_STUDY_NAME_PREFIX)
def take_action(self, parsed_args): # type: (Namespace) -> int config = optuna.config.load_optuna_config(self.app_args.config) storage_url = get_storage_url(self.app_args.storage, config) study = optuna.Study(storage=storage_url, study_name=parsed_args.study) # We force enabling the debug flag. As we are going to execute user codes, we want to show # exception stack traces by default. self.app.options.debug = True target_module = imp.load_source('optuna_target_module', parsed_args.file) try: target_method = getattr(target_module, parsed_args.method) except AttributeError: self.logger.error('Method {} not found in file {}.'.format( parsed_args.method, parsed_args.file)) return 1 study.optimize(target_method, n_trials=parsed_args.n_trials, timeout=parsed_args.timeout, n_jobs=parsed_args.n_jobs) return 0
def main(): parser = argparse.ArgumentParser() parser.add_argument("kifu") parser.add_argument("engine") parser.add_argument("value_ranges") parser.add_argument("out_dir") parser.add_argument("--trials", type=int, default=10) parser.add_argument("--resume", action="store_true") args = parser.parse_args() base_config["engine"] = yaml_load(args.engine) base_config["value_ranges"] = yaml_load(args.value_ranges) base_config["kifu"] = args.kifu base_config["out_dir"] = args.out_dir os.makedirs(base_config["out_dir"], exist_ok=True) study_name = 'next_move_evaluation_opt' # Unique identifier of the study. # sqliteの指定は絶対パスでもokのようだ if args.resume: study = optuna.Study(study_name=study_name, storage='sqlite:///' + os.path.join(base_config["out_dir"], "optuna.db")) else: study = optuna.create_study( study_name=study_name, storage='sqlite:///' + os.path.join(base_config["out_dir"], "optuna.db")) study.optimize(objective, n_trials=args.trials)
def take_action(self, parsed_args): # type: (Namespace) -> None config = optuna.config.load_optuna_config(self.app_args.config) storage_url = get_storage_url(self.app_args.storage, config) study = optuna.Study(storage=storage_url, study_name=parsed_args.study) study.set_user_attr(parsed_args.key, parsed_args.value) self.logger.info('Attribute successfully written.')
def take_action(self, parsed_args): # type: (Namespace) -> None config = optuna.config.load_optuna_config(self.app_args.config) storage_url = get_storage_url(self.app_args.storage, config) study = optuna.Study(storage=storage_url, study_name=parsed_args.study) if parsed_args.out is None: optuna.dashboard.serve(study) else: optuna.dashboard.write(study, parsed_args.out) self.logger.info('Report successfully written to: {}'.format(parsed_args.out))
def main(study_name, is_resume_study): df_train, _ = load_data(scaled=False) df_train = df_train.loc[:, sorted(df_train.columns)] X_train = df_train.drop([ 'HasDetections', 'MachineIdentifier', 'machine_id', 'AvSigVersion_1', 'test_probability' ], axis=1) y_train = df_train['HasDetections'] f = partial(objective, X_train, y_train, df_train) if is_resume_study: study = optuna.Study(study_name=study_name, storage='sqlite:///example.db') else: study = optuna.create_study(study_name=study_name, storage='sqlite:///example.db') study.optimize(f, n_trials=50) print('params:', study.best_params)
best_val_f_score = -np.inf for epoch, [val_loss, val_result] in enumerate(zip(val_losses, val_results)): if best_val_loss > val_loss: best_val_loss = val_loss best_val_loss_epoch = epoch best_val_loss_result = val_result if best_val_f_score < val_result["f_score"]: best_val_f_score = val_result["f_score"] best_val_f_score_loss = val_loss best_val_f_score_epoch = epoch best_val_f_score_result = val_result assert min(val_losses) == best_val_loss assert max(result["f_score"] for result in val_results) == best_val_f_score best_val_losses.append(best_val_loss) best_val_loss_results.append(best_val_loss_result) best_val_loss_epochs.append(best_val_loss_epoch) best_val_f_score_losses.append(best_val_f_score_loss) best_val_f_score_results.append(best_val_f_score_result) best_val_f_score_epochs.append(best_val_f_score_epoch) return best_val_losses, best_val_loss_results, best_val_loss_epochs, best_val_f_score_losses, best_val_f_score_results, best_val_f_score_epochs import optuna OPTUNA_STUDY_NAME = "model" OPTUNA_STORAGE = "sqlite:///opt.db" study = optuna.Study(study_name=OPTUNA_STUDY_NAME, storage=OPTUNA_STORAGE) study.optimize(wrapper, n_trials=1)
## Test ## with chainer.using_config('train', False): best_mic, best_mac = estimate_test(test_data, reader, epoch, file_path, args.model, model, args.multilabel, best_mic, best_mac) return 1 - best_mic ## microが最小になるように if __name__ == "__main__": num_of_trials = 20 ## n_trialsはhyper_parameterを探索する試行回数 args = parse_argument() reader, train_data, test_data = prepare() study = optuna.Study(study_name=args.dbname, storage=args.storagename) if args.model == "TRF-Delay-Multi" or args.model == "TRF-Sequential": study.optimize(objective, n_trials=num_of_trials) else: study.optimize(objective, n_trials=num_of_trials) with open(args.filepath + "/opt_result.txt", mode='w') as f: f.write('Number of finished trials: {}'.format(len(study.trials)) + "\n") f.write('Best trial:' + "\n") trial = study.best_trial f.write(' Value: {}'.format(trial.value) + "\n") f.write(' Params: ' + "\n") for key, value in trial.params.items():
report = evaluator() # The following line mitigates the memory problem in CircleCI # (see https://github.com/pfnet/optuna/pull/325 for more details). gc.collect() return 1.0 - report['main/accuracy'] if __name__ == '__main__': # Please make sure common study and storage are shared among nodes. study_name = sys.argv[1] storage_url = sys.argv[2] study = optuna.Study(study_name, storage_url, pruner=optuna.pruners.MedianPruner()) comm = chainermn.create_communicator('naive') if comm.rank == 0: print('Study name:', study_name) print('Storage URL:', storage_url) print('Number of nodes:', comm.size) # Run optimization! chainermn_study = optuna.integration.ChainerMNStudy(study, comm) chainermn_study.optimize(objective, n_trials=25) if comm.rank == 0: pruned_trials = [ t for t in study.trials if t.state == optuna.structs.TrialState.PRUNED
trainer.run() # Evaluate. evaluator = chainer.training.extensions.Evaluator(test_iter, model) evaluator = chainermn.create_multi_node_evaluator(evaluator, comm) report = evaluator() return 1.0 - report['main/accuracy'] if __name__ == '__main__': # Please make sure common study and storage are shared among nodes. study_name = sys.argv[1] storage_url = sys.argv[2] study = optuna.Study(study_name, storage_url) comm = chainermn.create_communicator('naive') if comm.rank == 0: print('Study name:', study_name) print('Storage URL:', storage_url) print('Number of nodes:', comm.size) # Run optimization! chainermn_study = optuna.integration.ChainerMNStudy(study, comm) chainermn_study.optimize(objective, n_trials=25) if comm.rank == 0: print('Number of finished trials: ', len(study.trials)) print('Best trial:') trial = study.best_trial print(' Value: ', trial.value)
def search(sim_sn_path, hsc_path, model_dir, batch_size, optimizer, adabound_gamma, adabound_final_lr, lr, seed, epochs, patience, n_trials, norm, input1, input2, threads, eval_frequency, task_name, remove_y, target_distmod): storage = 'sqlite:///{}/example.db'.format(model_dir) if not os.path.exists(model_dir): os.makedirs(model_dir) if platform.system() == 'Windows': tmp = (Path(__file__).parents[1] / 'mlruns' / 'search-hsc-redshift' / 'mlruns') uri = str(tmp.absolute().as_uri()) # uri = 'file://' + str(tmp.absolute()) else: tmp = (Path(__file__).parents[1] / 'mlruns' / 'search-hsc-redshift' / 'mlruns') uri = str(tmp.absolute().as_uri()) mlflow.set_tracking_uri(uri) print(model_dir) name = '{flag}-{input1}-{input2}'.format(flag=task_name, input1=input1, input2=input2) if remove_y: name += '-remove-y' name += '-{}'.format('distmod' if target_distmod else 'redshift') mlflow.set_experiment(name) db_path = os.path.join(model_dir, 'example.db') sampler = MyTPESampler() if os.path.exists(db_path): study = optuna.Study(study_name='study190513', storage=storage, sampler=sampler) else: study = optuna.create_study(study_name='study190513', storage=storage, sampler=sampler) input_setting = InputSetting(batch_size=batch_size, mixup='none', mixup_alpha=2, mixup_beta=2) input_data = InputData(training_data=None, validation_data=None, test_data=None, mean=None, std=None, input1=input1, input2=input2, remove_y=remove_y, is_hsc=True, n_classes=1, input_setting=input_setting) optimizer_setting = OptimizerSetting(name=optimizer, lr=lr, gamma=adabound_gamma, final_lr=adabound_final_lr) loop_setting = LoopSetting(epochs=epochs, patience=patience, eval_frequency=eval_frequency, end_by_epochs=False) print('loading data') sim_sn, _ = load_hsc_data(sim_sn_path=sim_sn_path, hsc_path=hsc_path, remove_y=input_data.remove_y) sim_sn = sklearn.utils.shuffle(sim_sn, random_state=seed) if target_distmod: lcdm = Cosmology() target = np.asarray( Parallel(n_jobs=threads)(delayed(lcdm.DistMod)(z) for z in sim_sn['redshift'])) else: target = sim_sn['redshift'] tmp_x, test_x, tmp_y, test_y = train_test_split(sim_sn, target, test_size=0.3, random_state=42) dev_x, val_x, dev_y, val_y = train_test_split(tmp_x, tmp_y, test_size=0.3, random_state=44) if norm: output_mean, output_std = np.mean(dev_y), np.std(dev_y) dev_y = ((dev_y - output_mean) / output_std).astype(np.float32) val_y = ((val_y - output_mean) / output_std).astype(np.float32) test_y = ((test_y - output_mean) / output_std).astype(np.float32) # ハイパーパラメータ探索の最中なので平均と分散は保存しない else: # output_mean, output_std = 0, 1 dev_y = dev_y.astype(np.float32) val_y = val_y.astype(np.float32) test_y = test_y.astype(np.float32) mean, std = compute_moments(train_data=dev_x, input1=input1, input2=input2, norm=norm, use_redshift=False, is_hsc=True, threads=threads) input_data.mean, input_data.std = mean, std training_data = Data(x=dev_x, y=dev_y) validation_data = Data(x=val_x, y=val_y) test_data = Data(x=test_x, y=test_y) input_data.training_data = training_data input_data.validation_data = validation_data input_data.test_data = test_data for i in range(n_trials): study.optimize( lambda trial: objective_hsc(trial=trial, sim_sn_path=sim_sn_path, hsc_path=hsc_path, optimizer_setting=optimizer_setting, seed=seed, loop_setting=loop_setting, normalization=norm, threads=threads, input_data=input_data), n_trials=1) df = study.trials_dataframe() df.to_csv(os.path.join(model_dir, 'result.csv'))
def train_and_generate_model(): #global log_fd global log_fd_opt global tr_input_arr global tr_angle_arr global val_input_arr global val_angle_arr data_len = len(exchange_rates) log_fd_tr = open("./train_progress_log_" + dt.now().strftime("%Y-%m-%d_%H-%M-%S") + ".txt", mode="w") # inner logger function for backtest def logfile_writeln_tr(log_str): nonlocal log_fd_tr log_fd_tr.write(log_str + "\n") log_fd_tr.flush() print("data size of rates: " + str(data_len)) print("num of rate datas for tarin: " + str(COMPETITION_TRAIN_DATA_NUM_AT_RATE_ARR)) print("input features sets for tarin: " + str(COMPETITION_TRAIN_DATA_NUM)) logfile_writeln_tr("data size of rates: " + str(data_len)) logfile_writeln_tr("num of rate datas for tarin: " + str(COMPETITION_TRAIN_DATA_NUM_AT_RATE_ARR)) tr_input_mat = [] tr_angle_mat = [] is_loaded_input_mat = False if os.path.exists("./tr_input_mat.pickle"): with open('./tr_input_mat.pickle', 'rb') as f: tr_input_mat = pickle.load(f) with open('./tr_angle_mat.pickle', 'rb') as f: tr_angle_mat = pickle.load(f) is_loaded_input_mat = True else: for i in range(DATA_HEAD_ASOBI, len(exchange_rates) - DATA_HEAD_ASOBI - OUTPUT_LEN, SLIDE_IDX_NUM_AT_GEN_INPUTS_AND_COLLECT_LABELS): tr_input_mat.append([ exchange_rates[i], (exchange_rates[i] - exchange_rates[i - 1]) / exchange_rates[i - 1], get_rsi(exchange_rates, i), get_ma(exchange_rates, i), get_ma_kairi(exchange_rates, i), get_bb_1(exchange_rates, i), get_bb_2(exchange_rates, i), get_ema(exchange_rates, i), get_ema_rsi(exchange_rates, i), get_cci(exchange_rates, i), get_mo(exchange_rates, i), get_lw(exchange_rates, i), get_ss(exchange_rates, i), get_dmi(exchange_rates, i), get_vorarity(exchange_rates, i), get_macd(exchange_rates, i), str(judge_chart_type(exchange_rates[i - CHART_TYPE_JDG_LEN:i])) ]) tr_input_mat.append([ reverse_exchange_rates[i], (reverse_exchange_rates[i] - reverse_exchange_rates[i - 1]) / reverse_exchange_rates[i - 1], get_rsi(reverse_exchange_rates, i), get_ma(reverse_exchange_rates, i), get_ma_kairi(reverse_exchange_rates, i), get_bb_1(reverse_exchange_rates, i), get_bb_2(reverse_exchange_rates, i), get_ema(reverse_exchange_rates, i), get_ema_rsi(reverse_exchange_rates, i), get_cci(reverse_exchange_rates, i), get_mo(reverse_exchange_rates, i), get_lw(reverse_exchange_rates, i), get_ss(reverse_exchange_rates, i), get_dmi(reverse_exchange_rates, i), get_vorarity(reverse_exchange_rates, i), get_macd(reverse_exchange_rates, i), str( judge_chart_type( reverse_exchange_rates[i - CHART_TYPE_JDG_LEN:i])) ]) tmp = exchange_rates[i + OUTPUT_LEN] - exchange_rates[i] if tmp >= 0: tr_angle_mat.append(1) else: tr_angle_mat.append(0) tmp = reverse_exchange_rates[ i + OUTPUT_LEN] - reverse_exchange_rates[i] if tmp >= 0: tr_angle_mat.append(1) else: tr_angle_mat.append(0) if is_loaded_input_mat == False: with open('tr_input_mat.pickle', 'wb') as f: pickle.dump(tr_input_mat, f) with open('tr_angle_mat.pickle', 'wb') as f: pickle.dump(tr_angle_mat, f) #log output for tensorboard #configure("logs/xgboost_trade_cpu_1") tr_input_arr = np.array(tr_input_mat[0:COMPETITION_TRAIN_DATA_NUM]) tr_angle_arr = np.array(tr_angle_mat[0:COMPETITION_TRAIN_DATA_NUM]) watchlist = None split_idx = COMPETITION_TRAIN_DATA_NUM + int( (len(tr_input_mat) - COMPETITION_TRAIN_DATA_NUM) * VALIDATION_DATA_RATIO) if VALIDATION_DATA_RATIO != 0.0: val_input_arr = np.array( tr_input_mat[COMPETITION_TRAIN_DATA_NUM:split_idx]) val_angle_arr = np.array( tr_angle_mat[COMPETITION_TRAIN_DATA_NUM:split_idx]) watchlist = [(tr_input_arr, tr_angle_arr), (val_input_arr, val_angle_arr)] else: watchlist = [(tr_input_arr, tr_angle_arr)] start = time.time() if is_param_tune_with_optuna: log_fd_opt = open("./tune_progress_log_" + dt.now().strftime("%Y-%m-%d_%H-%M-%S") + ".txt", mode="w") study = None if is_use_db_at_tune: study = optuna.Study(study_name='fxsystrade', storage='sqlite:///../fxsystrade.db') else: study = optuna.create_study() parallel_num = RAPTOP_THREAD_NUM * 2 if is_colab_cpu or is_exec_at_mba: parallel_num = COLAB_CPU_AND_MBA_THREAD_NUM * 2 if special_optuna_parallel_num != -1: parallel_num = special_optuna_parallel_num study.optimize(opt, n_trials=OPTUNA_TRIAL_NUM, n_jobs=parallel_num) process_time = time.time() - start logfile_writeln_opt("best_params: " + str(study.best_params)) logfile_writeln_opt("best_value: " + str(study.best_value)) logfile_writeln_opt("best_trial: " + str(study.best_trial)) logfile_writeln_opt("excecution time of tune: " + str(process_time)) log_fd_opt.flush() log_fd_opt.close() exit() param = {} n_thread = RAPTOP_THREAD_NUM if is_use_gpu: param['tree_method'] = 'gpu_hist' param['max_bin'] = 16 param['gpu_id'] = 0 n_thread = COLAB_CPU_AND_MBA_THREAD_NUM if is_colab_cpu or is_exec_at_mba: n_thread = COLAB_CPU_AND_MBA_THREAD_NUM logfile_writeln_tr("training parameters are below...") logfile_writeln_tr(str(param)) eval_result_dic = {} logfile_writeln_tr("num_round: " + str(NUM_ROUND)) clf = XGBClassifier(max_depth=MAX_DEPTH, random_state=42, n_estimators=NUM_ROUND, min_child_weight=18, subsample=0.9, colsample_bytree=0.6, eta=ETA, objective='binary:logistic', verbosity=0, n_thread=n_thread, **param) verbosity = True if is_use_gpu or is_colab_cpu: verbosity = False clf.fit(tr_input_arr, tr_angle_arr, eval_set=watchlist, verbose=verbosity) process_time = time.time() - start logfile_writeln_tr("excecution time of training: " + str(process_time)) clf.save_model('./xgb.model') booster = clf.get_booster() booster.dump_model('./xgb_model.raw.txt') eval_result_dic = clf.evals_result() for ii in range(len(eval_result_dic['validation_0']['error'])): if VALIDATION_DATA_RATIO != 0.0: logfile_writeln_tr( str(ii) + "," + str(eval_result_dic['validation_0']['error'][ii]) + "," + str(eval_result_dic['validation_1']['error'][ii])) else: logfile_writeln_tr( str(ii) + "," + str(eval_result_dic['validation_0']['error'][ii])) # Feature Importance fti = clf.feature_importances_ logfile_writeln_tr('Feature Importances:') for i, feat in enumerate(FEATURE_NAMES): logfile_writeln_tr('\t{0:20s} : {1:>.6f}'.format(feat, fti[i])) log_fd_tr.flush() log_fd_tr.close() print("finished training and saved model.")
def train_and_generate_model(): global log_fd global tr_input_arr global tr_angle_arr global val_input_arr global val_angle_arr data_len = len(exchange_rates) # if is_param_tune_with_optuna: # train_len = len(exchange_rates) - 1000 - OUTPUT_LEN # else: # train_len = int(len(exchange_rates)/TRAINDATA_DIV) log_fd = open("./train_progress_log_" + dt.now().strftime("%Y-%m-%d_%H-%M-%S") + ".txt", mode = "w") print("data size of rates: " + str(data_len)) print("num of rate datas for tarin: " + str(COMPETITION_TRAIN_DATA_NUM_AT_RATE_ARR)) print("input features sets for tarin: " + str(COMPETITION_TRAIN_DATA_NUM)) logfile_writeln("data size of rates: " + str(data_len)) logfile_writeln("num of rate datas for tarin: " + str(COMPETITION_TRAIN_DATA_NUM_AT_RATE_ARR)) tr_input_mat = [] tr_angle_mat = [] if os.path.exists("./tr_input_mat.pickle"): with open('./tr_input_mat.pickle', 'rb') as f: tr_input_mat = pickle.load(f) with open('./tr_angle_mat.pickle', 'rb') as f: tr_angle_mat = pickle.load(f) else: for i in range(DATA_HEAD_ASOBI, len(exchange_rates) - DATA_HEAD_ASOBI - OUTPUT_LEN, SLIDE_IDX_NUM_AT_GEN_INPUTS_AND_COLLECT_LABELS): # if "2006" in exchange_dates[i]: # print(len(tr_input_mat)) # print(str(DATA_HEAD_ASOBI + i - 1)) # quit() tr_input_mat.append( [exchange_rates[i], (exchange_rates[i] - exchange_rates[i - 1])/exchange_rates[i - 1], get_rsi(exchange_rates, i), get_ma(exchange_rates, i), get_ma_kairi(exchange_rates, i), get_bb_1(exchange_rates, i), get_bb_2(exchange_rates, i), get_ema(exchange_rates, i), get_ema_rsi(exchange_rates, i), get_cci(exchange_rates, i), get_mo(exchange_rates, i), get_lw(exchange_rates, i), get_ss(exchange_rates, i), get_dmi(exchange_rates, i), get_vorarity(exchange_rates, i), get_macd(exchange_rates, i), str(judge_chart_type(exchange_rates[i-CHART_TYPE_JDG_LEN:i])) ] ) tr_input_mat.append( [reverse_exchange_rates[i], (reverse_exchange_rates[i] - reverse_exchange_rates[i - 1])/reverse_exchange_rates[i - 1], get_rsi(reverse_exchange_rates, i), get_ma(reverse_exchange_rates, i), get_ma_kairi(reverse_exchange_rates, i), get_bb_1(reverse_exchange_rates, i), get_bb_2(reverse_exchange_rates, i), get_ema(reverse_exchange_rates, i), get_ema_rsi(reverse_exchange_rates, i), get_cci(reverse_exchange_rates, i), get_mo(reverse_exchange_rates, i), get_lw(reverse_exchange_rates, i), get_ss(reverse_exchange_rates, i), get_dmi(reverse_exchange_rates, i), get_vorarity(reverse_exchange_rates, i), get_macd(reverse_exchange_rates, i), str(judge_chart_type(reverse_exchange_rates[i-CHART_TYPE_JDG_LEN:i])) ] ) tmp = exchange_rates[i+OUTPUT_LEN] - exchange_rates[i] if tmp >= 0: tr_angle_mat.append(1) else: tr_angle_mat.append(0) tmp = reverse_exchange_rates[i+OUTPUT_LEN] - reverse_exchange_rates[i] if tmp >= 0: tr_angle_mat.append(1) else: tr_angle_mat.append(0) with open('tr_input_mat.pickle', 'wb') as f: pickle.dump(tr_input_mat, f) with open('tr_angle_mat.pickle', 'wb') as f: pickle.dump(tr_angle_mat, f) #log output for tensorboard #configure("logs/xgboost_trade_cpu_1") # if is_param_tune_with_optuna: # gen_data_len = int(((len(exchange_rates)/TRAINDATA_DIV))/5.0) # else: # gen_data_len = len(tr_input_mat) tr_input_arr = np.array(tr_input_mat[0:COMPETITION_TRAIN_DATA_NUM]) tr_angle_arr = np.array(tr_angle_mat[0:COMPETITION_TRAIN_DATA_NUM]) dtrain = xgb.DMatrix(tr_input_arr, label=tr_angle_arr) split_idx = COMPETITION_TRAIN_DATA_NUM + int((len(tr_input_mat) - COMPETITION_TRAIN_DATA_NUM) * VALIDATION_DATA_RATIO) if VALIDATION_DATA_RATIO != 0.0: val_input_arr = np.array(tr_input_mat[COMPETITION_TRAIN_DATA_NUM:split_idx]) val_angle_arr = np.array(tr_angle_mat[COMPETITION_TRAIN_DATA_NUM:split_idx]) dval = xgb.DMatrix(val_input_arr, label=val_angle_arr) watchlist = [(dtrain,'train'),(dval,'validation')] else: watchlist = [(dtrain,'train')] start = time.time() if is_param_tune_with_optuna: #study = optuna.create_study() study = optuna.Study(study_name='distributed-example', storage='sqlite:///example.db') study.optimize(opt, n_trials=100) process_time = time.time() - start logfile_writeln(str(study.best_params)) logfile_writeln(str(study.best_value)) logfile_writeln(str(study.best_trial)) logfile_writeln("excecution time of tune: " + str(process_time)) log_fd.flush() quit() param = {'max_depth':MAX_DEPTH, 'eta':ETA, 'objective':'binary:logistic', 'verbosity':0, 'n_thread':4,'random_state':42, 'n_estimators':NUM_ROUND, 'min_child_weight': 15, 'subsample': 0.7, 'colsample_bytree':0.7} #param = {'max_depth':6, 'learning_rate':0.1, 'subsumble':0.5, 'objective':'binary:logistic', 'verbosity':0, 'booster': 'dart', # 'sample_type': 'uniform', 'normalize_type': 'tree', 'rate_drop': 0.1, 'skip_drop': 0.5} #param = {'max_depth':3, 'eta':0.1, 'objective':'binary:logistic', 'verbosity':0, 'n_thread':4, # 'random_state':42, 'n_estimators':NUM_ROUND, 'min_child_weight': 15, 'subsample': 0.7, 'colsample_bytree':0.7, # 'booster': 'dart', 'sample_type': 'uniform', 'normalize_type': 'tree', 'rate_drop': 0.1, 'skip_drop': 0.5} if is_use_gpu: param['tree_method'] = 'gpu_hist' param['max_bin'] = 16 param['gpu_id'] = 0 param['n_thread'] = 2 if is_colab_cpu: param['n_thread'] = 2 logfile_writeln("training parameters are below...") logfile_writeln(str(param)) eval_result_dic = {} logfile_writeln("num_round: " + str(NUM_ROUND)) bst = xgb.train(param, dtrain, NUM_ROUND, evals=watchlist, evals_result=eval_result_dic, verbose_eval=int(NUM_ROUND/100)) process_time = time.time() - start logfile_writeln("excecution time of training: " + str(process_time)) bst.dump_model('./xgb_model.raw.txt') bst.save_model('./xgb.model') for ii in range(len(eval_result_dic['train']['error'])): if VALIDATION_DATA_RATIO != 0.0: logfile_writeln(str(ii) + "," + str(eval_result_dic['train']['error'][ii]) + "," + str(eval_result_dic['validation']['error'][ii])) else: logfile_writeln(str(ii) + "," + str(eval_result_dic['train']['error'][ii])) # feature importance create_feature_map() fti = bst.get_fscore(fmap='fx_systrade_xgb.fmap') logfile_writeln('Feature Importances:') logfile_writeln(str(fti)) # for feat in FEATURE_NAMES: # logfile_writeln('\t{0:10s} : {1:>12.4f}'.format(feat, fti[feat])) log_fd.flush() log_fd.close() print("finished training and saved model.")
def confirm(self, directory): self.study = optuna.Study( study_name=self.name, storage='sqlite:///{}/hypara_search.db'.format(directory)) self.df = self.study.trials_dataframe() """
arglist.append(f"--{arg}") continue arglist.append(f"--{arg}={value}") statement = " ".join(arglist) print(f"statement for this study is: ") print(statement) # calling through subprocess to ensure that all cuda memory is fully released between experiments subprocess.check_call(arglist) result_file = open("logs/" + opt.study_name + ".txt", "r+") score = float(result_file.read()) print(f"score for this study is {score}") return score # want to return a value to minimize try: # study = optuna.create_study(study_name=opt.study_name, storage="mysql+pymysql://root:[email protected]/optuna") study = optuna.create_study(study_name=opt.study_name) print("Created optuna study") except ValueError as e: if "Please use a different name" in str(e): # study = optuna.Study(study_name=opt.study_name, storage="mysql+pymysql://root:[email protected]/optuna") study = optuna.Study(study_name=opt.study_name) print("Joined existing optuna study") else: raise except: raise study.optimize(objective, n_trials=opt.num_trials)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--train', required=True, help='TSV file for training (.tsv)') parser.add_argument('--valid', required=True, help='TSV file for validation (.tsv)') parser.add_argument('--save-dir', required=True, help='Save directory') parser.add_argument('--encoder', default='LSTM', choices=['LSTM', 'BiLSTM', 'NSE'], help='Select type of encoder') parser.add_argument('--attn', default=False, action='store_true', help='Whether to add attention layer') parser.add_argument('--batchsize', type=int, default=64, help='Batch size') parser.add_argument('--embsize', type=int, default=128, help='Embedding size') parser.add_argument('--unit', type=int, default=128, help='Number of unit') parser.add_argument('--lr-schedule-gamma', type=float, default=0.95, help='Gamma of lr-scheduler') parser.add_argument('--clip', type=float, default=10, help='Clipping gradients') parser.add_argument('--epoch', type=int, default=25, help='Max epoch') parser.add_argument('--minfreq', type=int, default=2, help='Min word frequency') parser.add_argument('--maxlen', type=int, default=70, help='Max number of words for validation') parser.add_argument('--vocabsize', type=int, default=40000, help='vocabulary size') parser.add_argument( '--early-stop-n', type=int, default=2, help='Stop training if the best score does not update n epoch before') parser.add_argument('--n-trial', type=int, default=100, help='Number of trial') parser.add_argument('--study-name', default=None, help='Study name for sqlite') args = parser.parse_args() ### setup data ### print('setup data...\n') SRC = Field(init_token='<sos>', eos_token='<eos>', lower=True) TRG = Field(init_token='<sos>', eos_token='<eos>', lower=True) train_data, valid_data = TabularDataset.splits(path='.', train=args.train, validation=args.valid, format='tsv', fields=[('src', SRC), ('trg', TRG)]) def objective(trial): ### setup trials ### lr = trial.suggest_loguniform('lr', 1e-5, 1e-1) weight_decay = trial.suggest_loguniform('weight_decay', 1e-10, 1e-3) layer = trial.suggest_categorical('n_layer', [1, 2, 3]) dropout = trial.suggest_categorical('dropout', [0.1, 0.2, 0.3]) init_emb = trial.suggest_categorical('init-emb', ['fasttext', 'glove', 'none']) share_emb = trial.suggest_categorical('share-emb', [True, False]) ### setup vocabularies ### if init_emb == 'none': SRC.build_vocab(train_data, min_freq=args.minfreq, max_size=args.vocabsize) TRG.build_vocab(train_data, min_freq=args.minfreq, max_size=args.vocabsize) else: if init_emb == 'fasttext': vectors = FastText(language='en') elif init_emb == 'glove': vectors = GloVe() SRC.build_vocab(train_data, vectors=vectors, min_freq=args.minfreq, max_size=args.vocabsize) TRG.build_vocab(train_data, vectors=vectors, min_freq=args.minfreq, max_size=args.vocabsize) args.embsize = SRC.vocab.vectors.size()[1] vocabs = { 'src_stoi': SRC.vocab.stoi, 'src_itos': SRC.vocab.itos, 'trg_stoi': TRG.vocab.stoi, 'trg_itos': TRG.vocab.itos } train_iter, valid_iter = BucketIterator.splits( (train_data, valid_data), batch_size=args.batchsize, sort_within_batch=True, sort_key=lambda x: len(x.src), repeat=False, shuffle=True, device=device) train_size = len(train_data) valid_size = len(valid_data) src_vocabsize = len(SRC.vocab) trg_vocabsize = len(TRG.vocab) print(f'# training examples: {train_size}') print(f'# validation examples: {valid_size} \n') print(f'# unique tokens in source vocabulary: {src_vocabsize}') print(f'# unique tokens in target vocabulary: {trg_vocabsize} \n') ### setup model ### sos_id = TRG.vocab.stoi['<sos>'] eos_id = TRG.vocab.stoi['<eos>'] src_pad_id = SRC.vocab.stoi['<pad>'] trg_pad_id = TRG.vocab.stoi['<pad>'] if share_emb: vocabsize = max(src_vocabsize, trg_vocabsize) assert src_pad_id == trg_pad_id embedding = EmbeddingLayer(vocabsize, args.embsize, src_pad_id) encoder_embedding = embedding decoder_embedding = embedding else: encoder_embedding = EmbeddingLayer(src_vocabsize, args.embsize, src_pad_id) decoder_embedding = EmbeddingLayer(trg_vocabsize, args.embsize, trg_pad_id) bidirectional = True if args.encoder == 'BiLSTM' else False if args.encoder == 'NSE': encoder = NSE(encoder_embedding, args.unit, layer, dropout) else: encoder = LSTMEncoder(encoder_embedding, args.unit, layer, dropout, bidirectional) decoder = LSTMDecoder(decoder_embedding, args.unit, layer, dropout, args.attn, encoder.output_units) model = Seq2seq(encoder, decoder, sos_id, eos_id, device).to(device) parameter_num = count_parameters(model) print(model) # Multi GPU if device.__str__() == 'cuda': model = torch.nn.DataParallel(model) cudnn.benchmark = True criterion = nn.CrossEntropyLoss(ignore_index=TRG.vocab.stoi['<pad>']) optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) # lr scheduling with exponential curve scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=args.lr_schedule_gamma) ### make directory for saving ### if os.path.exists(args.save_dir): shutil.rmtree(args.save_dir) os.mkdir(args.save_dir) params = args.__dict__ params.update(lr=lr, weight_decay=weight_decay, layer=layer, dropout=dropout, init_emb=init_emb, share_emb=share_emb, train_size=train_size, valid_size=valid_size, src_vocabsize=src_vocabsize, trg_vocabsize=trg_vocabsize, parameter_num=parameter_num) json.dump(params, open(f'{args.save_dir}/params.json', 'w', encoding='utf-8'), ensure_ascii=False) pprint.pprint(params, indent=4) print() ### training and validation ### best_loss = float('inf') no_update_best_interval = 0 for epoch in range(args.epoch): is_first = True if epoch == 0 else False scheduler.step() # reduce lr train_loss = train(model, train_iter, optimizer, criterion, args.clip) valid_loss = eval(model, valid_iter, criterion, SRC.vocab.itos, TRG.vocab.itos, is_first) if valid_loss < best_loss: best_loss = valid_loss no_update_best_interval = 0 # save best model model_path = f'{args.save_dir}/model-best.pt' state = { 'vocabs': vocabs, 'params': params, 'state_dict': model.state_dict() } torch.save(state, model_path) else: no_update_best_interval += 1 # logging logs = f"Epoch: {epoch+1:02}\tTrain loss: {train_loss:.3f}\tVal. Loss: {valid_loss:.3f}\n" print(logs) # early stopping if no_update_best_interval >= args.early_stop_n: print('Early stopped in training') break return best_loss if args.study_name: # You have to do the following command in advance: # optuna create-study --study '{args.study_name}' --storage 'sqlite:///{args.study_name}.db' study = optuna.Study(study_name=args.study_name, storage=f'sqlite:///{args.study_name}.db') else: study = optuna.create_study() study.optimize(objective, n_trials=args.n_trial) print('\nbest params: ', study.best_params) print(f'best value: {study.best_value}') print(f'best trial: {study.best_trial.trial_id}')
def search_hsc(sim_sn_path, hsc_path, model_dir, batch_size, optimizer, adabound_gamma, adabound_final_lr, lr, seed, epochs, patience, n_trials, norm, input1, input2, mixup, threads, eval_frequency, binary, task_name, remove_y, mixup_alpha, mixup_beta): storage = 'sqlite:///{}/example.db'.format(model_dir) if not os.path.exists(model_dir): os.makedirs(model_dir) if platform.system() == 'Windows': tmp = (Path(__file__).parents[1] / 'mlruns' / 'search-hsc-classification' / 'mlruns') uri = str(tmp.absolute().as_uri()) # uri = 'file://' + str(tmp.absolute()) else: tmp = (Path(__file__).absolute().parents[1] / 'mlruns' / 'search-hsc-classification' / 'mlruns') uri = str(tmp.absolute().as_uri()) mlflow.set_tracking_uri(uri) mlflow.set_tracking_uri(uri) n_classes = 2 if binary else 3 name = '{n_classes}-{task_name}-{input1}-{input2}'.format( n_classes=n_classes, task_name=task_name, input1=input1, input2=input2 ) if remove_y: name += '-remove-y' mlflow.set_experiment(name) print(model_dir) db_path = os.path.join(model_dir, 'example.db') sampler = MyTPESampler() if os.path.exists(db_path): study = optuna.Study(study_name='study190513', storage=storage, sampler=sampler) else: study = optuna.create_study(study_name='study190513', storage=storage, sampler=sampler) input_setting = InputSetting( batch_size=batch_size, mixup=mixup, mixup_alpha=mixup_alpha, mixup_beta=mixup_beta ) input_data = InputData( training_data=None, validation_data=None, test_data=None, mean=None, std=None, input1=input1, input2=input2, remove_y=remove_y, is_hsc=True, n_classes=n_classes, input_setting=input_setting ) optimizer_setting = OptimizerSetting( name=optimizer, lr=lr, gamma=adabound_gamma, final_lr=adabound_final_lr ) loop_setting = LoopSetting(epochs=epochs, patience=patience, eval_frequency=eval_frequency, end_by_epochs=False) print('loading data') sim_sn, _ = load_hsc_data( sim_sn_path=sim_sn_path, hsc_path=hsc_path, remove_y=input_data.remove_y ) sim_sn = sklearn.utils.shuffle(sim_sn, random_state=seed) # クラスラベルを数字にする label_map = get_label_map(binary=binary) sim_sn_y = np.array([label_map[c] for c in sim_sn['sn_type']]) sim_x1, sim_x2, sim_y1, sim_y2 = train_test_split( sim_sn, sim_sn_y, test_size=0.3, random_state=42, stratify=sim_sn_y ) sim_dev_x, sim_val_x, sim_dev_y, sim_val_y = train_test_split( sim_x1, sim_y1, test_size=0.3, random_state=44, stratify=sim_y1 ) training_data = Data(x=sim_dev_x, y=sim_dev_y) validation_data = Data(x=sim_val_x, y=sim_val_y) test_data = Data(x=sim_x2, y=sim_y2) input_data.training_data = training_data input_data.validation_data = validation_data input_data.test_data = test_data mean, std = compute_moments( train_data=training_data.x, input1=input1, input2=input2, norm=norm, use_redshift=False, is_hsc=True, threads=threads ) input_data.mean, input_data.std = mean, std for i in range(n_trials): study.optimize( lambda trial: objective_hsc( trial=trial, sim_sn_path=sim_sn_path, hsc_path=hsc_path, optimizer_setting=optimizer_setting, seed=seed, loop_setting=loop_setting, normalization=norm, threads=threads, binary=binary, input_data=input_data ), n_trials=1 ) df = study.trials_dataframe() df.to_csv(os.path.join(model_dir, 'result.csv'))
def search_plasticc(sim_sn_path, training_cosmos_path, test_cosmos_path, model_dir, batch_size, optimizer, adabound_gamma, adabound_final_lr, lr, seed, epochs, patience, n_trials, norm, flux_err, input1, input2, mixup, threads, eval_frequency, binary, mixup_alpha, mixup_beta): storage = 'sqlite:///{}/example.db'.format(model_dir) if not os.path.exists(model_dir): os.makedirs(model_dir) if platform.system() == 'Windows': tmp = (Path(__file__).parents[1] / 'mlruns' / 'search-plasticc-classification' / 'mlruns') uri = str(tmp.absolute().as_uri()) # uri = 'file://' + str(tmp.absolute()) else: tmp = (Path(__file__).parents[1] / 'mlruns' / 'search-plasticc-classification' / 'mlruns') uri = str(tmp.absolute().as_uri()) mlflow.set_tracking_uri(uri) n_classes = 2 if binary else 3 name = '{n_classes}-{input1}-{input2}'.format( n_classes=n_classes, input1=input1, input2=input2 ) mlflow.set_experiment(name) db_path = os.path.join(model_dir, 'example.db') sampler = MyTPESampler() if os.path.exists(db_path): study = optuna.Study(study_name='study190513', storage=storage, sampler=sampler) else: study = optuna.create_study(study_name='study190513', storage=storage, sampler=sampler) input_setting = InputSetting( batch_size=batch_size, mixup=mixup, mixup_alpha=mixup_alpha, mixup_beta=mixup_beta, balance=False ) input_data = InputData( training_data=None, validation_data=None, test_data=None, mean=None, std=None, input1=input1, input2=input2, remove_y=False, is_hsc=False, n_classes=n_classes, input_setting=input_setting ) optimizer_setting = OptimizerSetting( name=optimizer, lr=lr, gamma=adabound_gamma, final_lr=adabound_final_lr ) loop_setting = LoopSetting(epochs=epochs, patience=patience, eval_frequency=eval_frequency, end_by_epochs=False) print('loading data') # 今までのflux_errなら1, 新しいflux_errなら2 sim_sn, training_cosmos, _ = load_plasticc_data( sim_sn_path=sim_sn_path, training_cosmos_path=training_cosmos_path, test_cosmos_path=test_cosmos_path, use_flux_err2=flux_err == 2 ) sim_sn = sklearn.utils.shuffle(sim_sn, random_state=seed) training_cosmos = sklearn.utils.shuffle(training_cosmos, random_state=seed + 1) for data in (sim_sn, training_cosmos): for key in ('flux', 'flux_err'): tmp = data[key] data[key][np.isnan(tmp)] = 0 # クラスラベルを数字にする label_map = get_label_map(binary=binary) sim_sn_y = np.array([label_map[c] for c in sim_sn['sn_type']]) training_cosmos_y = np.array([label_map[c] for c in training_cosmos['sn_type']]) sim_x1, sim_x2, sim_y1, sim_y2 = train_test_split( sim_sn, sim_sn_y, test_size=0.3, random_state=42, stratify=sim_sn_y ) cosmos_x1, cosmos_x2, cosmos_y1, cosmos_y2 = train_test_split( training_cosmos, training_cosmos_y, test_size=0.3, random_state=43, stratify=training_cosmos_y ) sim_dev_x, sim_val_x, sim_dev_y, sim_val_y = train_test_split( sim_x1, sim_y1, test_size=0.3, random_state=44, stratify=sim_y1 ) cosmos_dev_x, cosmos_val_x, cosmos_dev_y, cosmos_val_y = train_test_split( cosmos_x1, cosmos_y1, test_size=0.3, random_state=45, stratify=cosmos_y1 ) weight = np.asarray([0.9 / len(sim_dev_y)] * len(sim_dev_y) + [0.1 / len(cosmos_dev_y)] * len(cosmos_dev_y)) training_data = Data(x=np.hstack([sim_dev_x, cosmos_dev_x]), y=np.hstack([sim_dev_y, cosmos_dev_y]), weight=weight) validation_data = Data(x=np.hstack([sim_val_x, cosmos_val_x]), y=np.hstack([sim_val_y, cosmos_val_y])) test_data = Data(x=np.hstack([sim_x2, cosmos_x2]), y=np.hstack([sim_y2, cosmos_y2])) input_data.training_data = training_data input_data.validation_data = validation_data input_data.test_data = test_data mean, std = compute_moments( train_data=training_data.x, input1=input1, input2=input2, norm=norm, use_redshift=False, is_hsc=False, threads=threads ) input_data.mean, input_data.std = mean, std for i in range(n_trials): study.optimize( lambda trial: objective_plasticc( trial=trial, input_data=input_data, optimizer_setting=optimizer_setting, seed=seed, loop_setting=loop_setting, normalization=norm, threads=threads, binary=binary, sim_sn_path=sim_sn_path, training_cosmos_path=training_cosmos_path, flux_err=flux_err ), n_trials=1 ) df = study.trials_dataframe() df.to_csv(os.path.join(model_dir, 'result.csv'))
"bagging_fraction": trial.suggest_uniform("bagging_fraction", 0.4, 1.0), "bagging_freq": trial.suggest_int("bagging_freq", 1, 7), "min_data_in_leaf": trial.suggest_int("min_data_in_leaf", 1, 50), "min_child_samples": trial.suggest_int("min_child_samples", 5, 100), }, }, } evals_result = dict() model = init_instance_by_config(task["model"]) model.fit(dataset, evals_result=evals_result) return min(evals_result["valid"]) if __name__ == "__main__": provider_uri = "~/.qlib/qlib_data/cn_data" GetData().qlib_data(target_dir=provider_uri, region=REG_CN, exists_skip=True) qlib.init(provider_uri=provider_uri, region=REG_CN) dataset = init_instance_by_config(DATASET_CONFIG) study = optuna.Study(study_name="LGBM_360", storage="sqlite:///db.sqlite3") study.optimize(objective, n_jobs=6)
return history #print(time_callback.times) def objective(trial): # ハイパーパラメータ(オプティマイザーと学習率を調べる) # optimizer = trial.suggest_categorical("optimizer", ["sgd", "momentum", "rmsprop", "adam"]) optimizer = trial.suggest_categorical("optimizer", ["sgd", "adam"]) learning_rate = trial.suggest_loguniform("learning_rate", 1e-4, 1e0) K.clear_session() hist = train(optimizer, learning_rate, trial) # return 1.0 - np.max(hist["val_acc"]) return hist["val_loss"][-1] #study = optuna.create_study() study = optuna.Study(study_name='distributed-raccoon', storage='sqlite:///example.db') study.optimize(objective, n_trials=10) print(study.best_params) print(study.best_value) print(study.best_trial) trial_df = study.trials_dataframe() trial_df.to_csv("optuna_ssd_raccoon.csv")