def analyse_ray_dump(ray_directory, topn, metric="avg_inc_acc"): if metric not in ("avg_inc_acc", "last_acc"): raise NotImplementedError("Unknown metric {}.".format(metric)) ea = Analysis(ray_directory) trials_dataframe = ea.dataframe() trials_dataframe = trials_dataframe.sort_values(by=metric, ascending=False) mapping_col_to_index = {} result_index = -1 for index, col in enumerate(trials_dataframe.columns): if col.startswith("config:"): mapping_col_to_index[col[7:]] = index elif col == metric: result_index = index print("Ray config: {}".format(ray_directory)) print("Best Config:") print("{}: {} with {}.".format( metric, trials_dataframe.iloc[0][result_index], _get_line_results(trials_dataframe, 0, mapping_col_to_index))) print("\nFollowed by:") if topn < 0: topn = len(trials_dataframe) else: topn = min(topn - 1, len(trials_dataframe)) for i in range(1, topn): print("avg_inc_acc: {} with {}.".format( trials_dataframe.iloc[i][result_index], _get_line_results(trials_dataframe, i, mapping_col_to_index))) return _get_line_results(trials_dataframe, 0, mapping_col_to_index)
def main(): analysis = Analysis(TUNE_RESULTS_FOLDER) print("Best hyperparameter {}".format( analysis.get_best_config(metric="mean_reward", mode="max"))) best_model_path = analysis.get_best_logdir(metric="mean_reward", mode="max") print( "Best model found in {}, start rendering .gif".format(best_model_path)) best_model = SomeModelToTrain({ 'learning_rate': 0.1, 'batch_size': 1, 'target_update': 1 }) checkpoint_path = f'{best_model_path}/checkpoint_{MAX_TRAINING_ITERATION}' best_model.load(checkpoint_path + '/' + MODEL_FILENAME) # we got this part from https://stable-baselines.readthedocs.io/en/master/guide/examples.html and modified it env = gym.make('LunarLander-v2') images = [] state = env.reset() for j in range(210): action = best_model.agent.act(state) img = env.render(mode='rgb_array') images.append(img) state, reward, done, _ = env.step(action) if done: break env.close() imageio.mimsave( 'best_model.gif', [np.array(img) for i, img in enumerate(images) if i % 2 == 0], fps=29) optimize('best_model.gif')
def testBestLogdir(self): analysis = Analysis(self.test_dir) logdir = analysis.get_best_logdir(self.metric) self.assertTrue(logdir.startswith(self.test_dir)) logdir2 = analysis.get_best_logdir(self.metric, mode="min") self.assertTrue(logdir2.startswith(self.test_dir)) self.assertNotEquals(logdir, logdir2)
def collect(self): """ Collects and cleans data on the running Tune experiment from the Tune logs so that users can see this information in the front-end client """ sub_dirs = os.listdir(self._logdir) job_names = filter( lambda d: os.path.isdir(os.path.join(self._logdir, d)), sub_dirs) self._trial_records = {} # search through all the sub_directories in log directory for job_name in job_names: analysis = Analysis(str(os.path.join(self._logdir, job_name))) df = analysis.dataframe() if len(df) == 0: continue self._available = True # make sure that data will convert to JSON without error df["trial_id"] = df["trial_id"].astype(str) df = df.fillna(0) # convert df to python dict df = df.set_index("trial_id") trial_data = df.to_dict(orient="index") # clean data and update class attribute if len(trial_data) > 0: trial_data = self.clean_trials(trial_data, job_name) self._trial_records.update(trial_data)
def analyze_ray_experiment(exp_dir, default_metric, default_mode): from ray.tune import Analysis analysis = Analysis(exp_dir, default_metric=default_metric, default_mode=default_mode) topk_summary_plot_v2(analysis, 5, save_dir=exp_dir) summ, styled = summarize_top_k(analysis, k=10, save_dir=exp_dir)
def get_tune_dfs(path): """ Here we load the df which is produced/logged by Tune. For more details see here: https://docs.ray.io/en/latest/tune/api_docs/analysis.html#tune-analysis-docs """ analysis = Analysis(path) d = analysis.trial_dataframes return d
def collect(self): """ Collects and cleans data on the running Tune experiment from the Tune logs so that users can see this information in the front-end client """ sub_dirs = os.listdir(self._logdir) job_names = filter( lambda d: os.path.isdir(os.path.join(self._logdir, d)), sub_dirs) self._trial_records = {} # search through all the sub_directories in log directory for job_name in job_names: analysis = Analysis(str(os.path.join(self._logdir, job_name))) df = analysis.dataframe() if len(df) == 0 or "trial_id" not in df.columns: continue # start TensorBoard server if not started yet if not self._tensor_board_started: tb = program.TensorBoard() tb.configure(argv=[None, "--logdir", self._logdir]) tb.launch() self._tensor_board_started = True self._available = True # make sure that data will convert to JSON without error df["trial_id_key"] = df["trial_id"].astype(str) df = df.fillna(0) trial_ids = df["trial_id"] for i, value in df["trial_id"].iteritems(): if type(value) != str and type(value) != int: trial_ids[i] = int(value) df["trial_id"] = trial_ids # convert df to python dict df = df.set_index("trial_id_key") trial_data = df.to_dict(orient="index") # clean data and update class attribute if len(trial_data) > 0: trial_data = self.clean_trials(trial_data, job_name) self._trial_records.update(trial_data) self.collect_errors(job_name, df)
def get_best_specific_params_dir(opt, layer, model, att_type): analysis = Analysis("../ray_tune/{}".format(opt['folder'])) df = analysis.dataframe(metric=opt['metric'], mode='max') print(df) df.columns = [c.replace('config/', '') for c in df.columns] print(df) # # newdf = df.loc[(df.num_layers == layers) & (df.model == model) & (df.att_type == att_type)] print(layer) print(df['num_layers'].unique()) print(model) print(df['model'].unique()) print(att_type) print(df['att_type'].unique()) newdf = df.loc[(df['num_layers'] == layer) & (df['model'] == model) & (df['att_type'] == att_type)] print(newdf) best_params_dir = newdf.sort_values( 'accuracy', ascending=False)['logdir'].iloc[opt['index']] return best_params_dir
def meta_setup(self): from ray.tune import Analysis out_meta = ContextCompositeNode.meta_setup(self) if 'tune' in self.conf: if 'local_dir' in self.conf['tune']: path = self.conf['tune']['local_dir'] if 'name' in self.conf['tune']: exp = self.conf['tune']['name'] try: analysis = Analysis(path + '/' + exp) if 'best' in self.conf: best = analysis.get_best_config( **self.conf['best']) for key in best.keys(): self.conf['context'][key]['value'] = best[key] print('get best', best) out_meta.outports[self.OUTPUT_CONFIG] = self.conf except Exception: pass return out_meta
def collect(self): """ Collects and cleans data on the running Tune experiment from the Tune logs so that users can see this information in the front-end client """ self._trial_records = {} self._errors = {} if not self._logdir: return # search through all the sub_directories in log directory analysis = Analysis(str(self._logdir)) df = analysis.dataframe() if len(df) == 0 or "trial_id" not in df.columns: return self._trials_available = True # make sure that data will convert to JSON without error df["trial_id_key"] = df["trial_id"].astype(str) df = df.fillna(0) trial_ids = df["trial_id"] for i, value in df["trial_id"].iteritems(): if type(value) != str and type(value) != int: trial_ids[i] = int(value) df["trial_id"] = trial_ids # convert df to python dict df = df.set_index("trial_id_key") trial_data = df.to_dict(orient="index") # clean data and update class attribute if len(trial_data) > 0: trial_data = self.clean_trials(trial_data) self._trial_records.update(trial_data) self.collect_errors(df)
from ray.tune import Analysis import pandas as pd import os import numpy as np if __name__ == "__main__": analysis = Analysis( "/Users/shaobohu/Documents/我的坚果云/project/circles_experiment/TRY_ALL/Train") print(sorted(analysis.dataframe()['acc'].tolist())) print(analysis.get_best_config('acc', 'max'))
def main(args): ray.init(num_cpus=args.num_cpus, memory=3000 * 1024**2, object_store_memory=300 * 1024**2) def train_reg(config, reporter): sys.path.append(BASE_DIR) # 1) load / generate data from experiments.data_sim import provide_data data_train, data_valid, _ = provide_data(dataset=args.dataset) # 2) setup model from meta_learn.GPR_meta_vi import GPRegressionMetaLearnedVI torch.set_num_threads(N_THREADS_PER_RUN) model = GPRegressionMetaLearnedVI(data_train, **config) # 3) train and evaluate model eval_period = 2000 train_iter = 0 for i in range(config["num_iter_fit"] // eval_period): loss = model.meta_fit(verbose=False, log_period=2000, n_iter=eval_period) train_iter += eval_period ll, rmse, calib_err = model.eval_datasets(data_valid) reporter(timesteps_total=train_iter, loss=loss, test_rmse=rmse, test_ll=ll, calib_err=calib_err) @ray.remote def train_test(config): results_dict = config try: sys.path.append(BASE_DIR) # 1) load / generate data from experiments.data_sim import provide_data data_train, _, data_test = provide_data(dataset=args.dataset, seed=SEED) # 2) Fit model from meta_learn.GPR_meta_vi import GPRegressionMetaLearnedVI torch.set_num_threads(N_THREADS_PER_RUN) model = GPRegressionMetaLearnedVI(data_train, **config) model.meta_fit(data_test, log_period=5000) # 3) evaluate on test set ll, rmse, calib_err = model.eval_datasets(data_test) results_dict.update(ll=ll, rmse=rmse, calib_err=calib_err) except Exception as e: print(e) results_dict.update(ll=np.nan, rmse=np.nan, calib_err=np.nan) return results_dict if len(args.load_analysis_from) > 0: assert os.path.isdir(args.load_analysis_from ), 'load_analysis_from must be a valid directory' print('Loading existing tune analysis results from %s' % args.load_analysis_from) analysis = Analysis(args.load_analysis_from) exp_name = os.path.basename(args.load_analysis_from) else: space = { "weight_prior_std": hp.loguniform("weight_prior_std", math.log(5e-2), math.log(1.0)), "prior_factor": hp.loguniform("prior_factor", math.log(1e-5), math.log(1e-1)), "lr": hp.loguniform("lr", math.log(5e-4), math.log(5e-3)), "lr_decay": hp.loguniform("lr_decay", math.log(0.8), math.log(1.0)), "svi_batch_size": hp.choice("svi_batch_size", [10, 50]), "task_batch_size": hp.choice("task_batch_size", [4, 10]), } config = { "num_samples": 240, "config": { "num_iter_fit": 30000, 'kernel_nn_layers': [32, 32, 32, 32], 'mean_nn_layers': [32, 32, 32, 32], 'random_seed': SEED, 'mean_module': 'NN', 'covar_module': args.covar_module, 'normalize_data': True, 'cov_type': 'diag' }, "stop": { "timesteps_total": 30000 }, } # Run hyper-parameter search algo = HyperOptSearch(space, max_concurrent=args.num_cpus, metric="test_ll", mode="max") exp_name = 'tune_meta_vi_%s_kernel_%s' % (args.covar_module, args.dataset) analysis = tune.run(train_reg, name=exp_name, search_alg=algo, verbose=1, local_dir=HPARAM_EXP_DIR, **config) # Select N best configurations re-run train & test with 5 different seeds from experiments.hyperparam_search.util import select_best_configs if args.metric == 'test_ll': best_configs = select_best_configs(analysis, metric='test_ll', mode='max', N=args.n_test_runs) elif args.metric == 'test_rmse': best_configs = select_best_configs(analysis, metric='test_rmse', mode='min', N=args.n_test_runs) else: raise AssertionError('metric must be test_ll or test_rmse') test_configs = [] for config in best_configs: for seed in TEST_SEEDS: test_config = copy.deepcopy(config) test_config.update({'random_seed': seed}) test_configs.append(test_config) result_dicts = ray.get( [train_test.remote(config) for config in test_configs]) result_df = pd.DataFrame(result_dicts) print(result_df.to_string()) csv_file_name = os.path.join( HPARAM_EXP_DIR, '%s_%s.csv' % (exp_name, datetime.now().strftime("%b_%d_%Y_%H:%M:%S"))) result_df.to_csv(csv_file_name) print("\nSaved result csv to %s" % csv_file_name)
def main(args): ray.init(num_cpus=args.num_cpus, memory=3000 * 1024**2, object_store_memory=300 * 1024**2) def train_reg(config, reporter): sys.path.append(BASE_DIR) # 1) load / generate data from experiments.data_sim import provide_data data_train, data_valid, _ = provide_data(dataset=args.dataset, seed=SEED) # 2) setup model from meta_learn.GPR_meta_pac import GPRegressionMetaLearnedPAC torch.set_num_threads(N_THREADS_PER_RUN) model = GPRegressionMetaLearnedPAC(data_train, **config) # 3) train and evaluate model with gpytorch.settings.max_cg_iterations(300): log_period = 5000 train_iter = 0 loss = 0.0 diagnostics_dict = {} for i in range(config["num_iter_fit"] // log_period): loss, diagnostics_dict = model.meta_fit(verbose=False, log_period=1000, eval_period=100000, n_iter=log_period) train_iter += log_period if i < config["num_iter_fit"] // log_period - 1: reporter(timesteps_total=train_iter, loss=loss, test_rmse=math.nan, test_ll=math.nan, calib_err=math.nan, **diagnostics_dict) ll, rmse, calib_err = model.eval_datasets(data_valid, n_iter_meta_test=3000) reporter(timesteps_total=train_iter, loss=loss, test_rmse=rmse, test_ll=ll, calib_err=calib_err, **diagnostics_dict) @ray.remote def train_test(config): results_dict = config try: sys.path.append(BASE_DIR) # 1) load / generate data from experiments.data_sim import provide_data data_train, _, data_test = provide_data(dataset=args.dataset, seed=SEED) # 2) Fit model from meta_learn.GPR_meta_pac import GPRegressionMetaLearnedPAC torch.set_num_threads(N_THREADS_PER_RUN) with gpytorch.settings.max_cg_iterations(500): model = GPRegressionMetaLearnedPAC(data_train, **config) model.meta_fit( data_test, log_period=1000, eval_period=100000, ) # 3) evaluate on test set ll, rmse, calib_err = model.eval_datasets( data_test, n_iter_meta_test=3000) results_dict.update(ll=ll, rmse=rmse, calib_err=calib_err) except Exception as e: print(e) results_dict.update(ll=np.nan, rmse=np.nan, calib_err=np.nan) return results_dict assert args.metric in ['test_ll', 'test_rmse'] exp_name = 'tune_meta_pac_%s_kernel_%s' % (args.covar_module, args.dataset) if args.load_analysis: analysis_dir = os.path.join(HPARAM_EXP_DIR, exp_name) assert os.path.isdir( analysis_dir), 'load_analysis_from must be a valid directory' print('Loading existing tune analysis results from %s' % analysis_dir) analysis = Analysis(analysis_dir) else: space = { "task_kl_weight": hp.loguniform("task_kl_weight", math.log(5e-2), math.log(1e0)), "meta_kl_weight": hp.loguniform("meta_kl_weight", math.log(1e-7), math.log(1e0)), "lr": hp.loguniform("lr", math.log(1e-4), math.log(1e-3)), "lr_decay": hp.loguniform("lr_decay", math.log(0.92), math.log(0.97)), "posterior_lr_multiplier": hp.loguniform("posterior_lr_multiplier", math.log(1e0), math.log(10.)), "svi_batch_size": hp.choice("svi_batch_size", [5, 10]), "task_batch_size": hp.choice("task_batch_size", [5, 20]), } config = { "num_samples": 150, "config": { "num_iter_fit": 40000, 'kernel_nn_layers': [32, 32, 32, 32], 'mean_nn_layers': [32, 32, 32, 32], 'random_seed': SEED, 'mean_module': 'NN', 'covar_module': args.covar_module, 'normalize_data': True, 'cov_type': 'diag' }, "stop": { "timesteps_total": 100000 }, } config["config"].update() # configs_to_evaluate = [{ # "task_kl_weight": 1.0, # "meta_kl_weight": 1e-5, # "lr": 1e-3, # "lr_decay": 0.95, # "posterior_lr_multiplier": 5.0, # "svi_batch_size": 0, # "task_batch_size": 0, # }, # ] # Run hyper-parameter search algo = HyperOptSearch( space, max_concurrent=args.num_cpus, metric=args.metric, mode="max" if args.metric == 'test_ll' else "min", ) analysis = custom_tune.run(train_reg, name=exp_name, search_alg=algo, verbose=1, raise_on_failed_trial=False, local_dir=HPARAM_EXP_DIR, resume=args.resume, **config) # Select N best configurations re-run train & test with 5 different seeds from experiments.hyperparam_search.util import select_best_configs if args.metric == 'test_ll': best_configs = select_best_configs(analysis, metric='test_ll', mode='max', N=args.n_test_runs) elif args.metric == 'test_rmse': best_configs = select_best_configs(analysis, metric='test_rmse', mode='min', N=args.n_test_runs) else: raise AssertionError('metric must be test_ll or test_rmse') test_configs = [] for config in best_configs: for seed in TEST_SEEDS: test_config = copy.deepcopy(config) test_config.update({'random_seed': seed}) test_configs.append(test_config) result_dicts = ray.get( [train_test.remote(config) for config in test_configs]) result_df = pd.DataFrame(result_dicts) print(result_df.to_string()) csv_file_name = os.path.join( HPARAM_EXP_DIR, '%s_%s.csv' % (exp_name, datetime.now().strftime("%b_%d_%Y_%H:%M:%S"))) result_df.to_csv(csv_file_name) print("\nSaved result csv to %s" % csv_file_name)
project_dir = os.path.dirname(os.path.join(os.getcwd(), __file__)) sys.path.append(os.path.normpath(os.path.join(project_dir, '..', '..'))) from experiments.utils.constants import RESULTS_PATH series = 6 type_env = 2 dataset_id = 0 workload_id = 0 experiment_id = 0 algorithm = 'PPO' trial = 'PPO_CloudSimV2_847c5_00000_0_fcnet_hiddens_0=20,fcnet_hiddens_1=20_2021-02-14_22-46-46' experiment_folder_path = os.path.join(RESULTS_PATH, "series", str(series), "envs", str(type_env), "datasets", str(dataset_id), "workloads", str(workload_id), "experiments", str(experiment_id), str(algorithm), trial) analysis = Analysis(experiment_folder_path) df = analysis.trial_dataframes[experiment_folder_path] selected_stats = [ 'episode_reward_mean', 'episodes_this_iter', 'timesteps_total', 'episodes_total', 'experiment_id', 'custom_metrics/num_moves_mean', 'custom_metrics/num_consolidated_mean', 'custom_metrics/num_overloaded_mean' ] df = df[selected_stats] # each row of the df is the training iteration a = 1
from ray.tune import Analysis import torch import matplotlib.pyplot as plt from core.dataloader import * from core.model import * from core.training import * from core.evaluating import * from plots.plots import * from core.create_folder import * from core.pred_sequence import * folders = [] folders.append( "Run_w_11_timesteps_3_hiddenDim_1_layers_0.00021723989839730966_LR") configs = [] analysis = Analysis("~/ray_results/Jan21") configs.append(analysis.get_best_config(metric="error", mode="max")) for i in range(2, 17): analysis = Analysis("~/ray_results/" + str(i) + "forward") config = analysis.get_best_config(metric="error", mode="max") configs.append(config) print(config) folder_name = 'Predicting' + str(config["num_forward"]) + '_w_' + str( config["timesteps"]) + '_timesteps_' + str( config["hidden_dim"]) + '_hiddenDim_' + str( config["num_layers"]) + '_layers_' + str(config["lr"]) + "_LR" folders.append(folder_name) #print(folder_name) print(len(folders)) print(len(configs)) model_keys = []
def raytune_analysis(exp_dir, save, skip, mode, metric): analysis = Analysis(exp_dir, default_metric=metric, default_mode=mode) plot_ray_analysis(analysis, save=save, skip=skip) analyze_ray_experiment(exp_dir, default_metric=metric, default_mode=mode)
def get_best_info(exp_dir, metrics=['vl_score', 'vl_loss'], ascending=[False, True], mode='auto'): if mode == 'auto': analysis = Analysis(exp_dir, 'vl_score', 'max') df = analysis.dataframe() df = df.sort_values(metrics, ascending=ascending) trial_dir = df.iloc[0][-1] min_ = 10000 for f in os.listdir(trial_dir): if 'checkpoint' in f: idx = int(f.split('_')[1]) min_ = min(min_, idx) chk_file = os.path.join(trial_dir, f'checkpoint_{min_}', 'model.pth') with open(os.path.join(trial_dir, 'params.json')) as f: config = json.load(f) with open(os.path.join(trial_dir, 'result.json')) as f: res = [json.loads(i) for i in f] best_res = res[min_ - 1] return { 'trial_dir': trial_dir, 'chk_file': chk_file, 'config': config, 'tr_loss': best_res['tr_loss'], 'tr_score': best_res['tr_score'], 'vl_loss': best_res['vl_loss'], 'vl_score': best_res['vl_score'] } elif mode == 'manual': best_dict = { 'trial_dir': None, 'chk_file': None, 'config': None, 'tr_loss': float('inf'), 'tr_score': 0, 'vl_loss': float('inf'), 'vl_score': 0 } dirs = [ part_dir for part_dir in os.listdir(exp_dir) if os.path.isdir(os.path.join(exp_dir, part_dir)) ] for part_dir in dirs: trial_dir = os.path.join(exp_dir, part_dir) min_ = 400 for f in os.listdir(trial_dir): if 'checkpoint' in f: idx = int(f.split('_')[1]) min_ = min(min_, idx) with open(os.path.join(trial_dir, 'result.json')) as f: for i, d in enumerate(f): if i + 1 == min_: curr = json.loads(d) if best_dict['vl_score'] < curr[ 'vl_score']: #or (best_dict['vl_score'] == curr['vl_score'] and best_dict['vl_loss'] > curr['vl_loss']): with open(os.path.join(trial_dir, 'params.json')) as f: config = json.load(f) best_dict = { 'trial_dir': trial_dir, 'chk_file': os.path.join(trial_dir, f'checkpoint_{min_}/model.pth'), 'config': config, 'tr_loss': curr['tr_loss'], 'tr_score': curr['tr_score'], 'vl_loss': curr['vl_loss'], 'vl_score': curr['vl_score'] } return best_dict
def testDataframe(self): analysis = Analysis(self.test_dir) df = analysis.dataframe() self.assertTrue(isinstance(df, pd.DataFrame)) self.assertEquals(df.shape[0], self.num_samples * 2)
def get_best_params_dir(opt): analysis = Analysis("../ray_tune/{}".format(opt['folder'])) df = analysis.dataframe(metric=opt['metric'], mode='max') best_params_dir = df.sort_values( 'accuracy', ascending=False)['logdir'].iloc[opt['index']] return best_params_dir
def testBestConfigIsLogdir(self): analysis = Analysis(self.test_dir) for metric, mode in [(self.metric, "min"), (self.metric, "max")]: logdir = analysis.get_best_logdir(metric, mode=mode) best_config = analysis.get_best_config(metric, mode=mode) self.assertEquals(analysis.get_all_configs()[logdir], best_config)
""" from ray.tune import Analysis import torch import matplotlib.pyplot as plt from core.dataloader import * from core.model import * from core.training import * from core.evaluating import * from plots.plots import * from core.create_folder import * from core.pred_sequence import * configs = [] #analysis_GSPC = Analysis("~/ray_results/1forward_returns_GSPC") #analysis_IXIC = Analysis("~/ray_results/1forward_returns_IXIC") #analysis_N225 = Analysis("~/ray_results/1forward_returns_N225") analysis_DJI = Analysis("~/ray_results/1_forward_returns_DJI") #configs.append(analysis_GSPC.get_best_config(metric="error", mode="max")) #configs.append(analysis_IXIC.get_best_config(metric="error", mode="max")) #configs.append(analysis_N225.get_best_config(metric="error", mode="max")) configs.append(analysis_DJI.get_best_config(metric="error", mode="max")) print(configs) configs_np = np.empty(shape=(4, 6)) datasets = ["GSPC", "IXIC", "N225", "DJI"] for i in range(4): configs_np[i][0] = datasets[i] configs_np[i][1] = configs[i]["num_layers"] configs_np[i][2] = configs[i]["hidden_dim"] configs_np[i][3] = configs[i]["lr"] configs_np[i][4] = configs[i]["timesteps"] configs_np[i][5] = configs[i]["dropout"] configs_df = pd.DataFrame(configs_np,