def load_args(load_key='default'): """get train args of retore id""" load_data = Config.get_load_data(load_key) if load_data is None: return False args_dict = load_data['args'] #Config.parse_args_dict(args_dict) return args_dict
def load_datapoints(load_path=None, load_key=None): if load_path is None: load_data = Config.get_load_data(load_key) else: load_path = file_to_path(load_path) if os.path.exists(load_path): load_data = joblib.load(load_path) print('Load file', load_path) if load_data is None: return False return load_data['datapoints']
def load_params_for_scope(sess, scope, load_key='default', load_path=None): if load_path is None: load_data = Config.get_load_data(load_key) else: load_path = file_to_path(load_path) print('Load file', load_path) if os.path.exists(load_path): load_data = joblib.load(load_path) print('Load file', load_path) else: raise ValueError if load_data is None: return False params_dict = load_data['params'] if scope in params_dict: print('Loading saved file for scope', scope) loaded_params = params_dict[scope] loaded_params, params = get_savable_params(loaded_params, scope, keep_heads=True) restore_params(sess, loaded_params, params) return True
def main(): args = setup_and_load() comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() seed = int(time.time()) % 10000 utils.mpi_print(seed * 100 + rank) set_global_seeds(seed * 100 + rank) # For wandb package to visualize results curves config = Config.get_args_dict() config['global_seed'] = seed wandb.init(name=config["run_id"], project="coinrun", notes=" GARL generate seed", tags=["try"], config=config) utils.setup_mpi_gpus() utils.mpi_print('Set up gpu', args) config = tf.ConfigProto() config.gpu_options.allow_growth = True # pylint: disable=E1101 eval_limit = Config.EVAL_STEP * 10**6 phase_eval_limit = int(eval_limit // Config.TRAIN_ITER) total_timesteps = int(Config.TOTAL_STEP * 10**6) phase_timesteps = int((total_timesteps - eval_limit) // Config.TRAIN_ITER) with tf.Session(config=config): sess = tf.get_default_session() # init env nenv = Config.NUM_ENVS env = make_general_env(nenv, rand_seed=seed) utils.mpi_print('Set up env') policy = policies_back.get_policy() utils.mpi_print('Set up policy') optimizer = SeedOptimizer(env=env, logdir=Config.LOGDIR, spare_size=Config.SPA_LEVELS, ini_size=Config.INI_LEVELS, eval_limit=phase_eval_limit, train_set_limit=Config.NUM_LEVELS, load_seed=Config.LOAD_SEED, rand_seed=seed, rep=1, log=True) step_elapsed = 0 t = 0 if args.restore_id is not None: datapoints = Config.get_load_data('default')['datapoints'] step_elapsed = datapoints[-1][0] optimizer.load() seed = optimizer.hist[-1] env.set_seed(seed) t = 16 print('loadrestore') Config.RESTORE_ID = Config.get_load_data( 'default')['args']['run_id'] Config.RUN_ID = Config.get_load_data( 'default')['args']['run_id'].replace('-', '_') while (step_elapsed < (Config.TOTAL_STEP - 1) * 10**6): # ============ GARL ================= # optimize policy mean_rewards, datapoints = learn_func( sess=sess, policy=policy, env=env, log_interval=args.log_interval, save_interval=args.save_interval, nsteps=Config.NUM_STEPS, nminibatches=Config.NUM_MINIBATCHES, lam=Config.GAE_LAMBDA, gamma=Config.GAMMA, noptepochs=Config.PPO_EPOCHS, ent_coef=Config.ENTROPY_COEFF, vf_coef=Config.VF_COEFF, max_grad_norm=Config.MAX_GRAD_NORM, lr=lambda f: f * Config.LEARNING_RATE, cliprange=lambda f: f * Config.CLIP_RANGE, start_timesteps=step_elapsed, total_timesteps=phase_timesteps, index=t) # test catestrophic forgetting if 'Forget' in Config.RUN_ID: last_set = list(env.get_seed_set()) if t > 0: curr_set = list(env.get_seed_set()) last_scores, _ = eval_test(sess, nenv, last_set, train=True, idx=None, rep_count=len(last_set)) curr_scores, _ = eval_test(sess, nenv, curr_set, train=True, idx=None, rep_count=len(curr_set)) tmp = set(curr_set).difference(set(last_set)) mpi_print("Forgetting Exp") mpi_print("Last setsize", len(last_set)) mpi_print("Last scores", np.mean(last_scores), "Curr scores", np.mean(curr_scores)) mpi_print("Replace count", len(tmp)) # optimize env step_elapsed = datapoints[-1][0] if t < Config.TRAIN_ITER: best_rew_mean = max(mean_rewards) env, step_elapsed = optimizer.run(sess, env, step_elapsed, best_rew_mean) t += 1 save_final_test = True if save_final_test: final_test = {} final_test['step_elapsed'] = step_elapsed train_set = env.get_seed() final_test['train_set_size'] = len(train_set) eval_log = eval_test(sess, nenv, train_set, train=True, is_high=False, rep_count=1000, log=True) final_test['Train_set'] = eval_log eval_log = final_test(sess, nenv, None, train=False, is_high=True, rep_count=1000, log=True) final_test['Test_set'] = eval_log joblib.dump(final_test, setup_utils.file_to_path('final_test')) env.close()