def main(log_dir, easy, n_steps=450): exp_dir, seed_offset = get_exp_dir( os.path.join(log_dir, "reacher-obstacle-default" + ("-easy" if easy else "") + "-ppo")) print("Seed offset: " + str(seed_offset)) log_path = os.path.join(exp_dir, "ppo-reach-avoid.log") avg_log_path = exp_dir if not os.path.exists(log_path): n_envs = 8 env = VecNormalize( SubprocVecEnv([create_env_fn(seed_offset * n_envs + i, easy=easy) for i in range(0, n_envs)]), gamma=0.999) model = PPO2(policy='MlpPolicy', env=env, n_steps=n_steps, nminibatches=5, verbose=1, gamma=0.999, noptepochs=15, ent_coef=1e-3, lam=1, policy_kwargs=dict(layers=[164, 164])) average_rewards = [] def log_callback(local_vars, global_vars): avg_r = np.mean([ep_info['r'] for ep_info in local_vars["ep_info_buf"]]) average_rewards.append(avg_r) return True # 3067500 = 409 iterations (400 + 9 for buffer initialization) * 50 trajectories * 150 timesteps model.learn(3067500, seed=seed_offset, callback=log_callback) model.save(log_path) env.save_running_average(avg_log_path) np.save(os.path.join(exp_dir, "rewards.npy"), np.array(average_rewards))
class PPO2_SB(): def __init__(self): self.love = 'Ramona' self.env_fns = [] self.env_names = [] def make_env(self, env_id, rank, seed=0): """ Utility function for multiprocessed env. :param env_id: (str) the environment ID :param num_env: (int) the number of environment you wish to have in subprocesses :param seed: (int) the inital seed for RNG :param rank: (int) index of the subprocess """ def _init(): env = Template_Gym() env.seed(seed + rank) return env set_global_seeds(seed) return _init def train(self, num_e=1, n_timesteps=1000000, save_fraction=0.1, save='saves/aud5'): env_id = "default" num_e = 1 # Number of processes to use # Create the vectorized environment #env = DummyVecEnv([lambda: env]) #Ramona self.env = SubprocVecEnv([self.make_env(env_id, i) for i in range(num_e)]) #env = Template_Gym() #self.env = DummyVecEnv([lambda: env]) self.env = VecNormalize(self.env, norm_obs=True, norm_reward=True) self.model = PPO2(CustomPolicy_4, self.env, verbose=0, learning_rate=1e-4, nminibatches=1, tensorboard_log="./day1" ) #self.model = PPO2.load("default9", self.env, policy=CustomPolicy, tensorboard_log="./test/" ) n_timesteps = n_timesteps * save_fraction n_timesteps = int(n_timesteps) training_loop = 1 / save_fraction training_loop = int(training_loop) log_dir = "saves" for i in range(training_loop): self.model.learn(n_timesteps) self.model.save(save+str(i)) self.env.save_running_average(log_dir) self.env.save_running_average(log_dir) def evaluate(self, num_env=1, num_steps=21900, load="saves/aud5", runs=10): """ Evaluate a RL agent :param model: (BaseRLModel object) the RL Agent :param num_steps: (int) number of timesteps to evaluate it :return: (float) Mean reward """ env_id = 'default' num_e = 1 log_dir = "saves" self.env = SubprocVecEnv([self.make_env(env_id, i) for i in range(num_env)]) #self.model = PPO2(CustomPolicy, self.env, verbose=1, learning_rate=1e-5, tensorboard_log="./default" ) self.env = VecNormalize(self.env, norm_obs=True, norm_reward=True) self.env.load_running_average(log_dir) for i in range(runs): self.model = PPO2.load(load+str(i), self.env, policy=CustomPolicy_4, tensorboard_log="./default/" ) self.env.load_running_average(log_dir) episode_rewards = [[0.0] for _ in range(self.env.num_envs)] #self.total_pips = [] obs = self.env.reset() state = None # When using VecEnv, done is a vector done = [False for _ in range(env.num_envs)] for i in range(num_steps): # _states are only useful when using LSTM policies action, state = self.model.predict(obs, state=state, mask=done, deterministic=True) obs, rewards , dones, _ = self.env.step(action) #actions, _states = self.model.predict(obs) # # here, action, rewards and dones are arrays # # because we are using vectorized env #obs, rewards, dones, info = self.env.step(actions) #self.total_pips.append(self.env.player.placement) # Stats for i in range(self.env.num_envs): episode_rewards[i][-1] += rewards[i] if dones[i]: episode_rewards[i].append(0.0) mean_rewards = [0.0 for _ in range(self.env.num_envs)] n_episodes = 0 for i in range(self.env.num_envs): mean_rewards[i] = np.mean(episode_rewards[i]) n_episodes += len(episode_rewards[i]) # Compute mean reward mean_reward = np.mean(mean_rewards) print("Mean reward:", mean_reward, "Num episodes:", n_episodes) return mean_reward def pre_train(self, num_e=1, load="saves/m19"): env_id = 'default' num_e = 1 log_dir = "saves" # Usingenv = make_env() only one expert trajectory # you can specify `traj_limitation=-1` for using the whole dataset dataset = ExpertDataset(expert_path='default2.npz',traj_limitation=1, batch_size=128) self.env = SubprocVecEnv([self.make_env(env_id, i) for i in range(num_e)]) #env = Template_Gym() #self.env = DummyVecEnv([lambda: env]) self.env = VecNormalize(self.env, norm_obs=True, norm_reward=True) #env = make_env() #model = GAIL("MlpPolicy", env=env, expert_dataset=dataset, verbose=1) self.env.load_running_average("saves") self.model = PPO2(CustomPolicy, self.env, verbose=1, nminibatches=1, learning_rate=1e-5, tensorboard_log="./m1ln4" ) #self.model = PPO2.load("saves/m19", self.env, policy=CustomPolicy, tensorboard_log="./default/" ) self.env.load_running_average("saves") # Pretrain the PPO2 model self.model.pretrain(dataset, n_epochs=10000) # As an option, you can train the RL agent #self.model.learn(int(100000000)) # Test the pre-trained model self.env = self.model.get_env() self.env.load_running_average("saves") obs = self.env.reset() reward_sum = 0.0 for _ in range(1000000): action, _ = self.model.predict(obs) obs, reward, done, _ = self.env.step(action) reward_sum += reward #self.env.render() if done: print(reward_sum) reward_sum = 0.0 obs = self.env.reset() self.env.close() def gen_pre_train(self, num_e=1, save='default2', episodes=1000): #self.create_envs(game_name=game, state_name=state, num_env=num_e) #self.env=SubprocVecEnv(self.env_fns) env_id = 'default' num_e = 1 self.env = SubprocVecEnv([self.make_env(env_id, i) for i in range(num_e)]) #env = Template_Gym() #self.env = DummyVecEnv([lambda: env]) self.env = VecNormalize(self.env, norm_obs=True, norm_reward=True) #env = make_env() #model = GAIL("MlpPolicy", env=env, expert_dataset=dataset, verbose=1) self.env.load_running_average("saves") self.model = PPO2.load("saves/m19", self.env, policy=CustomPolicy, tensorboard_log="./default/" ) self.env.load_running_average("saves") #env = make_env() #self.expert_agent = generate_expert_traj(self.model, save, self.env, n_episodes=episodes)
class Optimization(): def __init__(self, config): self.reward_strategy = 'sortino2' #self.input_data_file = 'data/coinbase_hourly.csv' self.params_db_file = 'sqlite:///params.db' # number of parallel jobs self.n_jobs = 1 # maximum number of trials for finding the best hyperparams self.n_trials = 1000 #number of test episodes per trial self.n_test_episodes = 10 # number of evaluations for pruning per trial self.n_evaluations = 10 self.config = config #self.df = pd.read_csv(input_data_file) #self.df = df.drop(['Symbol'], axis=1) #self.df = df.sort_values(['Date']) #self.df = add_indicators(df.reset_index()) #self.train_len = int(len(df) * 0.8) #self.df = df[:train_len] #self.validation_len = int(train_len * 0.8) #self.train_df = df[:validation_len] #self.test_df = df[validation_len:] def make_env(self, env_id, rank, seed=0, eval=False,config=pc.configeurcad4h): """ Utility function for multiprocessed env. :param env_id: (str) the environment ID :param num_env: (int) the number of environment you wish to have in subprocesses :param seed: (int) the inital seed for RNG :param rank: (int) index of the subprocess """ def _init(): self.config = config self.eval= eval env = Template_Gym(config=self.config, eval=self.eval) env.seed(seed + rank) return env set_global_seeds(seed) return _init # Categorical parameter #optimizer = trial.suggest_categorical('optimizer', ['MomentumSGD', 'Adam']) # Int parameter #num_layers = trial.suggest_int('num_layers', 1, 3) # Uniform parameter #dropout_rate = trial.suggest_uniform('dropout_rate', 0.0, 1.0) # Loguniform parameter #learning_rate = trial.suggest_loguniform('learning_rate', 1e-5, 1e-2) # Discrete-uniform parameter #drop_path_rate = trial.suggest_discrete_uniform('drop_path_rate', 0.0, 1.0, 0.1) def optimize_envs(self, trial): return { 'reward_func': self.reward_strategy, 'forecast_len': int(trial.suggest_loguniform('forecast_len', 1, 200)), 'confidence_interval': trial.suggest_uniform('confidence_interval', 0.7, 0.99), } def optimize_config(self, trial): return { 'sl': trial.suggest_loguniform('sl', 1.0, 10.0), 'tp': trial.suggest_loguniform('tp', 1.0 ,10.0) } def optimize_ppo2(self,trial): return { 'n_steps': int(trial.suggest_int('n_steps', 16, 2048)), 'gamma': trial.suggest_loguniform('gamma', 0.9, 0.9999), 'learning_rate': trial.suggest_loguniform('learning_rate', 1e-5, 1.), 'ent_coef': trial.suggest_loguniform('ent_coef', 1e-8, 1e-1), 'cliprange': trial.suggest_uniform('cliprange', 0.1, 0.4), 'noptepochs': int(trial.suggest_int('noptepochs', 1, 48)), 'lam': trial.suggest_uniform('lam', 0.8, 1.) } def optimize_lstm(self, trial): return { 'lstm': trial.suggest_categorical('optimizer', ['lstm', 'mlp']) } def ob_types(self, trial): return { 'lstm': trial.suggest_categorical('optimizer', ['lstm', 'mlp']) } def optimize_agent(self,trial): #self.env_params = self.optimize_envs(trial) env_id = "default"+str() num_e = 1 # Number of processes to use #self.config_param = self.optimize_config(trial) #self.config.sl = self.config_param['sl'] #self.config.sl = self.config_param['tp'] #self.model_type = self.optimize_lstm(trial) #self.model_type = self.model_type['lstm'] self.model_type = "mlp" if self.model_type == 'mlp': self.policy = CustomPolicy_5 else: self.policy = CustomPolicy_4 self.train_env = SubprocVecEnv([self.make_env(env_id+str('train'), i, eval=False, config=self.config) for i in range(num_e)]) #self.train_env = SubprocVecEnv([self.make_env(env_id, i, eval=False) for i in range(num_e)]) self.train_env = VecNormalize(self.train_env, norm_obs=True, norm_reward=True) self.test_env =SubprocVecEnv([self.make_env(env_id+str("test"), i, eval=True, config=self.config) for i in range(num_e)]) #self.test_env = SubprocVecEnv([self.make_env(env_id, i, eval=True) for i in range(num_e)]) self.test_env = VecNormalize(self.test_env, norm_obs=True, norm_reward=True) try: self.test_env.load_running_average("saves") self.train_env.load_running_average("saves") except: print('cant load') self.model_params = self.optimize_ppo2(trial) self.model = PPO2(self.policy, self.train_env, verbose=0, nminibatches=1, tensorboard_log="./gbp_chf_single", **self.model_params ) #self.model = PPO2(CustomPolicy_2, self.env, verbose=0, learning_rate=1e-4, nminibatches=1, tensorboard_log="./min1" ) last_reward = -np.finfo(np.float16).max #evaluation_interval = int(len(train_df) / self.n_evaluations) evaluation_interval = 36525 for eval_idx in range(self.n_evaluations): try: self.model.learn(evaluation_interval) self.test_env.save_running_average("saves") self.train_env.save_running_average("saves") except: print('did not work') rewards = [] n_episodes, reward_sum = 0, 0.0 print('Eval') obs = self.test_env.reset() #state = None #done = [False for _ in range(self.env.num_envs)] while n_episodes < self.n_test_episodes: action, _ = self.model.predict(obs, deterministic=True) obs, reward, done, _ = self.test_env.step(action) reward_sum += reward if done: rewards.append(reward_sum) reward_sum = 0.0 n_episodes += 1 obs = self.test_env.reset() last_reward = np.mean(rewards) trial.report(-1 * last_reward, eval_idx) if trial.should_prune(eval_idx): raise optuna.structs.TrialPruned() return -1 * last_reward def optimize(self, config): self.config = config study_name = 'ppo2_single_ready' study_name = 'ppo2_single_ready_nosltp' study_name = 'ppo2_single_ready_nosltp_all_yeah' study_name = 'ppo2_eur_gbp_op' study_name = 'ppo2_gbp_chf_op' study_name = 'ppo2_gbp_chf_h1_new1' study_name = 'ppo2_gbp_chf_h4_r_new11' study_name = 'ppo2_gbp_chf_h4_r_withvolfixed' study_name = 'ppo2_gbp_chf_h4_r_withvolclosefix212' study_name = 'ppo2_gbp_chf_h4_loged_sortinonew' study = optuna.create_study( study_name=study_name, storage=self.params_db_file, load_if_exists=True) try: study.optimize(self.optimize_agent, n_trials=self.n_trials, n_jobs=self.n_jobs) except KeyboardInterrupt: pass print('Number of finished trials: ', len(study.trials)) print('Best trial:') trial = study.best_trial print('Value: ', trial.value) print('Params: ') for key, value in trial.params.items(): print(' {}: {}'.format(key, value)) return study.trials_dataframe() #if __name__ == '__main__': #optimize()
class PPO2_SB(): def __init__(self): self.love = 'Ramona' self.env_fns = [] self.env_names = [] def make_env(self, env_id, rank, seed=0, eval=False,config=pc.configeurcad4h): """ Utility function for multiprocessed env. :param env_id: (str) the environment ID :param num_env: (int) the number of environment you wish to have in subprocesses :param seed: (int) the inital seed for RNG :param rank: (int) index of the subprocess """ def _init(): self.config = config self.eval= eval env = Template_Gym(config=self.config, eval=self.eval) env.seed(seed + rank) return env set_global_seeds(seed) return _init def train(self, num_e=1, n_timesteps=1000000, save_fraction=0.0125, save='saves/audbuyh4120', config=pc.configgbpchf4h): env_id = "default" num_e = 1 # Number of processes to use # Create the vectorized environment #env = DummyVecEnv([lambda: env]) #Ramona self.config = config self.env = SubprocVecEnv([self.make_env(env_id, i, eval=False, config=self.config) for i in range(num_e)]) #env = Template_Gym() #self.env = DummyVecEnv([lambda: env]) self.env = VecNormalize(self.env, norm_obs=True, norm_reward=True) #self.model = PPO2(CustomPolicy_4, self.env, verbose=0, nminibatches=1, tensorboard_log="./gbp_chf_4h_r", **self.config.params ) #self.model = PPO2(CustomPolicy_5, self.env, verbose=0, nminibatches=1, tensorboard_log="./aud_chf", learning_rate=1e-5 )#**self.config.params #self.model = PPO2.load('saves/playerdetails39', self.env, policy=CustomPolicy, tensorboard_log="./playerdetailsex" ) self.model = PPO2.load(self.config.path+str(79)+'.pkl', self.env, policy=CustomPolicy_5, tensorboard_log="./default/" ) #self.model = PPO2.load("default9", self.env, policy=CustomPolicy, tensorboard_log="./test/" ) n_timesteps = n_timesteps * save_fraction n_timesteps = int(n_timesteps) training_loop = 1 / save_fraction training_loop = int(training_loop) log_dir = "saves" #self.env.load_running_average(log_dir) for i in range(training_loop): self.model.learn(n_timesteps) self.model.save(self.config.path+'8'+str(i)) self.env.save_running_average(log_dir) self.env.save_running_average(log_dir) def evaluate(self, num_env=1, num_steps=1461, load='saves/audbuyh1', runs=80, config=pc.configgbpchf4h): """ Evaluate a RL agent :param model: (BaseRLModel object) the RL Agent :param num_steps: (int) number of timesteps to evaluate it :return: (float) Mean reward """ env_id = config.year+config.pair num_e = 1 self.config = config log_dir = self.config.log #log_dir = self.config.norm #self.env = SubprocVecEnv([self.make_env(env_id, i, eval=True) for i in range(num_env)]) self.env = SubprocVecEnv([self.make_env(env_id, i, eval=True, config=self.config) for i in range(num_env)]) #self.model = PPO2(CustomPolicy, self.env, verbose=1, learning_rate=1e-5, tensorboard_log="./default" ) self.env = VecNormalize(self.env, norm_obs=True, norm_reward=True) try: self.env.load_running_average(log_dir) except: print('cant load') for i in range(runs): #self.model = PPO2(CustomPolicy, self.env, verbose=0, learning_rate=1e-5, tensorboard_log="./moose14" ) #self.model = PPO2.load(self.config.path, self.env, policy=CustomPolicy_2, tensorboard_log="./default/" ) self.model = PPO2.load(self.config.path+'8'+str(i)+'.pkl', self.env, policy=CustomPolicy_5, tensorboard_log="./default/" ) #self.env.load_running_average(log_dir) episode_rewards = [[0.0] for _ in range(self.env.num_envs)] #self.total_pips = [] obs = self.env.reset() state = None # When using VecEnv, done is a vector done = [False for _ in range(self.env.num_envs)] for i in range(num_steps): # _states are only useful when using LSTM policies action, state = self.model.predict(obs, state=state, mask=done, deterministic=True) obs, rewards , dones, _ = self.env.step(action) #actions, _states = self.model.predict(obs) # # here, action, rewards and dones are arrays # # because we are using vectorized env #obs, rewards, dones, info = self.env.step(actions) #self.total_pips.append(self.env.player.placement) # Stats for i in range(self.env.num_envs): episode_rewards[i][-1] += rewards[i] if dones[i]: episode_rewards[i].append(0.0) #self.env.save_running_average(log_dir) mean_rewards = [0.0 for _ in range(self.env.num_envs)] n_episodes = 0 for i in range(self.env.num_envs): mean_rewards[i] = np.mean(episode_rewards[i]) n_episodes += len(episode_rewards[i]) # Compute mean reward mean_reward = np.mean(mean_rewards) print("Mean reward:", mean_reward, "Num episodes:", n_episodes) #self.env.save(log_dir) return mean_reward def live(self, num_env=1, num_steps=1461, load='saves/gbp_usd_buy', runs=1, config=pc.configgbpcad4h): """ Evaluate a RL agent :param model: (BaseRLModel object) the RL Agent :param num_steps: (int) number of timesteps to evaluate it :return: (float) Mean reward """ self.config = config env_id = self.config.pair num_e = 1 log_dir = self.config.log self.config.live = True self.config.load = False self.env = SubprocVecEnv([self.make_env(env_id, i, eval=True, config=self.config) for i in range(num_env)]) self.env = VecNormalize(self.env, norm_obs=True, norm_reward=True) try: self.env.load_running_average(self.config.log) except: print('cant load') self.env.num_envs = 1 for i in range(runs): self.model = PPO2.load(self.config.path+str(self.config.best)+'.pkl', self.env, policy=CustomPolicy_5, tensorboard_log="./default/" ) episode_rewards = [[0.0] for _ in range(self.env.num_envs)] print(datetime.datetime.now()) print(time.ctime()) print('Market Check') print("Market time check") obs = self.env.reset() state = None # When using VecEnv, done is a vector done = [False for _ in range(self.env.num_envs)] for i in range(num_steps): # _states are only useful when using LSTM policies print("live step") action, state = self.model.predict(obs, state=state, mask=done, deterministic=True) obs, rewards , dones, _ = self.env.step(action) # # here, action, rewards and dones are arrays # # because we are using vectorized env #obs, rewards, dones, info = self.env.step(actions) #self.total_pips.append(self.env.player.placement) # Stats for i in range(self.env.num_envs): episode_rewards[i][-1] += rewards[i] if dones[i]: episode_rewards[i].append(0.0) print(datetime.datetime.now()) print(time.ctime()) mean_rewards = [0.0 for _ in range(self.env.num_envs)] n_episodes = 0 for i in range(self.env.num_envs): mean_rewards[i] = np.mean(episode_rewards[i]) n_episodes += len(episode_rewards[i]) # Compute mean reward mean_reward = np.mean(mean_rewards) print("Mean reward_gbp_buy:", mean_reward, "Num episodes:", n_episodes) return mean_reward # The two function below are not working atm def pre_train(self, num_e=1, load="saves/m19"): env_id = 'default' num_e = 1 log_dir = "saves" # Usingenv = make_env() only one expert trajectory # you can specify `traj_limitation=-1` for using the whole dataset dataset = ExpertDataset(expert_path='default2.npz',traj_limitation=1, batch_size=128) self.env = SubprocVecEnv([self.make_env(env_id, i) for i in range(num_e)]) #env = Template_Gym() #self.env = DummyVecEnv([lambda: env]) self.env = VecNormalize(self.env, norm_obs=True, norm_reward=True) #env = make_env() #model = GAIL("MlpPolicy", env=env, expert_dataset=dataset, verbose=1) self.env.save_running_average("saves"+self.config.pair) self.model = PPO2(CustomPolicy, self.env, verbose=1, nminibatches=1, learning_rate=1e-5, tensorboard_log="./m1ln4" ) #self.model = PPO2.load("saves/m19", self.env, policy=CustomPolicy, tensorboard_log="./default/" ) self.env.save_running_average("saves"+self.config.pair) # Pretrain the PPO2 model self.model.pretrain(dataset, n_epochs=10000) # As an option, you can train the RL agent #self.model.learn(int(100000000)) # Test the pre-trained model self.env = self.model.get_env() self.env.save_running_average("saves"+self.config.pair) obs = self.env.reset() reward_sum = 0.0 for _ in range(1000000): action, _ = self.model.predict(obs) obs, reward, done, _ = self.env.step(action) reward_sum += reward #self.env.render() if done: print(reward_sum) reward_sum = 0.0 obs = self.env.reset() self.env.close() def gen_pre_train(self, num_e=1, save='default2', episodes=1000): #self.create_envs(game_name=game, state_name=state, num_env=num_e) #self.env=SubprocVecEnv(self.env_fns) env_id = 'default' num_e = 1 self.env = SubprocVecEnv([self.make_env(env_id, i) for i in range(num_e)]) #env = Template_Gym() #self.env = DummyVecEnv([lambda: env]) self.env = VecNormalize(self.env, norm_obs=True, norm_reward=True) #env = make_env() #model = GAIL("MlpPolicy", env=env, expert_dataset=dataset, verbose=1) self.env.load_running_average("saves") self.model = PPO2.load("saves/m19", self.env, policy=CustomPolicy, tensorboard_log="./default/" ) self.env.load_running_average("saves") #env = make_env() #self.expert_agent = generate_expert_traj(self.model, save, self.env, n_episodes=episodes) def gen_pre_train_2(self, game, state, num_e=1, save='default2', episodes=10): self.create_envs(game_name=game, state_name=state, num_env=num_e) env=SubprocVecEnv(self.env_fns) self.expert_agent = "moose" self.generate_expert_traj(self.expert_agent, save, env, n_episodes=episodes)