def objective(trial): noise = trial.suggest_uniform('Noise', 0.1, 0.8) timesteps = trial.suggest_int('Timesteps', 10, 100) n_actions = env.action_space.shape[-1] action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(noise) * np.ones(n_actions)) model = DDPG('MlpPolicy', env, action_noise=action_noise) model.learn(total_timesteps=timesteps * 1000, log_interval=1000) return test_model(env, model, '')
def main(): """ # Example with a simple Dummy vec env env = gym.envs.make('panda-ip-reach-v0', renders= True) env = DummyVecEnv([lambda: env]) """ print("Env created !") env = PandaReachGymEnv(renders=True) env.render(mode='rgb_array') model = DDPG.load("ddpg_panda_reach") print("model loaded !") while True: obs, done = env.reset(), False print("===================================") print("obs") print(obs) episode_rew = 0 while not done: env.render(mode='rgb_array') action, _states = model.predict(obs) obs, rew, done, info = env.step(action) episode_rew += rew print("Episode reward", episode_rew)
def create_model(env, algorithm, save_path): # the noise object n_actions = env.action_space.shape[-1] action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.2) * np.ones(n_actions), theta=0.15) if algorithm == "ddpg": return DDPG(DDPG_MlpPolicy, env, learning_rate=0.001, buffer_size=1000000, batch_size=64, tau=0.001, gamma=0.99, train_freq=(10, "step"), action_noise=action_noise, policy_kwargs=dict(optimizer_class=th.optim.AdamW), tensorboard_log=save_path) elif algorithm == "td3": return TD3(TD3_MlpPolicy, env, action_noise=action_noise, tensorboard_log=save_path) elif algorithm == "sac": return SAC(SAC_MlpPolicy, env, action_noise=action_noise, tensorboard_log=save_path) else: raise Exception("--> Alican's LOG: Unknown agent type!")
def ddpg(env, hyper, policy="MlpPolicy", verbose=0, tensorboard_log=None, seed=0, use_sde=True, device="auto"): policy_kwargs = make_policy_kwargs(hyper, "ddpg") hyper = action_noise(hyper, "ddpg", n_actions=env.action_space.shape[0]) model = DDPG( 'MlpPolicy', env, verbose=verbose, tensorboard_log=tensorboard_log, seed=seed, gamma=hyper['params_gamma'], learning_rate=hyper['params_lr'], batch_size=np.int(hyper['params_batch_size']), buffer_size=np.int(hyper['params_buffer_size']), action_noise=hyper['params_action_noise'], train_freq=hyper['params_train_freq'], # gradient_steps = np.int(hyper['params_train_freq']), # n_episodes_rollout = np.int(hyper['params_n_episodes_rollout']), policy_kwargs=policy_kwargs, device=device) return model
def train_stable_baselines(submodule, flags): """Train policies using the PPO algorithm in stable-baselines.""" from stable_baselines3.common.vec_env import DummyVecEnv flow_params = submodule.flow_params # Path to the saved files exp_tag = flow_params['exp_tag'] result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S")) # Perform training. start_time = timeit.default_timer() # print experiment.json information print("=========================================") print('Beginning training.') print('Algorithm :', flags.algorithm) model = run_model_stablebaseline(flow_params, flags.num_cpus, flags.rollout_size, flags.num_steps, flags.algorithm, flags.exp_config) stop_time = timeit.default_timer() run_time = stop_time - start_time print("Training is Finished") print("total runtime: ", run_time) # Save the model to a desired folder and then delete it to demonstrate # loading. print('Saving the trained model!') path = os.path.realpath(os.path.expanduser('~/baseline_results')) ensure_dir(path) save_path = os.path.join(path, result_name) model.save(save_path) # dump the flow params with open(os.path.join(path, result_name) + '.json', 'w') as outfile: json.dump(flow_params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4) # Replay the result by loading the model print('Loading the trained model and testing it out!') if flags.exp_config.lower() == "ppo": from stable_baselines3 import PPO model = PPO.load(save_path) elif flags.exp_config.lower() == "ddpg": from stable_baselines3 import DDPG model = DDPG.load(save_path) flow_params = get_flow_params(os.path.join(path, result_name) + '.json') flow_params['sim'].render = True env = env_constructor(params=flow_params, version=0)() # The algorithms require a vectorized environment to run eval_env = DummyVecEnv([lambda: env]) obs = eval_env.reset() reward = 0 for _ in range(flow_params['env'].horizon): action, _states = model.predict(obs) obs, rewards, dones, info = eval_env.step(action) reward += rewards print('the final reward is {}'.format(reward))
def main(): # Create log dir log_dir = './ddpg_data' os.makedirs(log_dir, exist_ok=True) vix_env = trading_vix_env.trading_vix_env() env = Monitor(vix_env, log_dir) # Create action noise because TD3 and DDPG use a deterministic policy n_actions = env.action_space.shape[-1] action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions)) # Create the callback: check every 20000 steps callback = custom_call_back.CustomCallback(check_freq = 20000,log_dir = log_dir) # Create RL model model = DDPG('MlpPolicy',env,action_noise = action_noise, verbose=2,batch_size = 10000) # Train the agent model.learn(total_timesteps=int(5e9), callback=callback)
def train(): best_reward, best_reward_timesteps = None, None save_path = "model_save/"+MODEL_PATH+"/" if save_path is not None: os.makedirs(save_path, exist_ok=True) # log_dir = f"model_save/" log_dir = save_path env, env_eval = ENV(util='train', par=PARAM, dt=DT), ENV(util='val', par=PARAM, dt=DT) env, env_eval = Monitor(env, log_dir), Monitor(env_eval, log_dir) env, env_eval = DummyVecEnv([lambda: env]), DummyVecEnv([lambda: env_eval]) # env = VecNormalize(env, norm_obs=True, norm_reward=True, # clip_obs=10.) if PARAM['algo']=='td3': model = TD3('MlpPolicy', env, verbose=1, batch_size=PARAM['batch_size'], seed=PARAM['seed'], learning_starts=PARAM['learning_starts']) elif PARAM['algo']=='ddpg': model = DDPG('MlpPolicy', env, verbose=1, batch_size=PARAM['batch_size'], seed=PARAM['seed'], learning_starts=PARAM['learning_starts']) elif PARAM['algo']=='ppo': model = PPO('MlpPolicy', env, verbose=1, batch_size=PARAM['batch_size'], seed=PARAM['seed']) eval_callback = EvalCallback(env_eval, best_model_save_path=save_path+MODEL_PATH+'_best_model', log_path=log_dir, eval_freq=PARAM['eval_freq'], save_freq=PARAM['save_freq'], deterministic=True, render=False) model.learn(total_timesteps=int(PARAM['total_time_step']), callback=eval_callback, log_interval = 500) print("best mean reward:", eval_callback.best_mean_reward_overall, "timesteps:", eval_callback.best_mean_reward_timestep) model.save(save_path+MODEL_PATH+'_final_timesteps')
def load_model(env, algorithm, filename): if algorithm == "ddpg": return DDPG.load(filename, env=env) elif algorithm == "td3": return TD3.load(filename, env=env) elif algorithm == "sac": return SAC.load(filename, env=env) else: raise Exception("--> Alican's LOG: Unknown agent type!")
def __init__(self, env, hyperparameters=DEFAULT_HYPERPARAMETERS): self.P = hyperparameters if self.P["model_class"] == "dqn": from stable_baselines3 import DQN self.model = DQN('MlpPolicy', env, verbose=self.P["verbose"]) self.model_class = DQN elif self.P["model_class"] == "a2c": from stable_baselines3 import A2C from stable_baselines3.a2c import MlpPolicy self.model = A2C(MlpPolicy, env, verbose=self.P["verbose"]) self.model_class = A2C elif self.P["model_class"] == "ddpg": from stable_baselines3 import DDPG from stable_baselines3.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise n_actions = env.action_space.shape[-1] action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions)) self.model = DDPG('MlpPolicy', env, action_noise=action_noise, verbose=self.P["verbose"]) self.model_class = DDPG elif self.P["model_class"] == "td3": from stable_baselines3 import TD3 from stable_baselines3.td3.policies import MlpPolicy from stable_baselines3.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise n_actions = env.action_space.shape[-1] action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions)) self.model = TD3(MlpPolicy, env, action_noise=action_noise, verbose=self.P["verbose"]) self.model_class = TD3 elif self.P["model_class"] == "ppo": from stable_baselines3 import PPO from stable_baselines3.ppo import MlpPolicy self.model = PPO(MlpPolicy, env, verbose=self.P["verbose"]) self.model_class = PPO elif self.P["model_class"] == "sac": from stable_baselines3 import SAC from stable_baselines3.sac import MlpPolicy self.model = SAC(MlpPolicy, env, verbose=self.P["verbose"]) self.model_class = SAC else: raise NotImplementedError()
def test(MODEL_TEST): log_dir = "model_save/" + MODEL_PATH + "/" + MODEL_PATH + MODEL_TEST env = ENV(util='test', par=PARAM, dt=DT) env.render = True env = Monitor(env, log_dir) if PARAM['algo']=='td3': model = TD3.load(log_dir) elif PARAM['algo']=='ddpg': model = DDPG.load(log_dir) elif PARAM['algo']=='ppo': model = PPO.load(log_dir) # plot_results(f"model_save/") trade_dt = pd.DataFrame([]) # trade_dt: 所有股票的交易数据 result_dt = pd.DataFrame([]) # result_dt: 所有股票一年测试结果数据 for i in range(TEST_STOCK_NUM): state = env.reset() stock_bh_id = 'stock_bh_'+str(i) # 记录每个股票交易的buy_hold stock_port_id = 'stock_port_'+str(i) # 记录每个股票交易的portfolio stock_action_id = 'stock_action_' + str(i) # 记录每个股票交易的action flow_L_id = 'stock_flow_' + str(i) # 记录每个股票的流水 stock_bh_dt, stock_port_dt, action_policy_dt, flow_L_dt = [], [], [], [] day = 0 while True: action = model.predict(state) next_state, reward, done, info = env.step(action[0]) state = next_state # print("trying:",day,"reward:", reward,"now profit:",env.profit) # 测试每一步的交易policy stock_bh_dt.append(env.buy_hold) stock_port_dt.append(env.Portfolio_unit) action_policy_dt.append(action[0][0]) # 用于记录policy flow_L_dt.append(env.flow) day+=1 if done: print('stock: {}, total profit: {:.2f}%, buy hold: {:.2f}%, sp: {:.4f}, mdd: {:.2f}%, romad: {:.4f}' .format(i, env.profit*100, env.buy_hold*100, env.sp, env.mdd*100, env.romad)) # 交易完后记录:股票ID,利润(单位100%),buy_hold(单位100%),夏普率,最大回撤率(单位100%),romad result=pd.DataFrame([[i,env.profit*100,env.buy_hold*100,env.sp,env.mdd*100,env.romad]]) break trade_dt_stock = pd.DataFrame({stock_port_id: stock_port_dt, stock_bh_id: stock_bh_dt, stock_action_id: action_policy_dt, flow_L_id: flow_L_dt}) # 支股票的交易数据 trade_dt = pd.concat([trade_dt, trade_dt_stock], axis=1) # 所有股票交易数据合并(加行) result_dt = pd.concat([result_dt,result],axis=0) # 所有股票结果数据合并(加列) result_dt.columns = ['stock_id','prfit(100%)','buy_hold(100%)','sp','mdd(100%)','romad'] trade_dt.to_csv('out_dt/trade_'+MODEL_PATH+'.csv',index=False) result_dt.to_csv('out_dt/result_'+MODEL_PATH+'.csv',index=False)
def train_DDPG(self, model_name, model_params = config.DDPG_PARAMS): """DDPG model""" from stable_baselines3 import DDPG from stable_baselines3.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise env_train = self.env n_actions = env_train.action_space.shape[-1] action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1*np.ones(n_actions)) start = time.time() model = DDPG('MlpPolicy', env_train, batch_size=model_params['batch_size'], buffer_size=model_params['buffer_size'], action_noise=action_noise, verbose=model_params['verbose'], tensorboard_log = f"{config.TENSORBOARD_LOG_DIR}/{model_name}" ) model.learn(total_timesteps=model_params['timesteps'], tb_log_name = "DDPG_run") end = time.time() model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}") print('Training time (DDPG): ', (end-start)/60,' minutes') return model
def main(): args = parse_arguments() load_path = os.path.join("logs", args.env, args.agent, "best_model.zip") stats_path = os.path.join(args.log_dir, args.env, args.agent, "vec_normalize.pkl") if args.agent == 'ddpg': from stable_baselines3 import DDPG model = DDPG.load(load_path) elif args.agent == 'td3': from stable_baselines3 import TD3 model = TD3.load(load_path) elif args.agent == 'ppo': from stable_baselines3 import PPO model = PPO.load(load_path) env = make_vec_env(args.env, n_envs=1) env = VecNormalize.load(stats_path, env) # do not update them at test time env.training = False # reward normalization is not needed at test time env.norm_reward = False # env = gym.make(args.env) img = [] if args.render: env.render('human') done = False obs = env.reset() action = model.predict(obs) if args.gif: img.append(env.render('rgb_array')) if args.timesteps is None: while not done: action, _= model.predict(obs) obs, reward, done, info = env.step(action) if args.gif: img.append(env.render('rgb_array')) else: env.render() else: for i in range(args.timesteps): action, _= model.predict(obs) obs, reward, done, info = env.step(action) if args.gif: img.append(env.render('rgb_array')) else: env.render() if args.gif: imageio.mimsave(f'{os.path.join("logs", args.env, args.agent, "recording.gif")}', [np.array(img) for i, img in enumerate(img) if i%2 == 0], fps=29)
def main(): """ # Example with Vectorized env num_cpu = 4 # Number of processes to use my_env_kwargs={'renders': False} env = make_vec_env('panda-ip-reach-v0', n_envs=num_cpu, env_kwargs=my_env_kwargs) """ # Example with a simple Dummy vec env env = gym.envs.make('panda-ip-reach-v0', renders=False) env = DummyVecEnv([lambda: env]) #check_env(pandaenv) # The noise objects for DDPG n_actions = env.action_space.shape[-1] print("n_actions = {0}".format(n_actions)) #action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions)) action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions)) model = DDPG(policy='MlpPolicy', env=env, learning_rate=0.001, buffer_size=1000000, learning_starts=100, batch_size=100, tau=0.005, gamma=0.99, train_freq=1, gradient_steps=-1, action_noise=action_noise, optimize_memory_usage=False, tensorboard_log="./ddpg_panda_reach_tensorboard/", create_eval_env=False, policy_kwargs=None, verbose=1, seed=None, device='auto', _init_setup_model=True) """ print("start model evaluation without learning !") mean_reward_before, std_reward_before = evaluate_policy(model, env, n_eval_episodes=1) print("end model evaluation !") """ print("start model learning !") model.learn(total_timesteps=200000, log_interval=10) print("end model learning !") print("-> model saved !!") model.save("ddpg_panda_reach") """ print("start model evaluation with learning !") mean_reward_after, std_reward_after = evaluate_policy(model, env, n_eval_episodes=1) print("end model evaluation !") """ """
def create(self, n_envs=1): """Create the agent""" self.env = self.agent_helper.env log_dir = self.agent_helper.config_dir os.makedirs(log_dir, exist_ok=True) self.env = Monitor(self.env, log_dir) #TODO: # Create DDPG policy and define its hyper parameter here! even the action space and observation space. # add policy policy_name = self.agent_helper.config['policy'] self.policy = eval(policy_name) # action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions)) n_actions = int(self.agent_helper.env.action_space.shape[0]) action_noise = NormalActionNoise( mean=np.zeros(n_actions), sigma=self.agent_helper.config['rand_sigma'] * np.ones(n_actions)) #FIXME: test: # self.model = DDPG("MlpPolicy", self.env, action_noise=action_noise, verbose=1, tensorboard_log=self.agent_helper.graph_path) # TODO: fix the obvervation space and action space later. Test if the obervation space input is correct? Output action space is correct? # activ_function_name = self.agent_helper.config['nn_activ'] # activ_function = eval(activ_function_name) # policy_kwargs = dict(activation_fn=activ_function, # net_arch=[dict(pi=[32, 32], qf=[32, 32])]) logger.info("Create the DDPG model") policy_kwargs = dict(net_arch=self.agent_helper.config['layers']) self.model = DDPG( self.policy, self.env, learning_rate=self.agent_helper.config['learning_rate'], buffer_size=self.agent_helper.config['buffer_size'], batch_size=self.agent_helper.config['batch_size'], tau=self.agent_helper.config['tau'], gamma=self.agent_helper.config['gamma'], gradient_steps=self.agent_helper.config['gradient_steps'], action_noise=action_noise, optimize_memory_usage=self.agent_helper. config['optimize_memory_usage'], create_eval_env=self.agent_helper.config['create_eval_env'], policy_kwargs=policy_kwargs, verbose=self.agent_helper.config['verbose'], learning_starts=self.agent_helper.config['learning_starts'], tensorboard_log=self.agent_helper.graph_path, seed=self.agent_helper.seed) pass
def test_ddpg(): log_dir = f"model_save/best_model_ddpg_cnn" env = ENV(istest=True) env.render = True env = Monitor(env, log_dir) model = DDPG.load(log_dir) plot_results(f"model_save/") for i in range(10): state = env.reset() while True: action = model.predict(state) next_state, reward, done, info = env.step(action[0]) state = next_state # print("trying:",i,"action:", action,"now profit:",env.profit) if done: print('stock',i,' total profit=',env.profit,' buy hold=',env.buy_hold) break
def train_ddpg(): log_dir = f"model_save/" env = ENV(istest=False) env = Monitor(env, log_dir) env = DummyVecEnv([lambda: env]) # env = VecNormalize(env, norm_obs=True, norm_reward=True, # clip_obs=10.) model = DDPG("CnnPolicy", env, policy_kwargs=policy_kwargs, verbose=1, batch_size=2048, seed=1, learning_starts=500000) callback = SaveOnBestTrainingRewardCallback(check_freq=480, log_dir=log_dir) model.learn(total_timesteps=int(1000000), callback = callback, log_interval = 480) model.save('model_save/ddpg_cnn')
def run(env, algname, filename): if algname == "TD3": model = TD3.load(f"{algname}_pkl") elif algname == "SAC": if filename: model = SAC.load(f"{filename}") else: model = SAC.load(f"{algname}_pkl") elif algname == "DDPG": model = DDPG.load(f"{algname}_pkl") else: raise "Wrong algorithm name provided." obs = env.reset() while True: action, _states = model.predict(obs) obs, rewards, done, info = env.step(action) env.render() if done: break
def train_DDPG(env_train, model_name, timesteps=10000): """DDPG model""" # add the noise objects for DDPG n_actions = env_train.action_space.shape[-1] action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions)) start = time.time() model = DDPG('MlpPolicy', env_train, action_noise=action_noise) model.learn(total_timesteps=timesteps) end = time.time() model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}") print('Training time (DDPG): ', (end-start)/60,' minutes') return model
def test_ddpg(): log_dir = f"model_save/best_model_ddpg_sp2" env = ENV(istest=True) env.render = True env = Monitor(env, log_dir) model = DDPG.load(log_dir) plot_results(f"model_save/") for i in range(10): state = env.reset() day = 0 while True: action = model.predict(state) next_state, reward, done, info = env.step(action[0]) state = next_state # print("trying:",day,"reward:", reward,"now profit:",env.profit) day += 1 if done: print( 'stock: {}, total profit: {:.2f}%, buy hold: {:.2f}%, sp: {:.4f}, mdd: {:.2f}%, romad: {:.4f}' .format(i, env.profit * 100, env.buy_hold * 100, env.sp, env.mdd * 100, env.romad)) break
def train_DDPG(env_train, model_name, timesteps=10000): """DDPG model""" # the noise objects for DDPG n_actions = env_train.action_space.shape[-1] # action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions)) action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions)) start = time.time() param_noise = None # removed keyword "param_noise=param_noise" stable_baselines3 doesn't need this one model = DDPG('MlpPolicy', env_train, action_noise=action_noise) model.learn(total_timesteps=timesteps) end = time.time() model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}") print('Training time (DDPG): ', (end - start) / 60, ' minutes') return model
def train(): log_dir = f"model_save/" env = ENV(istest=False) env = Monitor(env, log_dir) env = DummyVecEnv([lambda: env]) # env = VecNormalize(env, norm_obs=True, norm_reward=True, # clip_obs=10.) model = DDPG('MlpPolicy', env, verbose=1, batch_size=PARAM['batch_size'], seed=PARAM['seed'], learning_starts=PARAM['learning_starts']) callback = SaveOnBestTrainingRewardCallback(check_freq=480, log_dir=log_dir) model.learn(total_timesteps=int(PARAM['total_time_step']), callback=callback, log_interval=480) model.save('model_save/' + MODEL_PATH)
def train_DDPG(env): print(f"action space shape -1:{env.action_space.shape[-1]}") # The noise objects for TD3 n_actions = env.action_space.shape[-1] action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.02 * np.ones(n_actions)) model = DDPG( 'MlpPolicy', env, learning_rate=0.0003, learning_starts=5, train_freq=10, n_episodes_rollout=-1, buffer_size=100000, action_noise=action_noise, batch_size=128, verbose=2, ) model.learn(total_timesteps=1000000, log_interval=1) model.save("DDPG_pkl")
if os.path.isfile(ARGS.exp + '/success_model.zip'): path = ARGS.exp + '/success_model.zip' elif os.path.isfile(ARGS.exp + '/best_model.zip'): path = ARGS.exp + '/best_model.zip' else: print("[ERROR]: no model under the specified path", ARGS.exp) if algo == 'a2c': model = A2C.load(path) if algo == 'ppo': model = PPO.load(path) if algo == 'sac': model = SAC.load(path) if algo == 'td3': model = TD3.load(path) if algo == 'ddpg': model = DDPG.load(path) #### Parameters to recreate the environment ################ env_name = ARGS.exp.split("-")[1] + "-aviary-v0" OBS = ObservationType.KIN if ARGS.exp.split( "-")[3] == 'kin' else ObservationType.RGB if ARGS.exp.split("-")[4] == 'rpm': ACT = ActionType.RPM elif ARGS.exp.split("-")[4] == 'dyn': ACT = ActionType.DYN elif ARGS.exp.split("-")[4] == 'pid': ACT = ActionType.PID elif ARGS.exp.split("-")[4] == 'vel': ACT = ActionType.VEL elif ARGS.exp.split("-")[4] == 'tun': ACT = ActionType.TUN
# A2C algorithm for i in range(n_tests): test_name = 'saved_models/a2c_soccer_actions_env_1_' + str(i) n_actions = env.action_space.shape[-1] model = A2C('MlpPolicy', env) model.learn(total_timesteps=25000, log_interval=1000) model.save(test_name) test_model(env, model, test_name) # DDPG algorithm for i in range(n_tests): test_name = 'saved_models/ddpg_soccer_actions_env_1_' + str(i) n_actions = env.action_space.shape[-1] action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.3) * np.ones(n_actions)) model = DDPG('MlpPolicy', env, action_noise=action_noise) model.learn(total_timesteps=10000, log_interval=1000) model.save(test_name) test_model(env, model, test_name) for i in range(n_tests): test_name = 'saved_models/ddpg_soccer_actions_env_2_' + str(i) n_actions = env.action_space.shape[-1] action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.3) * np.ones(n_actions)) policy_kwargs = dict(net_arch=[400, 300]) model = DDPG('MlpPolicy', env, action_noise=action_noise, policy_kwargs=policy_kwargs) model.learn(total_timesteps=10000, log_interval=1000) model.save(test_name) test_model(env, model, test_name) for i in range(n_tests):
check_env(env, warn=True, skip_render_check=True) #### action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(env.N_ACTIONS), sigma=0.1 * np.ones(env.N_ACTIONS), dt=0.005) #### Create the callback: check every 1000 steps callback = SaveOnBestTrainingRewardCallback(check_freq=1000, log_dir=log_dir) #### Train the model ############################################################################### model = DDPG(CustomPolicy, env, verbose=1, batch_size=64, action_noise=action_noise) for i in range(step_iters): # run for step_iters * training_timesteps model.learn(total_timesteps=training_timesteps) model.save("./models/ddpg" + str((i + 1) * training_timesteps)) model.save_replay_buffer("./experiences/ddpg_experience" + str((i + 1) * training_timesteps)) #### Show (and record a video of) the model's performance ########################################## env_test = RLTetherAviary(gui=False, record=True) obs = env_test.reset() start = time.time()
seed=args.seed, tensorboard_log=args.tensorboard) #--------------------------------------------------------# # DDPG # #--------------------------------------------------------# elif args.algorithm == 'DDPG': if args.sigma: # noise objects for DDPG n_actions = env.action_space.shape[-1] action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions)) model = DDPG("MlpPolicy", env, action_noise=action_noise, verbose=1, seed=args.seed, tensorboard_log=args.tensorboard) #--------------------------------------------------------# # A2C # #--------------------------------------------------------# elif args.algorithm == 'A2C': model = A2C('MlpPolicy', env, verbose=1, learning_rate=args.learning_rate, n_steps=args.n_steps, gamma=args.gamma, gae_lambda=args.gae_lambda, ent_coef=args.ent_coef, vf_coef=args.vf_coef,
return True if __name__ == '__main__': # Instantiate Environment env_id = 'gym_spm:spm-v0' env = gym.make('gym_spm:spm-v0') # HyperParameters lr = 3e-4 # Instantiate Model n_actions = env.action_space.shape[-1] action_noise = NormalActionNoise(mean=-30 * np.zeros(n_actions), sigma=.75 * np.ones(n_actions)) model = DDPG('MlpPolicy', env, action_noise=action_noise, verbose=1) # model = PPO('MlpPolicy', env, tensorboard_log=log_dir) # Train OR Load Model model.learn(total_timesteps=25000) # model.save(model_dir_description) mean_reward, std_reward = evaluate_policy(model, model.get_env(), n_eval_episodes=10) print("Mean Reward = ", mean_reward) epsi_sp_list = [] action_list = []
verbose=1) if ARGS.algo == 'td3': model = TD3(td3ddpgMlpPolicy, train_env, policy_kwargs=offpolicy_kwargs, tensorboard_log=filename + '/tb/', verbose=1) if ARGS.obs == ObservationType.KIN else TD3( td3ddpgCnnPolicy, train_env, policy_kwargs=offpolicy_kwargs, tensorboard_log=filename + '/tb/', verbose=1) if ARGS.algo == 'ddpg': model = DDPG(td3ddpgMlpPolicy, train_env, policy_kwargs=offpolicy_kwargs, tensorboard_log=filename + '/tb/', verbose=1) if ARGS.obs == ObservationType.KIN else DDPG( td3ddpgCnnPolicy, train_env, policy_kwargs=offpolicy_kwargs, tensorboard_log=filename + '/tb/', verbose=1) #### Create eveluation environment ######################### if ARGS.obs == ObservationType.KIN: eval_env = gym.make( env_name, aggregate_phy_steps=shared_constants.AGGR_PHY_STEPS, obs=ARGS.obs, act=ARGS.act)
def load_weights(self, weights_file): """ Load the model from a zip archive """ logger.info(f"load weight from file: {weights_file}") self.model = DDPG.load(weights_file, env=self.env) pass
deterministic=True, render=False) ### DDPG Noise ### Try increasing the noise when retraining. ### Try less noise based on the policy plot. n_actions = env.action_space.shape[-1] action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=1 * np.ones(n_actions)) # action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions)) model = DDPG( 'MlpPolicy', env, action_noise=action_noise, verbose=1, tensorboard_log="./h={}/".format(horizons[rank]), gamma=0.99, learning_rate=0.0003, ) # model = DDPG.load("Model_DDPG_FS_30.zip") # model.learning_rate = 0.0003 # model.gamma = 0.99 # action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=0.05*np.ones(n_actions)) # action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.075 * np.ones(n_actions)) # model.action_noise = action_noise trainer = Trainer(env) trainer.retrain_rl(model, episodes=20000, path="./h={}/".format(horizons[rank]))