def train():
best_reward, best_reward_timesteps = None, None
save_path = "model_save/"+MODEL_PATH+"/"
if save_path is not None:
os.makedirs(save_path, exist_ok=True)
# log_dir = f"model_save/"
log_dir = save_path
env, env_eval = ENV(util='train', par=PARAM, dt=DT), ENV(util='val', par=PARAM, dt=DT)
env, env_eval = Monitor(env, log_dir), Monitor(env_eval, log_dir)
env, env_eval = DummyVecEnv([lambda: env]), DummyVecEnv([lambda: env_eval])
# env = VecNormalize(env, norm_obs=True, norm_reward=True,
# clip_obs=10.)
if PARAM['algo']=='td3':
model = TD3('MlpPolicy', env, verbose=1, batch_size=PARAM['batch_size'], seed=PARAM['seed'],
learning_starts=PARAM['learning_starts'])
elif PARAM['algo']=='ddpg':
model = DDPG('MlpPolicy', env, verbose=1, batch_size=PARAM['batch_size'], seed=PARAM['seed'],
learning_starts=PARAM['learning_starts'])
elif PARAM['algo']=='ppo':
model = PPO('MlpPolicy', env, verbose=1, batch_size=PARAM['batch_size'], seed=PARAM['seed'])
eval_callback = EvalCallback(env_eval, best_model_save_path=save_path+MODEL_PATH+'_best_model',
log_path=log_dir, eval_freq=PARAM['eval_freq'], save_freq=PARAM['save_freq'],
deterministic=True, render=False)
model.learn(total_timesteps=int(PARAM['total_time_step']), callback=eval_callback, log_interval = 500)
print("best mean reward:", eval_callback.best_mean_reward_overall, "timesteps:", eval_callback.best_mean_reward_timestep)
model.save(save_path+MODEL_PATH+'_final_timesteps')
def train_DDPG(self, model_name, model_params = config.DDPG_PARAMS):
"""DDPG model"""
from stable_baselines3 import DDPG
from stable_baselines3.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise
env_train = self.env
n_actions = env_train.action_space.shape[-1]
action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1*np.ones(n_actions))
start = time.time()
model = DDPG('MlpPolicy',
env_train,
batch_size=model_params['batch_size'],
buffer_size=model_params['buffer_size'],
action_noise=action_noise,
verbose=model_params['verbose'],
tensorboard_log = f"{config.TENSORBOARD_LOG_DIR}/{model_name}"
)
model.learn(total_timesteps=model_params['timesteps'], tb_log_name = "DDPG_run")
end = time.time()
model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}")
print('Training time (DDPG): ', (end-start)/60,' minutes')
return model
def main():
"""
# Example with Vectorized env
num_cpu = 4 # Number of processes to use
my_env_kwargs={'renders': False}
env = make_vec_env('panda-ip-reach-v0', n_envs=num_cpu, env_kwargs=my_env_kwargs)
"""
# Example with a simple Dummy vec env
env = gym.envs.make('panda-ip-reach-v0', renders=False)
env = DummyVecEnv([lambda: env])
#check_env(pandaenv)
# The noise objects for DDPG
n_actions = env.action_space.shape[-1]
print("n_actions = {0}".format(n_actions))
#action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions))
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=0.1 * np.ones(n_actions))
model = DDPG(policy='MlpPolicy',
env=env,
learning_rate=0.001,
buffer_size=1000000,
learning_starts=100,
batch_size=100,
tau=0.005,
gamma=0.99,
train_freq=1,
gradient_steps=-1,
action_noise=action_noise,
optimize_memory_usage=False,
tensorboard_log="./ddpg_panda_reach_tensorboard/",
create_eval_env=False,
policy_kwargs=None,
verbose=1,
seed=None,
device='auto',
_init_setup_model=True)
"""
print("start model evaluation without learning !")
mean_reward_before, std_reward_before = evaluate_policy(model, env, n_eval_episodes=1)
print("end model evaluation !")
"""
print("start model learning !")
model.learn(total_timesteps=200000, log_interval=10)
print("end model learning !")
print("-> model saved !!")
model.save("ddpg_panda_reach")
"""
print("start model evaluation with learning !")
mean_reward_after, std_reward_after = evaluate_policy(model, env, n_eval_episodes=1)
print("end model evaluation !")
"""
"""
def train_ddpg():
log_dir = f"model_save/"
env = ENV(istest=False)
env = Monitor(env, log_dir)
env = DummyVecEnv([lambda: env])
# env = VecNormalize(env, norm_obs=True, norm_reward=True,
# clip_obs=10.)
model = DDPG("CnnPolicy", env, policy_kwargs=policy_kwargs, verbose=1, batch_size=2048, seed=1, learning_starts=500000)
callback = SaveOnBestTrainingRewardCallback(check_freq=480, log_dir=log_dir)
model.learn(total_timesteps=int(1000000), callback = callback, log_interval = 480)
model.save('model_save/ddpg_cnn')
def objective(trial):
noise = trial.suggest_uniform('Noise', 0.1, 0.8)
timesteps = trial.suggest_int('Timesteps', 10, 100)
n_actions = env.action_space.shape[-1]
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(noise) *
np.ones(n_actions))
model = DDPG('MlpPolicy', env, action_noise=action_noise)
model.learn(total_timesteps=timesteps * 1000, log_interval=1000)
return test_model(env, model, '')
def train_DDPG(env_train, model_name, timesteps=10000):
"""DDPG model"""
# add the noise objects for DDPG
n_actions = env_train.action_space.shape[-1]
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions))
start = time.time()
model = DDPG('MlpPolicy', env_train, action_noise=action_noise)
model.learn(total_timesteps=timesteps)
end = time.time()
model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}")
print('Training time (DDPG): ', (end-start)/60,' minutes')
return model
def main():
# Create log dir
log_dir = './ddpg_data'
os.makedirs(log_dir, exist_ok=True)
vix_env = trading_vix_env.trading_vix_env()
env = Monitor(vix_env, log_dir)
# Create action noise because TD3 and DDPG use a deterministic policy
n_actions = env.action_space.shape[-1]
action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions))
# Create the callback: check every 20000 steps
callback = custom_call_back.CustomCallback(check_freq = 20000,log_dir = log_dir)
# Create RL model
model = DDPG('MlpPolicy',env,action_noise = action_noise, verbose=2,batch_size = 10000)
# Train the agent
model.learn(total_timesteps=int(5e9), callback=callback)
def train_DDPG(env_train, model_name, timesteps=10000):
"""DDPG model"""
# the noise objects for DDPG
n_actions = env_train.action_space.shape[-1]
# action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions))
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(0.5) *
np.ones(n_actions))
start = time.time()
param_noise = None
# removed keyword "param_noise=param_noise" stable_baselines3 doesn't need this one
model = DDPG('MlpPolicy', env_train, action_noise=action_noise)
model.learn(total_timesteps=timesteps)
end = time.time()
model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}")
print('Training time (DDPG): ', (end - start) / 60, ' minutes')
return model
def train():
log_dir = f"model_save/"
env = ENV(istest=False)
env = Monitor(env, log_dir)
env = DummyVecEnv([lambda: env])
# env = VecNormalize(env, norm_obs=True, norm_reward=True,
# clip_obs=10.)
model = DDPG('MlpPolicy',
env,
verbose=1,
batch_size=PARAM['batch_size'],
seed=PARAM['seed'],
learning_starts=PARAM['learning_starts'])
callback = SaveOnBestTrainingRewardCallback(check_freq=480,
log_dir=log_dir)
model.learn(total_timesteps=int(PARAM['total_time_step']),
callback=callback,
log_interval=480)
model.save('model_save/' + MODEL_PATH)
def train_DDPG(env):
print(f"action space shape -1:{env.action_space.shape[-1]}")
# The noise objects for TD3
n_actions = env.action_space.shape[-1]
action_noise = NormalActionNoise(mean=np.zeros(n_actions),
sigma=0.02 * np.ones(n_actions))
model = DDPG(
'MlpPolicy',
env,
learning_rate=0.0003,
learning_starts=5,
train_freq=10,
n_episodes_rollout=-1,
buffer_size=100000,
action_noise=action_noise,
batch_size=128,
verbose=2,
)
model.learn(total_timesteps=1000000, log_interval=1)
model.save("DDPG_pkl")
# A2C algorithm
for i in range(n_tests):
test_name = 'saved_models/a2c_soccer_actions_env_1_' + str(i)
n_actions = env.action_space.shape[-1]
model = A2C('MlpPolicy', env)
model.learn(total_timesteps=25000, log_interval=1000)
model.save(test_name)
test_model(env, model, test_name)
# DDPG algorithm
for i in range(n_tests):
test_name = 'saved_models/ddpg_soccer_actions_env_1_' + str(i)
n_actions = env.action_space.shape[-1]
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.3) * np.ones(n_actions))
model = DDPG('MlpPolicy', env, action_noise=action_noise)
model.learn(total_timesteps=10000, log_interval=1000)
model.save(test_name)
test_model(env, model, test_name)
for i in range(n_tests):
test_name = 'saved_models/ddpg_soccer_actions_env_2_' + str(i)
n_actions = env.action_space.shape[-1]
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.3) * np.ones(n_actions))
policy_kwargs = dict(net_arch=[400, 300])
model = DDPG('MlpPolicy', env, action_noise=action_noise, policy_kwargs=policy_kwargs)
model.learn(total_timesteps=10000, log_interval=1000)
model.save(test_name)
test_model(env, model, test_name)
for i in range(n_tests):
test_name = 'saved_models/ddpg_soccer_actions_env_3_' + str(i)
dt=0.005)
#### Create the callback: check every 1000 steps
callback = SaveOnBestTrainingRewardCallback(check_freq=1000,
log_dir=log_dir)
#### Train the model ###############################################################################
model = DDPG(CustomPolicy,
env,
verbose=1,
batch_size=64,
action_noise=action_noise)
for i in range(step_iters): # run for step_iters * training_timesteps
model.learn(total_timesteps=training_timesteps)
model.save("./models/ddpg" + str((i + 1) * training_timesteps))
model.save_replay_buffer("./experiences/ddpg_experience" +
str((i + 1) * training_timesteps))
#### Show (and record a video of) the model's performance ##########################################
env_test = RLTetherAviary(gui=False, record=True)
obs = env_test.reset()
start = time.time()
for i in range(10 * env_test.SIM_FREQ):
action, _states = model.predict(obs, deterministic=True)
obs, reward, done, info = env_test.step(action)
if done: break
env_test.close()
# Instantiate Environment
env_id = 'gym_spm:spm-v0'
env = gym.make('gym_spm:spm-v0')
# HyperParameters
lr = 3e-4
# Instantiate Model
n_actions = env.action_space.shape[-1]
action_noise = NormalActionNoise(mean=-30 * np.zeros(n_actions),
sigma=.75 * np.ones(n_actions))
model = DDPG('MlpPolicy', env, action_noise=action_noise, verbose=1)
# model = PPO('MlpPolicy', env, tensorboard_log=log_dir)
# Train OR Load Model
model.learn(total_timesteps=25000)
# model.save(model_dir_description)
mean_reward, std_reward = evaluate_policy(model,
model.get_env(),
n_eval_episodes=10)
print("Mean Reward = ", mean_reward)
epsi_sp_list = []
action_list = []
soc_list = []
Concentration_list = []
Concentration_list1 = []
if __name__ == '__main__':
env_id = 'gym_spm:spm-v0'
num_cpu = 4 # Number of processes to use
env = gym.make('gym_spm:spm-v0')
# env = make_vec_env(env_id, n_envs=1, seed=0)
# env = VecCheckNan(env, raise_exception=True)
# env = check_env(env)
# The noise objects for DDPG
n_actions = env.action_space.shape[-1]
action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=.1 * np.ones(n_actions))
# model = TD3(MlpPolicy, env, action_noise=action_noise, verbose=1, tensorboard_log="./TD3_spm_v2_SOC_point5_two_state/")
model = DDPG(MlpPolicy, env, action_noise=action_noise, verbose=1, tensorboard_log="./DDPG_spm_v2_SOC_point5_two_state/")
model.learn(total_timesteps=25000, tb_log_name='DDPG_test_run_3_SOCpoint5_two_state')
# model.save('DDPG_test_3_SOC_point5_two_states')
#
#
# model.load('DDPG_test_2_SOC_point5_two_states')
mean_reward, std_reward = evaluate_policy(model, model.get_env(), n_eval_episodes=10)
print("Mean Reward = ", mean_reward)
epsi_sp_list = []
action_list = []
soc_list = []
Concentration_list = []
Concentration_list1 = []
obs = env.reset()
log_dir_description = log_dir + details
model_dir_description = model_dir + details
# Instantiate Model
n_actions = env.action_space.shape[-1]
action_noise = NormalActionNoise(mean=np.zeros(n_actions),
sigma=25.67 * np.ones(n_actions))
model = DDPG(MlpPolicy,
env,
action_noise=action_noise,
verbose=1,
tensorboard_log=log_dir)
# Train OR Load Model
if train_model:
model.learn(total_timesteps=25000, tb_log_name=details)
model.save(model_dir_description)
else:
model.load(model_dir_description)
mean_reward, std_reward = evaluate_policy(model,
model.get_env(),
n_eval_episodes=10)
print("Mean Reward = ", mean_reward)
epsi_sp_list = []
action_list = []
soc_list = []
Concentration_list = []
Concentration_list1 = []
#"params_n_episodes_rollout": 1,
"params_net_arch": "small",
"params_noise_type": 'normal',
"value": 0 # only in logs
}
# Alternately, load tuned values from log dir
#hyper = best_hyperpars(log_dir, env_id, algo)
# Default parameters not tuned
learning_starts=100
optimize_memory_usage=False
# Configured in hyperparams/ddpg.yml
use_sde = True
env = gym.make(env_id)
model = ddpg(env, hyper, 'MlpPolicy', verbose = verbose, tensorboard_log = tensorboard_log, seed = seed, use_sde = use_sde)
model.learn(total_timesteps = total_timesteps)
custom_eval(model, env_id, algo, seed = seed, outdir = outdir, value = hyper["value"])
## Compare to vanilla default execution. Vanilla is no action noise, but tuning always uses action noise(?)
model = DDPG('MlpPolicy', env, verbose = verbose, tensorboard_log=tensorboard_log,
seed = seed, use_sde = use_sde)
model.learn(total_timesteps = total_timesteps)
custom_eval(model, env_id, algo, seed, "vanilla")
