def objective(trial):
noise = trial.suggest_uniform('Noise', 0.1, 0.8)
timesteps = trial.suggest_int('Timesteps', 10, 100)
n_actions = env.action_space.shape[-1]
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(noise) *
np.ones(n_actions))
model = DDPG('MlpPolicy', env, action_noise=action_noise)
model.learn(total_timesteps=timesteps * 1000, log_interval=1000)
return test_model(env, model, '')
def main():
"""
# Example with a simple Dummy vec env
env = gym.envs.make('panda-ip-reach-v0', renders= True)
env = DummyVecEnv([lambda: env])
"""
print("Env created !")
env = PandaReachGymEnv(renders=True)
env.render(mode='rgb_array')
model = DDPG.load("ddpg_panda_reach")
print("model loaded !")
while True:
obs, done = env.reset(), False
print("===================================")
print("obs")
print(obs)
episode_rew = 0
while not done:
env.render(mode='rgb_array')
action, _states = model.predict(obs)
obs, rew, done, info = env.step(action)
episode_rew += rew
print("Episode reward", episode_rew)
def create_model(env, algorithm, save_path):
# the noise object
n_actions = env.action_space.shape[-1]
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(0.2) *
np.ones(n_actions),
theta=0.15)
if algorithm == "ddpg":
return DDPG(DDPG_MlpPolicy,
env,
learning_rate=0.001,
buffer_size=1000000,
batch_size=64,
tau=0.001,
gamma=0.99,
train_freq=(10, "step"),
action_noise=action_noise,
policy_kwargs=dict(optimizer_class=th.optim.AdamW),
tensorboard_log=save_path)
elif algorithm == "td3":
return TD3(TD3_MlpPolicy,
env,
action_noise=action_noise,
tensorboard_log=save_path)
elif algorithm == "sac":
return SAC(SAC_MlpPolicy,
env,
action_noise=action_noise,
tensorboard_log=save_path)
else:
raise Exception("--> Alican's LOG: Unknown agent type!")
def ddpg(env,
hyper,
policy="MlpPolicy",
verbose=0,
tensorboard_log=None,
seed=0,
use_sde=True,
device="auto"):
policy_kwargs = make_policy_kwargs(hyper, "ddpg")
hyper = action_noise(hyper, "ddpg", n_actions=env.action_space.shape[0])
model = DDPG(
'MlpPolicy',
env,
verbose=verbose,
tensorboard_log=tensorboard_log,
seed=seed,
gamma=hyper['params_gamma'],
learning_rate=hyper['params_lr'],
batch_size=np.int(hyper['params_batch_size']),
buffer_size=np.int(hyper['params_buffer_size']),
action_noise=hyper['params_action_noise'],
train_freq=hyper['params_train_freq'],
# gradient_steps = np.int(hyper['params_train_freq']),
# n_episodes_rollout = np.int(hyper['params_n_episodes_rollout']),
policy_kwargs=policy_kwargs,
device=device)
return model
def train_stable_baselines(submodule, flags):
"""Train policies using the PPO algorithm in stable-baselines."""
from stable_baselines3.common.vec_env import DummyVecEnv
flow_params = submodule.flow_params
# Path to the saved files
exp_tag = flow_params['exp_tag']
result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S"))
# Perform training.
start_time = timeit.default_timer()
# print experiment.json information
print("=========================================")
print('Beginning training.')
print('Algorithm :', flags.algorithm)
model = run_model_stablebaseline(flow_params, flags.num_cpus,
flags.rollout_size, flags.num_steps,
flags.algorithm, flags.exp_config)
stop_time = timeit.default_timer()
run_time = stop_time - start_time
print("Training is Finished")
print("total runtime: ", run_time)
# Save the model to a desired folder and then delete it to demonstrate
# loading.
print('Saving the trained model!')
path = os.path.realpath(os.path.expanduser('~/baseline_results'))
ensure_dir(path)
save_path = os.path.join(path, result_name)
model.save(save_path)
# dump the flow params
with open(os.path.join(path, result_name) + '.json', 'w') as outfile:
json.dump(flow_params,
outfile,
cls=FlowParamsEncoder,
sort_keys=True,
indent=4)
# Replay the result by loading the model
print('Loading the trained model and testing it out!')
if flags.exp_config.lower() == "ppo":
from stable_baselines3 import PPO
model = PPO.load(save_path)
elif flags.exp_config.lower() == "ddpg":
from stable_baselines3 import DDPG
model = DDPG.load(save_path)
flow_params = get_flow_params(os.path.join(path, result_name) + '.json')
flow_params['sim'].render = True
env = env_constructor(params=flow_params, version=0)()
# The algorithms require a vectorized environment to run
eval_env = DummyVecEnv([lambda: env])
obs = eval_env.reset()
reward = 0
for _ in range(flow_params['env'].horizon):
action, _states = model.predict(obs)
obs, rewards, dones, info = eval_env.step(action)
reward += rewards
print('the final reward is {}'.format(reward))
def main():
# Create log dir
log_dir = './ddpg_data'
os.makedirs(log_dir, exist_ok=True)
vix_env = trading_vix_env.trading_vix_env()
env = Monitor(vix_env, log_dir)
# Create action noise because TD3 and DDPG use a deterministic policy
n_actions = env.action_space.shape[-1]
action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions))
# Create the callback: check every 20000 steps
callback = custom_call_back.CustomCallback(check_freq = 20000,log_dir = log_dir)
# Create RL model
model = DDPG('MlpPolicy',env,action_noise = action_noise, verbose=2,batch_size = 10000)
# Train the agent
model.learn(total_timesteps=int(5e9), callback=callback)
def train():
best_reward, best_reward_timesteps = None, None
save_path = "model_save/"+MODEL_PATH+"/"
if save_path is not None:
os.makedirs(save_path, exist_ok=True)
# log_dir = f"model_save/"
log_dir = save_path
env, env_eval = ENV(util='train', par=PARAM, dt=DT), ENV(util='val', par=PARAM, dt=DT)
env, env_eval = Monitor(env, log_dir), Monitor(env_eval, log_dir)
env, env_eval = DummyVecEnv([lambda: env]), DummyVecEnv([lambda: env_eval])
# env = VecNormalize(env, norm_obs=True, norm_reward=True,
# clip_obs=10.)
if PARAM['algo']=='td3':
model = TD3('MlpPolicy', env, verbose=1, batch_size=PARAM['batch_size'], seed=PARAM['seed'],
learning_starts=PARAM['learning_starts'])
elif PARAM['algo']=='ddpg':
model = DDPG('MlpPolicy', env, verbose=1, batch_size=PARAM['batch_size'], seed=PARAM['seed'],
learning_starts=PARAM['learning_starts'])
elif PARAM['algo']=='ppo':
model = PPO('MlpPolicy', env, verbose=1, batch_size=PARAM['batch_size'], seed=PARAM['seed'])
eval_callback = EvalCallback(env_eval, best_model_save_path=save_path+MODEL_PATH+'_best_model',
log_path=log_dir, eval_freq=PARAM['eval_freq'], save_freq=PARAM['save_freq'],
deterministic=True, render=False)
model.learn(total_timesteps=int(PARAM['total_time_step']), callback=eval_callback, log_interval = 500)
print("best mean reward:", eval_callback.best_mean_reward_overall, "timesteps:", eval_callback.best_mean_reward_timestep)
model.save(save_path+MODEL_PATH+'_final_timesteps')
def load_model(env, algorithm, filename):
if algorithm == "ddpg":
return DDPG.load(filename, env=env)
elif algorithm == "td3":
return TD3.load(filename, env=env)
elif algorithm == "sac":
return SAC.load(filename, env=env)
else:
raise Exception("--> Alican's LOG: Unknown agent type!")
def __init__(self, env, hyperparameters=DEFAULT_HYPERPARAMETERS):
self.P = hyperparameters
if self.P["model_class"] == "dqn":
from stable_baselines3 import DQN
self.model = DQN('MlpPolicy', env, verbose=self.P["verbose"])
self.model_class = DQN
elif self.P["model_class"] == "a2c":
from stable_baselines3 import A2C
from stable_baselines3.a2c import MlpPolicy
self.model = A2C(MlpPolicy, env, verbose=self.P["verbose"])
self.model_class = A2C
elif self.P["model_class"] == "ddpg":
from stable_baselines3 import DDPG
from stable_baselines3.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise
n_actions = env.action_space.shape[-1]
action_noise = NormalActionNoise(mean=np.zeros(n_actions),
sigma=0.1 * np.ones(n_actions))
self.model = DDPG('MlpPolicy',
env,
action_noise=action_noise,
verbose=self.P["verbose"])
self.model_class = DDPG
elif self.P["model_class"] == "td3":
from stable_baselines3 import TD3
from stable_baselines3.td3.policies import MlpPolicy
from stable_baselines3.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise
n_actions = env.action_space.shape[-1]
action_noise = NormalActionNoise(mean=np.zeros(n_actions),
sigma=0.1 * np.ones(n_actions))
self.model = TD3(MlpPolicy,
env,
action_noise=action_noise,
verbose=self.P["verbose"])
self.model_class = TD3
elif self.P["model_class"] == "ppo":
from stable_baselines3 import PPO
from stable_baselines3.ppo import MlpPolicy
self.model = PPO(MlpPolicy, env, verbose=self.P["verbose"])
self.model_class = PPO
elif self.P["model_class"] == "sac":
from stable_baselines3 import SAC
from stable_baselines3.sac import MlpPolicy
self.model = SAC(MlpPolicy, env, verbose=self.P["verbose"])
self.model_class = SAC
else:
raise NotImplementedError()
def test(MODEL_TEST):
log_dir = "model_save/" + MODEL_PATH + "/" + MODEL_PATH + MODEL_TEST
env = ENV(util='test', par=PARAM, dt=DT)
env.render = True
env = Monitor(env, log_dir)
if PARAM['algo']=='td3':
model = TD3.load(log_dir)
elif PARAM['algo']=='ddpg':
model = DDPG.load(log_dir)
elif PARAM['algo']=='ppo':
model = PPO.load(log_dir)
# plot_results(f"model_save/")
trade_dt = pd.DataFrame([]) # trade_dt: 所有股票的交易数据
result_dt = pd.DataFrame([]) # result_dt: 所有股票一年测试结果数据
for i in range(TEST_STOCK_NUM):
state = env.reset()
stock_bh_id = 'stock_bh_'+str(i) # 记录每个股票交易的buy_hold
stock_port_id = 'stock_port_'+str(i) # 记录每个股票交易的portfolio
stock_action_id = 'stock_action_' + str(i) # 记录每个股票交易的action
flow_L_id = 'stock_flow_' + str(i) # 记录每个股票的流水
stock_bh_dt, stock_port_dt, action_policy_dt, flow_L_dt = [], [], [], []
day = 0
while True:
action = model.predict(state)
next_state, reward, done, info = env.step(action[0])
state = next_state
# print("trying:",day,"reward:", reward,"now profit:",env.profit) # 测试每一步的交易policy
stock_bh_dt.append(env.buy_hold)
stock_port_dt.append(env.Portfolio_unit)
action_policy_dt.append(action[0][0]) # 用于记录policy
flow_L_dt.append(env.flow)
day+=1
if done:
print('stock: {}, total profit: {:.2f}%, buy hold: {:.2f}%, sp: {:.4f}, mdd: {:.2f}%, romad: {:.4f}'
.format(i, env.profit*100, env.buy_hold*100, env.sp, env.mdd*100, env.romad))
# 交易完后记录：股票ID，利润（单位100%），buy_hold（单位100%），夏普率，最大回撤率（单位100%），romad
result=pd.DataFrame([[i,env.profit*100,env.buy_hold*100,env.sp,env.mdd*100,env.romad]])
break
trade_dt_stock = pd.DataFrame({stock_port_id: stock_port_dt,
stock_bh_id: stock_bh_dt,
stock_action_id: action_policy_dt,
flow_L_id: flow_L_dt}) # 支股票的交易数据
trade_dt = pd.concat([trade_dt, trade_dt_stock], axis=1) # 所有股票交易数据合并（加行）
result_dt = pd.concat([result_dt,result],axis=0) # 所有股票结果数据合并（加列）
result_dt.columns = ['stock_id','prfit(100%)','buy_hold(100%)','sp','mdd(100%)','romad']
trade_dt.to_csv('out_dt/trade_'+MODEL_PATH+'.csv',index=False)
result_dt.to_csv('out_dt/result_'+MODEL_PATH+'.csv',index=False)
def train_DDPG(self, model_name, model_params = config.DDPG_PARAMS):
"""DDPG model"""
from stable_baselines3 import DDPG
from stable_baselines3.common.noise import NormalActionNoise, OrnsteinUhlenbeckActionNoise
env_train = self.env
n_actions = env_train.action_space.shape[-1]
action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1*np.ones(n_actions))
start = time.time()
model = DDPG('MlpPolicy',
env_train,
batch_size=model_params['batch_size'],
buffer_size=model_params['buffer_size'],
action_noise=action_noise,
verbose=model_params['verbose'],
tensorboard_log = f"{config.TENSORBOARD_LOG_DIR}/{model_name}"
)
model.learn(total_timesteps=model_params['timesteps'], tb_log_name = "DDPG_run")
end = time.time()
model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}")
print('Training time (DDPG): ', (end-start)/60,' minutes')
return model
def main():
args = parse_arguments()
load_path = os.path.join("logs", args.env, args.agent, "best_model.zip")
stats_path = os.path.join(args.log_dir, args.env, args.agent, "vec_normalize.pkl")
if args.agent == 'ddpg':
from stable_baselines3 import DDPG
model = DDPG.load(load_path)
elif args.agent == 'td3':
from stable_baselines3 import TD3
model = TD3.load(load_path)
elif args.agent == 'ppo':
from stable_baselines3 import PPO
model = PPO.load(load_path)
env = make_vec_env(args.env, n_envs=1)
env = VecNormalize.load(stats_path, env)
# do not update them at test time
env.training = False
# reward normalization is not needed at test time
env.norm_reward = False
# env = gym.make(args.env)
img = []
if args.render:
env.render('human')
done = False
obs = env.reset()
action = model.predict(obs)
if args.gif:
img.append(env.render('rgb_array'))
if args.timesteps is None:
while not done:
action, _= model.predict(obs)
obs, reward, done, info = env.step(action)
if args.gif:
img.append(env.render('rgb_array'))
else:
env.render()
else:
for i in range(args.timesteps):
action, _= model.predict(obs)
obs, reward, done, info = env.step(action)
if args.gif:
img.append(env.render('rgb_array'))
else:
env.render()
if args.gif:
imageio.mimsave(f'{os.path.join("logs", args.env, args.agent, "recording.gif")}', [np.array(img) for i, img in enumerate(img) if i%2 == 0], fps=29)
def main():
"""
# Example with Vectorized env
num_cpu = 4 # Number of processes to use
my_env_kwargs={'renders': False}
env = make_vec_env('panda-ip-reach-v0', n_envs=num_cpu, env_kwargs=my_env_kwargs)
"""
# Example with a simple Dummy vec env
env = gym.envs.make('panda-ip-reach-v0', renders=False)
env = DummyVecEnv([lambda: env])
#check_env(pandaenv)
# The noise objects for DDPG
n_actions = env.action_space.shape[-1]
print("n_actions = {0}".format(n_actions))
#action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions))
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=0.1 * np.ones(n_actions))
model = DDPG(policy='MlpPolicy',
env=env,
learning_rate=0.001,
buffer_size=1000000,
learning_starts=100,
batch_size=100,
tau=0.005,
gamma=0.99,
train_freq=1,
gradient_steps=-1,
action_noise=action_noise,
optimize_memory_usage=False,
tensorboard_log="./ddpg_panda_reach_tensorboard/",
create_eval_env=False,
policy_kwargs=None,
verbose=1,
seed=None,
device='auto',
_init_setup_model=True)
"""
print("start model evaluation without learning !")
mean_reward_before, std_reward_before = evaluate_policy(model, env, n_eval_episodes=1)
print("end model evaluation !")
"""
print("start model learning !")
model.learn(total_timesteps=200000, log_interval=10)
print("end model learning !")
print("-> model saved !!")
model.save("ddpg_panda_reach")
"""
print("start model evaluation with learning !")
mean_reward_after, std_reward_after = evaluate_policy(model, env, n_eval_episodes=1)
print("end model evaluation !")
"""
"""
def create(self, n_envs=1):
"""Create the agent"""
self.env = self.agent_helper.env
log_dir = self.agent_helper.config_dir
os.makedirs(log_dir, exist_ok=True)
self.env = Monitor(self.env, log_dir)
#TODO:
# Create DDPG policy and define its hyper parameter here! even the action space and observation space.
# add policy
policy_name = self.agent_helper.config['policy']
self.policy = eval(policy_name)
# action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions))
n_actions = int(self.agent_helper.env.action_space.shape[0])
action_noise = NormalActionNoise(
mean=np.zeros(n_actions),
sigma=self.agent_helper.config['rand_sigma'] * np.ones(n_actions))
#FIXME: test:
# self.model = DDPG("MlpPolicy", self.env, action_noise=action_noise, verbose=1, tensorboard_log=self.agent_helper.graph_path)
# TODO: fix the obvervation space and action space later. Test if the obervation space input is correct? Output action space is correct?
# activ_function_name = self.agent_helper.config['nn_activ']
# activ_function = eval(activ_function_name)
# policy_kwargs = dict(activation_fn=activ_function,
# net_arch=[dict(pi=[32, 32], qf=[32, 32])])
logger.info("Create the DDPG model")
policy_kwargs = dict(net_arch=self.agent_helper.config['layers'])
self.model = DDPG(
self.policy,
self.env,
learning_rate=self.agent_helper.config['learning_rate'],
buffer_size=self.agent_helper.config['buffer_size'],
batch_size=self.agent_helper.config['batch_size'],
tau=self.agent_helper.config['tau'],
gamma=self.agent_helper.config['gamma'],
gradient_steps=self.agent_helper.config['gradient_steps'],
action_noise=action_noise,
optimize_memory_usage=self.agent_helper.
config['optimize_memory_usage'],
create_eval_env=self.agent_helper.config['create_eval_env'],
policy_kwargs=policy_kwargs,
verbose=self.agent_helper.config['verbose'],
learning_starts=self.agent_helper.config['learning_starts'],
tensorboard_log=self.agent_helper.graph_path,
seed=self.agent_helper.seed)
pass
def test_ddpg():
log_dir = f"model_save/best_model_ddpg_cnn"
env = ENV(istest=True)
env.render = True
env = Monitor(env, log_dir)
model = DDPG.load(log_dir)
plot_results(f"model_save/")
for i in range(10):
state = env.reset()
while True:
action = model.predict(state)
next_state, reward, done, info = env.step(action[0])
state = next_state
# print("trying:",i,"action:", action,"now profit:",env.profit)
if done:
print('stock',i,' total profit=',env.profit,' buy hold=',env.buy_hold)
break
def train_ddpg():
log_dir = f"model_save/"
env = ENV(istest=False)
env = Monitor(env, log_dir)
env = DummyVecEnv([lambda: env])
# env = VecNormalize(env, norm_obs=True, norm_reward=True,
# clip_obs=10.)
model = DDPG("CnnPolicy", env, policy_kwargs=policy_kwargs, verbose=1, batch_size=2048, seed=1, learning_starts=500000)
callback = SaveOnBestTrainingRewardCallback(check_freq=480, log_dir=log_dir)
model.learn(total_timesteps=int(1000000), callback = callback, log_interval = 480)
model.save('model_save/ddpg_cnn')
def run(env, algname, filename):
if algname == "TD3":
model = TD3.load(f"{algname}_pkl")
elif algname == "SAC":
if filename:
model = SAC.load(f"{filename}")
else:
model = SAC.load(f"{algname}_pkl")
elif algname == "DDPG":
model = DDPG.load(f"{algname}_pkl")
else:
raise "Wrong algorithm name provided."
obs = env.reset()
while True:
action, _states = model.predict(obs)
obs, rewards, done, info = env.step(action)
env.render()
if done:
break
def train_DDPG(env_train, model_name, timesteps=10000):
"""DDPG model"""
# add the noise objects for DDPG
n_actions = env_train.action_space.shape[-1]
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions))
start = time.time()
model = DDPG('MlpPolicy', env_train, action_noise=action_noise)
model.learn(total_timesteps=timesteps)
end = time.time()
model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}")
print('Training time (DDPG): ', (end-start)/60,' minutes')
return model
def test_ddpg():
log_dir = f"model_save/best_model_ddpg_sp2"
env = ENV(istest=True)
env.render = True
env = Monitor(env, log_dir)
model = DDPG.load(log_dir)
plot_results(f"model_save/")
for i in range(10):
state = env.reset()
day = 0
while True:
action = model.predict(state)
next_state, reward, done, info = env.step(action[0])
state = next_state
# print("trying:",day,"reward:", reward,"now profit:",env.profit)
day += 1
if done:
print(
'stock: {}, total profit: {:.2f}%, buy hold: {:.2f}%, sp: {:.4f}, mdd: {:.2f}%, romad: {:.4f}'
.format(i, env.profit * 100, env.buy_hold * 100, env.sp,
env.mdd * 100, env.romad))
break
def train_DDPG(env_train, model_name, timesteps=10000):
"""DDPG model"""
# the noise objects for DDPG
n_actions = env_train.action_space.shape[-1]
# action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions))
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=float(0.5) *
np.ones(n_actions))
start = time.time()
param_noise = None
# removed keyword "param_noise=param_noise" stable_baselines3 doesn't need this one
model = DDPG('MlpPolicy', env_train, action_noise=action_noise)
model.learn(total_timesteps=timesteps)
end = time.time()
model.save(f"{config.TRAINED_MODEL_DIR}/{model_name}")
print('Training time (DDPG): ', (end - start) / 60, ' minutes')
return model
def train():
log_dir = f"model_save/"
env = ENV(istest=False)
env = Monitor(env, log_dir)
env = DummyVecEnv([lambda: env])
# env = VecNormalize(env, norm_obs=True, norm_reward=True,
# clip_obs=10.)
model = DDPG('MlpPolicy',
env,
verbose=1,
batch_size=PARAM['batch_size'],
seed=PARAM['seed'],
learning_starts=PARAM['learning_starts'])
callback = SaveOnBestTrainingRewardCallback(check_freq=480,
log_dir=log_dir)
model.learn(total_timesteps=int(PARAM['total_time_step']),
callback=callback,
log_interval=480)
model.save('model_save/' + MODEL_PATH)
def train_DDPG(env):
print(f"action space shape -1:{env.action_space.shape[-1]}")
# The noise objects for TD3
n_actions = env.action_space.shape[-1]
action_noise = NormalActionNoise(mean=np.zeros(n_actions),
sigma=0.02 * np.ones(n_actions))
model = DDPG(
'MlpPolicy',
env,
learning_rate=0.0003,
learning_starts=5,
train_freq=10,
n_episodes_rollout=-1,
buffer_size=100000,
action_noise=action_noise,
batch_size=128,
verbose=2,
)
model.learn(total_timesteps=1000000, log_interval=1)
model.save("DDPG_pkl")
if os.path.isfile(ARGS.exp + '/success_model.zip'):
path = ARGS.exp + '/success_model.zip'
elif os.path.isfile(ARGS.exp + '/best_model.zip'):
path = ARGS.exp + '/best_model.zip'
else:
print("[ERROR]: no model under the specified path", ARGS.exp)
if algo == 'a2c':
model = A2C.load(path)
if algo == 'ppo':
model = PPO.load(path)
if algo == 'sac':
model = SAC.load(path)
if algo == 'td3':
model = TD3.load(path)
if algo == 'ddpg':
model = DDPG.load(path)
#### Parameters to recreate the environment ################
env_name = ARGS.exp.split("-")[1] + "-aviary-v0"
OBS = ObservationType.KIN if ARGS.exp.split(
"-")[3] == 'kin' else ObservationType.RGB
if ARGS.exp.split("-")[4] == 'rpm':
ACT = ActionType.RPM
elif ARGS.exp.split("-")[4] == 'dyn':
ACT = ActionType.DYN
elif ARGS.exp.split("-")[4] == 'pid':
ACT = ActionType.PID
elif ARGS.exp.split("-")[4] == 'vel':
ACT = ActionType.VEL
elif ARGS.exp.split("-")[4] == 'tun':
ACT = ActionType.TUN
# A2C algorithm
for i in range(n_tests):
test_name = 'saved_models/a2c_soccer_actions_env_1_' + str(i)
n_actions = env.action_space.shape[-1]
model = A2C('MlpPolicy', env)
model.learn(total_timesteps=25000, log_interval=1000)
model.save(test_name)
test_model(env, model, test_name)
# DDPG algorithm
for i in range(n_tests):
test_name = 'saved_models/ddpg_soccer_actions_env_1_' + str(i)
n_actions = env.action_space.shape[-1]
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.3) * np.ones(n_actions))
model = DDPG('MlpPolicy', env, action_noise=action_noise)
model.learn(total_timesteps=10000, log_interval=1000)
model.save(test_name)
test_model(env, model, test_name)
for i in range(n_tests):
test_name = 'saved_models/ddpg_soccer_actions_env_2_' + str(i)
n_actions = env.action_space.shape[-1]
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.3) * np.ones(n_actions))
policy_kwargs = dict(net_arch=[400, 300])
model = DDPG('MlpPolicy', env, action_noise=action_noise, policy_kwargs=policy_kwargs)
model.learn(total_timesteps=10000, log_interval=1000)
model.save(test_name)
test_model(env, model, test_name)
for i in range(n_tests):
check_env(env, warn=True, skip_render_check=True)
####
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(env.N_ACTIONS),
sigma=0.1 *
np.ones(env.N_ACTIONS),
dt=0.005)
#### Create the callback: check every 1000 steps
callback = SaveOnBestTrainingRewardCallback(check_freq=1000,
log_dir=log_dir)
#### Train the model ###############################################################################
model = DDPG(CustomPolicy,
env,
verbose=1,
batch_size=64,
action_noise=action_noise)
for i in range(step_iters): # run for step_iters * training_timesteps
model.learn(total_timesteps=training_timesteps)
model.save("./models/ddpg" + str((i + 1) * training_timesteps))
model.save_replay_buffer("./experiences/ddpg_experience" +
str((i + 1) * training_timesteps))
#### Show (and record a video of) the model's performance ##########################################
env_test = RLTetherAviary(gui=False, record=True)
obs = env_test.reset()
start = time.time()
seed=args.seed,
tensorboard_log=args.tensorboard)
#--------------------------------------------------------#
# DDPG #
#--------------------------------------------------------#
elif args.algorithm == 'DDPG':
if args.sigma:
# noise objects for DDPG
n_actions = env.action_space.shape[-1]
action_noise = NormalActionNoise(mean=np.zeros(n_actions),
sigma=0.1 *
np.ones(n_actions))
model = DDPG("MlpPolicy",
env,
action_noise=action_noise,
verbose=1,
seed=args.seed,
tensorboard_log=args.tensorboard)
#--------------------------------------------------------#
# A2C #
#--------------------------------------------------------#
elif args.algorithm == 'A2C':
model = A2C('MlpPolicy',
env,
verbose=1,
learning_rate=args.learning_rate,
n_steps=args.n_steps,
gamma=args.gamma,
gae_lambda=args.gae_lambda,
ent_coef=args.ent_coef,
vf_coef=args.vf_coef,
return True
if __name__ == '__main__':
# Instantiate Environment
env_id = 'gym_spm:spm-v0'
env = gym.make('gym_spm:spm-v0')
# HyperParameters
lr = 3e-4
# Instantiate Model
n_actions = env.action_space.shape[-1]
action_noise = NormalActionNoise(mean=-30 * np.zeros(n_actions),
sigma=.75 * np.ones(n_actions))
model = DDPG('MlpPolicy', env, action_noise=action_noise, verbose=1)
# model = PPO('MlpPolicy', env, tensorboard_log=log_dir)
# Train OR Load Model
model.learn(total_timesteps=25000)
# model.save(model_dir_description)
mean_reward, std_reward = evaluate_policy(model,
model.get_env(),
n_eval_episodes=10)
print("Mean Reward = ", mean_reward)
epsi_sp_list = []
action_list = []
verbose=1)
if ARGS.algo == 'td3':
model = TD3(td3ddpgMlpPolicy,
train_env,
policy_kwargs=offpolicy_kwargs,
tensorboard_log=filename + '/tb/',
verbose=1) if ARGS.obs == ObservationType.KIN else TD3(
td3ddpgCnnPolicy,
train_env,
policy_kwargs=offpolicy_kwargs,
tensorboard_log=filename + '/tb/',
verbose=1)
if ARGS.algo == 'ddpg':
model = DDPG(td3ddpgMlpPolicy,
train_env,
policy_kwargs=offpolicy_kwargs,
tensorboard_log=filename + '/tb/',
verbose=1) if ARGS.obs == ObservationType.KIN else DDPG(
td3ddpgCnnPolicy,
train_env,
policy_kwargs=offpolicy_kwargs,
tensorboard_log=filename + '/tb/',
verbose=1)
#### Create eveluation environment #########################
if ARGS.obs == ObservationType.KIN:
eval_env = gym.make(
env_name,
aggregate_phy_steps=shared_constants.AGGR_PHY_STEPS,
obs=ARGS.obs,
act=ARGS.act)
def load_weights(self, weights_file):
""" Load the model from a zip archive """
logger.info(f"load weight from file: {weights_file}")
self.model = DDPG.load(weights_file, env=self.env)
pass
deterministic=True,
render=False)
### DDPG Noise
### Try increasing the noise when retraining.
### Try less noise based on the policy plot.
n_actions = env.action_space.shape[-1]
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions),
sigma=1 * np.ones(n_actions))
# action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.1 * np.ones(n_actions))
model = DDPG(
'MlpPolicy',
env,
action_noise=action_noise,
verbose=1,
tensorboard_log="./h={}/".format(horizons[rank]),
gamma=0.99,
learning_rate=0.0003,
)
# model = DDPG.load("Model_DDPG_FS_30.zip")
# model.learning_rate = 0.0003
# model.gamma = 0.99
# action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=0.05*np.ones(n_actions))
# action_noise = NormalActionNoise(mean=np.zeros(n_actions), sigma=0.075 * np.ones(n_actions))
# model.action_noise = action_noise
trainer = Trainer(env)
trainer.retrain_rl(model,
episodes=20000,
path="./h={}/".format(horizons[rank]))
