def train_ddpg():
env = gimbal(5, 500)
env = DummyVecEnv([lambda: env])
eval_env = gimbal(5, 500)
eval_env = DummyVecEnv([lambda: eval_env])
# the noise objects for DDPG
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = None
model = DDPG(policy=MlpPolicy,
env=env,
gamma=0.99,
memory_policy=None,
eval_env=eval_env,
nb_train_steps=500,
nb_rollout_steps=500,
nb_eval_steps=500,
param_noise=param_noise,
action_noise=action_noise,
normalize_observations=False,
tau=0.001,
batch_size=128,
param_noise_adaption_interval=50,
normalize_returns=False,
enable_popart=False,
observation_range=(-5000.0, 5000.0),
critic_l2_reg=0.0,
return_range=(-inf, inf),
actor_lr=0.0001,
critic_lr=0.001,
clip_norm=None,
reward_scale=1.0,
render=False,
render_eval=False,
memory_limit=50000,
verbose=1,
tensorboard_log="./logs",
_init_setup_model=True,
policy_kwargs=None,
full_tensorboard_log=False)
#model = DDPG.load("./models/baseline_ddpg_t2")
#model.set_env(env)
model.learn(total_timesteps=1000000,
callback=None,
seed=None,
log_interval=100,
tb_log_name='DDPG',
reset_num_timesteps=True)
model.save("./models/baseline_ddpg_t2")
def train_ppo2_mlp():
n_cpu = 4
env = SubprocVecEnv([lambda: gimbal(5, 500) for i in range(n_cpu)])
model = PPO2(policy=MlpPolicy,
env=env,
gamma=0.99,
n_steps=100,
ent_coef=0.01,
learning_rate=0.00025,
vf_coef=0.5,
max_grad_norm=0.5,
lam=0.95,
nminibatches=4,
noptepochs=4,
cliprange=0.2,
verbose=1,
tensorboard_log="./logs",
_init_setup_model=True,
policy_kwargs=None,
full_tensorboard_log=False)
model.learn(total_timesteps=1000000,
callback=None,
seed=None,
log_interval=1,
tb_log_name='PPO2',
reset_num_timesteps=True)
model.save("./models/baseline_ppo2_t11_camshifted")
def objective(trial):
# Hyper-parameters to adjust
policy = trial.suggest_categorical('policy', ['MlpPolicy', 'MlpLnPolicy', 'MlpLstmPolicy', 'MlpLnLstmPolicy'])
gamma = trial.suggest_uniform('gamma', 0.10, 1.0)
ent_coef = trial.suggest_uniform('ent_coef', 0.01, 0.10)
learning_rate = trial.suggest_loguniform('learning_rate', 1e-5, 1e-2)
vf_coef = trial.suggest_uniform('vf_coef', 0.10, 1.0)
lam = trial.suggest_uniform('lam', 0.01, 0.95)
if policy == 'MlpPolicy':
policy = MlpPolicy
elif policy == 'MlpLnPolicy':
policy = MlpLnPolicy
elif policy == 'MlpLstmPolicy':
policy = MlpLstmPolicy
elif policy == 'MlpLnLstmPolicy':
policy = MlpLnLstmPolicy
# Train with those hyper-parameters
n_cpu = 4
env = SubprocVecEnv([lambda: gimbal(5, 500) for i in range(n_cpu)])
model = PPO2(policy=policy, env=env, gamma=gamma, n_steps=100, ent_coef=ent_coef, learning_rate=learning_rate,
vf_coef=vf_coef, max_grad_norm=0.5, lam=lam, nminibatches=4, noptepochs=4, cliprange=0.2,
verbose=0, tensorboard_log=None, _init_setup_model=True, policy_kwargs=None, full_tensorboard_log=False)
model.learn(total_timesteps=250000, callback=None, seed=None, log_interval=1, tb_log_name='PPO2', reset_num_timesteps=True)
# Calculate worth
env = gimbal(5, 500)
MAX_episodes = 25
reward_avg = 0
for episodes in range(MAX_episodes):
obs = env.reset()
r = 0
while True:
action, _states = model.predict(obs)
obs, rewards, dones, info = env.step(action)
r += rewards
#env.render()
if dones:
reward_avg += r
break
return - (reward_avg / MAX_episodes)
def run(self):
gm = gimbal(5, 500)
obs = gm.reset()
action = [5, -5]
for _ in range(50000):
gm.render()
print(gm.observation_space.shape)
#action = np.random.uniform(low=-6.13, high=6.13, size=2)
obs, rwd, done, info = gm.step(action)
if done:
obs = gm.reset()
gm.close()
def train_ppo2_mlplstm():
env = DummyVecEnv([lambda: gimbal(5, 500)])
#model = PPO2(policy=MlpLstmPolicy, env=env, gamma=0.99, n_steps=500, ent_coef=0.01, learning_rate=0.00025,
# vf_coef=0.5, max_grad_norm=0.5, lam=0.95, nminibatches=1, noptepochs=4, cliprange=0.2,
# verbose=1, tensorboard_log="./logs", _init_setup_model=True, policy_kwargs=None, full_tensorboard_log=False)
model = PPO2.load("./models/baseline_ppo2_t6_dynamicR", env=env)
model.learn(total_timesteps=500000,
callback=None,
seed=None,
log_interval=1,
tb_log_name='PPO2',
reset_num_timesteps=True)
model.save("./models/baseline_ppo2_t6_dynamicR")
def train_her():
env = gimbal(5, 500)
n_sampled_goal = 4
model = HER('MlpPolicy',
env,
SAC,
n_sampled_goal=n_sampled_goal,
goal_selection_strategy='future',
verbose=1,
buffer_size=int(1e6),
learning_rate=1e-3,
gamma=0.95,
batch_size=256,
policy_kwargs=dict(layers=[256, 256, 256]))
def view_ppo2_mlplstm():
env = gimbal(5, 500)
env = DummyVecEnv([lambda: gimbal(5, 500)])
model = PPO2.load("./models/baseline_ppo2_t6_dynamicR")
success_rate = 0
reward_avg = 0
for episodes in range(50):
obs = env.reset()
state = None
done = [False]
r = 0
while True:
action, state = model.predict(obs, state=state, mask=done)
obs, rewards, done, _ = env.step(action)
r += rewards
env.render()
if done:
if r > -100:
success_rate += 1
reward_avg += r
break
print("Success rate: ", success_rate, "Avg rewards: ",
(reward_avg / success_rate))
def view_ppo2_mlp():
env = gimbal(5, 500)
env = DummyVecEnv([lambda: gimbal(5, 500)])
model = PPO2.load("./models/baseline_ppo2_t7_prune1", env=env)
success_rate = 0
reward_avg = 0
for episodes in range(50):
obs = env.reset()
r = 0
while True:
action, _states = model.predict(obs)
obs, rewards, dones, info = env.step(action)
r += rewards
#env.render()
#env.target_ctrl()
if dones:
if r > -100:
success_rate += 1
reward_avg += r
break
print("Success rate: ", success_rate, "Avg rewards: ",
(reward_avg / success_rate))
def train_gail_withppo2():
env = gimbal(5, 500)
env = DummyVecEnv([lambda: env])
model = PPO2.load("./models/baseline_ppo2_t1")
generate_expert_traj(model,
'./models/baseline_expert_t1',
env,
n_timesteps=0,
n_episodes=100)
dataset = ExpertDataset(expert_path='./models/baseline_expert_t1.npz',
traj_limitation=-1,
verbose=1)
model = GAIL("MlpPolicy", env, dataset, verbose=1)
model.learn(total_timesteps=500000)
model.save("./models/baseline_gail_ppo2_t1")
def view_ddpg():
env = gimbal(5, 500)
model = DDPG.load("./models/baseline_ddpg_t2")
success_rate = 0
reward_avg = 0
for episodes in range(50):
obs = env.reset()
while True:
action, _states = model.predict(obs)
obs, rewards, dones, info = env.step(action)
env.render()
if dones:
if rewards > -100:
success_rate += 1
reward_avg += rewards
break
print("Success rate: ", success_rate, "Avg rewards: ",
reward_avg / success_rate)
def view(n_episodes, frame_skip, episode_length, success_threshold, model,
actions, features, high, low, actor_lr, critic_lr, rewardDecay,
priorityA):
test_rewards = []
dones_anyreward = 0
dones_goodreward = 0
env = gimbal(frame_skip, episode_length)
agent = DDPGAgent(env,
n_actions=actions,
n_features=features,
featurize=False,
action_high=high,
action_low=low,
actor_learning_rate=actor_lr,
critic_learning_rate=critic_lr,
reward_decay=rewardDecay,
priority_alpha=priorityA)
agent.saver.restore(agent.sess, model + str("_BEST.ckpt"))
for i_episode in range(n_episodes):
state = agent.env.reset()
r = 0
while True:
agent.env.render()
action = agent.choose_action([state], 0, agent.action_low,
agent.action_high)
next_state, reward, done, info = agent.env.step(action)
r += reward
state = next_state
if done:
if agent.env.timestep < agent.env.MAX_timestep:
dones_anyreward += 1
if r > success_threshold:
dones_goodreward += 1
print("episode:", i_episode + 1, "rewards: %.2f" % r, end="\r")
test_rewards += [r]
break
print("\n")
print("finished testing! Average reward: ",
np.sum(test_rewards) / n_episodes, "Dones (any reward): ",
dones_anyreward, "Dones (good reward)", dones_goodreward)
def conduct_test(agent, n_episodes, frame_skip, episode_length,
success_threshold, success_rate):
env = gimbal(frame_skip, episode_length)
successes = 0
rewards_sum = 0
for _ in range(n_episodes):
state = env.reset()
r = 0
while True:
action = agent.choose_action([state], 0, agent.action_low,
agent.action_high)
next_state, reward, done, info = env.step(action)
r += reward
state = next_state
if done:
if r >= success_threshold:
successes += 1
rewards_sum += r
break
env.close()
if successes >= success_rate:
return True, rewards_sum / successes
else:
return False, 0