def test_agent_performance(env: gym.Env,
sac_params: SacEntropyAdjustmentParams,
run_params: RunParams, writer,
test_episode_number: int,
scaler: sklearn.preprocessing.StandardScaler):
""" Tests the agent's performance by running the policy during a certain amount of episodes. The
average episode reward and episode length are logged on the console and optionally on Tensorboard"""
with torch.no_grad():
episode_rewards, episode_lengths = [], []
for j in range(sac_params.num_test_episodes):
state, done, episode_reward, episode_length = env.reset(
), False, 0, 0
while not done:
state_scaled = scale_state(
scaler, state) if run_params.should_scale_states else state
action = select_action_sac(
state_scaled, sac_params,
deterministic=True) # No noise, pure exploitation
state, reward, done, _ = env.step(action)
episode_reward += reward
episode_length += 1
episode_rewards.append(episode_reward)
episode_lengths.append(episode_length)
print(f"\tAverage total episode reward: {np.mean(episode_rewards):.3f}"
f"\tAverage episode length: {np.mean(episode_lengths):.3f}")
if run_params.use_tensorboard:
writer.add_scalar("Test Performance/Average Performance",
np.mean(episode_rewards), test_episode_number)
writer.add_scalar("Test Performance/Average Episode Steps",
np.mean(episode_lengths), test_episode_number)
def sac_entropy_adjustment_train(env: gym.Env, run_params: RunParams,
sac_params: SacEntropyAdjustmentParams):
""" Trains the soft actor critic (SAC) on the given environment. Training is done at the end of each episode.
Only continuous actions spaces are supported. Several features can be optionally enabled:
1) Scaling / normalizing the states / observations
2) Logging training statistics on Tensorboard
3) Render the environment periodically (pick render_frequency in the RunParams)
4) Testing the agent's performance periodically
5) Saving the policy and value estimators to disk periodically
6) During the first X steps, do random actions (see RunParams.num_random_action_steps)
"""
assert run_params.continuous_actions, "SAC implementation only implemented for continuous action spaces"
print(
f"The goal is a running reward of at least {env.spec.reward_threshold}."
)
# Optimization for speed: don't compute gradients for the target networks, since we will never use them
for network in [
sac_params.policy_target, sac_params.value_estimator1_target,
sac_params.value_estimator2_target
]:
for parameter in network.parameters():
parameter.requires_grad = False
# Setup tensorboard
writer = run_params.get_tensorboard_writer(
env) if run_params.use_tensorboard else None
# Setup scaler, training info and replay buffer
scaler = setup_observation_scaler(
env) if run_params.should_scale_states else None
training_info = TrainingInfo(GAMMA=run_params.gamma)
replay_buffer = ReplayBuffer(sac_params.replay_buffer_size)
training_step_number, step_number, test_episode_num = 0, 0, 0
max_episode_steps = env.spec.max_episode_steps
for episode_number in range(run_params.maximum_episodes):
state = env.reset()
episode_length = 0
# Update policy bounds
# sac_params.policy.action_high = sac_params.policy_target.action_high = torch.tensor(env.action_space.high)
# sac_params.policy.action_low = sac_params.policy_target.action_low = torch.tensor(env.action_space.low)
# Do a whole episode
for t in range(max_episode_steps):
if run_params.should_scale_states:
state = scale_state(scaler, state)
# Pick an action, execute and observe the results
# Note: in the first start_steps steps, we randomly pick actions from
# the action space (uniformly) to have better exploration.
if step_number >= sac_params.num_random_action_steps:
action, log_prob = select_action_sac(state,
sac_params,
compute_log_prob=True)
else:
action = env.action_space.sample()
log_prob = -1
# To be sure that actions are in the action space (see watershed.py)
action = np.clip(action, env.action_space.low,
env.action_space.high)
# For debugging, log the Q-values
if run_params.use_tensorboard:
if random.random(
) < 0.02: # Don't log too often to avoid slowing things down
s, a = torch.tensor(state).float(), torch.tensor(
action).float()
value1 = sac_params.value_estimator1.forward(s, a)
value2 = sac_params.value_estimator2.forward(s, a)
value1_target = sac_params.value_estimator1_target.forward(
s, a)
value2_target = sac_params.value_estimator2_target.forward(
s, a)
for action_index in range(a.shape[0]):
writer.add_scalar(f"Action/{action_index}",
a[action_index], step_number)
writer.add_scalar("Q-values/Normal Network 1", value1,
step_number)
writer.add_scalar("Q-values/Normal Network 2", value2,
step_number)
writer.add_scalar("Q-values/Target Network 1",
value1_target, step_number)
writer.add_scalar("Q-values/Target Network 2",
value2_target, step_number)
writer.add_scalar("Action/Log prob action", log_prob,
step_number)
new_state, reward, done, _ = env.step(action)
# Render the environment if wanted
if run_params.should_render(episode_number):
env.render()
# Store reward and updates the running reward
training_info.record_step(state, action, reward)
# Add the transition to the replay buffer
new_state_scaled = scale_state(
scaler,
new_state) if run_params.should_scale_states else new_state
replay_buffer.store(state, action, reward, new_state_scaled, done
and t < max_episode_steps - 1)
state = new_state
if done:
break
step_number += 1
episode_length += 1
# Training at the end of the episode approach taken from
# https://github.com/createamind/DRL/blob/master/spinup/algos/sac1/sac1_BipedalWalker-v2_200ep.py
for update_step in range(int(episode_length * 1.5)):
batch_transitions = replay_buffer.sample_batch(
sac_params.batch_size)
update_models(batch_transitions, sac_params, run_params, writer,
training_step_number)
training_step_number += 1
if (episode_number + 0) % sac_params.test_frequency == 0:
test_agent_performance(env, sac_params, run_params, writer,
test_episode_num, scaler)
test_episode_num += 1
if run_params.should_save_model(episode_number):
save_model_sac(env, sac_params, scaler)
training_info.update_running_reward(rate=0.01)
# Add some logging
log_on_console(env, episode_number, reward, run_params, t + 1,
training_info)
log_on_tensorboard(env, episode_number, reward, run_params, t + 1,
training_info, writer)
# Check if we have solved the environment reliably
if run_params.stop_at_threshold and env.spec.reward_threshold is not None and training_info.running_reward > env.spec.reward_threshold:
print(
f"Solved! The running reward is {training_info.running_reward:.2f}, which is above the threshold of "
f"{env.spec.reward_threshold}. The last episode ran for {t} steps."
)
save_model_sac(env, sac_params, scaler)
break
training_info.reset()
close_tensorboard(run_params, writer)
def actor_critic_train_per_episode(
policy: SimplePolicyContinuous,
critic: SimpleCritic,
env: gym.Env,
optimizer: Optimizer,
run_params: RunParams,
lr_scheduler: torch.optim.lr_scheduler._LRScheduler = None):
""" Trains the actor critic on the given environment. Training is done at the end of each episode, instead
of at the end of each step of an episode. This means the agent trains much more frequently.
Both discrete and continuous actions spaces are supported. Several features can be optionally enabled:
1) Scaling / normalizing the states / observations
2) Logging training statistics on Tensorboard
3) Render the environment periodically (pick render_frequency in the RunParams)
4) Using a learning rate scheduler
"""
training_info = TrainingInfo(GAMMA=run_params.gamma)
print(
f"The goal is a running reward of at least {env.spec.reward_threshold}."
)
# https://medium.com/@asteinbach/actor-critic-using-deep-rl-continuous-mountain-car-in-tensorflow-4c1fb2110f7c
# says it's crucial to scale the state
if run_params.should_scale_states:
scaler = setup_observation_scaler(env)
writer = run_params.get_tensorboard_writer(
env) if run_params.use_tensorboard else None
for episode_number in itertools.count(
): # itertools.count() is basically range(+infinity)
state = env.reset()
# Do a whole episode (upto 10000 steps, don't want infinite steps)
for t in range(env.spec.max_episode_steps):
if run_params.should_scale_states:
state = scale_state(scaler, state)
if run_params.continuous_actions:
action = select_action_continuous(state, policy, training_info,
env)
else:
action = select_action_discrete(state, policy, training_info)
state_value = get_state_value(state, critic)
new_state, reward, done, _ = env.step(action)
if run_params.should_render(episode_number):
env.render()
training_info.record_step(
state, action, reward,
state_value) # Store reward and updates the running reward
state = new_state
if done:
break
training_info.update_running_reward()
# Add some logging
log_on_console(env, episode_number, reward, run_params, t,
training_info)
log_on_tensorboard(env, episode_number, reward, run_params, t,
training_info, writer)
# Check if we have solved the environment reliably
if env.spec.reward_threshold is not None and training_info.running_reward > env.spec.reward_threshold:
print(
f"Solved! The running reward is {training_info.running_reward:.2f}, which is above the threshold of "
f"{env.spec.reward_threshold}. The last episode ran for {t} steps."
)
break
train_policy_on_episode(optimizer, training_info, episode_number)
if lr_scheduler:
lr_scheduler.step(episode_number)
close_tensorboard(run_params, writer)
def reinforceTraining(policy: SimplePolicyDiscrete, env: gym.Env,
optimizer: Optimizer, run_params: RunParams):
""" Trains the policy using the REINFORCE algorithm. Training is done at the end of each episode, and can
be done either using all transitions at once, or over many training iterations over mini-batches of transitions.
Both discrete and continuous actions spaces are supported. Several features can be optionally enabled:
1) Scaling / normalizing the states / observations
2) Logging training statistics on Tensorboard
3) Render the environment periodically (pick render_frequency in the RunParams)
4) Save the policy (and optionally the observation / state scaler) periodically (see RunParams.save_model_frequency)
"""
training_info = TrainingInfo(GAMMA=run_params.gamma)
print(
f"The goal is a running reward of at least {env.spec.reward_threshold}."
)
scaler = setup_observation_scaler(
env) if run_params.should_scale_states else None
writer = run_params.get_tensorboard_writer(
env) if run_params.use_tensorboard else None
for episode_number in itertools.count(
): # itertools.count() is basically range(+infinity)
state = env.reset()
# Do a whole episode (upto 10000 steps, don't want infinite steps)
for t in range(env.spec.max_episode_steps):
if run_params.should_scale_states:
state = scale_state(scaler, state)
if run_params.continuous_actions:
action = select_action_continuous(state, policy, training_info,
env)
else:
action = select_action_discrete(state, policy, training_info)
new_state, reward, done, _ = env.step(action)
if run_params.should_render(episode_number):
env.render()
training_info.record_step(
state, action,
reward) # Store reward and updates the running reward
state = new_state
if done:
break
training_info.update_running_reward()
# Add some logging
log_on_console(env, episode_number, reward, run_params, t,
training_info)
log_on_tensorboard(env, episode_number, reward, run_params, t,
training_info, writer)
# Check if we have solved the environment reliably
if env.spec.reward_threshold is not None and training_info.running_reward > env.spec.reward_threshold:
print(
f"Solved! The running reward is {training_info.running_reward:.2f}, which is above the threshold of "
f"{env.spec.reward_threshold}. The last episode ran for {t} steps."
)
break
if run_params.train_with_batches:
train_policy_batches(policy, optimizer, training_info, run_params)
else:
train_policy(optimizer, training_info, run_params)
if run_params.should_save_model(episode_number):
save_model(policy, env, "policy.data")
if scaler is not None:
save_scaler(scaler, env, "scaler.data")
close_tensorboard(run_params, writer)
def ddpg_train(env: gym.Env, run_params: RunParams, ddpg_params: DDPGParams):
"""
:param env: the OpenAI gym environment
:param run_params: the general training parameters shared by all training algorithm
:param ddpg_params: the DDPG-specific information (networks, optimizers, parameters)
"""
assert run_params.continuous_actions, "DDPG implementation only implemented for continuous action spaces"
print(f"The goal is a running reward of at least {env.spec.reward_threshold}.")
# Optimization for speed: don't compute gradients for the target networks, since we will never use them
for network in [ddpg_params.policy_target, ddpg_params.value_estimator_target]:
for parameter in network.parameters():
parameter.requires_grad = False
# Setup tensorboard
writer = run_params.get_tensorboard_writer(env) if run_params.use_tensorboard else None
# Setup scaler, training info and replay buffer
scaler = setup_observation_scaler(env) if run_params.should_scale_states else None
training_info = TrainingInfo(GAMMA=run_params.gamma)
replay_buffer = ReplayBuffer(ddpg_params.replay_buffer_size)
step_number, test_episode_num = 0, 0
max_episode_steps = env.spec.max_episode_steps
value_time_step = 0
for episode_number in range(run_params.maximum_episodes):
state = env.reset()
# Do a whole episode
for t in range(max_episode_steps):
if run_params.should_scale_states:
state = scale_state(scaler, state)
# Pick an action, execute and observe the results
# Note: in the first start_steps steps, we randomly pick actions from
# the action space (uniformly) to have better exploration.
if step_number >= ddpg_params.num_random_action_steps:
action = select_action_ddpg(state, ddpg_params, env, ddpg_params.noise_coeff * 0.995 ** episode_number)
else:
action = env.action_space.sample()
# For debugging, log the Q-values
if run_params.use_tensorboard:
s, a = torch.tensor(state).float(), torch.tensor(action).float()
value = ddpg_params.value_estimator.forward(s, a)
value_target = ddpg_params.value_estimator_target.forward(s, a)
for action_index in range(a.shape[0]):
writer.add_scalar(f"Action/{action_index}", a[action_index], value_time_step)
writer.add_scalar("Q-values/Normal Network", value, value_time_step)
writer.add_scalar("Q-values/Target Network", value_target, value_time_step)
value_time_step += 1
new_state, reward, done, _ = env.step(action)
# Render the environment if wanted
if run_params.should_render(episode_number):
env.render()
# Store reward and updates the running reward
training_info.record_step(state, action, reward)
# Add the transition to the replay buffer
new_state_scaled = scale_state(scaler, new_state) if run_params.should_scale_states else new_state
replay_buffer.store(state, action, reward, new_state_scaled, done and t < max_episode_steps - 1)
state = new_state
if done:
break
if step_number >= ddpg_params.update_start and step_number % ddpg_params.update_frequency == 0:
for update_step in range(ddpg_params.update_frequency):
batch_transitions = replay_buffer.sample_batch(ddpg_params.batch_size)
update_models(batch_transitions, ddpg_params, run_params, writer, step_number)
step_number += 1
if episode_number % ddpg_params.test_frequency == 0:
test_agent_performance(env, ddpg_params, run_params, writer, test_episode_num, scaler)
test_episode_num += 1
if run_params.should_save_model(episode_number):
save_model_ddpg(ddpg_params, env, scaler)
training_info.update_running_reward()
# Add some logging
log_on_console(env, episode_number, reward, run_params, t, training_info)
log_on_tensorboard(env, episode_number, reward, run_params, t, training_info, writer)
# Check if we have solved the environment reliably
if run_params.stop_at_threshold and env.spec.reward_threshold is not None and training_info.running_reward > env.spec.reward_threshold:
print(f"Solved! The running reward is {training_info.running_reward:.2f}, which is above the threshold of "
f"{env.spec.reward_threshold}. The last episode ran for {t} steps.")
break
training_info.reset()
ddpg_params.noise_source.reset()
close_tensorboard(run_params, writer)
