def test_extend_prioritized():
nvals = 16
states = [np.random.rand(2, 2) for _ in range(nvals)]
actions = [np.random.rand(2) for _ in range(nvals)]
rewards = [np.random.rand() for _ in range(nvals)]
newstate = [np.random.rand(2, 2) for _ in range(nvals)]
done = [np.random.randint(0, 2) for _ in range(nvals)]
size = 32
alpha = 0.99
baseline = PrioritizedReplayBuffer(size, alpha)
ext = PrioritizedReplayBuffer(size, alpha)
for data in zip(states, actions, rewards, newstate, done):
baseline.add(*data)
states, actions, rewards, newstates, done = map(
np.array, [states, actions, rewards, newstate, done])
ext.extend(states, actions, rewards, newstates, done)
assert len(baseline) == len(ext)
# Check buffers have same values
for i in range(nvals):
for j in range(5):
condition = (baseline.storage[i][j] == ext.storage[i][j])
if isinstance(condition, np.ndarray):
# for obs, obs_t1
assert np.all(condition)
else:
# for done, reward action
assert condition
# assert priorities
assert (baseline._it_min._value == ext._it_min._value).all()
assert (baseline._it_sum._value == ext._it_sum._value).all()
def learn(self, total_timesteps, callback=None, log_interval=100, tb_log_name="DQN",
reset_num_timesteps=True, replay_wrapper=None):
new_tb_log = self._init_num_timesteps(reset_num_timesteps)
callback = self._init_callback(callback)
with SetVerbosity(self.verbose), TensorboardWriter(self.graph, self.tensorboard_log, tb_log_name, new_tb_log) \
as writer:
self._setup_learn()
# Create the replay buffer
if self.prioritized_replay:
self.replay_buffer = PrioritizedReplayBuffer(self.buffer_size, alpha=self.prioritized_replay_alpha)
if self.prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
else:
prioritized_replay_beta_iters = self.prioritized_replay_beta_iters
self.beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=self.prioritized_replay_beta0,
final_p=1.0)
else:
self.replay_buffer = ReplayBuffer(self.buffer_size)
self.beta_schedule = None
if replay_wrapper is not None:
assert not self.prioritized_replay, "Prioritized replay buffer is not supported by HER"
self.replay_buffer = replay_wrapper(self.replay_buffer)
# Create the schedule for exploration starting from 1.
self.exploration = LinearSchedule(schedule_timesteps=int(self.exploration_fraction * total_timesteps),
initial_p=self.exploration_initial_eps,
final_p=self.exploration_final_eps)
episode_rewards = [0.0]
episode_successes = []
callback.on_training_start(locals(), globals())
callback.on_rollout_start()
reset = True
obs = self.env.reset()
# Retrieve unnormalized observation for saving into the buffer
if self._vec_normalize_env is not None:
obs_ = self._vec_normalize_env.get_original_obs().squeeze()
for _ in range(total_timesteps):
# Take action and update exploration to the newest value
kwargs = {}
if not self.param_noise:
update_eps = self.exploration.value(self.num_timesteps)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = \
-np.log(1. - self.exploration.value(self.num_timesteps) +
self.exploration.value(self.num_timesteps) / float(self.env.action_space.n))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
with self.sess.as_default():
action = self.act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0]
env_action = action
reset = False
new_obs, rew, done, info = self.env.step(env_action)
self.num_timesteps += 1
# Stop training if return value is False
if callback.on_step() is False:
break
# Store only the unnormalized version
if self._vec_normalize_env is not None:
new_obs_ = self._vec_normalize_env.get_original_obs().squeeze()
reward_ = self._vec_normalize_env.get_original_reward().squeeze()
else:
# Avoid changing the original ones
obs_, new_obs_, reward_ = obs, new_obs, rew
# Store transition in the replay buffer.
self.replay_buffer.add(obs_, action, reward_, new_obs_, float(done))
if self.expert_exp is not None:
self.add_expert_exp()
obs = new_obs
# Save the unnormalized observation
if self._vec_normalize_env is not None:
obs_ = new_obs_
if writer is not None:
ep_rew = np.array([reward_]).reshape((1, -1))
ep_done = np.array([done]).reshape((1, -1))
tf_util.total_episode_reward_logger(self.episode_reward, ep_rew, ep_done, writer,
self.num_timesteps)
episode_rewards[-1] += reward_
if done:
maybe_is_success = info.get('is_success')
if maybe_is_success is not None:
episode_successes.append(float(maybe_is_success))
if not isinstance(self.env, VecEnv):
obs = self.env.reset()
episode_rewards.append(0.0)
reset = True
# Do not train if the warmup phase is not over
# or if there are not enough samples in the replay buffer
can_sample = self.replay_buffer.can_sample(self.batch_size)
if can_sample and self.num_timesteps > self.learning_starts \
and self.num_timesteps % self.train_freq == 0:
callback.on_rollout_end()
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
# pytype:disable=bad-unpacking
if self.prioritized_replay:
assert self.beta_schedule is not None, \
"BUG: should be LinearSchedule when self.prioritized_replay True"
experience = self.replay_buffer.sample(self.batch_size,
beta=self.beta_schedule.value(self.num_timesteps),
env=self._vec_normalize_env)
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = self.replay_buffer.sample(self.batch_size,
env=self._vec_normalize_env)
weights, batch_idxes = np.ones_like(rewards), None
# pytype:enable=bad-unpacking
if writer is not None:
# run loss backprop with summary, but once every 100 steps save the metadata
# (memory, compute time, ...)
if (1 + self.num_timesteps) % 100 == 0:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
summary, td_errors = self._train_step(obses_t, actions, rewards, obses_tp1, obses_tp1,
dones, weights, sess=self.sess, options=run_options,
run_metadata=run_metadata)
writer.add_run_metadata(run_metadata, 'step%d' % self.num_timesteps)
else:
summary, td_errors = self._train_step(obses_t, actions, rewards, obses_tp1, obses_tp1,
dones, weights, sess=self.sess)
writer.add_summary(summary, self.num_timesteps)
else:
_, td_errors = self._train_step(obses_t, actions, rewards, obses_tp1, obses_tp1, dones, weights,
sess=self.sess)
if self.prioritized_replay:
new_priorities = np.abs(td_errors) + self.prioritized_replay_eps
assert isinstance(self.replay_buffer, PrioritizedReplayBuffer)
self.replay_buffer.update_priorities(batch_idxes, new_priorities)
callback.on_rollout_start()
if can_sample and self.num_timesteps > self.learning_starts and \
self.num_timesteps % self.target_network_update_freq == 0:
# Update target network periodically.
self.update_target(sess=self.sess)
if len(episode_rewards[-101:-1]) == 0:
mean_100ep_reward = -np.inf
else:
mean_100ep_reward = round(float(np.mean(episode_rewards[-101:-1])), 1)
num_episodes = len(episode_rewards)
if self.verbose >= 1 and done and log_interval is not None and len(episode_rewards) % log_interval == 0:
logger.record_tabular("steps", self.num_timesteps)
logger.record_tabular("episodes", num_episodes)
if len(episode_successes) > 0:
logger.logkv("success rate", np.mean(episode_successes[-100:]))
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * self.exploration.value(self.num_timesteps)))
logger.dump_tabular()
callback.on_training_end()
return self
def learn(self, total_timesteps, callback=None, log_interval=4, tb_log_name="BDQ",
reset_num_timesteps=True, replay_wrapper=None):
new_tb_log = self._init_num_timesteps(reset_num_timesteps)
callback = self._init_callback(callback)
with SetVerbosity(self.verbose), TensorboardWriter(self.graph, self.tensorboard_log, tb_log_name, new_tb_log) \
as writer:
self._setup_learn()
# Create the replay buffer
if self.prioritized_replay:
self.replay_buffer = PrioritizedReplayBuffer(self.buffer_size, alpha=self.prioritized_replay_alpha)
if self.prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
else:
prioritized_replay_beta_iters = self.prioritized_replay_beta_iters
self.beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=self.prioritized_replay_beta0,
final_p=1.0)
else:
self.replay_buffer = ReplayBuffer(self.buffer_size)
self.beta_schedule = None
if replay_wrapper is not None:
assert not self.prioritized_replay, "Prioritized replay buffer is not supported by HER"
self.replay_buffer = replay_wrapper(self.replay_buffer)
if self.epsilon_greedy:
approximate_num_iters = 2e6 / 4
# TODO Decide which schedule type to use
# self.exploration = PiecewiseSchedule([(0, 1.0),
# (approximate_num_iters / 50, 0.1),
# (approximate_num_iters / 5, 0.01)
# ], outside_value=0.01)
self.exploration = LinearSchedule(schedule_timesteps=int(self.exploration_fraction * total_timesteps),
initial_p=self.exploration_initial_eps,
final_p=self.exploration_final_eps)
else:
self.exploration = ConstantSchedule(value=0.0) # greedy policy
std_schedule = LinearSchedule(schedule_timesteps=self.timesteps_std,
initial_p=self.initial_std,
final_p=self.final_std)
episode_rewards = [0.0]
episode_successes = []
callback.on_training_start(locals(), globals())
callback.on_rollout_start()
obs = self.env.reset()
reset = True
self.episode_reward = np.zeros((1,))
# Retrieve unnormalized observation for saving into the buffer
if self._vec_normalize_env is not None:
obs_ = self._vec_normalize_env.get_original_obs().squeeze()
for _ in range(total_timesteps):
# Take action and update exploration to the newest value
kwargs = {}
if not self.param_noise:
update_eps = self.exploration.value(self.num_timesteps)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = \
-np.log(1. - self.exploration.value(self.num_timesteps) +
self.exploration.value(self.num_timesteps) / float(self.env.action_space.n))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
with self.sess.as_default():
# action = self.act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0]
# print("time step {} and update eps {}".format(self.num_timesteps, update_eps))
action_idxes = np.array(self.act(np.array(obs)[None], update_eps=update_eps, **kwargs)) #update_eps=exploration.value(t)))
action = action_idxes / self.num_action_grains * self.actions_range + self.low
if not self.epsilon_greedy: # Gaussian noise
actions_greedy = action
action_idx_stoch = []
action = []
for index in range(len(actions_greedy)):
a_greedy = actions_greedy[index]
out_of_range_action = True
while out_of_range_action:
# Sample from a Gaussian with mean at the greedy action and a std following a schedule of choice
a_stoch = np.random.normal(loc=a_greedy, scale=std_schedule.value(self.num_timesteps))
# Convert sampled cont action to an action idx
a_idx_stoch = np.rint((a_stoch + self.high[index]) / self.actions_range[index] * self.num_action_grains)
# Check if action is in range
if a_idx_stoch >= 0 and a_idx_stoch < self.num_actions_pad:
action_idx_stoch.append(a_idx_stoch)
action.append(a_stoch)
out_of_range_action = False
action_idxes = action_idx_stoch
env_action = action
reset = False
new_obs, rew, done, info = self.env.step(env_action)
self.num_timesteps += 1
# Stop training if return value is False
if callback.on_step() is False:
break
# Store only the unnormalized version
if self._vec_normalize_env is not None:
new_obs_ = self._vec_normalize_env.get_original_obs().squeeze()
reward_ = self._vec_normalize_env.get_original_reward().squeeze()
else:
# Avoid changing the original ones
obs_, new_obs_, reward_ = obs, new_obs, rew
# Store transition in the replay buffer.
self.replay_buffer.add(obs_, action_idxes, reward_, new_obs_, float(done))
obs = new_obs
# Save the unnormalized observation
if self._vec_normalize_env is not None:
obs_ = new_obs_
if writer is not None:
ep_rew = np.array([reward_]).reshape((1, -1))
ep_done = np.array([done]).reshape((1, -1))
tf_util.total_episode_reward_logger(self.episode_reward, ep_rew, ep_done, writer,
self.num_timesteps)
# self.episode_reward = total_episode_reward_logger(self.episode_reward, ep_rew, ep_done, writer,
# self.num_timesteps)
# episode_rewards[-1] += rew
episode_rewards[-1] += reward_
if done:
# print("ep number", len(episode_rewards))
maybe_is_success = info.get('is_success')
if maybe_is_success is not None:
episode_successes.append(float(maybe_is_success))
if not isinstance(self.env, VecEnv):
obs = self.env.reset()
episode_rewards.append(0.0)
reset = True
# Do not train if the warmup phase is not over
# or if there are not enough samples in the replay buffer
can_sample = self.replay_buffer.can_sample(self.batch_size)
if can_sample and self.num_timesteps > self.learning_starts \
and self.num_timesteps % self.train_freq == 0:
callback.on_rollout_end()
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
# pytype:disable=bad-unpacking
if self.prioritized_replay:
assert self.beta_schedule is not None, \
"BUG: should be LinearSchedule when self.prioritized_replay True"
experience = self.replay_buffer.sample(self.batch_size,
beta=self.beta_schedule.value(self.num_timesteps))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = self.replay_buffer.sample(self.batch_size)
weights, batch_idxes = np.ones_like(rewards), None
# pytype:enable=bad-unpacking
if writer is not None:
# run loss backprop with summary, but once every 100 steps save the metadata
# (memory, compute time, ...)
if (1 + self.num_timesteps) % 100 == 0:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
summary, td_errors, mean_loss = self._train_step(obses_t, actions, rewards, obses_tp1, obses_tp1,
dones, weights, sess=self.sess, options=run_options,
run_metadata=run_metadata)
writer.add_run_metadata(run_metadata, 'step%d' % self.num_timesteps)
else:
summary, td_errors, mean_loss = self._train_step(obses_t, actions, rewards, obses_tp1, obses_tp1,
dones, weights, sess=self.sess)
writer.add_summary(summary, self.num_timesteps)
else:
_, td_errors, mean_loss = self._train_step(obses_t, actions, rewards, obses_tp1, obses_tp1, dones, weights,
sess=self.sess)
if self.prioritized_replay:
new_priorities = np.abs(td_errors) + self.prioritized_replay_eps
assert isinstance(self.replay_buffer, PrioritizedReplayBuffer)
self.replay_buffer.update_priorities(batch_idxes, new_priorities)
callback.on_rollout_start()
if can_sample and self.num_timesteps > self.learning_starts and \
self.num_timesteps % self.target_network_update_freq == 0:
# Update target network periodically.
self.update_target(sess=self.sess)
if len(episode_rewards[-101:-1]) == 0:
mean_100ep_reward = -np.inf
else:
mean_100ep_reward = round(float(np.mean(episode_rewards[-101:-1])), 1)
num_episodes = len(episode_rewards)
# Log training infos
kvs = {}
if self.verbose >= 1 and done and log_interval is not None \
and len(episode_rewards) % log_interval == 0 \
and self.num_timesteps > self.train_freq \
and self.num_timesteps > self.learning_starts:
if self.log_dir is not None:
kvs["episodes"] = num_episodes
kvs["mean_100rew"] = mean_100ep_reward
kvs["current_lr"] = self.learning_rate
kvs["success_rate"] = np.mean(episode_successes[-100:])
kvs["total_timesteps"] = self.num_timesteps
kvs["mean_loss"] = mean_loss
kvs["mean_td_errors"] = np.mean(td_errors)
kvs["time_spent_exploring"] = int(100 * self.exploration.value(self.num_timesteps))
self.log_csv.writekvs(kvs)
logger.record_tabular("steps", self.num_timesteps)
logger.record_tabular("episodes", num_episodes)
if len(episode_successes) > 0:
logger.logkv("success rate", np.mean(episode_successes[-100:]))
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * self.exploration.value(self.num_timesteps)))
logger.dump_tabular()
callback.on_training_end()
return self
def learn(
self,
total_timesteps,
model_coworker,
role,
callback=None,
log_interval=100,
tb_log_name="DQN",
reset_num_timesteps=True,
replay_wrapper=None,
clipping_during_training=True,
):
new_tb_log = self._init_num_timesteps(reset_num_timesteps)
callback = self._init_callback(callback)
with SetVerbosity(self.verbose), TensorboardWriter(
self.graph, self.tensorboard_log, tb_log_name,
new_tb_log) as writer:
self._setup_learn()
# Create the replay buffer
if self.prioritized_replay:
self.replay_buffer = PrioritizedReplayBuffer(
self.buffer_size, alpha=self.prioritized_replay_alpha)
if self.prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
else:
prioritized_replay_beta_iters = self.prioritized_replay_beta_iters
self.beta_schedule = LinearSchedule(
prioritized_replay_beta_iters,
initial_p=self.prioritized_replay_beta0,
final_p=1.0,
)
else:
if self.replay_buffer is None:
self.replay_buffer = ReplayBuffer(self.buffer_size)
self.beta_schedule = None
if replay_wrapper is not None:
assert (not self.prioritized_replay
), "Prioritized replay buffer is not supported by HER"
self.replay_buffer = replay_wrapper(self.replay_buffer)
# Create the schedule for exploration starting from 1.
self.exploration = LinearSchedule(
schedule_timesteps=int(self.exploration_fraction *
total_timesteps),
initial_p=self.exploration_initial_eps,
final_p=self.exploration_final_eps,
)
episode_rewards = [0.0]
episode_successes = []
callback.on_training_start(locals(), globals())
callback.on_rollout_start()
reset = True
obs = self.env.reset()
# Retrieve unnormalized observation for saving into the buffer
if self._vec_normalize_env is not None:
obs_ = self._vec_normalize_env.get_original_obs().squeeze()
for _ in range(total_timesteps):
# Take action and update exploration to the newest value
kwargs = {}
if not self.param_noise:
update_eps = self.exploration.value(self.num_timesteps)
update_param_noise_threshold = 0.0
else:
update_eps = 0.0
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(
1.0 - self.exploration.value(self.num_timesteps) +
self.exploration.value(self.num_timesteps) /
float(self.env.action_space.n))
kwargs["reset"] = reset
kwargs[
"update_param_noise_threshold"] = update_param_noise_threshold
kwargs["update_param_noise_scale"] = True
with self.sess.as_default():
action = self.act(np.array(obs)[None],
update_eps=update_eps,
**kwargs)[0]
turn, speed = None, None
if role == "turn":
turn = action
speed, nothing = model_coworker.predict(np.array(obs))
else:
turn, nothing = model_coworker.predict(np.array(obs))
speed = action
if clipping_during_training:
# check if next state (after action) would be outside of fish tank (CLIPPING)
env_state = self.env.get_state()
turn_speed = self.env.action([turn, speed])
global_turn = env_state[0][2] + turn_speed[0]
coords = np.array([
env_state[0][0] + turn_speed[1] * np.cos(global_turn),
env_state[0][1] + turn_speed[1] * np.sin(global_turn),
])
changed = False
if coords[0] < -0.49:
coords[0] = -0.47
changed = True
elif coords[0] > 0.49:
coords[0] = 0.47
changed = True
if coords[1] < -0.49:
coords[1] = -0.47
changed = True
elif coords[1] > 0.49:
coords[1] = 0.47
changed = True
if changed:
diff = coords - env_state[0, :2]
speed = np.linalg.norm(diff)
angles = np.arctan2(diff[1], diff[0])
turn = angles - env_state[0, 2]
turn = turn - 2 * np.pi if turn > np.pi else turn
turn = turn + 2 * np.pi if turn < -np.pi else turn
# convert to DQN output
dist_turn = np.abs(self.env.turn_rate_bins - turn)
dist_speed = np.abs(self.env.speed_bins - speed)
# convert to bins
turn = np.argmin(dist_turn, axis=0)
speed = np.argmin(dist_speed, axis=0)
if role == "turn":
action = turn
else:
action = speed
reset = False
new_obs, rew, done, info = self.env.step([turn, speed])
self.num_timesteps += 1
# Stop training if return value is False
if callback.on_step() is False:
break
# Store only the unnormalized version
if self._vec_normalize_env is not None:
new_obs_ = self._vec_normalize_env.get_original_obs(
).squeeze()
reward_ = self._vec_normalize_env.get_original_reward(
).squeeze()
else:
# Avoid changing the original ones
obs_, new_obs_, reward_ = obs, new_obs, rew
# Store transition in the replay buffer, but change reward to 0 (use it for plot later though)
self.replay_buffer.add(obs_, action, 0, new_obs_, float(done))
# Also give transition to model coworker
if model_coworker.replay_buffer is None:
model_coworker.replay_buffer = ReplayBuffer(
self.buffer_size)
if role == "turn":
model_coworker.replay_buffer.add(obs_, speed, 0, new_obs_,
float(done))
else:
model_coworker.replay_buffer.add(obs_, turn, 0, new_obs_,
float(done))
obs = new_obs
# Save the unnormalized observation
if self._vec_normalize_env is not None:
obs_ = new_obs_
if writer is not None:
ep_rew = np.array([reward_]).reshape((1, -1))
ep_done = np.array([done]).reshape((1, -1))
tf_util.total_episode_reward_logger(
self.episode_reward, ep_rew, ep_done, writer,
self.num_timesteps)
episode_rewards[-1] += reward_
if done:
maybe_is_success = info.get("is_success")
if maybe_is_success is not None:
episode_successes.append(float(maybe_is_success))
if not isinstance(self.env, VecEnv):
obs = self.env.reset()
episode_rewards.append(0.0)
reset = True
# Do not train if the warmup phase is not over
# or if there are not enough samples in the replay buffer
can_sample = self.replay_buffer.can_sample(self.batch_size)
if (can_sample and self.num_timesteps > self.learning_starts
and self.num_timesteps % self.train_freq == 0):
callback.on_rollout_end()
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
# pytype:disable=bad-unpacking
if self.prioritized_replay:
assert (
self.beta_schedule is not None
), "BUG: should be LinearSchedule when self.prioritized_replay True"
experience = self.replay_buffer.sample(
self.batch_size,
beta=self.beta_schedule.value(self.num_timesteps),
env=self._vec_normalize_env,
)
(
obses_t,
actions,
rewards,
obses_tp1,
dones,
weights,
batch_idxes,
) = experience
else:
(
obses_t,
actions,
rewards,
obses_tp1,
dones,
) = self.replay_buffer.sample(
self.batch_size, env=self._vec_normalize_env)
# also sample from expert buffer
(
obses_t_exp,
actions_exp,
rewards_exp,
obses_tp1_exp,
dones_exp,
) = self.expert_buffer.sample(
self.batch_size, env=self._vec_normalize_env)
weights, batch_idxes = np.ones_like(rewards), None
weights_exp, batch_idxes_exp = np.ones_like(
rewards_exp), None
# pytype:enable=bad-unpacking
if writer is not None:
# run loss backprop with summary, but once every 100 steps save the metadata
# (memory, compute time, ...)
if (1 + self.num_timesteps) % 100 == 0:
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
summary, td_errors = self._train_step(
np.append(obses_t, obses_t_exp, axis=0),
np.append(actions,
actions_exp.flatten(),
axis=0),
np.append(rewards,
rewards_exp.flatten(),
axis=0),
np.append(obses_tp1, obses_tp1_exp, axis=0),
np.append(obses_tp1, obses_tp1_exp, axis=0),
np.append(dones.flatten(),
dones_exp.flatten(),
axis=0),
np.append(weights, weights_exp),
sess=self.sess,
options=run_options,
run_metadata=run_metadata,
)
writer.add_run_metadata(
run_metadata, "step%d" % self.num_timesteps)
else:
summary, td_errors = self._train_step(
np.append(obses_t, obses_t_exp, axis=0),
np.append(actions,
actions_exp.flatten(),
axis=0),
np.append(rewards,
rewards_exp.flatten(),
axis=0),
np.append(obses_tp1, obses_tp1_exp, axis=0),
np.append(obses_tp1, obses_tp1_exp, axis=0),
np.append(dones.flatten(),
dones_exp.flatten(),
axis=0),
np.append(weights, weights_exp),
sess=self.sess,
options=run_options,
run_metadata=run_metadata,
)
writer.add_summary(summary, self.num_timesteps)
else:
_, td_errors = self._train_step(
np.append(obses_t, obses_t_exp, axis=0),
np.append(actions, actions_exp.flatten(), axis=0),
np.append(rewards, rewards_exp.flatten(), axis=0),
np.append(obses_tp1, obses_tp1_exp, axis=0),
np.append(obses_tp1, obses_tp1_exp, axis=0),
np.append(dones.flatten(),
dones_exp.flatten(),
axis=0),
np.append(weights, weights_exp),
sess=self.sess,
)
if self.prioritized_replay:
new_priorities = np.abs(
td_errors) + self.prioritized_replay_eps
assert isinstance(self.replay_buffer,
PrioritizedReplayBuffer)
self.replay_buffer.update_priorities(
batch_idxes, new_priorities)
callback.on_rollout_start()
if (can_sample and self.num_timesteps > self.learning_starts
and self.num_timesteps %
self.target_network_update_freq == 0):
# Update target network periodically.
self.update_target(sess=self.sess)
if len(episode_rewards[-101:-1]) == 0:
mean_100ep_reward = -np.inf
else:
mean_100ep_reward = round(
float(np.mean(episode_rewards[-101:-1])), 1)
num_episodes = len(episode_rewards)
if (self.verbose >= 1 and done and log_interval is not None
and len(episode_rewards) % log_interval == 0):
logger.record_tabular("steps", self.num_timesteps)
logger.record_tabular("episodes", num_episodes)
if len(episode_successes) > 0:
logger.logkv("success rate",
np.mean(episode_successes[-100:]))
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular(
"% time spent exploring",
int(100 * self.exploration.value(self.num_timesteps)),
)
logger.dump_tabular()
callback.on_training_end()
return self
def learn(self,
total_timesteps,
callback=None,
log_interval=100,
tb_log_name="DQN",
reset_num_timesteps=True,
replay_wrapper=None):
new_tb_log = self._init_num_timesteps(reset_num_timesteps)
with SetVerbosity(self.verbose), TensorboardWriter(self.graph, self.tensorboard_log, tb_log_name, new_tb_log) \
as writer:
self._setup_learn()
# Create the replay buffer
if self.prioritized_replay:
self.replay_buffer = PrioritizedReplayBuffer(
self.buffer_size, alpha=self.prioritized_replay_alpha)
if self.prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
else:
prioritized_replay_beta_iters = self.prioritized_replay_beta_iters
self.beta_schedule = LinearSchedule(
prioritized_replay_beta_iters,
initial_p=self.prioritized_replay_beta0,
final_p=1.0)
else:
self.replay_buffer = ReplayBuffer(self.buffer_size)
self.beta_schedule = None
if replay_wrapper is not None:
assert not self.prioritized_replay, "Prioritized replay buffer is not supported by HER"
self.replay_buffer = replay_wrapper(self.replay_buffer)
# Create the schedule for exploration starting from 1.
self.exploration = LinearSchedule(
schedule_timesteps=int(self.exploration_fraction *
total_timesteps),
initial_p=self.exploration_initial_eps,
final_p=self.exploration_final_eps)
episode_rewards = [0.0]
episode_successes = []
obs = self.env.reset()
obs_hdqn_old = None
action_hdqn = None
reset = True
F = 0
for _ in range(total_timesteps):
if callback is not None:
# Only stop training if return value is False, not when it is None. This is for backwards
# compatibility with callbacks that have no return statement.
if callback(locals(), globals()) is False:
break
# Take action and update exploration to the newest value
kwargs = {}
if not self.param_noise:
update_eps = self.exploration.value(self.num_timesteps)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
update_param_noise_threshold = \
-np.log(1. - self.exploration.value(self.num_timesteps) +
self.exploration.value(self.num_timesteps) / float(self.env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
# Check if agent is busy or idle
OBS_IS_IDLE = True
if (OBS_IS_IDLE):
if not reset:
# Store HDQN transition
self.replay_buffer.add(obs_hdqn_old, action_hdqn, F,
obs, float(done))
# Select new goal for the agent using the current Q function
action = self.act(np.array(obs)[None],
update_eps=update_eps,
**kwargs)[0]
env_action = action
# Update bookkeepping for next HDQN buffer update
obs_hdqn_old = obs
action_hdqn = env_action
F = 0.
else:
# Agent is busy, so select a dummy action (it will be ignored anyway)
env_action = 0
reset = False
new_obs, rew, done, info = self.env.step(env_action)
F = F + rew
if writer is not None:
ep_rew = np.array([rew]).reshape((1, -1))
ep_done = np.array([done]).reshape((1, -1))
total_episode_reward_logger(self.episode_reward, ep_rew,
ep_done, writer,
self.num_timesteps)
episode_rewards[-1] += rew
if done:
# Store HDQN transition
self.replay_buffer.add(obs_hdqn_old, action_hdqn, F, obs,
float(done))
maybe_is_success = info.get('is_success')
if maybe_is_success is not None:
episode_successes.append(float(maybe_is_success))
if not isinstance(self.env, VecEnv):
obs = self.env.reset()
episode_rewards.append(0.0)
reset = True
# Do not train if the warmup phase is not over
# or if there are not enough samples in the replay buffer
can_sample = self.replay_buffer.can_sample(self.batch_size)
if can_sample and self.num_timesteps > self.learning_starts \
and self.num_timesteps % self.train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
# pytype:disable=bad-unpacking
if self.prioritized_replay:
assert self.beta_schedule is not None, \
"BUG: should be LinearSchedule when self.prioritized_replay True"
experience = self.replay_buffer.sample(
self.batch_size,
beta=self.beta_schedule.value(self.num_timesteps))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = self.replay_buffer.sample(
self.batch_size)
weights, batch_idxes = np.ones_like(rewards), None
# pytype:enable=bad-unpacking
if writer is not None:
# run loss backprop with summary, but once every 100 steps save the metadata
# (memory, compute time, ...)
if (1 + self.num_timesteps) % 100 == 0:
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
summary, td_errors = self._train_step(
obses_t,
actions,
rewards,
obses_tp1,
obses_tp1,
dones,
weights,
sess=self.sess,
options=run_options,
run_metadata=run_metadata)
writer.add_run_metadata(
run_metadata, 'step%d' % self.num_timesteps)
else:
summary, td_errors = self._train_step(
obses_t,
actions,
rewards,
obses_tp1,
obses_tp1,
dones,
weights,
sess=self.sess)
writer.add_summary(summary, self.num_timesteps)
else:
_, td_errors = self._train_step(obses_t,
actions,
rewards,
obses_tp1,
obses_tp1,
dones,
weights,
sess=self.sess)
if self.prioritized_replay:
new_priorities = np.abs(
td_errors) + self.prioritized_replay_eps
assert isinstance(self.replay_buffer,
PrioritizedReplayBuffer)
self.replay_buffer.update_priorities(
batch_idxes, new_priorities)
if can_sample and self.num_timesteps > self.learning_starts and \
self.num_timesteps % self.target_network_update_freq == 0:
# Update target network periodically.
self.update_target(sess=self.sess)
if len(episode_rewards[-101:-1]) == 0:
mean_100ep_reward = -np.inf
else:
mean_100ep_reward = round(
float(np.mean(episode_rewards[-101:-1])), 1)
num_episodes = len(episode_rewards)
if self.verbose >= 1 and done and log_interval is not None and len(
episode_rewards) % log_interval == 0:
logger.record_tabular("steps", self.num_timesteps)
logger.record_tabular("episodes", num_episodes)
if len(episode_successes) > 0:
logger.logkv("success rate",
np.mean(episode_successes[-100:]))
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular(
"% time spent exploring",
int(100 * self.exploration.value(self.num_timesteps)))
logger.dump_tabular()
self.num_timesteps += 1
return self