def __init__(
self,
ppo_model,
sess,
info,
is_continuous,
use_observations,
use_states,
training):
self.history_dict = ppo_hist.history_keys
self.is_continuous = is_continuous
self.is_training = training
self.model = ppo_model
self.reset_buffers(info, total=True)
self.sess = sess
self.stats = {'cumulative_reward': [],
'entropy': [],
'episode_length': [],
'learning_rate': [],
'policy_loss': [],
'value_estimate': [],
'value_loss': []}
self.training_buffer = ppo_hist.vectorize_history(ppo_hist.empty_local_history({}))
self.use_observations = use_observations
self.use_states = use_states
def process_experiences(self, info, time_horizon, gamma, lambd):
"""
Checks agent histories for processing condition, and processes them as necessary.
Processing involves calculating value and advantage targets for model updating step.
gamma:
Discount factor.
info:
Current BrainInfo
lambd:
GAE factor.
time_horizon:
Max steps for individual agent history before processing.
"""
for l in range(len(info.agents)):
if (info.local_done[l]
or len(self.history_dict[info.agents[l]]['actions']) >
time_horizon):
if len(self.history_dict[info.agents[l]]['actions']) > 0:
if info.local_done[l]:
value_next = 0.0
else:
feed_dict = {self.model.batch_size: len(info.states)}
if self.use_observations:
feed_dict[self.model.observation_in] = np.vstack(
info.observations)
if self.use_states:
feed_dict[self.model.state_in] = info.states
value_next = self.sess.run(self.model.value,
feed_dict)[l]
history = ppo_hist.vectorize_history(
self.history_dict[info.agents[l]])
history['advantages'] = ppo_hist.get_gae(
history['rewards'], history['value_estimates'], gamma,
lambd, value_next)
history['discounted_returns'] = history[
'advantages'] + history['value_estimates']
if len(self.training_buffer['actions']) > 0:
ppo_hist.append_history(
global_buffer=self.training_buffer,
local_buffer=history)
else:
ppo_hist.set_history(
global_buffer=self.training_buffer,
local_buffer=history)
self.history_dict[
info.agents[l]] = ppo_hist.empty_local_history(
self.history_dict[info.agents[l]])
if info.local_done[l]:
self.stats['cumulative_reward'].append(
history['cumulative_reward'])
self.stats['episode_length'].append(
history['episode_steps'])
history['cumulative_reward'] = 0
history['episode_steps'] = 0
def reset_buffers(self, brain_info=None, total=False):
"""
Resets either all training buffers or local training buffers
brain_info:
The BrainInfo object containing agent ids.
total:
Whether to completely clear buffer.
"""
if not total:
for key in self.history_dict:
self.history_dict[key] = ppo_hist.empty_local_history(self.history_dict[key])
else:
self.history_dict = ppo_hist.empty_all_history(agent_info=brain_info)
def update_model(self, batch_size, num_epoch):
"""
Uses training_buffer to update model.
batch_size:
Size of each mini-batch update.
num_epoch:
How many passes through data to update model for.
"""
total_v, total_p = 0, 0
advantages = self.training_buffer['advantages']
self.training_buffer['advantages'] = (
advantages - advantages.mean()) / advantages.std()
for _ in range(num_epoch):
training_buffer = ppo_hist.shuffle_buffer(self.training_buffer)
for l in range(len(training_buffer['actions']) // batch_size):
start = l * batch_size
end = (l + 1) * batch_size
feed_dict = {
self.model.returns_holder:
training_buffer['discounted_returns'][start:end],
self.model.advantage:
np.vstack(training_buffer['advantages'][start:end]),
self.model.old_probs:
np.vstack(training_buffer['action_probs'][start:end])
}
if self.is_continuous:
feed_dict[self.model.epsilon] = np.vstack(
training_buffer['epsilons'][start:end])
else:
feed_dict[self.model.action_holder] = np.hstack(
training_buffer['actions'][start:end])
if self.use_states:
feed_dict[self.model.state_in] = np.vstack(
training_buffer['states'][start:end])
if self.use_observations:
feed_dict[self.model.observation_in] = np.vstack(
training_buffer['observations'][start:end])
v_loss, p_loss, _ = self.sess.run([
self.model.value_loss, self.model.policy_loss,
self.model.update_batch
],
feed_dict=feed_dict)
total_v += v_loss
total_p += p_loss
self.stats['value_loss'].append(total_v)
self.stats['policy_loss'].append(total_p)
self.training_buffer = ppo_hist.vectorize_history(
ppo_hist.empty_local_history({}))