def reward_estimation(self, states, rewards, terminals):
"""Process rewards according to the configuration.
Args:
states:
rewards:
terminals:
Returns:
"""
discounted_rewards = util.cumulative_discount(
values=rewards,
terminals=terminals,
discount=self.discount
)
if self.baseline:
state_values = list()
for name, state in states.items():
state_value = self.baseline[name].predict(states=state)
state_values.append(state_value)
state_values = np.mean(state_values, axis=0)
if self.gae_rewards:
td_residuals = rewards + np.array(
[self.discount * state_values[n + 1] - state_values[n] if (n < len(state_values) - 1 and not terminal) else 0.0 for n, terminal in enumerate(terminals)])
rewards = util.cumulative_discount(
values=td_residuals,
terminals=terminals,
discount=(self.discount * self.gae_lambda)
)
else:
rewards = discounted_rewards - state_values
else:
rewards = discounted_rewards
mean = rewards.mean()
stddev = rewards.std()
self.logger.debug('Reward mean {} and variance {}.'.format(mean, stddev * stddev))
if self.normalize_rewards:
rewards = (rewards - mean) / max(stddev, util.epsilon)
self.logger.debug('First ten rewards: {}.'.format(rewards[:10]))
return rewards, discounted_rewards
def advantage_estimation(self, batch):
"""Expects a batch, returns advantages according to config.
Args:
batch:
Returns:
"""
if not self.baseline:
return batch['returns']
estimates = self.baseline.predict(states=batch['states'])
if self.generalized_advantage_estimation:
deltas = np.array([
self.discount * estimates[n + 1] - estimates[n] if
(n < len(estimates) - 1 and not terminal) else 0.0
for n, terminal in enumerate(batch['terminals'])
])
deltas += batch['rewards']
advantage = util.cumulative_discount(rewards=deltas,
terminals=batch['terminals'],
discount=(self.discount *
self.gae_lambda))
else:
advantage = np.array(batch['returns']) - estimates
if self.normalize_advantage:
advantage -= advantage.mean()
advantage /= advantage.std() + 1e-8
return advantage
def reward_estimation(self, states, rewards, terminals):
"""Process rewards according to the configuration.
Args:
states:
rewards:
terminals:
Returns:
"""
discounted_rewards = util.cumulative_discount(
values=rewards,
terminals=terminals,
discount=self.discount
)
if self.baseline:
state_values = []
for name, state in states.items():
state_values.append(self.baseline[name].predict(states=state))
state_values = np.mean(state_values, axis=0)
if self.gae_rewards:
td_residuals = rewards + np.array(
[self.discount * state_values[n + 1] - state_values[n] if (n < len(state_values) - 1 and not terminal) else 0.0 for n, terminal in enumerate(terminals)])
rewards = util.cumulative_discount(
values=td_residuals,
terminals=terminals,
discount=(self.discount * self.gae_lambda)
)
else:
rewards = discounted_rewards - state_values
else:
rewards = discounted_rewards
if self.normalize_rewards:
rewards = (rewards - rewards.mean()) / max(rewards.std(), util.epsilon)
return rewards, discounted_rewards
def update(self, batch):
"""Generic policy gradient update on a batch of experiences. Each model needs to update its specific
logic.
Args:
batch:
Returns:
"""
batch['returns'] = util.cumulative_discount(rewards=batch['rewards'], terminals=batch['terminals'], discount=self.discount)
batch['rewards'] = self.advantage_estimation(batch)
if self.baseline:
self.baseline.update(states=batch['states'], returns=batch['returns'])
super(PolicyGradientModel, self).update(batch)
def update_feed_dict(self, batch):
# assume temporally consistent sequence
# get state time + 1
feed_dict = {
next_state: [batch['states'][name][-1]]
for name, next_state in self.next_state.items()
}
feed_dict.update({
internal: [batch['internals'][n][-1]]
for n, internal in enumerate(self.next_internal_inputs,
self.network_internal_index)
})
# calcualte nstep rewards
target_q_vals = self.session.run(self.target_values,
feed_dict=feed_dict)
nstep_rewards = dict()
for name, value in target_q_vals.items():
nstep_rewards[name] = util.cumulative_discount(
values=batch['rewards'][:-1],
terminals=batch['terminals'][:-1],
discount=self.discount,
cumulative_start=target_q_vals[name][0])
# create update feed dict
feed_dict = {
state: batch['states'][name][:-1]
for name, state in self.state.items()
}
feed_dict.update({
action: batch['actions'][name][:-1]
for name, action in self.action.items()
})
feed_dict.update({
internal: batch['internals'][n][:-1]
for n, internal in enumerate(self.internal_inputs)
})
feed_dict.update({
self.nstep_rewards[name]: nstep_rewards[name]
for name, reward in nstep_rewards.items()
})
return feed_dict
def advantage_estimation(self, batch):
"""Expects a batch, returns advantages according to config.
Args:
batch:
Returns:
"""
if not self.baseline:
return batch['returns']
estimates = self.baseline.predict(states=batch['states'])
if self.generalized_advantage_estimation:
deltas = np.array(
self.discount * estimates[n + 1] -
estimates[n] if n < len(estimates) - 1 or terminal else 0.0
for n, terminal in enumerate(batch['terminals']))
deltas += batch['rewards']
# if terminals[-1]:
# adjusted_estimate = np.append(estimate, [0])
# else:
# adjusted_estimate = np.append(estimate, estimate[-1])
# deltas = batch['rewards'] + self.discount * adjusted_estimate[1:] - adjusted_estimate[:-1]
advantage = util.cumulative_discount(rewards=deltas,
terminals=batch['terminals'],
discount=(self.discount *
self.gae_lambda))
else:
advantage = batch['returns'] - estimates
if self.normalize_advantage:
advantage -= advantage.mean()
advantage /= advantage.std() + 1e-8
return advantage