def actor_optimizer(self):
action = K.placeholder(shape=[None, self.action_size])
advantages = K.placeholder(shape=[
None,
])
#advatages -> *multi-step*
policy = self.actor.output
action_prob = K.sum(action * policy, axis=1)
cross_entropy = K.log(action_prob + 1e-10) * advantages
cross_entropy = -K.mean(cross_entropy)
# add (-entropy) to loss function, for enthusiastic search
minus_entropy = K.sum(policy * K.log(policy + 1e-10), axis=1)
minus_entropy = K.mean(minus_entropy)
# optimizing loss minimizes cross_entropy, maximizes entropy
loss = cross_entropy #+ 0.01 * minus_entropy
optimizer = Adam(lr=self.actor_lr)
updates = optimizer.get_updates(loss, self.actor.trainable_weights)
train = K.function([self.actor.input, action, advantages], [loss],
updates=updates)
return train
def critic_optimizer(self):
discounted_prediction = K.placeholder(shape=(None, ))
value = self.critic.output
# loss = MSE(discounted_prediction, value)
loss = K.mean(K.square(discounted_prediction - value))
optimizer = Adam(lr=self.critic_lr)
updates = optimizer.get_updates(loss, self.critic.trainable_weights)
train = K.function([self.critic.input, discounted_prediction], [loss],
updates=updates)
return train
