def actor_optimizer(self):
action = K.placeholder(shape=[None, self.action_size])
advantages = K.placeholder(shape=[
None,
])
policy = self.actor.output
good_prob = K.sum(action * policy, axis=1)
eligibility = K.log(good_prob + 1e-10) * advantages
actor_loss = -K.mean(eligibility)
entropy = K.mean(policy * K.log(policy + 1e-10), axis=1)
entropy = K.mean(entropy)
loss = actor_loss + 0.01 * entropy
optimizer = RMSprop(lr=self.actor_lr,
rho=0.99,
epsilon=0.00001,
decay=0.99,
clipnorm=0.5)
# optimizer = Adam(lr=self.actor_lr)
updates = optimizer.get_updates(self.actor.trainable_weights, [], loss)
train = K.function([self.actor.input, action, advantages], [loss],
updates=updates)
return train
def actor_optimizer(self):
"""
This method updates actor network (policy network)
"""
action = K.placeholder(shape=[None, self.action_size])
advantages = K.placeholder(shape=[
None,
])
policy = self.actor.output
action_prob = K.sum(action * policy, axis=1)
# Cross entropy loss function about policy
cross_entropy = K.log(action_prob + 1e-10) * advantages
cross_entropy = -K.sum(cross_entropy)
# Remained actor network agents should continuously interact environment
# This is entropy loss function for continuous exploration
entropy = K.sum(policy * K.log(policy + 1e-10), axis=1)
entropy = K.sum(entropy)
# You create final loss function by adding two loss functions
loss = cross_entropy + 0.01 * entropy
optimizer = RMSprop(lr=self.actor_lr, rho=0.99, epsilon=0.01)
updates = optimizer.get_updates(self.actor.trainable_weights, [], loss)
train = K.function([self.actor.input, action, advantages], [loss],
updates=updates)
return train
def optimizer(self):
a = K.placeholder(shape=(None, ), dtype='int32')
y = K.placeholder(shape=(None, ), dtype='float32')
# the output tensor for the state-action pairs
py_x = self.q_duelling_part.output
a_one_hot = K.one_hot(a, 3)
q_value = K.sum(py_x * a_one_hot, axis=1)
error = K.abs(y - q_value)
quadratic_part = K.clip(error, 0.0, 1.0)
linear_part = error - quadratic_part
loss = K.mean(0.5 * K.square(quadratic_part) + linear_part)
optimizer = RMSprop(lr=0.00025, epsilon=0.01)
updates = optimizer.get_updates(self.q_duelling_part.trainable_weights,
[], loss)
train = K.function([self.q_duelling_part.input, a, y], [loss],
updates=updates)
return train
def actor_optimizer(self):
action = K.placeholder(shape=[None, self.action_size])
advantages = K.placeholder(shape=[
None,
])
policy = self.actor.output
# actor loss
good_prob = K.sum(action * policy, axis=1)
eligibility = K.log(good_prob + 1e-10) * advantages
actor_loss = -K.sum(eligibility)
# entropy-loss to encourage exploration
entropy = K.sum(policy * K.log(policy + 1e-10), axis=1) # None,1
entropy = K.sum(entropy) # scalar
# total loss
loss = actor_loss + 0.01 * entropy
optimizer = RMSprop(lr=self.actor_lr, rho=0.99, epsilon=0.01)
updates = optimizer.get_updates(self.actor.trainable_weights, [], loss)
train = K.function([self.actor.input, action, advantages], [loss],
updates=updates)
return train
def optimizer(self):
a = K.placeholder(shape=(None, ), dtype='int32')
y = K.placeholder(shape=(None, ), dtype='float32')
prediction = self.model.output
a_one_hot = K.one_hot(a, self.action_size)
q_value = K.sum(prediction * a_one_hot, axis=1)
error = K.abs(y - q_value)
quadratic_part = K.clip(error, 0.0, 1.0)
linear_part = error - quadratic_part
loss = K.mean(0.5 * K.square(quadratic_part) + linear_part)
optimizer = RMSprop(lr=0.00025, epsilon=0.01)
updates = optimizer.get_updates(self.model.trainable_weights, [], loss)
train = K.function([self.model.input, a, y], [loss], updates=updates)
return train
def actor_optimizer(self):
action = K.placeholder(shape=[None, self.action_size])
advantages = K.placeholder(shape=[
None,
])
policy = self.actor.output
# Policy cross entropy error function
action_prob = K.sum(action * policy, axis=1)
cross_entropy = K.log(action_prob + 1e-10) * advantages
cross_entropy = -K.sum(cross_entropy)
# Entropy error for exploring steadily
entropy = K.sum(policy * K.log(policy + 1e-10), axis=1)
entropy = K.sum(entropy)
# Generation of final error function by adding two error functions
loss = cross_entropy + 0.01 * entropy
optimizer = RMSprop(lr=self.actor_lr, rho=0.99, epsilon=0.01)
updates = optimizer.get_updates(self.actor.trainable_weights, [], loss)
train = K.function([self.actor.input, action, advantages], [loss],
updates=updates)
return train
def actor_optimizer(self):
action = K.placeholder(shape=[None, self.action_size])
advantages = K.placeholder(shape=[
None,
])
policy = self.actor.output
# Policy cross-entropy loss function
action_prob = K.sum(action * policy, axis=1)
cross_entropy = K.log(action_prob + 1e-10) * advantages
cross_entropy = -K.sum(cross_entropy)
# Entropy loss for continuous exploration
entropy = K.sum(policy * K.log(policy + 1e-10), axis=1)
entropy = K.sum(entropy)
# Final loss function using both entropy
loss = cross_entropy + 0.01 * entropy
optimizer = RMSprop(lr=self.actor_lr,
rho=0.99,
epsilon=0.01,
clipnorm=40)
updates = optimizer.get_updates(self.actor.trainable_weights, [], loss)
train = K.function([self.actor.input, action, advantages], [loss],
updates=updates)
return train
def actor_optimizer(self):
action = K.placeholder(shape=[None, self.action_size])
advantages = K.placeholder(shape=[
None,
])
policy = self.actor.output
# 정책 크로스 엔트로피 오류함수
action_prob = K.sum(action * policy, axis=1)
cross_entropy = K.log(action_prob + 1e-10) * advantages
cross_entropy = -K.sum(cross_entropy)
# 탐색을 지속적으로 하기 위한 엔트로피 오류
entropy = K.sum(policy * K.log(policy + 1e-10), axis=1)
entropy = K.sum(entropy)
# 두 오류함수를 더해 최종 오류함수를 만듬
loss = cross_entropy + 0.01 * entropy
optimizer = RMSprop(lr=self.actor_lr, rho=0.99, epsilon=0.01)
updates = optimizer.get_updates(self.actor.trainable_weights, [], loss)
train = K.function([self.actor.input, action, advantages], [loss],
updates=updates)
return train
def optimizer(self):
a = K.placeholder(shape=(None, ), dtype='int32')
y = K.placeholder(shape=(None, ), dtype='float32')
py_x = self.model.output #output is the Q value of each action
a_one_hot = K.one_hot(a, self.action_size)
q_value = K.sum(py_x * a_one_hot,
axis=1) #one hot encoding Q value of actions
error = K.abs(
y - q_value
) #looks like MAE for Huber Loss, caring about all losses equally
quadratic_part = K.clip(error, 0.0, 1.0)
linear_part = error - quadratic_part
loss = K.mean(0.5 * K.square(quadratic_part) +
linear_part) #the MSE for low error values
optimizer = RMSprop(lr=0.00025, epsilon=0.01) #
#get_updates is tf (loss, param)
updates = optimizer.get_updates(self.model.trainable_weights, [], loss)
#instantiate Keras fxn but args are(list placholder tensors, list output tensor, list update ops)
#why output tensor is loss?
train = K.function([self.model.input, a, y], [loss],
updates=updates) #GD step?
return train
def wgan_loss(inputshape, noiseshape, generator, discriminator, K ):
opt = RMSprop(lr=5e-5, clipvalue=0.01)
realimg = Input(shape=imageshape)
noise = Input(shape=noiseshape)
fakeimg = generator(noise)
d_real = discriminator(realimg)
d_fake = discriminator(fakeimg)
d_loss1 = K.mean(d_real, axis=-1)
d_loss2 = K.mean(d_fake, axis=-1)
d_loss = - d_loss1 + d_loss2
d_training_updates = opt.get_updates(discriminator.trainable_weights,[], d_loss)
d_train = K.function([realimg, noise], [d_loss], d_training_updates)
g_loss = - K.mean(d_fake, axis=-1)
g_training_updates = opt.get_updates(generator.trainable_weights,[], g_loss)
g_train = K.function([noise], [g_loss], g_training_updates)
return d_train, g_train
def _build_model(graph, state_size, skip_frames, action_size, learning_rate):
__keras_imports()
INPUT_SHAPE = (state_size, ) # input image size to model
ACTION_SIZE = action_size
# With the functional API we need to define the inputs.
LInput = layers.Input(INPUT_SHAPE, name='inputs')
# "The final hidden layer is fully-connected and consists of 256 rectifier units."
h1 = layers.Dense(64, activation='relu')(LInput)
h2 = layers.Dense(128, activation='relu')(h1)
rms_opt = RMSprop(lr=learning_rate, epsilon=0.1, rho=0.99)
x1 = layers.Dense(128, activation='relu')(h2)
x2 = layers.Dense(128, activation='relu')(h2)
# "The output layer is a fully-connected linear layer with a single output for each valid action."
output_actions = layers.Dense(ACTION_SIZE,
activation='softmax',
name='out1')(x1)
output_value = layers.Dense(1, activation='linear', name='out2')(x2)
pmodel = Model(inputs=[LInput], outputs=[output_actions])
vmodel = Model(inputs=[LInput], outputs=[output_value])
action_pl = K.placeholder(shape=(None, action_size))
advantages_pl = K.placeholder(shape=(None, ))
discounted_r = K.placeholder(shape=(None, ))
weighted_actions = K.sum(action_pl * pmodel.output, axis=1)
eligibility = K.log(weighted_actions +
1e-10) * K.stop_gradient(advantages_pl)
entropy = K.sum(pmodel.output * K.log(pmodel.output + 1e-10), axis=1)
ploss = 0.001 * entropy - K.sum(eligibility)
updates = rms_opt.get_updates(pmodel.trainable_weights, [], ploss)
optimizer = K.function([pmodel.input, action_pl, advantages_pl], [],
updates=updates)
closs = K.mean(K.square(discounted_r - vmodel.output))
updates2 = rms_opt.get_updates(vmodel.trainable_weights, [], closs)
optimizer2 = K.function([vmodel.input, discounted_r], [], updates=updates2)
return (pmodel, vmodel, optimizer, optimizer2)
def _compile_learning(self):
# Tensor Variables
s = K.placeholder(shape=tuple([None] + [self.history_len] +
self.state_shape))
a = K.placeholder(ndim=1, dtype='int32')
r = K.placeholder(ndim=1, dtype='float32')
s2 = K.placeholder(shape=tuple([None] + [self.history_len] +
self.state_shape))
t = K.placeholder(ndim=1, dtype='float32')
# Q(s, a)
q = self.network(s / self.normalize)
preds = slice_tensor_tensor(q, a)
# r + (1 - t) * gamma * max_a(Q'(s'))
q2 = self.target_network(s2 / self.normalize)
if self.ddqn:
q2_net = K.stop_gradient(self.network(s2 / self.normalize))
a_max = K.argmax(q2_net, axis=1)
q2_max = slice_tensor_tensor(q2, a_max)
else:
q2_max = K.max(q2, axis=1)
# over-estimation correction
if len(self.bootstrap_corr) > 0:
q2_max -= (q2_max - np.float32(self.bootstrap_corr[1])) * (
q2_max > self.bootstrap_corr[1])
q2_max -= (q2_max - np.float32(self.bootstrap_corr[0])) * (
q2_max < self.bootstrap_corr[0])
targets = r + (np.float32(1) - t) * self.gamma * q2_max
# Loss and Updates
cost = clipped_sum_error(y_true=targets, y_pred=preds)
optimizer = RMSprop(lr=self.learning_rate, rho=.95, epsilon=1e-7)
updates = optimizer.get_updates(params=self.network.trainable_weights,
loss=cost,
constraints={})
# Update Target Network
target_updates = []
for target_weight, network_weight in zip(
self.target_network.trainable_weights,
self.network.trainable_weights):
target_updates.append(K.update(target_weight, network_weight))
# Compiled Functions
self._train_on_batch = K.function(inputs=[s, a, r, s2, t],
outputs=[cost],
updates=updates)
self.predict_network = K.function(inputs=[s], outputs=[q])
self.predict_target = K.function(inputs=[s2], outputs=[q2])
self.update_weights = K.function(inputs=[],
outputs=[],
updates=target_updates)
def __update_critic__(self):
discounted_rewards = K.placeholder(shape=(None, ))
value = self.critic_.output
loss = K.mean(K.square(discounted_rewards - value))
gradient = RMSprop()
updates = gradient.get_updates(self.critic_.trainable_weights, [], loss)
train = K.function([self.critic_.input, discounted_rewards],[self.critic_.output],updates=updates)
return train
def critic_optimizer(self):
discounted_reward = K.placeholder(shape=(None, ))
value = self.critic.output
loss = K.mean(K.square(discounted_reward - value))
optimizer = RMSprop(lr=self.critic_lr, rho=0.99, epsilon=0.01)
updates = optimizer.get_updates(self.critic.trainable_weights, [], loss)
train = K.function([self.critic.input, discounted_reward], [loss], updates=updates)
return train
def critic_optimizer(self):
discounted_reward = K.placeholder(shape=(None, ))
value = self.critic.output
loss = K.mean(K.square(discounted_reward - value))
optimizer = RMSprop(lr=self.critic_lr, rho=0.99, epsilon=0.01)
updates = optimizer.get_updates(self.critic.trainable_weights, [], loss)
train = K.function([self.critic.input, discounted_reward], [loss], updates=updates)
return train
def _compile_learning(self):
s = K.placeholder(shape=tuple([None] + [self.history_len] + self.state_shape))
a = K.placeholder(ndim=1, dtype='int32')
r = K.placeholder(ndim=2, dtype='float32')
s2 = K.placeholder(shape=tuple([None] + [self.history_len] + self.state_shape))
t = K.placeholder(ndim=1, dtype='float32')
updates = []
costs = 0
qs = []
q2s = []
for i in range(len(self.networks)):
local_s = s
local_s2 = s2
if self.remove_features:
local_s = self._remove_features(local_s, i)
local_s2 = self._remove_features(local_s2, i)
qs.append(self.networks[i](local_s))
q2s.append(self.target_networks[i](local_s2))
if self.use_hra:
cost = self._compute_cost(qs[-1], a, r[:, i], t, q2s[-1])
optimizer = RMSprop(lr=self.learning_rate, rho=.95, epsilon=1e-7)
updates += optimizer.get_updates(params=self.networks[i].trainable_weights, loss=cost, constraints={})
costs += cost
if not self.use_hra:
q = sum(qs)
q2 = sum(q2s)
summed_reward = K.sum(r, axis=-1)
cost = self._compute_cost(q, a, summed_reward, t, q2)
optimizer = RMSprop(lr=self.learning_rate, rho=.95, epsilon=1e-7)
updates += optimizer.get_updates(params=self.all_params, loss=cost, constraints={})
costs += cost
target_updates = []
for network, target_network in zip(self.networks, self.target_networks):
for target_weight, network_weight in zip(target_network.trainable_weights, network.trainable_weights):
target_updates.append(K.update(target_weight, network_weight))
self._train_on_batch = K.function(inputs=[s, a, r, s2, t], outputs=[costs], updates=updates)
self.predict_network = K.function(inputs=[s], outputs=qs)
self.update_weights = K.function(inputs=[], outputs=[], updates=target_updates)
def _critic_optimizer(self):
discount_prediction = K.placeholder(shape=(None,))
value = self.cric.output
# [반환값 - 가치]의 제곱을 오류함수로 함.
loss = K.mean(K.square(discount_prediction - value))
optimizer = RMSprop(lr=2.5e-4, rho=0.99, epsilon=0.01)
updates = optimizer.get_updates(self.cric.trainable_weights, [], loss)
train = K.function([self.cric.input, discount_prediction], [loss], updates=updates)
return train
def critic_optimize(self):
target = K.placeholder(shape=(None, ))
loss = K.mean(K.square(target - self.critic.output))
optimizer = RMSprop(lr=self.critic_learning_rate,
rho=0.99,
epsilon=0.01)
update = optimizer.get_updates(self.critic.trainable_weights, [], loss)
train = K.function([self.critic.input, target], [loss], updates=update)
return train
def critic_optimizer(): R = K.placeholder(shape=(None,)) critic = model_critic.output critic = K.print_tensor(critic, message='critic: ') Lv = K.mean(K.square(R - critic)) Lv = K.sum(Lv) Lv = K.print_tensor(Lv, message='Lv: ') optimizer = RMSprop(lr=2.5e-4, rho=0.99, epsilon=0.01) #optimizer = my_optimizer_critic updates = optimizer.get_updates(model_critic.trainable_weights, [], Lv) train = K.function([model_critic.input, R], [Lv], updates=updates) return train
def critic_optimizer2(self):
target = K.placeholder(shape=[
None,
])
loss = K.mean(K.square(target - self.critic.output))
optimizer = RMSprop(lr=self.critic_lr * 50)
updates = optimizer.get_updates(self.critic.trainable_weights, [],
loss)
train = K.function([self.critic.input, target], [], updates=updates)
return train
def optimizer(self):
""" The critic loss: mean squared error over discounted rewards """
#Placeholders
discounted_returns_placeholder = K.placeholder(name='discounted_return',shape=(None,))
critic_loss = K.mean(K.square(discounted_returns_placeholder - self.model.output))
#Define optimizer
adam_critic = RMSprop(lr = self.lr, epsilon = 0.1, rho = 0.99) #arbitray
pars = self.model.trainable_weights
updates = adam_critic.get_updates(params=pars,loss=critic_loss)
return K.function([self.model.input, discounted_returns_placeholder], [], updates=updates)
def actor_optimizer(): a_t = K.placeholder(shape=[None, ACTION_COUNT]) A = K.placeholder(shape=(None, )) policy = model_actor.output Lpi = -K.sum(K.log(K.sum(policy*a_t, axis=1) + 1e-10) * A) LH = K.sum(K.sum(pi * K.log(policy + 1e-10), axis=1)) L = Lpi + 0.01 * LH optimizer = RMSprop(lr=2.5e-4, rho=0.99, epsilon=0.01) #optimizer = my_optimizer updates = optimizer.get_updates(model_actor.trainable_weights, [], L) train = K.function([model_actor.input, a_t, A], [L], updates=updates) return train
def __update_actor__(self):
action = K.placeholder(shape=(None, self.output_shape_))
advantages = K.placeholder(shape=(None, ))
policy = self.actor_.output
good_prob = K.sum(action * policy, axis=1)
eligibility = K.log(good_prob+1e-10) * K.stop_gradient(advantages)
loss = -K.sum(eligibility)
gradient = RMSprop()
updates = gradient.get_updates(self.actor_.trainable_weights,[],loss)
train = K.function([self.actor_.input,action,advantages],[self.actor_.output],updates=updates)
return train
def actor_optimizer(self):
action = K.placeholder(shape=[None, self.action_size])
advantage = K.placeholder(shape=[
None,
])
action_prob = K.sum(action * self.actor.output, axis=1)
cross_entropy = K.log(action_prob) * advantage
loss = -K.sum(cross_entropy)
optimizer = RMSprop(lr=self.actor_lr)
updates = optimizer.get_updates(self.actor.trainable_weights, [], loss)
train = K.function([self.actor.input, action, advantage], [],
updates=updates)
return train
def critic_optimizer(self):
"""
This method updates critic network (value network)
"""
discounted_prediction = K.placeholder(shape=(None, ))
value = self.critic.output
# You use loss function as mean squre error
loss = K.mean(K.square(discounted_prediction - value))
optimizer = RMSprop(lr=self.critic_lr, rho=0.99, epsilon=0.01)
updates = optimizer.get_updates(self.critic.trainable_weights, [],
loss)
train = K.function([self.critic.input, discounted_prediction], [loss],
updates=updates)
return train
def __mse(self):
'''
Mean squared error loss
:return: Keras function
'''
q_values = self._online_model.output
# trace of taken actions
target = K.placeholder(shape=(None, ), name='target_value')
a_1_hot = K.placeholder(shape=(None, self._action_dim), name='chosen_actions')
q_value = K.sum(q_values * a_1_hot, axis=1)
squared_error = K.square(target - q_value)
mse = K.mean(squared_error)
optimizer = RMSprop(lr=self._lr)
updates = optimizer.get_updates(loss=mse, params=self._online_model.trainable_weights)
return K.function(inputs=[self._online_model.input, target, a_1_hot], outputs=[], updates=updates)
def __build_train_fn(self):
action_Q_placeholder = self.policy_network.output
action_onehot_placeholder = K.placeholder(shape=(None, self.n_action))
target_Q_placeholder = K.placeholder(shape=(None, ))
action_Qvalue = K.sum(action_Q_placeholder * action_onehot_placeholder,
axis=1)
loss = K.max(K.square(action_Qvalue - target_Q_placeholder))
adam = Adam(lr=0.0001)
rmsprop = RMSprop(lr=0.0001, rho=0.99)
updates = rmsprop.get_updates(
params=self.policy_network.trainable_weights, loss=loss)
self.train_fn = K.function(inputs=[
self.policy_network.input, action_onehot_placeholder,
target_Q_placeholder
],
outputs=[],
updates=updates)
def build_functions(self):
S = Input(shape=self.state_size)
NS = Input(shape=self.state_size)
A = Input(shape=(1, ), dtype='int32')
R = Input(shape=(1, ), dtype='float32')
T = Input(shape=(1, ), dtype='int32')
self.build_model()
self.value_fn = K.function([S], self.model(S))
VS = self.model(S)
VNS = disconnected_grad(self.model(NS))
future_value = (1 - T) * VNS.max(axis=1, keepdims=True)
discounted_future_value = self.discount * future_value
target = R + discounted_future_value
cost = ((VS[:, A] - target)**2).mean()
opt = RMSprop(0.0001)
params = self.model.trainable_weights
updates = opt.get_updates(params, [], cost)
self.train_fn = K.function([S, NS, A, R, T], cost, updates=updates)
def actor_optimizer(self):
action = K.placeholder(shape=(None, self.output_pa))
advantages = K.placeholder(shape=(None, ))
policy = self.actor.output
good_prob = K.sum(action * policy, axis=1)
eligibility = K.log(good_prob + 1e-10) * K.stop_gradient(advantages)
loss = -K.sum(eligibility)
entropy = K.sum(policy * K.log(policy + 1e-10), axis=1)
actor_loss = loss + 0.01*entropy
# optimizer = Adam(lr=0.01)
optimizer = RMSprop(lr=2.5e-4, rho=0.99, epsilon=0.0001)
updates = optimizer.get_updates(self.actor.trainable_weights, [], actor_loss)
train = K.function([self.actor.input, action, advantages], [], updates=updates)
return train
def build_functions(self):
S = Input(shape=self.state_size)
NS = Input(shape=self.state_size)
A = Input(shape=(1,), dtype='int32')
R = Input(shape=(1,), dtype='float32')
T = Input(shape=(1,), dtype='int32')
self.build_model()
self.value_fn = K.function([S], self.model(S))
VS = self.model(S)
VNS = disconnected_grad(self.model(NS))
future_value = (1-T) * VNS.max(axis=1, keepdims=True)
discounted_future_value = self.discount * future_value
target = R + discounted_future_value
cost = ((VS[:, A] - target)**2).mean()
opt = RMSprop(0.0001)
params = self.model.trainable_weights
updates = opt.get_updates(params, [], cost)
self.train_fn = K.function([S, NS, A, R, T], cost, updates=updates)
def actor_optimizer(self):
action = K.placeholder(shape=[None, self.action_size])
advantages = K.placeholder(shape=[None, ])
policy = self.actor.output
good_prob = K.sum(action * policy, axis=1)
eligibility = K.log(good_prob + 1e-10) * advantages
actor_loss = -K.sum(eligibility)
entropy = K.sum(policy * K.log(policy + 1e-10), axis=1)
entropy = K.sum(entropy)
loss = actor_loss + 0.01*entropy
optimizer = RMSprop(lr=self.actor_lr, rho=0.99, epsilon=0.01)
updates = optimizer.get_updates(self.actor.trainable_weights, [], loss)
train = K.function([self.actor.input, action, advantages], [loss], updates=updates)
return train
def optimizer(self):
a = K.placeholder(shape=(None, ), dtype='int32')
y = K.placeholder(shape=(None, ), dtype='float32')
py_x = self.model.output
a_one_hot = K.one_hot(a, self.action_size)
q_value = K.sum(py_x * a_one_hot, axis=1)
error = K.abs(y - q_value)
quadratic_part = K.clip(error, 0.0, 1.0)
linear_part = error - quadratic_part
loss = K.mean(0.5 * K.square(quadratic_part) + linear_part)
optimizer = RMSprop(lr=0.00025, epsilon=0.01)
updates = optimizer.get_updates(self.model.trainable_weights, [], loss)
train = K.function([self.model.input, a, y], [loss], updates=updates)
return train
def ActorOptimizer(self):
action = K.placeholder(shape=[None, self.actionSpace])
advantages = K.placeholder(shape=[None, ])
policy = self.actor.output
good_prob = K.sum(action * policy, axis=1)
eligibility = K.log(good_prob + 1e-10) * advantages
actor_loss = -K.sum(eligibility)
entropy = K.sum(policy * K.log(policy + 1e-10), axis=1)
entropy = K.sum(entropy)
loss = actor_loss + 0.01*entropy
optimizer = RMSprop(lr=self.actorLearningrate, rho=0.99, epsilon=0.01)
updates = optimizer.get_updates(self.actor.trainable_weights, [], loss)
train = K.function([self.actor.input, action, advantages], [loss], updates=updates)
return train
def optimizer(self):
a = K.placeholder(shape=(None,), dtype='int32')
y = K.placeholder(shape=(None,), dtype='float32')
# history에서 예측한 q value
prediction = self.model.output
# history에서 취한 action을 one_hot
# action이 0일 때 [1,0,0]
a_one_hot = K.one_hot(a, self.action_size)
# [1,0,0] * [0.32113, 0.1123, 0.00123] = [0.32113,0,0]과 같이 된다.
# 위의 sum하면 0.32113
q_value = K.sum(prediction * a_one_hot, axis=1)
# target(reward + dicount_factor * np.max(model.predict(next_history)) )으로 받은 값이 y
# 즉 정답인 y - q_value만큼의 오차가 발생하는데 현재 q_value의 값이 잘 못 되었을 경우 오차가 크게 발생
# 위에서 q_value는 우리가 예측한 값이다. (q hat)
error = K.abs(y - q_value)
quadratic_part = K.clip(error, 0.0, 1.0)
linear_part = error - quadratic_part
loss = K.mean(0.5 * K.square(quadratic_part) + linear_part)
optimizer = RMSprop(lr=0.00025, epsilon=0.01)
# updates는 list
updates = optimizer.get_updates(self.model.trainable_weights, [], loss)
train = K.function([self.model.input, a, y], [loss], updates=updates)
"""Instantiates a Keras function.
# Arguments
inputs: List of placeholder tensors.
outputs: List of output tensors.
updates: List of update ops.
**kwargs: Passed to `tf.Session.run`.
# Returns
Output values as Numpy arrays.
# Raises
ValueError: if invalid kwargs are passed in.
"""
return train
def build_functions(self):
self.build_model()
S = Input(shape=(self.state_size,))
NS = Input(shape=(self.state_size,))
A = Input(shape=(1,), dtype='int32')
R = Input(shape=(1,), dtype='float32')
T = Input(shape=(1,), dtype='int32')
self.value_fn = kb.function([S], [self.model(S)])
values = self.model(S)
next_values = self.model(NS) #disconnected_grad(self.model(NS))
future_value = kb.cast((1-T), dtype='float32') * kb.max(next_values, axis=1, keepdims=True)
discounted_future_value = self.discount * future_value
target = R + discounted_future_value
cost = kb.mean(kb.pow(values - target, 2))
opt = RMSprop(0.0001)
params = self.model.trainable_weights
updates = opt.get_updates(params, [], cost)
self.train_fn = kb.function([S, NS, A, R, T], [cost], updates=updates)
params = layer4.params + layer3.params + layer2.params + layer1.params + layer0.params
grads = T.grad(cost, params)
# learning_rate = numpy.float32(0.01)
# updates = [
# (param_i, param_i - learning_rate * grad_i)
# for param_i, grad_i in zip(params, grads)
# ]
constraints_list = []
for param in params:
constraints_list.append(identity())
rms = RMSprop()
updates = rms.get_updates(params, constraints_list, cost)
def read_image(address):
img = Image.open(open(address))
img = numpy.asarray(img, dtype='float32') / 256.
# put image in 4D tensor of shape (1, 3, height, width)
img = img.transpose(2, 0, 1).reshape(1, 3, 128, 48)
return img
# img1 = read_image('/home/austin/Documents/Datasets/VIPeR/cam_a/001_45.bmp')
# img2 = read_image('/home/austin/Documents/Datasets/VIPeR/cam_b/091_90.bmp')
# f = theano.function([X1, X2, Y], [cost, layer2.similarity])
# y = numpy.asarray([-1], dtype='int32')
# [tmp, sim] = f(img1, img2, y)
def __init__(self):
rng = numpy.random.RandomState(23455)
self.X1 = T.tensor4('X1', dtype='float32')
self.X2 = T.tensor4('X2', dtype='float32')
self.Y = T.ivector('Y')
self.layer0 = Layer.ConvMaxPool2Layer(
rng,
input1=self.X1,
input2=self.X2,
filter_shape=[25, 3, 5, 5],
poolsize=[2, 2]
)
self.layer1 = Layer.ConvMaxPool2Layer(
rng,
input1=self.layer0.output1,
input2=self.layer0.output2,
filter_shape=[25, 25, 3, 3],
poolsize=[2, 2]
)
self.layer2 = Layer.SecretLayer(
rng,
input1=self.layer1.output1,
input2=self.layer1.output2,
filter_shape=[25, 25, 5, 5]
)
# self.layer3 = Layer.MultiConvMaxPoolLayer(
# rng,
# input=self.layer2.results,
# filter_shape=[25, 25, 3, 3],
# poolsize=(2, 2)
# )
self.layer3 = Layer.LocalCovLayerDropout(
rng,
input=self.layer2.results,
n_in=18*9*25,
n_out=200
)
self.layer4 = Layer.HiddenLayerDropout(
rng,
train_input=self.layer3.train_output,
test_input=self.layer3.test_output,
# n_in=25*24*3,
n_in=800,
n_out=200
)
# self.layer2 = Layer.ConvMaxPoolLayer(
# rng,
# input=T.abs_(self.layer1.output1 - self.layer1.output2),
# filter_shape=[25, 25, 3, 3],
# poolsize=[2, 2]
# )
#
# self.layer3 = Layer.HiddenLayer(
# rng,
# input=self.layer2.output,
# n_in=25*18*5,
# n_out=500
# )
# self.layer5 = Layer.LogisticRegression(self.layer4.output, 500, 2)
# self.cost = self.layer5.negative_log_likelihood(self.Y)
self.layer5 = Layer.LogisticRegressionDropout(
train_input=self.layer4.train_output,
test_input=self.layer4.test_output,
n_in=200,
n_out=2
)
self.cost = self.layer5.negative_log_likelihood_train(self.Y)
self.params = self.layer5.params + self.layer4.params + self.layer3.params + self.layer2.params + self.layer1.params + self.layer0.params
self.grads = T.grad(self.cost, self.params)
# learning_rate = numpy.float32(0.01)
# updates = [
# (param_i, param_i - learning_rate * grad_i)
# for param_i, grad_i in zip(params, grads)
# ]
constraints_list = []
for param in self.params:
constraints_list.append(identity())
rms = RMSprop()
self.updates = rms.get_updates(self.params, constraints_list, self.cost)
regularizers += _regularizers
constraints += _consts
updates += _updates
print('parameters:')
print(params)
print('regularizers:')
print(regularizers)
print('constrains:')
print(constraints)
print('updates:')
print(updates)
'''updates'''
optimizer = RMSprop()
_updates = optimizer.get_updates(params, constraints, train_loss)
updates += _updates
print('after RMSprop, updates:')
for update in updates:
print(update)
train_ins = [X_train, y, weights]
test_ins = [X_test, y, weights]
predict_ins = [X_test]
'''Get functions'''
_train = K.function(train_ins, [train_loss], updates=updates)
_train_with_acc = K.function(train_ins, [train_loss, train_accuracy], updates=updates)
_predict = K.function(predict_ins, [y_test], updates=state_updates)
_test = K.function(test_ins, [test_loss])
