def train_via_reinforce(self, sess, image_caption):
image, _ = image_caption
feed_dict = {
self.ResNet50.imgs: image,
self.generator.drop_out: 1.0,
self.generator.train: 1.0,
K.learning_phase(): 0
}
caption = sess.run(self.generator.gen_x, feed_dict)
reward = self.get_reward(self.generator,
self.discriminator,
sess,
caption,
image,
self.rollout_num,
apply_BRA=False)
feed_dict = {
self.ResNet50.imgs: image,
self.generator.drop_out: 1.0,
self.generator.x: caption,
self.generator.reward: reward,
K.learning_phase(): 0
}
_ = sess.run(self.generator.manager_updates, feed_dict)
_ = sess.run(self.generator.worker_updates, feed_dict)
def train_discriminator(self, sess, image_caption):
image, caption = image_caption
feed_dict = {
self.ResNet50.imgs: image,
self.generator.drop_out: 1.0,
self.generator.train: 1.0,
K.learning_phase(): 0
}
fake_caption = sess.run(self.generator.gen_x, feed_dict)
label = np.array([[0, 1]] * int(self.batch_size / 2) +
[[1, 0]] * int(self.batch_size / 2))
segment0 = np.concatenate([
caption[0:int(self.batch_size / 2)],
fake_caption[0:int(self.batch_size / 2)]
])
segment1 = np.concatenate([
caption[int(self.batch_size / 2):self.batch_size],
fake_caption[int(self.batch_size / 2):self.batch_size]
])
feed_dict_0 = {
self.discriminator.D_input_x: segment0,
self.discriminator.D_input_y: label,
self.discriminator.dropout_keep_prob: 0.75
}
feed_dict_1 = {
self.discriminator.D_input_x: segment1,
self.discriminator.D_input_y: label,
self.discriminator.dropout_keep_prob: 0.75
}
_ = sess.run(self.discriminator.D_train_op, feed_dict_0)
_ = sess.run(self.discriminator.D_train_op, feed_dict_1)
def build_model(self):
l2_regularization_kernel = 1e-5
# Input Layer
input = layers.Input(shape=(self.state_size,), name='input_states')
# Hidden Layers
model = layers.Dense(units=300, kernel_regularizer=regularizers.l2(l2_regularization_kernel))(input)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(1e-2)(model)
model = layers.Dense(units=400, kernel_regularizer=regularizers.l2(l2_regularization_kernel))(model)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(1e-2)(model)
model = layers.Dense(units=200, kernel_regularizer=regularizers.l2(l2_regularization_kernel))(model)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(1e-2)(model)
# Our output layer - a fully connected layer
output = layers.Dense(units=self.action_size, activation='tanh', kernel_regularizer=regularizers.l2(l2_regularization_kernel),
kernel_initializer=initializers.RandomUniform(minval=-3e-3, maxval=3e-3), name='output_actions')(model)
# Keras model
self.model = models.Model(inputs=input, outputs=output)
# Define loss and optimizer
action_gradients = layers.Input(shape=(self.action_size,))
loss = K.mean(-action_gradients * output)
optimizer = optimizers.Adam(lr=1e-4)
update_operation = optimizer.get_updates(params=self.model.trainable_weights, loss=loss)
self.train_fn = K.function(inputs=[self.model.input, action_gradients, K.learning_phase()],
outputs=[], updates=update_operation)
def build_model(self):
l2_kernel_regularization = 1e-5
# Define input layers
input_states = layers.Input(shape=(self.state_size, ),
name='input_states')
input_actions = layers.Input(shape=(self.action_size, ),
name='input_actions')
# Hidden layers for states
model_states = layers.Dense(
units=32,
kernel_regularizer=regularizers.l2(l2_kernel_regularization))(
input_states)
model_states = layers.BatchNormalization()(model_states)
model_states = layers.LeakyReLU(1e-2)(model_states)
model_states = layers.Dense(
units=64,
kernel_regularizer=regularizers.l2(l2_kernel_regularization))(
model_states)
model_states = layers.BatchNormalization()(model_states)
model_states = layers.LeakyReLU(1e-2)(model_states)
# Hidden layers for actions
model_actions = layers.Dense(
units=64,
kernel_regularizer=regularizers.l2(l2_kernel_regularization))(
input_actions)
model_actions = layers.BatchNormalization()(model_actions)
model_actions = layers.LeakyReLU(1e-2)(model_actions)
# Both models merge here
model = layers.add([model_states, model_actions])
# Fully connected and batch normalization
model = layers.Dense(units=32,
kernel_regularizer=regularizers.l2(
l2_kernel_regularization))(model)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(1e-2)(model)
# Q values / output layer
Q_values = layers.Dense(
units=1,
activation=None,
kernel_regularizer=regularizers.l2(l2_kernel_regularization),
kernel_initializer=initializers.RandomUniform(minval=-5e-3,
maxval=5e-3),
name='output_Q_values')(model)
# Keras wrap the model
self.model = models.Model(inputs=[input_states, input_actions],
outputs=Q_values)
optimizer = optimizers.Adam(lr=1e-2)
self.model.compile(optimizer=optimizer, loss='mse')
action_gradients = K.gradients(Q_values, input_actions)
self.get_action_gradients = K.function(
inputs=[*self.model.input, K.learning_phase()],
outputs=action_gradients)
def get_feed(self, batch, is_train, learn_rate=1e-4):
feed = {}
feed[self.input_img] = batch[0]
feed[self.input_label] = batch[1]
feed[K.learning_phase()] = int(is_train)
feed[self.input_lr] = learn_rate
return feed
def train_discriminator(self, sess, image_caption):
image, caption = image_caption
feed_dict = {self.ResNet50.imgs: image, K.learning_phase(): 0}
fake_caption = sess.run(self.model.gen_x, feed_dict)
feed_dict = {self.model.x : caption, self.model.fake_x: fake_caption}
_ = sess.run(self.model.d_update, feed_dict)
def build_model(self):
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size, ), name='states')
'''# Add hidden layers
net = layers.Dense(units=32, activation='relu')(states)
net = layers.Dense(units=64, activation='relu')(net)
net = layers.Dense(units=32, activation='relu')(net)
# Try different layer sizes, activations, add batch normalization, regularizers, etc.
# Add final output layer with sigmoid activation
raw_actions = layers.Dense(units=self.action_size, activation='sigmoid',
name='raw_actions')(net)
'''
###################################
# Add hidden layers
net = layers.Dense(units=400,
kernel_regularizer=regularizers.l2(1e-6))(states)
net = layers.BatchNormalization()(net)
net = layers.LeakyReLU(1e-2)(net)
net = layers.Dense(units=300,
kernel_regularizer=regularizers.l2(1e-6))(net)
net = layers.BatchNormalization()(net)
net = layers.LeakyReLU(1e-2)(net)
# Add final output layer with sigmoid activation
raw_actions = layers.Dense(
units=self.action_size,
activation='sigmoid',
name='raw_actions',
kernel_initializer=initializers.RandomUniform(minval=-0.003,
maxval=0.003))(net)
#######################################
# Scale [0, 1] output for each action dimension to proper range
actions = layers.Lambda(lambda x:
(x * self.action_range) + self.action_low,
name='actions')(raw_actions)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
# Define loss function using action value (Q value) gradients
action_gradients = layers.Input(shape=(self.action_size, ))
loss = K.mean(-action_gradients * actions)
# Incorporate any additional losses here (e.g. from regularizers)
# Define optimizer and training function
optimizer = optimizers.Adam(lr=1e-6)
updates_op = optimizer.get_updates(params=self.model.trainable_weights,
loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients,
K.learning_phase()],
outputs=[],
updates=updates_op)
def generate_caption(self, sess, image_caption, is_train=1.0):
image, _ = image_caption
feed_dict = {
self.ResNet50.imgs: image,
self.generator.drop_out: 1.0,
self.generator.train: is_train,
K.learning_phase(): 0
}
caption = sess.run(self.generator.gen_x, feed_dict)
return self.ind_to_str(caption)
def get_attention(self, sess, image_caption):
image, seq = image_caption
feed_dict = {
self.ResNet50.imgs: image,
self.generator.x: seq,
self.generator.drop_out: 1.0,
self.generator.train: 1.0,
K.learning_phase(): 0
}
alpha = sess.run(self.generator.alphas, feed_dict)
return alpha
def build_model(self):
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size, ), name='states')
#--------- copy from DDPG quadcopter -----------
net = layers.Dense(units=400)(states)
# net = layers.BatchNormalization()(net)
net = layers.Activation("relu")(net)
net = layers.Dense(units=200)(net)
# net = layers.BatchNormalization()(net)
net = layers.Activation("relu")(net)
actions = layers.Dense(units=self.action_size,
activation='softmax',
name='actions',
kernel_initializer=initializers.RandomUniform(
minval=-1, maxval=1))(net)
# actions = layers.Dense(units=self.action_size, activation='sigmoid', name='actions',
# kernel_initializer=initializers.RandomUniform(minval=-0.001, maxval=0.001))(net)
# Add hidden layers
# net = layers.Dense(units=16,activation=activations.sigmoid)(states)
# net = layers.BatchNormalization()(net)
# net = layers.Dense(units=16,activation=activations.sigmoid)(net)
# net = layers.BatchNormalization()(net)
# net = layers.Dense(units=128,activation=activations.relu)(net)
# net = layers.BatchNormalization()(net)
# Add final output layer with sigmoid activation
# actions = layers.Dense(units=self.action_size, activation='linear', # sigmoid
# name='raw_actions' )(net)
# Scale [0, 1] output for each action dimension to proper range
# actions = layers.Lambda(lambda x: (x * self.action_range) + self.action_low,
# name='actions')(raw_actions)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
action_gradients = layers.Input(shape=(self.action_size, ))
loss = K.mean(-action_gradients * actions)
# Define optimizer and training function
optimizer = optimizers.Adam(lr=.0001)
updates_op = optimizer.get_updates(params=self.model.trainable_weights,
loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients,
K.learning_phase()],
outputs=[],
updates=updates_op)
def get_feed(self, batches, is_train, drop_rate, learn_rate=None):
feed = {}
for i in range(args.num_gpu):
submodel = self.models[i]
batch = batches[i]
feed[submodel.input_img] = batch[0]
feed[submodel.input_label] = batch[1]
feed[submodel.input_droprate] = drop_rate
feed[K.learning_phase()] = int(is_train)
if learn_rate is not None:
feed[self.input_learnrate] = learn_rate
return feed
def train_via_MLE(self, sess, image_caption):
image, caption = image_caption
feed_dict = {
self.ResNet50.imgs: image,
self.generator.x: caption,
self.generator.drop_out: 1.0,
K.learning_phase(): 0,
self.generator.train: 1.0
}
_ = sess.run(self.generator.pretrain_worker_updates, feed_dict)
_ = sess.run(self.generator.pretrain_manager_updates, feed_dict)
def run_epoch(epoch, model, sess, data, is_train=True):
logger.info('begin epoch {}'.format(epoch))
if is_train:
train_op = model.train_op
else:
train_op = tf.no_op()
learn_rate = get_learn_rate(epoch)
data.reset()
sum_acc = 0
sum_loss = 0
st_time = time.time()
while not data.is_end():
batch = data.get_batch()
feed = {model.input_img:batch[0],model.input_label:batch[1]}
feed[model.input_lr] = learn_rate
feed[K.learning_phase()] = int(is_train)
calc_obj = [train_op,model.acc_num,model.loss]
calc_ans = sess.run(calc_obj, feed_dict=feed)
sum_acc += calc_ans[1]
sum_loss += calc_ans[2]
if data.batch_cnt % 20 == 0:
s = '[batch {}]acc:{:.3} loss:{:.3} avg-time:{:.5}'
avg_acc = sum_acc/data.data_cnt
avg_loss = sum_loss/data.batch_cnt
avg_time = (time.time()-st_time)/data.batch_cnt
s = s.format(data.batch_cnt,avg_acc,avg_loss,avg_time)
logger.info(s)
print(s)
logger.info('[epoch {}]End..batch_num:{},data_num:{}'.format(epoch,data.batch_cnt,data.data_cnt))
avg_acc = sum_acc / data.data_cnt
avg_loss = sum_loss / data.batch_cnt
return avg_acc,avg_loss
def build_model(self):
#Define input layers
inputStates = layers.Input(shape=(self.state_size, ),
name='inputStates')
inputActions = layers.Input(shape=(self.action_size, ),
name='inputActions')
# Hidden layers for states
modelS = layers.Dense(units=128, activation='linear')(inputStates)
modelS = layers.BatchNormalization()(modelS)
modelS = layers.LeakyReLU(0.01)(modelS)
modelS = layers.Dropout(0.3)(modelS)
modelS = layers.Dense(units=256, activation='linear')(modelS)
modelS = layers.BatchNormalization()(modelS)
modelS = layers.LeakyReLU(0.01)(modelS)
modelS = layers.Dropout(0.3)(modelS)
modelA = layers.Dense(units=256, activation='linear')(inputActions)
modelA = layers.LeakyReLU(0.01)(modelA)
modelA = layers.BatchNormalization()(modelA)
modelA = layers.Dropout(0.5)(modelA)
#Merging the models
model = layers.add([modelS, modelA])
model = layers.Dense(units=256, activation='linear')(model)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(0.01)(model)
#Q Layer
Qvalues = layers.Dense(units=1, activation=None,
name='outputQvalues')(model)
#Keras model
self.model = models.Model(inputs=[inputStates, inputActions],
outputs=Qvalues)
optimizer = optimizers.Adam()
self.model.compile(optimizer=optimizer, loss='mse')
actionGradients = K.gradients(Qvalues, inputActions)
self.get_action_gradients = K.function(
inputs=[*self.model.input, K.learning_phase()],
outputs=actionGradients)
def build_model(self):
states = layers.Input(shape=(self.state_size,), name='inputStates')
# Hidden Layers
model = layers.Dense(units=128, activation='linear')(states)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(0.01)(model)
model = layers.Dropout(0.3)(model)
model = layers.Dense(units=256, activation='linear')(model)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(0.01)(model)
model = layers.Dropout(0.3)(model)
model = layers.Dense(units=512, activation='linear')(model)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(0.01)(model)
model = layers.Dropout(0.3)(model)
model = layers.Dense(units=128, activation='linear')(model)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(0.01)(model)
model = layers.Dropout(0.3)(model)
output = layers.Dense(
units=self.action_size,
activation='tanh',
kernel_regularizer=regularizers.l2(0.01),
name='outputActions')(model)
#Keras
self.model = models.Model(inputs=states, outputs=output)
#Definint Optimizer
actionGradients = layers.Input(shape=(self.action_size,))
loss = K.mean(-actionGradients * output)
optimizer = optimizers.Adam()
update_operation = optimizer.get_updates(params=self.model.trainable_weights, loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, actionGradients, K.learning_phase()],
outputs=[],
updates=update_operation)
def get_reward(self,
gen,
dis,
sess,
input_x,
img,
rollout_num,
apply_BRA=True):
rewards = []
for i in range(rollout_num):
for given_num in range(1, gen.sequence_length / gen.step_size):
real_given_num = given_num * gen.step_size
feed = {
gen.x: input_x,
gen.given_num: real_given_num,
gen.drop_out: 1.0,
gen.ResNet_model.imgs: img,
K.learning_phase(): 0
}
samples = sess.run(gen.gen_for_reward, feed)
# print samples.shape
feed = {dis.D_input_x: samples, dis.dropout_keep_prob: 1.0}
ypred_for_auc = sess.run(dis.ypred_for_auc, feed)
ypred = np.array([item[1] for item in ypred_for_auc])
if i == 0:
rewards.append(ypred)
else:
rewards[given_num - 1] += ypred
# the last token reward
feed = {dis.D_input_x: input_x, dis.dropout_keep_prob: 1.0}
ypred_for_auc = sess.run(dis.ypred_for_auc, feed)
ypred = np.array([item[1] for item in ypred_for_auc])
if i == 0:
rewards.append(ypred)
else:
rewards[gen.sequence_length / gen.step_size - 1] += ypred
rewards = np.transpose(np.array(rewards)) / (
1.0 * rollout_num) # batch_size x seq_length
return rewards
def __init__(self, input_tensor, losses, input_range=(0, 255), wrt_tensor=None, norm_grads=True):
"""Creates an optimizer that minimizes weighted loss function.
Args:
input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
losses: List of ([Loss](vis.losses#Loss), weight) tuples.
input_range: Specifies the input range as a `(min, max)` tuple. This is used to rescale the
final optimized input to the given range. (Default value=(0, 255))
wrt_tensor: Short for, with respect to. This instructs the optimizer that the aggregate loss from `losses`
should be minimized with respect to `wrt_tensor`.
`wrt_tensor` can be any tensor that is part of the model graph. Default value is set to None
which means that loss will simply be minimized with respect to `input_tensor`.
norm_grads: True to normalize gradients. Normalization avoids very small or large gradients and ensures
a smooth gradient gradient descent process. If you want the actual gradient
(for example, visualizing attention), set this to false.
"""
self.input_tensor = input_tensor
self.input_range = input_range
self.loss_names = []
self.loss_functions = []
self.wrt_tensor = self.input_tensor if wrt_tensor is None else wrt_tensor
overall_loss = None
for loss, weight in losses:
# Perf optimization. Don't build loss function with 0 weight.
if weight != 0:
loss_fn = weight * loss.build_loss()
overall_loss = loss_fn if overall_loss is None else overall_loss + loss_fn
self.loss_names.append(loss.name)
self.loss_functions.append(loss_fn)
# Compute gradient of overall with respect to `wrt` tensor.
grads = K.gradients(overall_loss, self.wrt_tensor)[0]
if norm_grads:
grads = grads / (K.sqrt(K.mean(K.square(grads))) + K.epsilon())
# The main function to compute various quantities in optimization loop.
self.compute_fn = K.function([self.input_tensor, K.learning_phase()],
self.loss_functions + [overall_loss, grads, self.wrt_tensor])
def build_model(self):
"""Build a critic (value) network that maps (state, action) pairs -> Q-values."""
# Define input layers
states = layers.Input(shape=(self.state_size, ), name='states')
actions = layers.Input(shape=(self.action_size, ), name='actions')
# Add hidden layer(s) for state pathway
net_states = layers.Dense(
units=32,
activation='relu',
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.LeakyReLU(1e-2)(net_states)
net_states = layers.Dense(
units=64,
activation='relu',
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net_states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.LeakyReLU(1e-2)(net_states)
net_states = layers.Dense(
units=128,
activation='relu',
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net_states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.LeakyReLU(1e-2)(net_states)
# Add hidden layer(s) for action pathway
net_actions = layers.Dense(
units=32,
activation='relu',
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.LeakyReLU(1e-2)(net_actions)
net_actions = layers.Dense(
units=64,
activation='relu',
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net_actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.LeakyReLU(1e-2)(net_actions)
net_actions = layers.Dense(
units=128,
activation='relu',
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net_actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.LeakyReLU(1e-2)(net_actions)
# Try different layer sizes, activations, add batch normalization, regularizers, etc.
# Combine state and action pathways
net = layers.Add()([net_states, net_actions])
net = layers.Activation('relu')(net)
# Add more layers to the combined network if needed
# Add final output layer to prduce action values (Q values)
Q_values = layers.Dense(units=1, name='q_values')(net)
# Create Keras model
self.model = models.Model(inputs=[states, actions], outputs=Q_values)
# Define optimizer and compile model for training with built-in loss function
optimizer = optimizers.Adam()
self.model.compile(optimizer=optimizer, loss='mse')
# Compute action gradients (derivative of Q values w.r.t. to actions)
action_gradients = K.gradients(Q_values, actions)
# Define an additional function to fetch action gradients (to be used by actor model)
self.get_action_gradients = K.function(
inputs=[*self.model.input, K.learning_phase()],
outputs=action_gradients)
def run_convnet(params):
HYPER = make_hyper(params)
image_folder = params.image_folder
raw_data_folder = params.raw_data_folder
image_features_folder = params.vgg16_folder
logger = HYPER.logger
df = get_df(params)
if df.shape[0] == 0:
logger.info('No images remaining to process. All done.')
else:
logger.info('Processing %d images.', df.shape[0])
logger.info('\n#################### Args: ####################\n%s',
params.pformat())
logger.info(
'##################################################################\n')
logger.info(
'\n######################### HYPER Params: #########################\n%s',
HYPER.pformat())
logger.info(
'##################################################################\n')
b_it = BatchImageIterator2(raw_data_folder,
image_folder,
HYPER,
image_processor=ImagenetProcessor(HYPER),
df=df,
num_steps=params.num_steps,
num_epochs=params.num_epochs)
graph = tf.Graph()
with graph.as_default():
config = tf.ConfigProto(log_device_placement=False)
config.gpu_options.allow_growth = True
tf_session = tf.Session(config=config)
with tf_session.as_default():
K.set_session(tf_session)
tf_im = tf.placeholder(dtype=HYPER.dtype,
shape=((HYPER.B, ) + HYPER.image_shape),
name='image')
with tf.device(
'/gpu:1'): # change this to gpu:0 if you only have one gpu
tf_a_batch = build_vgg_context(HYPER, tf_im)
tf_a_list = tf.unstack(tf_a_batch, axis=0)
t_n = tfc.printVars('Trainable Variables',
tf.trainable_variables())
g_n = tfc.printVars('Global Variables', tf.global_variables())
l_n = tfc.printVars('Local Variables', tf.local_variables())
assert t_n == g_n
assert g_n == l_n
print '\nUninitialized params'
print tf_session.run(tf.report_uninitialized_variables())
print 'Flushing graph to disk'
tf_sw = tf.summary.FileWriter(tfc.makeTBDir(HYPER.tb), graph=graph)
tf_sw.flush()
print '\n'
start_time = time.clock()
for step, b in enumerate(b_it, start=1):
# if b.epoch > 1 or (params.num_steps >= 0 and step > params.num_steps):
# break
feed_dict = {tf_im: b.im, K.learning_phase(): 0}
a_list = tf_session.run(tf_a_list, feed_dict=feed_dict)
assert len(a_list) == len(b.image_name)
for i, a in enumerate(a_list):
## print 'Writing %s, shape=%s'%(b.image_name[i], a.shape)
with open(
os.path.join(
image_features_folder,
os.path.splitext(b.image_name[i])[0] + '.pkl'),
'wb') as f:
pickle.dump(a, f, pickle.HIGHEST_PROTOCOL)
if step % 10 == 0:
print('Elapsed time for %d steps = %d seconds' %
(step, time.clock() - start_time))
print('Elapsed time for %d steps = %d seconds' %
(step, time.clock() - start_time))
print 'done'
def run_test(epoch, model, sess, times, len_accs):
logger.info('begin test epoch {}'.format(epoch))
st_time = time.time()
with open(args.testid_path) as f:###
ids = f.readline().strip().split(',')
#get test probe
probe_features = []
for id_ in ids:
#path = os.path.join(args.probe_dir, id_.zfill(3), 'slice233-feature.mat')
#if os.path.exists(path):
if False:
mat = sio.loadmat(path)
features = mat['features']
else:
path = os.path.join(args.probe_dir, id_.zfill(3), 'slice0.1.mat')
mat = sio.loadmat(path)
imgs = mat['slice']
imgs = dataset.preprocess_image(imgs)
feed = {model.input_img:imgs}
feed[K.learning_phase()] = 0
calc_obj = [model.output_feature]
calc_ans = sess.run(calc_obj, feed_dict=feed)
features = calc_ans[0]
#path = os.path.join(args.probe_dir, id_.zfill(3), 'slice233-100-feature.mat')
#sio.savemat(path, {'features':features}, do_compression=True)
probe_features.append(features)
logger.info('get features of probe end')
# get gallery feature
ixs = []
imgs = []
for id_ in ids:
path = os.path.join(args.gallery_dir, id_.zfill(3), 'slice0.1.mat')
mat = sio.loadmat(path)
len_ = len(mat['slice'])
id_ixs = []
id_imgs = []
for i in range(times):
ix = np.random.randint(len_)
img = mat['slice'][ix]
id_imgs.append(img)
id_ixs.append(ix)
ixs.append(id_ixs)
imgs.append(id_imgs)
ixs = list(zip(*ixs))
imgs = list(zip(*imgs))
gallery_features = []
for i in range(times):
img = imgs[i]
img = np.array(img)
img = dataset.preprocess_image(img)
feed = {model.input_img:img}
feed[K.learning_phase()] = 0
calc_obj = [model.output_feature]
calc_ans = sess.run(calc_obj, feed_dict=feed)
features = calc_ans[0]
gallery_features.append(features)
#logger.info('{}'.format(str(ixs)))
#calc acc
sum_imgs = 0
for id_features in probe_features:
sum_imgs += len(id_features)
sum_accs = np.zeros(len_accs)
p_args = []
for i in range(times):
p_args.append([ids,probe_features,gallery_features[i],len_accs])
pool = Pool(times)
res = pool.map(proc_calc, p_args)
pool.close()
pool.join()
for one_accs in res:
#print(one_accs/sum_imgs)
sum_accs += one_accs
avg_accs = sum_accs / (sum_imgs*times)
print(avg_accs)
logger.info('[epoch {}]{}'.format(epoch,avg_accs))
return avg_accs[0],int(time.time()-st_time)
img = tf.placeholder(tf.float32, shape=(None, 784))
x = layers.Dense(128, activation='relu')(img)
print x.trainable_weights
x = layers.Dropout(0.5)(x)
x = layers.Dense(128, activation='relu')(x)
x = layers.Dropout(0.5)(x)
preds = layers.Dense(10, activation='softmax')(x)
labels = tf.placeholder(tf.int32)
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=preds, labels=labels))
mnist_data = input_data.read_data_sets("MNIST_data", one_hot=False)
optimize = tf.train.RMSPropOptimizer(0.001).minimize(loss)
acc = tf.reduce_mean(tf.cast(tf.not_equal(tf.cast(tf.argmax(preds, 1), tf.int32), labels), tf.float32))
init_op = tf.global_variables_initializer()
with tf.Session().as_default() as sess:
sess.run(init_op)
for i in range(10):
for j in range(100):
batch = mnist_data.train.next_batch(100)
sess.run(optimize, feed_dict={img: batch[0], labels: batch[1], K.learning_phase(): 1})
print sess.run(acc, feed_dict={img: mnist_data.test.images, labels: mnist_data.test.labels, K.learning_phase(): 0})
def build_model(self):
# Define input layers
input_states = layers.Input(shape=(self.state_size, ),
name='input_states')
input_actions = layers.Input(shape=(self.action_size, ),
name='input_actions')
#---------- copy from DDPG quadcopter ---------
# Add hidden layer(s) for state pathway
net_states = layers.Dense(units=400)(input_states)
# net_states = layers.BatchNormalization()(net_states)
net_states = layers.Activation("relu")(net_states)
net_states = layers.Dense(units=300)(net_states)
net_states = layers.Activation("relu")(net_states)
# Add hidden layer(s) for action pathway
net_actions = layers.Dense(units=300)(input_actions)
net_actions = layers.Activation("relu")(net_actions)
# net_actions = layers.Dense(units=250,kernel_regularizer=regularizers.l2(1e-7))(net_actions)
# net_actions = layers.BatchNormalization()(net_actions)
# net_actions = layers.Activation("relu")(net_actions)
# Combine state and action pathways
net = layers.Add()([net_states, net_actions])
net = layers.Activation('relu')(net)
net = layers.Dense(units=200,
kernel_initializer=initializers.RandomUniform(
minval=-0.5, maxval=0.5))(net)
net = layers.Activation('relu')(net)
# Add final output layer to prduce action values (Q values)
Q_values = layers.Dense(units=1, name='q_values')(net)
# ---------------- Hidden layers for states ----------------
# model_states = layers.Dense(units=32, activation=activations.sigmoid)(input_states)
# # model_states = layers.BatchNormalization()(model_states)
# model_states = layers.Dense(units=16, activation=activations.sigmoid)(model_states)
# # model_states = layers.BatchNormalization()(model_states)
# # model_states = layers.Dense(units=64)(model_states)
# # model_states = layers.BatchNormalization()(model_states)
# # ---------------- Hidden layers for actions ----------------
# model_actions = layers.Dense(units=16, activation=activations.sigmoid)(input_actions)
# # model_actions = layers.BatchNormalization()(model_actions)
# model_actions = layers.Dense(units=16, activation=activations.sigmoid)(model_actions)
# # model_actions = layers.BatchNormalization()(model_actions)
# # Both models merge here
# model = layers.add([model_states, model_actions])
# # Fully connected and batch normalization
# model = layers.Dense(units=8, activation=activations.sigmoid)(model)
# # model = layers.BatchNormalization()(model)
# # model = layers.Dense(units=64, activation=activations.relu)(model)
# # model = layers.BatchNormalization()(model)
# # Q values / output layer
# Q_values = layers.Dense(units=1, name='Q_s_a')(model)
# # model = layers.BatchNormalization()(model)
# Keras wrap the model
self.model = models.Model(inputs=[input_states, input_actions],
outputs=Q_values)
optimizer = optimizers.Adam(lr=0.0001)
self.model.compile(optimizer=optimizer, loss='mse')
action_gradients = K.gradients(Q_values, input_actions)
self.get_action_gradients = K.function(
inputs=[*self.model.input, K.learning_phase()],
outputs=action_gradients)
def get_guidance(self, sess, image_caption):
image, seq = image_caption
feed_dict = {self.ResNet50.imgs: image, self.model.x: seq, K.learning_phase(): 0}
guidance = sess.run(self.model.goals, feed_dict)
return guidance
def get_attention(self, sess, image_caption):
image, seq = image_caption
feed_dict = {self.ResNet50.imgs: image, self.model.x: seq, K.learning_phase(): 0}
alpha = sess.run(self.model.alphas, feed_dict)
return alpha
def generate_caption(self, sess, image_caption, is_train=1.0, is_realized=True):
image, _ = image_caption
feed_dict = {self.ResNet50.imgs: image, K.learning_phase(): 0}
caption = sess.run(self.model.gen_x, feed_dict)
return self.ind_to_str(caption) if is_realized else caption
def build_model(self):
kernel_l2_reg = 1e-5
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size, ), name='states')
# size_repeat = 30
# block_size = size_repeat*self.state_size
# print("Actor block size = {}".format(block_size))
#
# net = layers.concatenate([states]*size_repeat)
# # net = layers.Dense(block_size,
# # # kernel_initializer=initializers.RandomNormal(mean=1.0, stddev=0.1),
# # # bias_initializer=initializers.RandomNormal(mean=0.0, stddev=0.01),
# # activation=None,
# # use_bias=False)(states)
# net = layers.BatchNormalization()(net)
# net = layers.Dropout(0.2)(net)
# # net = layers.LeakyReLU(1e-2)(net)
#
# for _ in range(5):
# net = res_block(net, block_size)
# Add hidden layers
net = layers.Dense(
units=300,
kernel_regularizer=regularizers.l2(kernel_l2_reg))(states)
net = layers.BatchNormalization()(net)
net = layers.LeakyReLU(1e-2)(net)
net = layers.Dense(
units=400, kernel_regularizer=regularizers.l2(kernel_l2_reg))(net)
net = layers.BatchNormalization()(net)
net = layers.LeakyReLU(1e-2)(net)
net = layers.Dense(
units=200, kernel_regularizer=regularizers.l2(kernel_l2_reg))(net)
net = layers.BatchNormalization()(net)
net = layers.LeakyReLU(1e-2)(net)
# Try different layer sizes, activations, add batch normalization, regularizers, etc.
# # Add final output layer with sigmoid activation
# raw_actions = layers.Dense(units=self.action_size,
# activation='sigmoid',
# # kernel_regularizer=regularizers.l2(kernel_l2_reg),
# kernel_initializer=initializers.RandomUniform(minval=-3e-3, maxval=3e-3),
# # bias_initializer=initializers.RandomUniform(minval=-3e-3, maxval=3e-3),
# name='raw_actions')(net)
#
# # Scale [0, 1] output for each action dimension to proper range
# actions = layers.Lambda(lambda x: (x * self.action_range) + self.action_low, name='actions')(raw_actions)
actions = layers.Dense(
units=self.action_size,
activation='tanh',
kernel_regularizer=regularizers.l2(kernel_l2_reg),
kernel_initializer=initializers.RandomUniform(minval=-3e-3,
maxval=3e-3),
name='actions')(net)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
# Define loss function using action value (Q value) gradients
action_gradients = layers.Input(shape=(self.action_size, ))
loss = K.mean(-action_gradients * actions)
# Incorporate any additional losses here (e.g. from regularizers)
# Define optimizer and training function
optimizer = optimizers.Adam(lr=1e-4)
updates_op = optimizer.get_updates(params=self.model.trainable_weights,
loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients,
K.learning_phase()],
outputs=[],
updates=updates_op)
def train_via_MLE(self, sess, image_caption):
image, caption = image_caption
feed_dict = {self.ResNet50.imgs: image, self.model.x: caption, K.learning_phase(): 0}
_ = sess.run(self.model.pretrain_update, feed_dict)
def build_model(self):
kernel_l2_reg = 1e-5
# Dense Options
# units = 200,
# activation='relu',
# activation = None,
# activity_regularizer=regularizers.l2(0.01),
# kernel_regularizer=regularizers.l2(kernel_l2_reg),
# bias_initializer=initializers.Constant(1e-2),
# use_bias = True
# use_bias=False
"""Build a critic (value) network that maps (state, action) pairs -> Q-values."""
# Define input layers
states = layers.Input(shape=(self.state_size, ), name='states')
actions = layers.Input(shape=(self.action_size, ), name='actions')
# size_repeat = 30
# state_size = size_repeat*self.state_size
# action_size = size_repeat*self.action_size
# block_size = size_repeat*self.state_size + size_repeat*self.action_size
# print("Critic block size = {}".format(block_size))
#
# net_states = layers.concatenate(size_repeat * [states])
# net_states = layers.BatchNormalization()(net_states)
# net_states = layers.Dropout(0.2)(net_states)
#
# net_actions = layers.concatenate(size_repeat * [actions])
# net_actions = layers.BatchNormalization()(net_actions)
# net_actions = layers.Dropout(0.2)(net_actions)
#
# # State pathway
# for _ in range(3):
# net_states = res_block(net_states, state_size)
#
# # Action pathway
# for _ in range(2):
# net_actions = res_block(net_actions, action_size)
#
# # Merge state and action pathways
# net = layers.concatenate([net_states, net_actions])
#
# # Final blocks
# for _ in range(3):
# net = res_block(net, block_size)
# Add hidden layer(s) for state pathway
net_states = layers.Dense(
units=300,
kernel_regularizer=regularizers.l2(kernel_l2_reg))(states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.LeakyReLU(1e-2)(net_states)
net_states = layers.Dense(
units=400,
kernel_regularizer=regularizers.l2(kernel_l2_reg))(net_states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.LeakyReLU(1e-2)(net_states)
# Add hidden layer(s) for action pathway
net_actions = layers.Dense(
units=400,
kernel_regularizer=regularizers.l2(kernel_l2_reg))(actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.LeakyReLU(1e-2)(net_actions)
# Merge state and action pathways
net = layers.add([net_states, net_actions])
net = layers.Dense(
units=200, kernel_regularizer=regularizers.l2(kernel_l2_reg))(net)
net = layers.BatchNormalization()(net)
net = layers.LeakyReLU(1e-2)(net)
# Add final output layer to prduce action values (Q values)
Q_values = layers.Dense(
units=1,
activation=None,
kernel_regularizer=regularizers.l2(kernel_l2_reg),
kernel_initializer=initializers.RandomUniform(minval=-5e-3,
maxval=5e-3),
# bias_initializer=initializers.RandomUniform(minval=-3e-3, maxval=3e-3),
name='q_values')(net)
# Create Keras model
self.model = models.Model(inputs=[states, actions], outputs=Q_values)
# Define optimizer and compile model for training with built-in loss function
optimizer = optimizers.Adam(lr=1e-2)
self.model.compile(optimizer=optimizer, loss='mse')
# Compute action gradients (derivative of Q values w.r.t. to actions)
action_gradients = K.gradients(Q_values, actions)
# Define an additional function to fetch action gradients (to be used by actor model)
self.get_action_gradients = K.function(
inputs=[*self.model.input, K.learning_phase()],
outputs=action_gradients)
