def sg_hinge(tensor, opt):
assert opt.target is not None, 'target is mandatory.'
# default margin
opt += tf.sg_opt(margin=1)
# reshape target
shape = tensor.get_shape().as_list()
broadcast_shape = [-1] + [1] * (len(shape) - 2) + [shape[-1]]
target = tf.cast(tf.reshape(opt.target, broadcast_shape), tf.sg_floatx)
# hinge loss
out = tf.identity(tf.maximum(opt.margin - target * tensor, 0), 'hinge')
# add summary
tf.sg_summary_loss(out)
return out
def iou(self, boxes1, boxes2):
"""
Note: Modified from https://github.com/nilboy/tensorflow-yolo/blob/python2.7/yolo/net/yolo_net.py
calculate ious
Args:
boxes1: 4-D tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL, 4] ====> (x_center, y_center, w, h)
boxes2: 1-D tensor [4] ===> (x_center, y_center, w, h)
Return:
iou: 3-D tensor [CELL_SIZE, CELL_SIZE, BOXES_PER_CELL]
"""
boxes1 = tf.stack([
boxes1[:, 0] - boxes1[:, 2] / 2, boxes1[:, 1] - boxes1[:, 3] / 2,
boxes1[:, 0] + boxes1[:, 2] / 2, boxes1[:, 1] + boxes1[:, 3] / 2
])
boxes2 = tf.stack([
boxes2[:, 0] - boxes2[:, 2] / 2, boxes2[:, 1] - boxes2[:, 3] / 2,
boxes2[:, 0] + boxes2[:, 2] / 2, boxes2[:, 1] + boxes2[:, 3] / 2
])
#calculate the left up point
lu = tf.maximum(boxes1[0:2], boxes2[0:2])
rd = tf.minimum(boxes1[2:], boxes2[2:])
#intersection
intersection = rd - lu
inter_square = tf.multiply(intersection[0], intersection[1])
mask = tf.cast(intersection[0] > 0, tf.float32) * tf.cast(
intersection[1] > 0, tf.float32)
inter_square = tf.multiply(mask, inter_square)
#calculate the boxs1 square and boxs2 square
square1 = tf.multiply((boxes1[2] - boxes1[0]), (boxes1[3] - boxes1[1]))
square2 = tf.multiply((boxes2[2] - boxes2[0]), (boxes2[3] - boxes2[1]))
return inter_square / (square1 + square2 - inter_square +
1e-6), inter_square
def sg_hinge(tensor, opt):
r"""Returns hinge loss between `tensor` and `target`.
Args:
tensor: A `Tensor`.
opt:
target: A `Tensor`. Labels.
margin: An int. Maximum margin. Default is 1.
name: A `string`. A name to display in the tensor board web UI.
Returns:
A `Tensor`.
For example,
```
tensor = [[30, 10, 40], [13, 30, 42]]
target = [[0, 0, 1], [0, 1, 0]]
tensor.sg_hinge(target=target, one_hot=True) => [[ 1. 1. 0.]
[ 1. 0. 1.]]
```
"""
assert opt.target is not None, 'target is mandatory.'
# default margin
opt += tf.sg_opt(margin=1)
# reshape target
shape = tensor.get_shape().as_list()
broadcast_shape = [-1] + [1] * (len(shape) - 2) + [shape[-1]]
target = tf.cast(tf.reshape(opt.target, broadcast_shape), tf.sg_floatx)
# hinge loss
out = tf.identity(tf.maximum(opt.margin - target * tensor, 0), 'hinge')
# add summary
tf.sg_summary_loss(out, name=opt.name)
return out
sample = gen.sg_flatten()
nom = tf.matmul(sample, tf.transpose(sample, perm=[1, 0]))
denom = tf.reduce_sum(tf.square(sample), reduction_indices=[1], keep_dims=True)
pt = tf.square(nom / denom)
pt -= tf.diag(tf.diag_part(pt))
pt = tf.reduce_sum(pt) / (batch_size * (batch_size - 1))
#
# loss & train ops
#
# mean squared errors
mse = tf.reduce_mean(tf.square(disc - xx), reduction_indices=[1, 2, 3])
mse_real, mse_fake = mse[:batch_size], mse[batch_size:]
loss_disc = mse_real + tf.maximum(margin - mse_fake, 0) # discriminator loss
loss_gen = mse_fake + pt * pt_weight # generator loss + PT regularizer
train_disc = tf.sg_optim(loss_disc, lr=0.001,
category='discriminator') # discriminator train ops
train_gen = tf.sg_optim(loss_gen, lr=0.001,
category='generator') # generator train ops
#
# add summary
#
tf.sg_summary_loss(tf.identity(loss_disc, name='disc'))
tf.sg_summary_loss(tf.identity(loss_gen, name='gen'))
tf.sg_summary_image(gen)
pt = tf.square(nom / denom)
pt -= tf.diag(tf.diag_part(pt))
pt = tf.reduce_sum(pt) / (batch_size * (batch_size - 1))
#
# loss & train ops
#
# mean squared errors
mse_real = tf.reduce_mean(tf.square(disc_real - x),
reduction_indices=[1, 2, 3])
mse_fake = tf.reduce_mean(tf.square(disc_fake - gen),
reduction_indices=[1, 2, 3])
# discriminator loss
loss_disc = mse_real + tf.maximum(margin - mse_fake, 0)
# generator loss + PT regularizer
loss_gen = mse_fake + pt * pt_weight
train_disc = tf.sg_optim(loss_disc, lr=0.001,
category='discriminator') # discriminator train ops
train_gen = tf.sg_optim(loss_gen, lr=0.001,
category='generator') # generator train ops
# add summary
tf.sg_summary_loss(loss_disc, name='disc')
tf.sg_summary_loss(loss_gen, name='gen')
#
# training
#
def train(self):
''' Network '''
batch_pred_feats, batch_pred_coords, batch_pred_confs, self.final_state = self.LSTM(
'lstm', self.x)
iou_predict_truth, intersection = self.iou(batch_pred_coords,
self.y[:, 0:4])
should_exist = I = tf.cast(
tf.reduce_sum(self.y[:, 0:4], axis=1) > 0., tf.float32)
no_I = tf.ones_like(I, dtype=tf.float32) - I
object_loss = tf.nn.l2_loss(
I * (batch_pred_confs - iou_predict_truth)) * self.object_scale
noobject_loss = tf.nn.l2_loss(
no_I *
(batch_pred_confs - iou_predict_truth)) * self.noobject_scale
p_sqrt_w = tf.sqrt(
tf.minimum(1.0, tf.maximum(0.0, batch_pred_coords[:, 2])))
p_sqrt_h = tf.sqrt(
tf.minimum(1.0, tf.maximum(0.0, batch_pred_coords[:, 3])))
sqrt_w = tf.sqrt(tf.abs(self.y[:, 2]))
sqrt_h = tf.sqrt(tf.abs(self.y[:, 3]))
loss = (tf.nn.l2_loss(I * (batch_pred_coords[:, 0] - self.y[:, 0])) +
tf.nn.l2_loss(I * (batch_pred_coords[:, 1] - self.y[:, 1])) +
tf.nn.l2_loss(I * (p_sqrt_w - sqrt_w)) +
tf.nn.l2_loss(I * (p_sqrt_h - sqrt_h))) * self.coord_scale
#max_iou = tf.nn.l2_loss(I*(tf.ones_like(iou_predict_truth, dtype=tf.float32) - iou_predict_truth))
total_loss = loss + object_loss + noobject_loss #+ max_iou
''' Optimizer '''
optimizer = tf.train.AdamOptimizer(
learning_rate=self.learning_rate).minimize(
total_loss) # Adam Optimizer
''' Summary for tensorboard analysis '''
dataset_loss = -1
dataset_loss_best = 100
test_writer = tf.summary.FileWriter('summary/test')
tf.summary.scalar('dataset_loss', dataset_loss)
summary_op = tf.summary.merge_all()
''' Initializing the variables '''
self.saver = tf.train.Saver()
batch_states = np.zeros((self.batchsize, 2 * self.len_vec))
# TODO: make this a command line argument, etc.
# training set loader
batch_loader = BatchLoader(
"./DATA/TRAINING/",
seq_len=self.nsteps,
batch_size=self.batchsize,
step_size=1,
folders_to_use=[
"GOPR0005", "GOPR0006", "GOPR0008", "GOPR0008_2", "GOPR0009",
"GOPR0009_2", "GOPR0010", "GOPR0011", "GOPR0012", "GOPR0013",
"GOPR0014", "GOPR0015", "GOPR0016", "MVI_8607", "MVI_8609",
"MVI_8610", "MVI_8612", "MVI_8614", "MVI_8615", "MVI_8616"
])
validation_set_loader = BatchLoader(
"./DATA/VALID/",
seq_len=self.nsteps,
batch_size=self.batchsize,
step_size=1,
folders_to_use=[
"bbd_2017__2017-01-09-21-40-02_cam_flimage_raw",
"bbd_2017__2017-01-09-21-44-31_cam_flimage_raw",
"bbd_2017__2017-01-09-21-48-46_cam_flimage_raw",
"bbd_2017__2017-01-10-16-07-49_cam_flimage_raw",
"bbd_2017__2017-01-10-16-21-01_cam_flimage_raw",
"bbd_2017__2017-01-10-16-31-57_cam_flimage_raw",
"bbd_2017__2017-01-10-21-43-03_cam_flimage_raw",
"bbd_2017__2017-01-11-20-21-32_cam_flimage_raw",
"bbd_2017__2017-01-11-21-02-37_cam_flimage_raw"
])
print("%d available training batches" % len(batch_loader.batches))
print("%d available validation batches" %
len(validation_set_loader.batches))
''' Launch the graph '''
with tf.Session() as sess:
if self.restore_weights == True and os.path.isfile(
self.rolo_current_save + ".index"):
# sess.run(init)
tf.sg_init(sess)
self.saver.restore(sess, self.rolo_current_save)
print("Weight loaded, finetuning")
else:
# sess.run(init)
tf.sg_init(sess)
print("Training from scratch")
epoch_loss = []
for self.iter_id in range(self.n_iters):
''' Load training data & ground truth '''
batch_id = self.iter_id - self.batch_offset
batch_xs, batch_ys, _ = batch_loader.load_batch(batch_id)
''' Update weights by back-propagation '''
sess.run(optimizer,
feed_dict={
self.x: batch_xs,
self.y: batch_ys
})
if self.iter_id % self.display_step == 0:
''' Calculate batch loss '''
batch_loss = sess.run(total_loss,
feed_dict={
self.x: batch_xs,
self.y: batch_ys
})
epoch_loss.append(batch_loss)
print("Total Batch loss for iteration %d: %.9f" %
(self.iter_id, batch_loss))
if self.iter_id % self.display_step == 0:
''' Calculate batch loss '''
batch_loss = sess.run(loss,
feed_dict={
self.x: batch_xs,
self.y: batch_ys
})
print(
"Bounding box coord error loss for iteration %d: %.9f"
% (self.iter_id, batch_loss))
if self.display_object_loss and self.iter_id % self.display_step == 0:
''' Calculate batch object loss '''
batch_o_loss = sess.run(object_loss,
feed_dict={
self.x: batch_xs,
self.y: batch_ys
})
print("Object loss for iteration %d: %.9f" %
(self.iter_id, batch_o_loss))
if self.display_object_loss and self.iter_id % self.display_step == 0:
''' Calculate batch object loss '''
batch_noo_loss = sess.run(noobject_loss,
feed_dict={
self.x: batch_xs,
self.y: batch_ys
})
print("No Object loss for iteration %d: %.9f" %
(self.iter_id, batch_noo_loss))
if self.iou_with_ground_truth and self.iter_id % self.display_step == 0:
''' Calculate batch object loss '''
batch_o_loss = sess.run(tf.reduce_mean(iou_predict_truth),
feed_dict={
self.x: batch_xs,
self.y: batch_ys
})
print("Average IOU with ground for iteration %d: %.9f" %
(self.iter_id, batch_o_loss))
if self.display_coords is True and self.iter_id % self.display_step == 0:
''' Caculate predicted coordinates '''
coords_predict = sess.run(batch_pred_coords,
feed_dict={
self.x: batch_xs,
self.y: batch_ys
})
print("predicted coords:" + str(coords_predict[0]))
print("ground truth coords:" + str(batch_ys[0]))
''' Save model '''
if self.iter_id % self.save_step == 1:
self.saver.save(sess, self.rolo_current_save)
print("\n Model saved in file: %s" %
self.rolo_current_save)
''' Validation '''
if self.validate == True and self.iter_id % self.validate_step == 0 and self.iter_id > 0:
# Run validation set
dataset_loss = self.test(sess, total_loss,
validation_set_loader,
batch_pred_feats,
batch_pred_coords,
batch_pred_confs,
self.final_state)
''' Early-stop regularization '''
if dataset_loss <= dataset_loss_best:
dataset_loss_best = dataset_loss
self.saver.save(sess, self.rolo_weights_file)
print("\n Better Model saved in file: %s" %
self.rolo_weights_file)
''' Write summary for tensorboard '''
summary = sess.run(summary_op,
feed_dict={
self.x: batch_xs,
self.y: batch_ys
})
test_writer.add_summary(summary, self.iter_id)
print("Average total loss %f" % np.mean(epoch_loss))
return