def get_data(path):
# load data
Fish = {}
# read big matlab files
# imdb_5indiv_13000_9000_50_s1
# imdb_5indiv_15000_1000_s1
with h5py.File(path, 'r') as f:
Fish['data'] = f['images']['data'][()]
Fish['labels'] = f['images']['labels'][()]
print 'database loaded'
data = Fish['data'].astype(np.float32)
# get image dimensions
imsize = data[0].shape
label = Fish['labels'].astype(np.int32)
label = np.squeeze(label)
# compute number of individuals
numIndiv = len(list(set(label)))
# get labels in {0,...,numIndiv-1}
label = np.subtract(label,1)
print label
print 'splitting data in train, test and validation'
N = 3000*numIndiv # of training data
N_test = 500*numIndiv # of test data
N_val = 500*numIndiv # validation data
X_train = data[:N]
X_test = data[N:N+N_test]
X_valid = data[N+N_test:N+N_test+N_val]
y_train = label[:N]
y_test = label[N:N+N_test]
y_valid = label[N+N_test:N+N_test+N_val]
resolution = np.prod(imsize)
X_train = np.reshape(X_train, [N, resolution])
X_test = np.reshape(X_test, [N_test, resolution])
X_valid = np.reshape(X_valid, [N_val, resolution])
# dense to one hot, i.e. [i]-->[0,0,...0,1 (ith position),0,..,0]
Y_train = dense_to_one_hot(y_train, n_classes=numIndiv)
Y_valid = dense_to_one_hot(y_valid, n_classes=numIndiv)
Y_test = dense_to_one_hot(y_test, n_classes=numIndiv)
return X_train, X_test, X_valid, Y_train, Y_test, Y_valid, resolution, numIndiv
def dataHelper(path, num_train, num_test, num_valid):
Fish = {}
with h5py.File(path, 'r') as f:
Fish['data'] = f['images']['data'][()]
Fish['labels'] = f['images']['labels'][()]
print 'database loaded'
data = Fish['data'].astype(np.float32)
# get image dimensions
imsize = data[0].shape
resolution = np.prod(imsize)
# get labels
label = Fish['labels'].astype(np.int32)
label = np.squeeze(label)
# compute number of individuals (classes)
numIndiv = len(list(set(label)))
# get labels in {0,...,numIndiv-1}
label = np.subtract(label,1)
print 'splitting data in train, test and validation'
N = num_train*numIndiv # of training data
N_test = num_test*numIndiv # of test data
N_val = num_valid*numIndiv # validation data
X_train = data[:N]
X_test = data[N:N+N_test]
X_valid = data[N+N_test:N+N_test+N_val]
y_train = label[:N]
y_test = label[N:N+N_test]
y_valid = label[N+N_test:N+N_test+N_val]
# reshape images
X_train = np.reshape(X_train, [N, resolution])
X_test = np.reshape(X_test, [N_test, resolution])
X_valid = np.reshape(X_valid, [N_val, resolution])
# dense to one hot, i.e. [i]-->[0,0,...0,1 (ith position),0,..,0]
Y_train = dense_to_one_hot(y_train, n_classes=numIndiv)
Y_valid = dense_to_one_hot(y_valid, n_classes=numIndiv)
Y_test = dense_to_one_hot(y_test, n_classes=numIndiv)
return numIndiv, imsize, X_train, X_valid, X_test, Y_train, Y_valid, Y_test
import numpy as np
import matplotlib.pyplot as plt
from tf_utils import conv2d, linear, weight_variable, bias_variable, dense_to_one_hot
# %% Load data
mnist_cluttered = np.load('./data/mnist_sequence1_sample_5distortions5x5.npz')
X_train = mnist_cluttered['X_train']
y_train = mnist_cluttered['y_train']
X_valid = mnist_cluttered['X_valid']
y_valid = mnist_cluttered['y_valid']
X_test = mnist_cluttered['X_test']
y_test = mnist_cluttered['y_test']
# % turn from dense to one hot representation
Y_train = dense_to_one_hot(y_train, n_classes=10)
Y_valid = dense_to_one_hot(y_valid, n_classes=10)
Y_test = dense_to_one_hot(y_test, n_classes=10)
# %% Graph representation of our network
# %% Placeholders for 40x40 resolution
x = tf.placeholder(tf.float32, [None, 1600])
y = tf.placeholder(tf.float32, [None, 10])
# %% Since x is currently [batch, height*width], we need to reshape to a
# 4-D tensor to use it in a convolutional graph. If one component of
# `shape` is the special value -1, the size of that dimension is
# computed so that the total size remains constant. Since we haven't
# defined the batch dimension's shape yet, we use -1 to denote this
# dimension should not change size.
def main(_):
input_size = (80, 120)
num_subj = 15
mu = np.array([123.68, 116.779, 103.939], dtype=np.float32).reshape(
(1, 1, 3))
keep_prob = tf.placeholder(tf.float32)
is_train = tf.placeholder(tf.bool)
is_aug = tf.placeholder(tf.bool)
# input image
x_f = tf.placeholder(tf.float32, [None, input_size[1], input_size[1]])
x_l = tf.placeholder(tf.float32, [None, input_size[0], input_size[1]])
x_r = tf.placeholder(tf.float32, [None, input_size[0], input_size[1]])
y_ = tf.placeholder(tf.float32, [None, 2]) # output label
subj_id = tf.placeholder(tf.float32, [None, 2 * num_subj]) # subject index
# online data augmentation
f_processed = tf.where(is_aug, pre_process_face_images(x_f, True),
pre_process_face_images(x_f, False))
face = f_processed - mu
l_processed = tf.where(is_aug, pre_process_eye_images(x_l, True),
pre_process_eye_images(x_l, False))
left_eye = l_processed - mu
r_processed = tf.where(is_aug, pre_process_eye_images(x_r, True),
pre_process_eye_images(x_r, False))
right_eye = r_processed - mu
g_hat, t_hat, bias_W_fc, l2_loss = dilatedNet(face,
left_eye,
right_eye,
keep_prob,
is_train,
subj_id,
vgg_path=FLAGS.vgg_dir)
est_loss = tf.reduce_sum((g_hat - y_ * 10.)**2, axis=1, keep_dims=True)
total_loss = (tf.reduce_mean(est_loss) + 1e-3 * l2_loss +
tf.abs(tf.reduce_mean(bias_W_fc[:num_subj, 0])) +
tf.abs(tf.reduce_mean(bias_W_fc[num_subj:, 0])) +
tf.abs(tf.reduce_mean(bias_W_fc[:num_subj, 1])) +
tf.abs(tf.reduce_mean(bias_W_fc[num_subj:, 1])))
cls1_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = "face") + \
tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = "eye") + \
tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = "combine") + \
tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = "bias")
vgg_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope="transfer")
# optimizer
with tf.variable_scope("LearnRate"):
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(2e-3,
global_step,
8000,
0.1,
staircase=True)
opt1 = tf.train.AdamOptimizer(learning_rate)
opt2 = tf.train.AdamOptimizer(1e-1 * learning_rate)
update_ops1 = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope = "face") + \
tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope = "eye") + \
tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope = "combine")
with tf.control_dependencies(update_ops1):
optimizer1 = opt1.minimize(total_loss,
var_list=cls1_vars,
global_step=global_step)
optimizer2 = opt2.minimize(total_loss, var_list=vgg_vars)
batch_size = 100
train_batch_size = 64
batch_subj100 = np.zeros((batch_size, 2 * num_subj))
saver = tf.train.Saver(max_to_keep=100)
# load data
for fold in range(1, 16):
dataset = spio.loadmat(FLAGS.data_dir + str(fold) + 'train.mat')
face_train = dataset['face_img']
left_train = dataset['left_eye_img']
right_train = dataset['right_eye_img']
eye_train = dataset['eye_angle']
train_index = np.arange(eye_train.shape[0])
vec_train = _2d2vec(eye_train)
# generate subject index
subj_train = np.array([]).reshape((0, 1))
subj_train2 = np.array([]).reshape((0, 1))
for ii in range(eye_train.shape[0] // 3000):
subj_train = np.vstack([subj_train, ii * np.ones((3000, 1))])
subj_train2 = np.vstack([subj_train2, ii * np.ones((1500, 1))])
subj_train2 = np.vstack(
[subj_train2, (ii + num_subj) * np.ones((1500, 1))])
subj_train = np.concatenate([subj_train, subj_train2], axis=1)
dataset = spio.loadmat(FLAGS.data_dir + str(fold) + 'test.mat')
face_test = dataset['face_img']
left_test = dataset['left_eye_img']
right_test = dataset['right_eye_img']
eye_test = dataset['eye_angle']
vec_test = _2d2vec(eye_test)
del dataset
train_index = np.arange(eye_train.shape[0])
train_iter, train_element = creatIter(train_index,
train_batch_size,
isShuffle=True)
test_iter, test_element = creatIter(np.arange(eye_test.shape[0]),
batch_size)
sample_size = (eye_train.shape[0], eye_test.shape[0])
if ~osp.exists(str(fold)):
cmd = 'mkdir ' + str(fold)
os.system(cmd)
model_path = osp.join(str(fold), 'models')
if ~osp.exists(model_path):
cmd = 'mkdir ' + model_path
os.system(cmd)
text_file = open(str(fold) + '/Result.txt', 'w')
text_file.write('fold = ' + str(fold) + '\n')
for nSample in sample_size:
text_file.write('%g ' % nSample)
batch_counter = 0
save_index = 0
test_result = np.array([]).reshape((0, 5, sample_size[1]))
train_loss_list = []
test_loss_list = []
result_path = osp.join(str(fold), '/training_results')
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
""" training """
for epoch in range(15):
sess.run(train_iter.initializer)
for _ in range(sample_size[0] // train_batch_size):
try:
batch_index = sess.run(train_element)
except tf.errors.OutOfRangeError:
break
# train generator
batch_face = np.array(face_train[batch_index])
batch_left = np.array(left_train[batch_index])
batch_right = np.array(right_train[batch_index])
batch_vec = np.array(vec_train[batch_index])
batch_face, batch_eye = randomRotate(batch_face, batch_vec)
batch_subj = np.array(subj_train[batch_index])
batch_face, batch_left, batch_right, batch_eye, batch_subj_f = flip_images(
batch_face, batch_left, batch_right, batch_eye,
batch_subj, 0.5, num_subj)
# check training loss
if batch_counter % FLAGS.train_check_step == 0:
res_loss = sess.run(est_loss,
feed_dict={
x_f:
batch_face,
x_l:
batch_left,
x_r:
batch_right,
y_:
batch_eye,
keep_prob:
1.0,
is_train:
False,
subj_id:
dense_to_one_hot(
batch_subj_f[:, 1:2],
n_classes=2 * num_subj),
is_aug:
True
})
print('Epoch %d %d, batch accuracy %g' %
(epoch, batch_counter, np.mean(res_loss)))
text_file.write(
'Epoch %d %d, batch accuracy %g\n' %
(epoch, batch_counter, np.mean(res_loss)))
train_loss_list.append(np.mean(res_loss))
# update network
if batch_counter < FLAGS.warm_up:
sess.run(optimizer1,
feed_dict={
x_f:
batch_face,
x_l:
batch_left,
x_r:
batch_right,
y_:
batch_eye,
keep_prob:
0.5,
is_train:
True,
subj_id:
dense_to_one_hot(batch_subj_f[:, 1:2],
n_classes=2 * num_subj),
is_aug:
True
})
else:
sess.run(
[optimizer1, optimizer2],
feed_dict={
x_f:
batch_face,
x_l:
batch_left,
x_r:
batch_right,
y_:
batch_eye,
keep_prob:
0.5,
is_train:
True,
subj_id:
dense_to_one_hot(batch_subj_f[:, 1:2],
n_classes=2 * num_subj),
is_aug:
True
})
batch_counter += 1
""" testing accuracy """
if batch_counter % FLAGS.test_check_step == 0:
test_result = np.concatenate(
[test_result,
np.zeros((1, 5, sample_size[1]))],
axis=0)
test_loss = np.array([], dtype=np.float32)
result_list1 = np.array([], dtype=np.float32).reshape(
(2, 0))
result_gt = np.array([], dtype=np.float32).reshape(
(2, 0))
sess.run(test_iter.initializer)
for batch_step in range(eye_test.shape[0] //
batch_size + 1):
try:
batch_index = sess.run(test_element)
except tf.errors.OutOfRangeError:
break
batch_face = np.array(face_test[batch_index])
batch_left = np.array(left_test[batch_index])
batch_right = np.array(right_test[batch_index])
batch_eye = np.array(eye_test[batch_index])
res_loss, res_t_hat = sess.run(
[est_loss, t_hat],
feed_dict={
x_f: batch_face,
x_l: batch_left,
x_r: batch_right,
y_: batch_eye,
keep_prob: 1.0,
is_train: False,
subj_id: batch_subj100,
is_aug: False
})
res_t_hat /= 10.
test_loss = np.append(test_loss, np.mean(res_loss))
result_list1 = np.hstack(
[result_list1,
res_t_hat.transpose()])
result_gt = np.hstack(
[result_gt, batch_eye.transpose()])
test_result[save_index, 0:2, :] = result_gt
test_result[save_index, 2:4, :] = result_list1
angle_result = _angle2error(
_2d2vec(result_list1.transpose()), vec_test)
test_result[save_index, 4, :] = angle_result
text_file.write(
'testing accuracy %g %g\n' %
(np.mean(test_loss), np.mean(angle_result)))
print('testing accuracy %g %g' %
(np.mean(test_loss), np.mean(angle_result)))
test_loss_list.append(np.mean(test_loss))
save_index += 1
if batch_counter % FLAGS.save_interval == 0:
print('save_index: %g' % save_index)
print(' ')
#if np.mean(valid_loss) < prev_min:
# save_path = saver.save(sess, str(fold)+"/models/model"+str(save_index)+".ckpt")
#prev_min = np.mean(valid_loss)
save_path = saver.save(
sess,
osp.join(model_path,
'model{}.ckpt'.format(save_index)))
b_W_fc = bias_W_fc.eval()
np.savez(result_path,
fold=fold,
test_result=test_result,
train_loss_list=train_loss_list,
test_loss_list=test_loss_list,
b_W_fc=b_W_fc)
save_path = saver.save(
sess, osp.join(model_path, 'model{}.ckpt'.format(save_index)))
save_index += 1
b_W_fc = bias_W_fc.eval()
np.savez(result_path,
fold=fold,
test_result=test_result,
train_loss_list=train_loss_list,
test_loss_list=test_loss_list,
b_W_fc=b_W_fc)
N_val = 100*numIndiv # validation data X_train = data[:N] X_test = data[N:N+N_test] X_valid = data[N+N_test:N+N_test+N_val] y_train = label[:N] y_test = label[N:N+N_test] y_valid = label[N+N_test:N+N_test+N_val] resolution = np.prod(imsize) X_train = np.reshape(X_train, [N, resolution]) X_test = np.reshape(X_test, [N_test, resolution]) X_valid = np.reshape(X_valid, [N_val, resolution]) # dense to one hot, i.e. [i]-->[0,0,...0,1 (ith position),0,..,0] Y_train = dense_to_one_hot(y_train, n_classes=numIndiv) Y_valid = dense_to_one_hot(y_valid, n_classes=numIndiv) Y_test = dense_to_one_hot(y_test, n_classes=numIndiv) # %% Graph representation of our network # %% Placeholders for imsize = height*width resolution and labels x = tf.placeholder(tf.float32, [None, resolution]) y = tf.placeholder(tf.float32, [None, numIndiv]) # %% Since x is currently [batch, height*width], we need to reshape to a # 4-D tensor to use it in a convolutional graph. If one component of # `shape` is the special value -1, the size of that dimension is # computed so that the total size remains constant. Since we haven't # defined the batch dimension's shape yet, we use -1 to denote this # dimension should not change size.
import tensorflow as tf
from spatial_transformer import transformer
from tf_utils import weight_variable, bias_variable, dense_to_one_hot
import numpy as np
import matplotlib.pyplot as plt
import os
print("Using tensorflow version", tf.__version__)
#### Load MNIST Data ####
mnist_cluttered = np.load(os.path.join('stn_code','data','mnist_sequence1_sample_5distortions5x5.npz'))
# Get Train and Test Sets from MNIST array
# Change labels from dense to one hot encoding
X_train = mnist_cluttered['X_train']
y_train = dense_to_one_hot(mnist_cluttered['y_train'], n_classes=10)
X_valid = mnist_cluttered['X_valid']
y_valid = dense_to_one_hot(mnist_cluttered['y_valid'], n_classes=10)
X_test = mnist_cluttered['X_test']
y_test = dense_to_one_hot(mnist_cluttered['y_test'], n_classes=10)
# Show an image to make sure it is working
plt.imshow(np.reshape(X_train[15], (40,40)))
plt.show()
#### Create Graph representation of network ####
# Placeholders for 40x40 resolution image
x = tf.placeholder(tf.float32, [None, 1600])
y = tf.placeholder(tf.float32, [None, 1600])
import numpy as np
import matplotlib.pyplot as plt
from tf_utils import conv2d, linear, weight_variable, bias_variable, dense_to_one_hot
# %% Load the data
mnist_cluttered = np.load('./data/mnist_sequence1_sample_5distortions5x5.npz')
X_train = mnist_cluttered['X_train']
y_train = mnist_cluttered['y_train']
X_valid = mnist_cluttered['X_valid']
y_valid = mnist_cluttered['y_valid']
X_test = mnist_cluttered['X_test']
y_test = mnist_cluttered['y_test']
# % turn from dense to one hot representation
Y_train = dense_to_one_hot(y_train, n_classes=10)
Y_valid = dense_to_one_hot(y_valid, n_classes=10)
Y_test = dense_to_one_hot(y_test, n_classes=10)
# %% Graph representation of our network
# %% Placeholders for 40x40 resolution
x = tf.placeholder(tf.float32, [None, 1600])
y = tf.placeholder(tf.float32, [None, 10])
# %% Since x is currently [batch, height*width], we need to reshape to a
# 4-D tensor to use it in a convolutional graph. If one component of
# `shape` is the special value -1, the size of that dimension is
# computed so that the total size remains constant. Since we haven't
# defined the batch dimension's shape yet, we use -1 to denote this
# dimension should not change size.
