def train():
images, labels = inputs(FLAGS.batch, FLAGS.train, FLAGS.train_labels)
tf.summary.image('labels', labels)
one_hot_labels = classifier.one_hot(labels)
autoencoder = utils.get_autoencoder(config.autoencoder,
config.working_dataset, config.strided)
logits = autoencoder.inference(images)
accuracy_op = accuracy(logits, one_hot_labels)
loss_op = loss(logits, one_hot_labels)
tf.summary.scalar('accuracy', accuracy_op)
tf.summary.scalar(loss_op.op.name, loss_op)
optimizer = tf.train.AdamOptimizer(1e-04)
train_step = optimizer.minimize(loss_op)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=config.gpu_memory_fraction)
session_config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
ckpt = tf.train.get_checkpoint_state(FLAGS.ckpt_dir)
tf.initialize_all_variables()
with tf.Session(config=session_config) as sess:
tf.initialize_all_variables()
ckpt = tf.train.get_checkpoint_state(FLAGS.ckpt_dir)
if not ckpt:
print('No checkpoint file found. Initializing...')
global_step = 0
sess.run(init)
else:
global_step = len(ckpt.all_model_checkpoint_paths) * FLAGS.steps
ckpt_path = ckpt.model_checkpoint_path
saver.restore(sess, ckpt_path)
summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.train_logs, sess.graph)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for step in tqdm(range(FLAGS.steps + 1)):
sess.run(train_step)
if step % FLAGS.summary_step == 0:
summary_str = sess.run(summary)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
if step % FLAGS.batch == 0:
saver.save(sess, FLAGS.train_ckpt, global_step=global_step)
coord.request_stop()
coord.join(threads)
def test():
images, labels = inputs(FLAGS.batch, FLAGS.test, FLAGS.test_labels)
tf.summary.image('labels', labels)
one_hot_labels = classifier.one_hot(labels)
autoencoder = SegNetAutoencoder(4, strided=FLAGS.strided)
logits = autoencoder.inference(images)
accuracy_op = accuracy(logits, one_hot_labels, FLAGS.batch)
tf.summary.scalar('accuracy', accuracy_op)
saver = tf.train.Saver(tf.global_variables())
summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.test_logs)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=config.gpu_memory_fraction)
session_config = tf.ConfigProto(allow_soft_placement=True, gpu_options=gpu_options)
with tf.Session(config=session_config) as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
ckpt = tf.train.get_checkpoint_state(FLAGS.ckpt_dir)
if not (ckpt and ckpt.model_checkpoint_path):
print('No checkpoint file found')
return
ckpt_path = ckpt.model_checkpoint_path
saver.restore(sess, ckpt_path)
summary_str = sess.run(summary)
summary_writer.add_summary(summary_str)
summary_writer.flush()
coord.request_stop()
coord.join(threads)
tf.float32,
[FLAGS.batch, 48, 80, 3
]) # was time_Step, input_ve # swapped inpput vec and time step
labels = tf.placeholder(tf.float32, [FLAGS.batch, 48, 80, 3]) # was 10
one_hot_labels = classifier.one_hot(labels) # was labels
autoencoder = SegNetAutoencoder(32, strided=FLAGS.strided)
logits = autoencoder.inference(images) # was images
#accuracy_op = intersect_over_union(logits, one_hot_labels, FLAGS.batch)
#accuracy_op = imagewise_iou(logits,one_hot_labels,FLAGS.batch)
logits = tf.slice(logits, [0, 8, 0, 0], [-1, 48, -1, -1])
accuracy_op = accuracy(logits, one_hot_labels)
loss_op = loss(logits, one_hot_labels)
#tf.summary.scalar('accuracy', accuracy_op)
#tf.summary.scalar(loss_op.op.name, loss_op)
optimizer = tf.train.AdamOptimizer(
1e-04) # it was 9e-04,epsilon=1e-05 with smaller dataset
train_step = optimizer.minimize(loss_op)
tf.initialize_all_variables()
saver = tf.train.Saver()
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=config.gpu_memory_fraction)
session_config = tf.ConfigProto(
def testing_code(currentTestingItr, newDirPath, dirPath, images, labels, one_hot_labels, autoencoder, logits, FLAGS, trX, trY, valX, valY, teX, teY):
num_channels = 3
os.chdir(dirPath + 'segnet/train_logs/')
newDirPath = newDirPath + "/" + str(currentTestingItr + 1)
os.mkdir(dirPath + 'segnet/train_logs/' + newDirPath)
os.mkdir(dirPath + 'segnet/train_logs/' + newDirPath + "/train")
os.mkdir(dirPath + 'segnet/train_logs/' + newDirPath + "/test")
# tr_range = np.arange(trX.shape[0])
# np.random.shuffle(tr_range)
# trX = trX[tr_range[:budget]]
# trY = trY[tr_range[:budget]]
# TAKE SINGLE CHANNEL ENDS
trX = trX.astype(np.float32)
trY = trY.astype(np.float32)
valX = valX.astype(np.float32)
valY = valY.astype(np.float32)
teX = teX.astype(np.float32)
teY = teY.astype(np.float32)
print "****************************"
print trX.shape
print trY.shape
print valX.shape
print valY.shape
print teX.shape
print teY.shape
print "****************************"
mode = 'all'
#rootdir = './ckpts-explicit-'+mode+'-'
folder = FLAGS.ckpt_dir;
#accuracy_op = intersect_over_union(logits, one_hot_labels, FLAGS.batch)
#accuracy_op = imagewise_iou(logits,one_hot_labels,FLAGS.batch)
# logits = tf.slice(logits,[0,8,8,0],[-1,48,80,-1])
accuracy_op = accuracy(logits,one_hot_labels)
loss_op = loss(logits, one_hot_labels)
#tf.summary.scalar('accuracy', accuracy_op)
#tf.summary.scalar(loss_op.op.name, loss_op)
optimizer = tf.train.AdamOptimizer(1e-04) # it was 9e-04,epsilon=1e-05 with smaller dataset
train_step = optimizer.minimize(loss_op)
tf.initialize_all_variables()
saver = tf.train.Saver()
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=config.gpu_memory_fraction)
session_config = tf.ConfigProto(gpu_options=gpu_options) # allow_soft_placement=True
ckpt = tf.train.get_checkpoint_state(FLAGS.ckpt_dir)
with tf.Session(config=session_config) as sess:
ckpt = tf.train.get_checkpoint_state(FLAGS.ckpt_dir)
if not ckpt:
print('No checkpoint file found. Initializing...')
global_step = 0
#sess.run(init)
tf.initialize_all_variables().run()
else:
global_step = len(ckpt.all_model_checkpoint_paths) * FLAGS.steps
ckpt_path = ckpt.model_checkpoint_path
saver.restore(sess, ckpt_path)
#summary = tf.merge_all_summaries()
#summary_writer = tf.train.SummaryWriter(FLAGS.train_logs, sess.graph)
max_val_accuracy = 0
max_traing_accuracy = 0
logits_value_for_mat = np.full((FLAGS.batch,48,80,2), 0.0)
file_itr = open(dirPath + 'segnet/train_logs/' + newDirPath + "/accuracy.txt",'w', 0)
for step in range(FLAGS.steps):
current_train_accuracy_log = []
current_val_accuracy_log = []
for i in range(int(trX.shape[0]/FLAGS.batch)):
_, current_accuracy,curr_logits, current_loss = sess.run([train_step,accuracy_op,logits,loss_op],feed_dict={images:trX[i*FLAGS.batch:(i+1)*FLAGS.batch],labels:trY[i*FLAGS.batch:(i+1)*FLAGS.batch]})
#current_accuracy = imagewise_iou_np(curr_logits,trY[i*FLAGS.batch:(i+1)*FLAGS.batch],FLAGS.batch)
if max_traing_accuracy < current_accuracy:
max_traing_accuracy = current_accuracy
for count in range(0,FLAGS.batch):
np_image = np.ones((48,80, 3),dtype=np.float)
np_merge = np.ones((48 , 80*3 + 20, 3),dtype=np.float)
for j in range(0, 48):
for k in range(0, 80):
logits_value_for_mat[count][j][k][0] = curr_logits[count][j][k][0]
logits_value_for_mat[count][j][k][1] = curr_logits[count][j][k][1]
if curr_logits[count][j][k][0] > curr_logits[count][j][k][1]:
np_image[j][k] = 0
else:
np_image[j][k] = 255
np_merge[: , 0 : 80] = trX[i * FLAGS.batch + count]
np_merge[: , 80 : 80 + 10] = 224
np_merge[: , 80 + 10 : 80*2 + 10] = trY[i * FLAGS.batch + count]
np_merge[: , 80*2 + 10 : 80*2 + 20] = 224
np_merge[: , 80*2 + 20: 80*3 + 20] = np_image
cv2.imwrite(dirPath + 'segnet/train_logs/' + newDirPath + "/train/" + str(count) + "_training.png", np_merge)
# sio.savemat(dirPath + 'segnet/logitst_mat', {"logits":logits_value_for_mat})
current_train_accuracy_log.append(current_accuracy)
file_itr.write("Training accuracy: " + str(sum(current_train_accuracy_log)/len(current_train_accuracy_log)) + "\n")
print "Training accuracy: " + str(sum(current_train_accuracy_log)/len(current_train_accuracy_log))
for i in range(int(valX.shape[0]/FLAGS.batch)):
current_accuracy,curr_logits = sess.run([accuracy_op,logits],feed_dict={images:valX[i*FLAGS.batch:(i+1)*FLAGS.batch],labels:valY[i*FLAGS.batch:(i+1)*FLAGS.batch]})
#current_accuracy = imagewise_iou_np(curr_logits,valY[i*FLAGS.batch:(i+1)*FLAGS.batch],FLAGS.batch)
current_val_accuracy_log.append(current_accuracy)
# print str(i)+" Current validation accuracy: "+str(current_accuracy)
print "Validation accuracy: " + str(sum(current_val_accuracy_log) / len(current_val_accuracy_log))
mean_val_accuracy = sum(current_val_accuracy_log)/len(current_val_accuracy_log)
#if step % FLAGS.summary_step == 0:
# summary_str = sess.run(summary)
# summary_writer.add_summary(summary_str, step)
if mean_val_accuracy > max_val_accuracy: #and abs(mean_val_accuracy - current_val_accuracy) < 0.1:#step % FLAGS.batch == 0:
max_val_accuracy = mean_val_accuracy
test_accuracy_log = []
# preds_log = np.zeros((teX.shape[0],720,960,32),dtype=np.float32)
for i in range(int(teX.shape[0] / FLAGS.batch)):
[curr_logits, current_accuracy] = sess.run([logits, accuracy_op],
feed_dict={images: teX[i * FLAGS.batch : (i + 1) * FLAGS.batch],
labels: teY[i * FLAGS.batch : (i + 1) * FLAGS.batch]})
for count in range(0,FLAGS.batch):
np_image = np.ones((48,80, 3),dtype=np.float)
np_merge = np.ones((48 , 80*3 + 20, 3),dtype=np.float)
for j in range(0, 48):
for k in range(0, 80):
if curr_logits[count][j][k][0] > curr_logits[count][j][k][1]:
np_image[j][k] = 0
else:
np_image[j][k] = 255
np_merge[: , 0 : 80] = teX[i * FLAGS.batch + count]
np_merge[: , 80 : 80 + 10] = 224
np_merge[: , 80 + 10 : 80*2 + 10] = teY[i * FLAGS.batch + count]
np_merge[: , 80*2 + 10 : 80*2 + 20] = 224
np_merge[: , 80*2 + 20: 80*3 + 20] = np_image
cv2.imwrite(dirPath + 'segnet/train_logs/' + newDirPath + "/test/" + str(count) + "_testing.png", np_merge)
# cv2.imwrite(dirPath + 'segnet/train_logs/' + str(i) + "_originalSegmentation.png", teY[i * FLAGS.batch + count])
# cv2.imwrite(dirPath + 'segnet/train_logs/' + str(i) + "_original.png", teX[i * FLAGS.batch + count])
# cv2.imwrite(dirPath + 'segnet/train_logs/' + str(i) + "_generatedSegmentation.png", np_image)
# current_accuracy = imagewise_iou_np(curr_logits,teY[i*FLAGS.batch:(i+1)*FLAGS.batch],FLAGS.batch)
test_accuracy_log.append(current_accuracy)
print "Test accuracy: " + str(sum(test_accuracy_log) / len(test_accuracy_log))
file_itr.write("Test accuracy: " + str(sum(test_accuracy_log)/len(test_accuracy_log)) + "\n")
else:
file_itr.write("Empty" + "\n")
#min_loss = current_loss
#if save:
# saver.save(sess, FLAGS.train_ckpt, global_step=global_step)
print "Step done!"
file_itr.write("Final test accuracy: " + str(sum(test_accuracy_log) / len(test_accuracy_log)))
print "Final test accuracy: " + str(sum(test_accuracy_log) / len(test_accuracy_log))
# f.write(str(sum(test_accuracy_log)/len(test_accuracy_log)))
file_itr.close()
return sum(test_accuracy_log) / len(test_accuracy_log)
loss, logits = tower_loss(scope, image_batch,
one_hot_label_batch, autoencoder)
tf.get_variable_scope().reuse_variables()
inference_op = autoencoder.inference(image_batch)
#summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope)
# Calculate the gradients for the batch of data on this CIFAR tower.
grads = optimizer.compute_gradients(loss)
# Keep track of the gradients across all towers.
tower_grads.append(grads)
#tower_logits.append(logits)
#tower_labels.append(one_hot_label_batch)
accuracy_vals.append(accuracy(logits, one_hot_label_batch))
pass
accuracy_vals = tf.stack(accuracy_vals)
accuracy_op = tf.reduce_mean(accuracy_vals)
grads = average_gradients(tower_grads)
train_op = optimizer.apply_gradients(grads)
#logits = autoencoder.inference(images)
#logits = tf.slice(logits, [0, 8, 0, 0], [-1, 720, -1, -1])
#one_hot_labels = classifier.one_hot(labels)
#accuracy_op = accuracy(logits,one_hot_labels)
def test():
if config.working_dataset == 'UBC_easy' or config.working_dataset == 'forced_UBC_easy' \
or config.working_dataset == 'UBC_interpolated' or config.working_dataset == 'UBC_hard':
if config.working_dataset == 'UBC_easy' or config.working_dataset == 'UBC_hard':
gt_folder = 'groundtruth'
elif config.working_dataset == 'forced_UBC_easy':
gt_folder = 'forced_groundtruth'
elif config.working_dataset == 'UBC_interpolated':
gt_folder = 'interpol_groundtruth'
images = tf.placeholder(tf.float32, shape=[224, 224, 4])
image = tf.slice(images, [0, 0, 0], [224, 224, 3])
alpha = tf.slice(images, [0, 0, 2], [224, 224, 1])
alpha = tf.cast(alpha * 255, tf.uint8)
image /= 255
image = tf.reshape(image, [-1, 224, 224, 3])
labels = tf.placeholder(tf.float32, shape=[224, 224, 4])
label = tf.slice(labels, [0, 0, 0], [224, 224, 3])
label /= 255
label = tf.reshape(label, [-1, 224, 224, 3])
tf.summary.image('labels', label)
one_hot_labels = classifier.one_hot(label)
autoencoder = utils.get_autoencoder(config.autoencoder,
config.working_dataset,
config.strided)
logits = autoencoder.inference(image)
rgb_image = classifier.rgb(logits)
tf.summary.image('output', rgb_image, max_outputs=3)
rgb_image = tf.reshape(tf.cast(rgb_image * 255, tf.uint8),
[224, 224, 3])
rgba_image = tf.concat([rgb_image, alpha], 2)
# Calculate Accuracy
accuracy_op = accuracy(logits, one_hot_labels)
tf.summary.scalar('accuracy', accuracy_op)
pc_accuracy = per_class_accuracy(logits, one_hot_labels)
pc_size = pc_accuracy.get_shape().as_list()
for k in range(pc_size[0]):
tf.summary.scalar('accuracy_class%02d' % (k + 1), pc_accuracy[k])
tf.summary.tensor_summary('class_wise_accuracy', pc_accuracy)
saver = tf.train.Saver(tf.global_variables())
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=config.gpu_memory_fraction)
session_config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
with tf.Session(config=session_config) as sess:
# File paths
test_logs, ckpt_dir, main_dir, main_output_dir = config.get_directories(
config.working_dataset)
# Store Summaries
summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(test_logs)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
if not (ckpt and ckpt.model_checkpoint_path):
print('No checkpoint file found')
return
ckpt_path = ckpt.model_checkpoint_path
saver.restore(sess, ckpt_path)
# Start Testing on the set
train_set = 20
cam_range = 3
for num_test in range(1, train_set + 1):
cwa = np.array([])
for num_cam in range(1, cam_range + 1):
print 'Currently processing subject numbeer ' + str(
num_test) + ' camera no- ' + str(num_cam)
main_input_dir = '%s%d/images' % (main_dir, num_test)
depth_file_list = glob.glob(
os.path.join(main_input_dir, 'depthRender',
'Cam%d' % num_cam, '*.png'))
label_file_list = glob.glob(
os.path.join(main_input_dir, gt_folder,
'Cam%d' % num_cam, '*.png'))
output_dir = '%s%d/Cam%d' % (main_output_dir, num_test,
num_cam)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
for depth_file_name, label_file_name in zip(
depth_file_list, label_file_list):
output_file_name = os.path.join(
output_dir,
depth_file_name.split('/')[-1])
# print output_file_name
if not os.path.exists(output_file_name):
depth_image = Image.open(depth_file_name)
label_image = Image.open(label_file_name)
# Find Min-max non zero pixels in the label image
label_image_array = np.array(label_image).sum(
axis=2)
itemidx = np.where(
label_image_array.sum(axis=0) != 0)
itemidy = np.where(
label_image_array.sum(axis=1) != 0)
# Crop and Resize Test Depth Image
cropped_depth_image = depth_image.crop(
(min(itemidx[0]), min(itemidy[0]),
max(itemidx[0]), max(itemidy[0])))
cropped_depth_image = cropped_depth_image.resize(
(224, 224), Image.NEAREST)
# Crop and Resize Test Label Image
cropped_label_image = label_image.crop(
(min(itemidx[0]), min(itemidy[0]),
max(itemidx[0]), max(itemidy[0])))
cropped_label_image = cropped_label_image.resize(
(224, 224), Image.NEAREST)
# Infer body-part labels from the learned model
summary_str, inferred_image, pc_acc = sess.run(
[summary, rgba_image, pc_accuracy],
feed_dict={
images: cropped_depth_image,
labels: cropped_label_image
})
cwa = np.concatenate(
[cwa, np.expand_dims(pc_acc, 0)],
axis=0) if cwa.size else np.expand_dims(
pc_acc, 0)
# Restore the original size of the inferred label image
inferred_image = Image.fromarray(
inferred_image.astype('uint8'))
# Reshape to original size and aspect ratio
resized_inferred_image = inferred_image.resize(
(max(itemidx[0]) - min(itemidx[0]),
max(itemidy[0]) - min(itemidy[0])),
Image.NEAREST)
resized_inferred_image = np.array(
resized_inferred_image)
Pad = np.zeros((424, 512, 4))
Pad[min(itemidy[0]):max(itemidy[0]),
min(itemidx[0]):max(
itemidx[0]), :] = resized_inferred_image
resized_inferred_image = Image.fromarray(
Pad.astype('uint8'))
# Save the restored image
resized_inferred_image.save(output_file_name)
# Write the summary collected from the session
summary_writer.add_summary(summary_str)
summary_writer.flush()
sub_dir = '%s%d/' % (main_output_dir, num_test)
if not config.working_dataset == 'MHAD_UBC':
hdf5storage.write(np.transpose(cwa),
path='/cwa',
filename=sub_dir +
'class_wise_accuracy.mat',
matlab_compatible=True)
coord.request_stop()
coord.join(threads)
elif config.working_dataset == 'MHAD' or config.working_dataset == 'UBC_MHAD':
images = tf.placeholder(tf.float32, shape=[224, 224, 3])
image = tf.slice(images, [0, 0, 0], [224, 224, 3])
image /= 255
image = tf.reshape(image, [-1, 224, 224, 3])
labels = tf.placeholder(tf.float32, shape=[224, 224, 3])
label = tf.slice(labels, [0, 0, 0], [224, 224, 3])
label /= 255
label = tf.reshape(label, [-1, 224, 224, 3])
tf.summary.image('labels', label)
autoencoder = utils.get_autoencoder(config.autoencoder,
config.working_dataset,
config.strided)
logits = autoencoder.inference(image)
rgb_image = classifier.rgb(logits)
tf.summary.image('output', rgb_image, max_outputs=3)
one_hot_labels = classifier.one_hot(label)
rgb_image = tf.reshape(tf.cast(rgb_image * 255, tf.uint8),
[224, 224, 3])
pc_accuracy = per_class_accuracy(logits, one_hot_labels)
pc_size = pc_accuracy.get_shape().as_list()
for k in range(pc_size[0]):
tf.summary.scalar('accuracy_class%02d' % (k + 1), pc_accuracy[k])
tf.summary.tensor_summary('class_wise_accuracy', pc_accuracy)
saver = tf.train.Saver(tf.global_variables())
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=config.gpu_memory_fraction)
session_config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
with tf.Session(config=session_config) as sess:
if FLAGS.random_split_mode:
# File paths
test_logs, ckpt_dir, main_input_dir, output_sub_dir = config.get_directories(
config.working_dataset)
else:
test_logs, ckpt_dir, main_input_dir, output_sub_dir = config.get_directories(
config.working_dataset)
test_logs = os.path.join(test_logs, str(
FLAGS.test_subject)) + '/'
ckpt_dir = os.path.join(ckpt_dir, str(
FLAGS.test_subject)) + '/'
output_sub_dir = os.path.join(output_sub_dir,
str(FLAGS.test_subject)) + '/'
# Store Summaries and Initialize threads
summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(test_logs)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
if not (ckpt and ckpt.model_checkpoint_path):
print('No checkpoint file found')
return
ckpt_path = ckpt.model_checkpoint_path
saver.restore(sess, ckpt_path)
subjects = 12
cameras = 2
actions = 11
recordings = 5
for cam in range(1, cameras + 1):
for sub in range(1, subjects + 1):
for act in range(1, actions + 1):
cwa = np.array([])
for rec in range(1, recordings + 1):
print 'Currently processing subject no- ' + str(
sub) + ' action no- ' + str(
act) + ' recording no = ' + str(rec)
depth_file_list = glob.glob(
os.path.join(
main_input_dir,
'Kin%02d/S%02d/A%02d/R%02d/depth_png' %
(cam, sub, act, rec), '*.png'))
label_file_list = glob.glob(
os.path.join(
main_input_dir,
'Kin%02d/S%02d/A%02d/R%02d/labels' %
(cam, sub, act, rec), '*.png'))
output_dir = os.path.join(
main_input_dir, output_sub_dir,
'Kin%02d/S%02d/A%02d/R%02d/' %
(cam, sub, act, rec))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
for depth_file_name, label_file_name in zip(
depth_file_list, label_file_list):
output_file_name = os.path.join(
output_dir,
depth_file_name.split('/')[-1])
if not os.path.exists(output_file_name):
depth_image = cv.imread(depth_file_name)
label_image = cv.imread(label_file_name)
depth_image_array = np.array(depth_image)
label_image_array = np.array(label_image)
# Find Min-max non zero pixels in the label image
itemidx = np.where(
depth_image_array.sum(axis=0) != 0)
itemidy = np.where(
depth_image_array.sum(axis=1) != 0)
# print itemidx
# Crop and Resize Test Depth Image
try:
cropped_image = depth_image_array[
min(itemidy[0]):max(itemidy[0]),
min(itemidx[0]):max(itemidx[0]), :]
resized_depth_image = cv.resize(
cropped_image, (224, 224),
interpolation=cv.INTER_LINEAR)
cropped_label_image = label_image_array[
min(itemidy[0]):max(itemidy[0]),
min(itemidx[0]):max(itemidx[0]), :]
resized_label_image = cv.resize(
cropped_label_image, (224, 224),
interpolation=cv.INTER_LINEAR)
# Infer body-part labels from the learned model
summary_str, inferred_image, pc_acc = sess.run(
[summary, rgb_image, pc_accuracy],
feed_dict={
images: resized_depth_image,
labels: resized_label_image
})
cwa = np.concatenate(
[cwa,
np.expand_dims(pc_acc, 0)],
axis=0
) if cwa.size else np.expand_dims(
pc_acc, 0)
# Reshape to original size and aspect ratio
resized_inferred_image = cv.resize(
inferred_image,
((max(itemidx[0]) -
min(itemidx[0])),
(max(itemidy[0]) -
min(itemidy[0]))),
interpolation=cv.INTER_NEAREST)
resized_inferred_image = np.array(
resized_inferred_image)
Pad = np.zeros((480, 640, 3))
Pad[min(itemidy[0]):max(itemidy[0]),
min(itemidx[0]):
max(itemidx[0]
), :] = resized_inferred_image
cv.imwrite(output_file_name,
Pad[..., ::-1])
# Write the summary collected from the session
summary_writer.add_summary(summary_str)
summary_writer.flush()
except:
print 'Skipping the image name %s' % depth_file_name
sub_dir = os.path.join(
main_input_dir, output_sub_dir,
'Kin%02d/S%02d/A%02d/R%02d_cwa/' %
(cam, sub, act, rec))
if not os.path.exists(sub_dir):
os.makedirs(sub_dir)
hdf5storage.write(np.transpose(cwa),
path='/cwa',
filename=sub_dir +
'class_wise_accuracy.mat',
matlab_compatible=True)
coord.request_stop()
coord.join(threads)
elif config.working_dataset == 'MHAD_UBC':
images = tf.placeholder(tf.float32, shape=[224, 224, 4])
image = tf.slice(images, [0, 0, 0], [224, 224, 3])
alpha = tf.slice(images, [0, 0, 2], [224, 224, 1])
alpha = tf.cast(alpha * 255, tf.uint8)
image /= 255
image = tf.reshape(image, [-1, 224, 224, 3])
autoencoder = utils.get_autoencoder(config.autoencoder,
config.working_dataset,
config.strided)
logits = autoencoder.inference(image)
rgb_image = classifier.rgb(logits)
tf.summary.image('output', rgb_image, max_outputs=3)
rgb_image = tf.reshape(tf.cast(rgb_image * 255, tf.uint8),
[224, 224, 3])
rgba_image = tf.concat([rgb_image, alpha], 2)
saver = tf.train.Saver(tf.global_variables())
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=config.gpu_memory_fraction)
session_config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
with tf.Session(config=session_config) as sess:
# File paths
test_logs, ckpt_dir, main_dir, main_output_dir = config.get_directories(
config.working_dataset)
# Store Summaries
summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(test_logs)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
if not (ckpt and ckpt.model_checkpoint_path):
print('No checkpoint file found')
return
ckpt_path = ckpt.model_checkpoint_path
saver.restore(sess, ckpt_path)
# Start Testing on the set
train_set = 20
cam_range = 3
for num_test in range(1, train_set + 1):
cwa = np.array([])
for num_cam in range(1, cam_range + 1):
print 'Currently processing subject number ' + str(
num_test) + ' camera no- ' + str(num_cam)
main_input_dir = '%s%d/images' % (main_dir, num_test)
depth_file_list = glob.glob(
os.path.join(main_input_dir, 'depthRender',
'Cam%d' % num_cam, '*.png'))
output_dir = '%s%d/Cam%d' % (main_output_dir, num_test,
num_cam)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
for depth_file_name in depth_file_list:
output_file_name = os.path.join(
output_dir,
depth_file_name.split('/')[-1])
if not os.path.exists(output_file_name):
depth_image = Image.open(depth_file_name)
# Find Min-max non zero pixels in the label image
depth_image_array = np.array(depth_image).sum(
axis=2)
itemidx = np.where(
depth_image_array.sum(axis=0) != 0)
itemidy = np.where(
depth_image_array.sum(axis=1) != 0)
# Crop and Resize Test Depth Image
cropped_depth_image = depth_image.crop(
(min(itemidx[0]), min(itemidy[0]),
max(itemidx[0]), max(itemidy[0])))
cropped_depth_image = cropped_depth_image.resize(
(224, 224), Image.NEAREST)
# Infer body-part labels from the learned model
summary_str, inferred_image = sess.run(
[summary, rgba_image],
feed_dict={images: cropped_depth_image})
# Restore the original size of the inferred label image
inferred_image = Image.fromarray(
inferred_image.astype('uint8'))
# Reshape to original size and aspect ratio
resized_inferred_image = inferred_image.resize(
(max(itemidx[0]) - min(itemidx[0]),
max(itemidy[0]) - min(itemidy[0])),
Image.NEAREST)
resized_inferred_image = np.array(
resized_inferred_image)
Pad = np.zeros((424, 512, 4))
Pad[min(itemidy[0]):max(itemidy[0]),
min(itemidx[0]):max(
itemidx[0]), :] = resized_inferred_image
resized_inferred_image = Image.fromarray(
Pad.astype('uint8'))
# Save the restored image
resized_inferred_image.save(output_file_name)
# Write the summary collected from the session
summary_writer.add_summary(summary_str)
summary_writer.flush()
sub_dir = '%s%d/' % (main_output_dir, num_test)
if not config.working_dataset == 'MHAD_UBC':
hdf5storage.write(np.transpose(cwa),
path='/cwa',
filename=sub_dir +
'class_wise_accuracy.mat',
matlab_compatible=True)
coord.request_stop()
coord.join(threads)
def test():
images, labels = inputs(FLAGS.batch, FLAGS.test, FLAGS.test_labels)
tf.summary.image('labels', labels)
one_hot_labels = classifier.one_hot(labels)
autoencoder = utils.get_autoencoder(config.autoencoder, config.working_dataset, config.strided)
logits = autoencoder.inference(images)
accuracy_op = accuracy(logits, one_hot_labels)
tf.summary.scalar('accuracy', accuracy_op)
saver = tf.train.Saver(tf.global_variables())
summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.test_logs)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=config.gpu_memory_fraction)
session_config = tf.ConfigProto(allow_soft_placement=True, gpu_options=gpu_options)
with tf.Session(config=session_config) as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
ckpt = tf.train.get_checkpoint_state(FLAGS.ckpt_dir)
if not (ckpt and ckpt.model_checkpoint_path):
print('No checkpoint file found')
return
ckpt_path = ckpt.model_checkpoint_path
saver.restore(sess, ckpt_path)
summary_str = sess.run(summary)
summary_writer.add_summary(summary_str)
summary_writer.flush()
coord.request_stop()
coord.join(threads)
# extract output label and save green pixel OR coordinate bounds
summary_pb = tf.summary.Summary()
summary_pb.ParseFromString(summary_str)
for val in summary_pb.value:
if 'output/image/' in val.tag:
summary_img = val.image
break
output = sess.run(tf.image.decode_png(summary_img.encoded_image_string))
'''
from matplotlib import pyplot as plt
plt.imshow(output, interpolation='nearest')
plt.show()
'''
img = Image.fromarray(output)
img.save(os.path.join('results', 'output.png'))
greenpixelcount = 0
with open(os.path.join('results', 'samples.txt'), 'w+') as sfile, open(os.path.join('results', 'numgreenpixels.txt'), 'w+') as gpfile:
for width in xrange(img.size[0]):
for length in xrange(img.size[1]):
pixel = img.getpixel((width, length))
if pixel == (0, 255, 0):
greenpixelcount += 1
pbounds = pixelBounds((width, length))
pbstring = str(pbounds[0][0]) + ' ' + str(pbounds[0][1]) + ' ' + str(pbounds[1][0]) + ' ' + str(pbounds[1][1])
sfile.write(pbstring + '\n')
gpfile.write(str(greenpixelcount))
def train(train_filename, train_ckpt, train_logs, ckpt_dir):
images, labels = inputs(FLAGS.batch, train_filename)
tf.summary.image('labels', labels)
one_hot_labels = classifier.one_hot(labels)
autoencoder = utils.get_autoencoder(config.autoencoder,
config.working_dataset, config.strided)
logits = autoencoder.inference(images)
# Store output images
rgb_image = classifier.rgb(logits)
tf.summary.image('output', rgb_image, max_outputs=3)
# Loss/Accuracy Metrics
accuracy_op = accuracy(logits, one_hot_labels)
loss_op = loss(logits, one_hot_labels)
pc_accuracy = per_class_accuracy(logits, one_hot_labels)
pc_size = pc_accuracy.get_shape().as_list()
for k in range(pc_size[0]):
tf.summary.scalar('accuracy_class%02d' % (k + 1), pc_accuracy[k])
tf.summary.tensor_summary('class_wise_accuracy', pc_accuracy)
tf.summary.scalar('accuracy', accuracy_op)
tf.summary.scalar(loss_op.op.name, loss_op)
optimizer = tf.train.AdamOptimizer(1e-04)
train_step = optimizer.minimize(loss_op)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=config.gpu_memory_fraction)
session_config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
with tf.Session(config=session_config) as sess:
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
if not ckpt:
print('No checkpoint file present. Initializing...')
global_step = 0
sess.run(init)
else:
saver.restore(sess, tf.train.latest_checkpoint(ckpt_dir))
global_step = int(
tf.train.latest_checkpoint(ckpt_dir).split('/')[-1].split('-')
[-1])
print 'Restoring the training from step- ' + str(global_step)
summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(train_logs, sess.graph)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for step in tqdm(range(global_step, FLAGS.steps + 1)):
sess.run(train_step)
if step % FLAGS.summary_step == 0:
summary_str = sess.run(summary)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
print '\ntraining step ' + str(step)
print('\nAccuracy= %06f , Loss= %06f' %
(accuracy_op.eval(), loss_op.eval()))
if step == 0:
saver.save(sess, train_ckpt, global_step=step)
elif step == 100 or step == 500 or step == 1000 or step == 5000 or step == 7500 or step == 10000 \
or step == 15000 or step == FLAGS.steps:
saver.save(sess,
train_ckpt,
global_step=step,
write_meta_graph=False)
coord.request_stop()
coord.join(threads)