def main():
"""Create the model and start the training."""
args = get_arguments()
IMG_MEAN = np.zeros(3)
image_std = [1.0, 1.0, 1.0]
# parameters of building data set
citylist = [
'Norfolk', 'Arlington', 'Atlanta', 'Austin', 'Seekonk', 'NewHaven'
]
image_mean_list = {
'Norfolk': [127.07435926, 129.40160709, 128.28713284],
'Arlington': [88.30304996, 94.97338776, 93.21268212],
'Atlanta': [101.997014375, 108.42171833, 110.044871],
'Austin': [97.0896012682, 102.94697026, 100.7540157],
'Seekonk': [86.67800904, 93.31221168, 92.1328146],
'NewHaven': [106.7092798, 111.4314, 110.74903832]
} # BGR mean for the training data for each city
image_std_list = {
'Norfolk': [28.615469420031832, 32.662536832452886, 37.64149854207523],
'Arlington':
[30.40903039206398, 37.973725024862595, 43.58402191634698],
'Atlanta': [36.93662467838125, 39.43470059838385, 41.74732676809388],
'Austin': [42.81337177109884, 43.71071321350751, 44.440517007230675],
'Seekonk': [25.506449467410715, 32.46885262572024, 37.76814267502958],
'NewHaven': [33.05784541012469, 36.62685162291547, 37.686084270914435]
}
# set evaluation data
if args.training_data == 'SP':
IMG_MEAN = np.array(
(121.68045527, 132.14961763, 129.30317439),
dtype=np.float32) # mean of solar panel data in BGR order
elif args.training_data in citylist:
print("Training on {} data".format(args.training_data))
IMG_MEAN = image_mean_list[args.training_data]
if args.unit_std:
image_std = image_std_list[args.training_data]
else:
print("Wrong data option: {}".format(args.data_option))
# setup used GPU
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = args.GPU
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
tf.set_random_seed(args.random_seed)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(args.data_dir, args.data_list, input_size,
args.random_scale, args.random_mirror,
args.random_rotate, args.ignore_label, IMG_MEAN,
coord, image_std)
image_batch, label_batch = reader.dequeue(args.batch_size)
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=args.is_training,
num_classes=args.num_classes)
# For a small batch size, it is better to keep
# the statistics of the BN layers (running means and variances)
# frozen, and to not update the values provided by the pre-trained model.
# If is_training=True, the statistics will be updated during the training.
# Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
# if they are presented in var_list of the optimiser definition.
# Predictions.
raw_output = net.layers['fc1_voc12']
# Which variables to load. Running means and variances are not trainable,
# thus all_variables() should be restored.
restore_var = [
v for v in tf.global_variables()
if 'fc' not in v.name or not args.not_restore_last
]
all_trainable = [
v for v in tf.trainable_variables()
if 'beta' not in v.name and 'gamma' not in v.name
]
fc_trainable = [v for v in all_trainable if 'fc' in v.name]
conv_trainable = [v for v in all_trainable
if 'fc' not in v.name] # lr * 1.0
fc_w_trainable = [v for v in fc_trainable
if 'weights' in v.name] # lr * 10.0
fc_b_trainable = [v for v in fc_trainable
if 'biases' in v.name] # lr * 20.0
assert (len(all_trainable) == len(fc_trainable) + len(conv_trainable))
assert (len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))
# Predictions: ignoring all predictions with labels greater or equal than n_classes
raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
label_proc = prepare_label(label_batch,
tf.stack(raw_output.get_shape()[1:3]),
num_classes=args.num_classes,
one_hot=False) # [batch_size, h, w]
raw_gt = tf.reshape(label_proc, [
-1,
])
indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)),
1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
prediction = tf.gather(raw_prediction, indices)
# Pixel-wise softmax loss.
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction,
labels=gt)
# loss = tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
l2_losses = [
tf.nn.l2_loss(v) for v in tf.trainable_variables()
if 'weights' in v.name
]
reduced_loss = tf.reduce_mean(
loss) + args.weight_decay * tf.add_n(l2_losses)
# Processed predictions: for visualisation.
raw_output_up = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Define loss and optimisation parameters.
base_lr = tf.constant(args.learning_rate)
# step_ph = tf.placeholder(dtype=tf.float32, shape=())
global_step = tf.Variable(0,
dtype=tf.int32,
trainable=False,
name='global_step')
global_epoch = tf.cast(global_step * args.batch_size / args.data_size,
tf.int8)
# learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
if (args.decay_step <= 0):
learning_rate = base_lr
else:
learning_rate = tf.train.exponential_decay(
base_lr,
global_step,
tf.cast(args.data_size / args.batch_size * args.decay_step,
tf.int32),
args.decay_rate,
staircase=True)
opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)
grads = tf.gradients(reduced_loss,
conv_trainable + fc_w_trainable + fc_b_trainable)
grads_conv = grads[:len(conv_trainable)]
grads_fc_w = grads[len(conv_trainable):(len(conv_trainable) +
len(fc_w_trainable))]
grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):]
train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable),
global_step=global_step)
train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))
train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)
# IOU of training batch
# IOU = IOU_tf(pred, label_batch)
# Image summary.
images_summary = tf.py_func(
inv_preprocess,
[image_batch, args.save_num_images, IMG_MEAN, image_std], tf.uint8)
labels_summary = tf.py_func(
decode_labels, [label_batch, args.save_num_images, args.num_classes],
tf.uint8)
preds_summary = tf.py_func(decode_labels,
[pred, args.save_num_images, args.num_classes],
tf.uint8)
image_summary = tf.summary.image(
'images',
tf.concat(axis=2,
values=[images_summary, labels_summary, preds_summary]),
max_outputs=args.save_num_images) # Concatenate row-wise.
loss_summary = tf.summary.scalar("loss", reduced_loss)
entropy_summary = tf.summary.scalar("entropy", tf.reduce_mean(loss))
l2_loss_summary = tf.summary.scalar("L2_loss", tf.add_n(l2_losses))
learning_rate_summary = tf.summary.scalar(
"learning_rate", learning_rate) # summary recording learning rate
# IOU_summary = tf.summary.scalar("IOU", IOU)
summary_writer = tf.summary.FileWriter(args.snapshot_dir,
graph=tf.get_default_graph())
print("Setting up summary op...")
total_summary = tf.summary.merge_all()
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(),
max_to_keep=2) # saver for save/load checkpoint
# load weights from saved checkpoint or initial pre-trained model
if os.path.isdir(args.snapshot_dir):
# search checkpoint at given path
ckpt = tf.train.get_checkpoint_state(args.snapshot_dir)
if ckpt and ckpt.model_checkpoint_path:
# load checkpoint file
load(saver, sess, ckpt.model_checkpoint_path)
elif os.path.isfile(args.snapshot_dir):
# load checkpoint file
load(saver, sess, args.snapshot_dir)
elif args.restore_from is not None:
loader = tf.train.Saver(
var_list=restore_var) # loader for part of pre-trained model
load(loader, sess, args.restore_from)
print("Load weights from{}".format(args.restore_from))
else:
print("No model found at{}".format(args.restore_from))
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Initial status
loss_value, entropy_loss, summary, itr, epoch = sess.run([
reduced_loss,
tf.reduce_mean(loss), total_summary, global_step, global_epoch
])
print('step {:d} \t loss = {:.3f}, entropy_loss = {:.3f})'.format(
itr, loss_value, entropy_loss))
summary_writer.add_summary(summary, itr)
# Iterate over training steps.
while (epoch < args.num_epochs):
start_time = time.time()
# feed_dict = { step_ph : step }
_, itr, epoch = sess.run([train_op, global_step, global_epoch])
# save summary file
if itr % 100 == 0:
duration = time.time() - start_time
pred_temp, truth_temp, loss_value, entropy_loss, summary, itr = sess.run(
[
pred, label_batch, reduced_loss,
tf.reduce_mean(loss), total_summary, global_step
])
summary_writer.add_summary(summary, itr)
IOU_temp = IOU_(pred_temp, truth_temp)
print(
'Epoch {:d} step {:d} \t loss = {:.3f}, entropy_loss = {:.3f}, IOU = {:.3f}, ({:.3f} sec/step)'
.format(epoch, itr, loss_value, entropy_loss, IOU_temp,
duration))
# print('Epoch {:d} step {:d} \t loss = {:.3f}, entropy_loss = {:.3f}, ({:.3f} sec/step)'.format(
# epoch, itr, loss_value, entropy_loss, duration))
# save checkpoint
if itr % args.save_pred_every == 0:
# images, labels, preds = sess.run([image_batch, label_batch, pred])
save(saver, sess, args.snapshot_dir, global_step)
# final status
loss_value, entropy_loss, summary, itr = sess.run(
[reduced_loss,
tf.reduce_mean(loss), total_summary, global_step])
print('step {:d} \t loss = {:.3f}, entropy_loss = {:.3f}'.format(
itr, loss_value, entropy_loss))
save(saver, sess, args.snapshot_dir, global_step)
summary_writer.add_summary(summary, itr)
coord.request_stop()
coord.join(threads)
sess.close()
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
#image list/ label list
f = open(DATA_ID_PATH, 'r')
maskslist = []
for line in f:
mask = line.strip("\n")
maskslist.append(mask)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
[321, 321], # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord)
image, label = reader.image, reader.label
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(
label, dim=0) # Add one batch dimension.
# Create network.
#net = DeepLabResNetModel({'data': image_batch}, is_training=False, num_classes=args.num_classes)
net = DeepLabResNetModelOri50({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output = tf.argmax(raw_output, dimension=3)
pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
# # mIoU first convert pred and gt to vector,then compute mIoU
# pred = tf.reshape(pred, [-1, ])
# gt = tf.reshape(label_batch, [-1, ])
# # tensorflow 1.3.0 conflict
# # weights = tf.cast(tf.less_equal(gt, args.num_classes - 1), tf.int32) # Ignoring all labels greater than or equal to n_classes.
# # mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred, gt, num_classes=args.num_classes, weights=weights)
# indices = tf.squeeze(tf.where(tf.less_equal(gt, args.num_classes - 1)), 1) # ignore all labels >= num_classes
# gt = tf.cast(tf.gather(gt, indices), tf.int32)
# pred = tf.gather(pred, indices)
# mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred, gt, num_classes=args.num_classes)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
start = time.time()
for step in range(args.num_steps):
# preds, _ = sess.run([pred, update_op])
preds = sess.run(pred)
# save predicted label.png
# msk = decode_labels(preds, num_classes=args.num_classes)
mask = preds
# n, h, w, c = mask.shape
# num_classes=21
# num_images = 1
# assert (n >= num_images), 'Batch size %d should be greater or equal than number of images to save %d.' % (n, num_images)
# outputs = np.zeros((1, h, w), dtype=np.uint8)
outputs = np.array(mask[0, :, :, 0], dtype=np.uint8)
# for i in range(num_images):
# img = Image.new('L', (len(mask[i, 0]), len(mask[i])))
# pixels = img.load()
# pixels = mask[i, :, :, 0]
# outputs[i] = np.array(img)
msk = outputs
im = Image.fromarray(msk)
# im = Image.fromarray(msk[0])
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
im.save(args.save_dir + maskslist[step] + '.png')
print('The output file has been saved to {}'.format(args.save_dir +
maskslist[step] +
'.png'))
end = time.time()
avgfps = (step + 1) / (end - start)
print('frame , %s //s' % avgfps)
if step % 100 == 0:
print('step {:d}'.format(step))
# print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
def main():
args, preds = get_arguments(), []
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord)
image_orig, label = reader.image, reader.label
for rots in range(4):
image = tf.image.rot90(image_orig, k=rots)
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(label, dim=0) # Add one batch dimension.
# Create network.
net = DeepLabResNetModel({'data': image_batch}, is_training=False, num_classes=args.num_classes)
tf.get_variable_scope().reuse_variables()
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3,])
# CRF.
if args.crf:
inv_image = tf.py_func(inv_preprocess, [image_batch, 1, IMG_MEAN], tf.uint8)
raw_output = tf.py_func(dense_crf, [tf.nn.softmax(raw_output), inv_image], tf.float32)
# Rotate to original
raw_output = tf.image.rot90(tf.squeeze(raw_output), k=(4-rots))
raw_output = tf.expand_dims(raw_output, dim=0)
preds.append(raw_output)
if not args.augment:
break
pred = tf.reduce_mean(tf.concat(preds, axis=0), axis=0)
# Set class based on threshold
if args.thresh:
pred = tf.py_func(threshold, [tf.nn.softmax(pred), int(args.thresh[0]), float(args.thresh[1])], tf.int32)
pred = tf.expand_dims(pred, dim=0)
else:
pred = tf.argmax(tf.expand_dims(pred, dim=0), dimension=3)
pred = tf.cast(tf.expand_dims(pred, dim=3), tf.int32) # create 4D tensor
# mIoU
pred = tf.reshape(pred, [-1,])
gt = tf.reshape(label_batch, [-1,])
weights = tf.cast(tf.less_equal(gt, args.num_classes - 1), tf.int32) # Ignoring all labels greater than or equal to n_classes.
mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred, gt, num_classes=args.num_classes, weights=weights)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in tqdm(range(args.num_steps)):
preds, _ = sess.run([pred, update_op])
print('Mean IoU: {:.10f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the training."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
tf.set_random_seed(args.random_seed)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(args.data_dir, args.data_list, input_size,
args.random_scale, args.random_mirror,
args.ignore_label, IMG_MEAN, coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
# Create network.
sys.stdout.flush()
sys.stderr.flush()
net = DeepLabResNetModel({'data': image_batch},
is_training=args.is_training,
num_classes=args.num_classes)
sys.stdout.flush()
sys.stderr.flush()
# For a small batch size, it is better to keep
# the statistics of the BN layers (running means and variances)
# frozen, and to not update the values provided by the pre-trained model.
# If is_training=True, the statistics will be updated during the training.
# Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
# if they are presented in var_list of the optimiser definition.
# Predictions.
raw_output = net.layers['fc1_voc12']
# Which variables to load. Running means and variances are not trainable,
# thus all_variables() should be restored.
all_trainable = [
v for v in tf.trainable_variables()
if 'beta' not in v.name and 'gamma' not in v.name
]
fc_trainable = [v for v in all_trainable if 'fc' in v.name]
conv_trainable = [v for v in all_trainable
if 'fc' not in v.name] # lr * 1.0
fc_w_trainable = [v for v in fc_trainable
if 'weights' in v.name] # lr * 10.0
fc_b_trainable = [v for v in fc_trainable
if 'biases' in v.name] # lr * 20.0
if args.freeze_convolutions:
print("freezing backbone")
all_trainable = [v for v in all_trainable if v not in conv_trainable]
conv_trainable = []
assert (len(all_trainable) == len(fc_trainable) + len(conv_trainable))
assert (len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))
print("training {} variables out of total {} trainable variables".format(
len(all_trainable), len(tf.trainable_variables())))
# Predictions: ignoring all predictions with labels greater or equal than n_classes
raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
label_proc = prepare_label(label_batch,
tf.stack(raw_output.get_shape()[1:3]),
num_classes=args.num_classes,
one_hot=False) # [batch_size, h, w]
raw_gt = tf.reshape(label_proc, [
-1,
])
indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)),
1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
prediction = tf.gather(raw_prediction, indices)
# Pixel-wise softmax loss.
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction,
labels=gt)
l2_losses = [
args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables()
if 'weights' in v.name
]
reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses)
# Processed predictions: for visualisation.
raw_output_up = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Image summary.
images_summary = tf.py_func(inv_preprocess,
[image_batch, args.save_num_images, IMG_MEAN],
tf.uint8)
labels_summary = tf.py_func(
decode_labels, [label_batch, args.save_num_images, args.num_classes],
tf.uint8)
preds_summary = tf.py_func(decode_labels,
[pred, args.save_num_images, args.num_classes],
tf.uint8)
if not os.path.exists(args.snapshot_dir):
os.mkdir(args.snapshot_dir)
total_summary = tf.summary.image(
'images',
tf.concat(axis=2,
values=[images_summary, labels_summary, preds_summary]),
max_outputs=args.save_num_images) # Concatenate row-wise.
summary_writer = tf.summary.FileWriter(args.snapshot_dir,
graph=tf.get_default_graph())
# Define loss and optimisation parameters.
base_lr = tf.constant(args.learning_rate)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(
base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)
grads = tf.gradients(reduced_loss,
conv_trainable + fc_w_trainable + fc_b_trainable)
grads_conv = grads[:len(conv_trainable)]
grads_fc_w = grads[len(conv_trainable):(len(conv_trainable) +
len(fc_w_trainable))]
grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):]
train_op_group_list = []
if len(conv_trainable) > 0:
train_op_conv = opt_conv.apply_gradients(
zip(grads_conv, conv_trainable))
train_op_group_list.append(train_op_conv)
train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
train_op_group_list.append(train_op_fc_w)
train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))
train_op_group_list.append(train_op_fc_b)
train_op = tf.group(*train_op_group_list)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
snapshot_dir = args.snapshot_dir
skip_next_arg_print = False
for i in range(1, len(sys.argv)):
if skip_next_arg_print:
skip_next_arg_print = False
continue
if sys.argv[i].startswith('--snapshot-dir') or sys.argv[i].startswith(
'--data-dir') or sys.argv[i].startswith('--restore-from'):
skip_next_arg_print = True
continue
if sys.argv[i].startswith('--'):
words = sys.argv[i].replace('--', '').split('-')
snapshot_dir += '_' + ''.join([w[0] for w in words]) + "="
else:
snapshot_dir += sys.argv[i].replace('/', '-')
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=50)
start_from_step = 0
# Load variables if the checkpoint is provided.
if args.restore_from is not None:
if args.restore_from == RESTORE_FROM and os.path.exists(
snapshot_dir) and len(os.listdir(snapshot_dir)) >= 3:
restore_path = tf.train.latest_checkpoint(snapshot_dir)
print("Auto restoring weights from " + str(restore_path))
else:
restore_path = args.restore_from
try:
start_from_step = int(restore_path.split("-")[-1])
except ValueError:
start_from_step = 0
print("Auto starting from step " + str(start_from_step) +
" (detected from checkpoint file)")
vars_in_checkpoint = get_tensors_in_checkpoint_file(
file_name=restore_path)
loadable_tensors = match_loaded_and_memory_tensors(vars_in_checkpoint)
loadable_tensors = [
v for v in loadable_tensors
if 'fc' not in v.name or not args.not_restore_last
]
loader = tf.train.Saver(var_list=loadable_tensors)
load(loader, sess, restore_path)
sys.stdout.flush()
sys.stderr.flush()
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
os.makedirs(snapshot_dir, exist_ok=True)
with open(os.path.join(snapshot_dir, "args.json"), "+w") as f:
f.write(json.dumps(args.__dict__, indent=2))
def should_print(step):
if step < 100:
return True
elif step < 1000:
return step % 100 == 0
elif step < 10000:
return step % 1000 == 0
else:
return step % 10000 == 0
loss_sum = 0
num_loss_sum = 0
# Iterate over training steps.
for step in range(start_from_step, args.num_steps):
start_time = time.time()
feed_dict = {step_ph: step}
if step % args.save_pred_every == 0 or step == args.num_steps - 1:
loss_value, imgs, lbls, preds, summary, _ = sess.run(
[
reduced_loss, image_batch, label_batch, pred,
total_summary, train_op
],
feed_dict=feed_dict)
summary_writer.add_summary(summary, step)
save(saver, sess, snapshot_dir, step)
else:
loss_value, _ = sess.run([reduced_loss, train_op],
feed_dict=feed_dict)
loss_sum += loss_value
num_loss_sum += 1
duration = time.time() - start_time
if should_print(step) or should_print(
step - start_from_step) or step == args.num_steps - 1:
print(
'{:2.2f}% step {:d}/{:d} \t loss = {:.3f} , ({:.3f} sec/step)'.
format(
float(step / args.num_steps) * 100., step, args.num_steps,
loss_sum / num_loss_sum, duration))
loss_sum = 0
num_loss_sum = 0
sys.stdout.flush()
sys.stderr.flush()
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
coord)
image, label = reader.image, reader.label
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(label, dim=0) # Add one batch dimension.
h_orig, w_orig = tf.to_float(tf.shape(image_batch)[1]), tf.to_float(tf.shape(image_batch)[2])
image_batch075 = tf.image.resize_images(image_batch, tf.stack([tf.to_int32(tf.multiply(h_orig, 0.75)), tf.to_int32(tf.multiply(w_orig, 0.75))]))
image_batch05 = tf.image.resize_images(image_batch, tf.stack([tf.to_int32(tf.multiply(h_orig, 0.5)), tf.to_int32(tf.multiply(w_orig, 0.5))]))
# Create network.
with tf.variable_scope('', reuse=False):
net = DeepLabResNetModel({'data': image_batch}, is_training=False)
with tf.variable_scope('', reuse=True):
net075 = DeepLabResNetModel({'data': image_batch075}, is_training=False)
with tf.variable_scope('', reuse=True):
net05 = DeepLabResNetModel({'data': image_batch05}, is_training=False)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output100 = net.layers['fc1_voc12']
raw_output075 = tf.image.resize_images(net075.layers['fc1_voc12'], tf.shape(raw_output100)[1:3,])
raw_output05 = tf.image.resize_images(net05.layers['fc1_voc12'], tf.shape(raw_output100)[1:3,])
raw_output = tf.reduce_max(tf.stack([raw_output100, raw_output075, raw_output05]), axis=0)
raw_output = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3,])
raw_output = tf.argmax(raw_output, dimension=3)
pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
# mIoU
pred = tf.reshape(pred, [-1,])
gt = tf.reshape(label_batch, [-1,])
weights = tf.cast(tf.less_equal(gt, n_classes - 1), tf.int32) # Ignoring all labels greater than or equal to n_classes.
mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred, gt, num_classes=n_classes, weights=weights)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
preds, _ = sess.run([pred, update_op])
if step % 100 == 0:
print('step {:d}'.format(step))
print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
def inference_for_keypoints(image_dir, image_list, checkpoint, output_dir):
# Create directory for output.
if not os.path.exists(output_dir):
os.system('mkdir ' + output_dir)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
image_dir,
image_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
IGNORE_LABEL,
IMG_MEAN,
coord)
image, label = reader.image, reader.label
# Add one batch dimension.
image_batch, label_batch = tf.expand_dims(image,
dim=0), tf.expand_dims(label,
dim=0)
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=NUM_CLASSES)
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output = tf.argmax(raw_output, dimension=3)
pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
tf.train.Saver(var_list=restore_var).restore(sess, checkpoint)
print('Successfully restored model parameters from {}'.format(checkpoint))
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Read list text file.
with open(image_list, 'r') as txtfile:
files = txtfile.readlines()
# Iterate for all images.
start = time.time()
for idx, file in enumerate(files):
# Print status.
if (idx + 1) % 1000 == 0 or (idx + 1) == len(files):
print(str(idx + 1) + ' / ' + str(len(files)))
# Inference for human keypoints, save the label image.
preds = sess.run(pred)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
io.imsave(output_dir + '/' + file.split('.')[0] + '.png',
np.uint8(np.squeeze(preds)))
print('Successfully processed all the images in %.2f seconds.' %
(time.time() - start))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
print(args)
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_mask
try:
os.makedirs('eval/mhdout')
except:
pass
event_end = Event()
queue_proc = Queue()
with open(args.data_list, 'r') as f:
list_of_all_lines = f.readlines()
f.seek(0)
dict = {}
for line in f:
if re.match(".*\\/(.*)\\.mhd.*", line).group(1) not in dict:
dict[re.match(".*\\/(.*)\\.mhd.*", line).group(1)] = []
dict[re.match(".*\\/(.*)\\.mhd.*", line).group(1)].append(line)
with tf.Graph().as_default():
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
(512, 512), # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord,
shuffle=False)
image_batch, label_batch = reader.dequeue(args.batch_size)
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output = tf.argmax(raw_output, dimension=3)
pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
# mIoU
pred = tf.reshape(pred, [
-1,
])
gt = tf.reshape(label_batch, [
-1,
])
# weights = tf.cast(tf.less_equal(gt, args.num_classes - 1),
# tf.int32) # Ignoring all labels greater than or equal to n_classes.
correct_pred = tf.equal(tf.cast(pred, tf.uint8), gt)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
accuracy_per_class = []
for i in xrange(0, args.num_classes):
curr_class = tf.constant(i, tf.uint8)
accuracy_per_class.append(
tf.reduce_mean(
tf.cast(
tf.gather(correct_pred,
tf.where(tf.equal(gt, curr_class))),
tf.float32)))
sess = tf.Session()
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
proc = Process(target=saving_process,
args=(queue_proc, event_end, args.num_classes,
args.data_dir, args.post_processing))
proc.start()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
acc_per_class = np.zeros(args.num_classes)
for sublist in [
list_of_all_lines[i:i + args.batch_size]
for i in xrange(0, len(list_of_all_lines), args.batch_size)
]:
preds, labels, acc, acc_per_class[0], acc_per_class[1], \
acc_per_class[2], acc_per_class[3], acc_per_class[4] = sess.run(
[raw_output, label_batch, accuracy, accuracy_per_class[0],
accuracy_per_class[1], accuracy_per_class[2], accuracy_per_class[3],
accuracy_per_class[4]])
for i, thing in enumerate(sublist):
regex_match = re.match(".*\\/(.*)\\.mhd_([0-9]+).*", thing)
# print(regex_match.group(1) + ' ' + str(regex_match.group(2)))
queue_proc.put(
(regex_match.group(1), int(regex_match.group(2)),
preds[i], labels[i], acc_per_class, acc,
len(dict[regex_match.group(1)])))
coord.request_stop()
coord.join(threads)
event_end.set()
proc.join()
def main():
"""Create the model and start the training."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
tf.set_random_seed(args.random_seed)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
input_size,
args.random_scale,
args.random_mirror,
args.ignore_label,
IMG_MEAN,
coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
# Create network.
net = DeepLabResNetModel({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes)
# For a small batch size, it is better to keep
# the statistics of the BN layers (running means and variances)
# frozen, and to not update the values provided by the pre-trained model.
# If is_training=True, the statistics will be updated during the training.
# Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
# if they are presented in var_list of the optimiser definition.
# Predictions.
raw_output = net.layers['fc1_voc12']
# Which variables to load. Running means and variances are not trainable,
# thus all_variables() should be restored.
# Restore all variables, or all except the last ones.
restore_var = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
trainable = [v for v in tf.trainable_variables() if 'fc1_voc12' in v.name] # Fine-tune only the last layers.
prediction = tf.reshape(raw_output, [-1, args.num_classes])
label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes)
gt = tf.reshape(label_proc, [-1, args.num_classes])
# Pixel-wise softmax loss.
loss = tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
reduced_loss = tf.reduce_mean(loss)
# Processed predictions.
raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3,])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Image summary.
images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images, IMG_MEAN], tf.uint8)
labels_summary = tf.py_func(decode_labels, [label_batch, args.save_num_images, args.num_classes], tf.uint8)
preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images, args.num_classes], tf.uint8)
total_summary = tf.summary.image('images',
tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]),
max_outputs=args.save_num_images) # Concatenate row-wise.
summary_writer = tf.summary.FileWriter(args.snapshot_dir,
graph=tf.get_default_graph())
# Define loss and optimisation parameters.
optimiser = tf.train.AdamOptimizer(learning_rate=args.learning_rate)
optim = optimiser.minimize(reduced_loss, var_list=trainable)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=40)
# Load variables if the checkpoint is provided.
if args.restore_from is not None:
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
if step % args.save_pred_every == 0:
loss_value, images, labels, preds, summary, _ = sess.run([reduced_loss, image_batch, label_batch, pred, total_summary, optim])
summary_writer.add_summary(summary, step)
save(saver, sess, args.snapshot_dir, step)
else:
loss_value, _ = sess.run([reduced_loss, optim])
duration = time.time() - start_time
print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
num_steps = file_len(os.path.join(args.img_path, args.data_list))
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
os.path.join(args.img_path, "texture"),
os.path.join(args.img_path, args.data_list),
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
255,
IMG_MEAN,
coord,
)
image, label = reader.image, reader.label
title = reader.queue[0]
image_batch, label_batch = (
tf.expand_dims(image, axis=0),
tf.expand_dims(label, axis=0),
) # Add one batch dimension.
# Create network.
net = DeepLabResNetModel({"data": image_batch},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers["fc1_voc12"]
before_argmax = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output_up = tf.argmax(before_argmax, dimension=3)
pred = tf.expand_dims(raw_output_up, axis=3)
hw_only = pred[0, :, :, 0]
class_0 = tf.where(tf.equal(hw_only, 0))
class_1 = tf.where(tf.equal(hw_only, 1))
class_2 = tf.where(tf.equal(hw_only, 2))
class_3 = tf.where(tf.equal(hw_only, 3))
class_4 = tf.where(tf.equal(hw_only, 4))
class_5 = tf.where(tf.equal(hw_only, 5))
class_6 = tf.where(tf.equal(hw_only, 6))
# Set up TF session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, args.model_weights)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
start_time = time.time()
os.makedirs(os.path.join(args.img_path, args.body_dir), exist_ok=True)
os.makedirs(os.path.join(args.img_path, args.vis_dir), exist_ok=True)
# write the header
rois_file = os.path.join(args.img_path, "rois.csv")
if os.path.isfile(rois_file):
print(f"The rois file {rois_file} already exists...")
ans = None
while all(ans != choice for choice in ("a", "o", "q")):
ans = input("Do you want to (a)ppend, (o)verwrite, or (q)uit? ")
if ans == "o":
print("Overwriting existing rois file...")
write_header(rois_file)
elif ans == "q":
sys.exit(1)
else:
write_header(rois_file)
# Perform inference.
t = trange(num_steps, desc="Inference progress", unit="img")
for step in t:
# run through the model
jpg_path, c0, c1, c2, c3, c4, c5, c6, raw_output_up_ = sess.run([
title,
class_0,
class_1,
class_2,
class_3,
class_4,
class_5,
class_6,
raw_output_up,
])
# == First, save the body segmentation ==
if not args.no_body:
# convert to a 2D compressed matrix, because we have a lot of 0's for the
# background
compressed = sparse.csr_matrix(np.squeeze(raw_output_up_))
fname = os.path.splitext(os.path.basename(str(jpg_path)))[0]
out = os.path.join(args.img_path, args.body_dir, fname)
sparse.save_npz(out, compressed)
# == Next, save the ROIs ==
if not args.no_rois:
img_id = extract_nums_only(fname)
for c in (c0, c1, c2, c3, c4, c5, c6):
try:
min_x = np.min(c[:, 1])
except ValueError:
min_x = None
try:
min_y = np.min(c[:, 0])
except ValueError:
min_y = None
try:
max_x = np.max(c[:, 1])
except ValueError:
max_x = None
try:
max_y = np.max(c[:, 0])
except ValueError:
max_y = None
# write out the stuff
with open(rois_file, "a") as f:
f.write(",".join((img_id, str(min_x), str(min_y),
str(max_x), str(max_y), "\n")))
# Save an image of the mask for our own reference every 1000 steps
if not args.no_vis and step % args.visualize_step == 0:
preds = np.expand_dims(raw_output_up_, axis=3)
msk = decode_labels(preds, num_classes=args.num_classes)
# the mask
im = Image.fromarray(msk[0])
# # Save the mask separately
# jpg_path = str(jpg_path).split('/')[-1].split('.')[0]
# out = os.path.join(args.vis_dir, jpg_path + '.png')
# im.save(out)
# Save the mask with background
img_orig = Image.open(jpg_path)
# create the final result using the mask and the original
img = np.array(im) * 0.9 + np.array(img_orig) * 0.7
# clip surpassed colors
img[img > 255] = 255
img = Image.fromarray(np.uint8(img))
out = os.path.join(args.img_path, args.vis_dir, fname + ".png")
img.save(out)
# # print('Image processed {}.png'.format(jpg_path))
t.set_description("Finished " + fname)
total_time = time.time() - start_time
print(
f"The output files have been saved to {args.img_path}/{args.body_dir}")
print(f"It took {total_time / num_steps} sec on each image.")
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '3'
"""Create the model and start the training."""
args = get_arguments()
# Get width (w) and height (h) of an input image
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
# Set random seed for reproducibility of the results
tf.set_random_seed(args.random_seed)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
input_size,
args.random_scale,
args.random_mirror,
args.ignore_label,
IMG_MEAN,
coord)
image_batch, label_batch, _ = reader.dequeue(args.batch_size) # 1st = images, 2nd = gt labels, 3rd = imu (we do not need IMU here)
# Create network.
with tf.variable_scope('', reuse=False):
net = wasr_NOIMU2({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes)
# For a small batch size, it is better to keep
# the statistics of the BN layers (running means and variances)
# frozen, and to not update the values provided by the pre-trained model.
# If is_training=True, the statistics will be updated during the training.
# Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
# if they are presented in var_list of the optimiser definition.
# Predictions.
# The layer at which the final output is located
raw_output = net.layers['fc1_voc12']
# The layer from which we extract features for computing water-obstacle separation loss
inthemiddle_output = net.layers['res4b20']
# Which variables to load. Running means and variances are not trainable,
# thus all_variables() should be restored.
restore_var = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
#all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma'] # all trainable variables
all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name] #new all trainable
fc_trainable = [v for v in all_trainable if 'fc' in v.name] # only variables from the ASPP module
arm_trainable = [v for v in all_trainable if 'arm_conv' in v.name] # only variables from the ARM module
ffm_trainable = [v for v in all_trainable if 'ffm_conv' in v.name] # only variables from the FFM module
batch_trainable = [v for v in tf.trainable_variables() if 'beta' in v.name or 'gamma' in v.name] # for batchnorms
conv_trainable = [v for v in all_trainable if 'fc' not in v.name and 'arm_conv' not in v.name and 'ffm_conv' not in v.name] # lr * 1.0
fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0
fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0
# Check if everything sums up correctly. Do the neccessary assertions
print("----")
print("Number of all trainable: {:d}\nNumber of fc trainable: {:d}\nNumber of conv trainable: {:d}\nNumber of ARM trainable: {:d}\nNumber of FFM trainable: {:d}\n".format(len(all_trainable), len(fc_trainable), len(conv_trainable), len(arm_trainable), len(ffm_trainable)))
assert(len(all_trainable) == len(fc_trainable) + len(conv_trainable) + len(arm_trainable) + len(ffm_trainable))
print("----")
print("Number of fc trainable: {:d}\nNumber of fc_w trainable: {:d}\nNumber of fc_b trainable: {:d}\n".format(len(fc_trainable), len(fc_w_trainable), len(fc_b_trainable)))
assert(len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))
# Predictions: ignoring all predictions with labels greater or equal than n_classes
raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False)
raw_gt = tf.reshape(label_proc, [-1,])
indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
prediction = tf.gather(raw_prediction, indices)
# Features loss from somewhere in the middle. This forces the network to separate water pixels from obstacles
loss_0 = cost_function_separate_water_obstacle(inthemiddle_output, label_batch)
#loss_0 = tf.Print(loss_0, [loss_0], 'Water separation loss ')
# Pixel-wise softmax cross entropy loss (This is the tensorflow implementation)
ce_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt))
ce_loss = tf.Print(ce_loss, [ce_loss], 'Default TF crossentropy loss ')
# Weight decay losses (l2 regularization)
l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
added_l2_losses = 10.e-2 * tf.add_n(l2_losses) # add together all l2 losses
added_l2_losses = tf.Print(added_l2_losses, [added_l2_losses], message="l2 losses ")
# Focal loss
focal_loss = focal_loss_cost(labels=gt, logits=prediction)
focal_loss = tf.Print(focal_loss, [focal_loss], message="Focal loss ")
# Add together all of the losses (focal loss, weight decay and water-separation loss)
reduced_loss = added_l2_losses + focal_loss + loss_0 #(10.e-2 * loss_0) # focal loss
#reduced_loss = added_l2_losses + ce_loss + loss_0 #(10.e-2 * loss_0) # normal cross entropy
# Define loss and optimisation parameters.
base_lr = tf.constant(args.learning_rate)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
# Learning rate modified based on the the current step
learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power)) # version 1
#learning_rate = tf.train.exponential_decay(base_lr, step_ph, 750, 0.7, staircase=True) # version 2
# RMSProp optimizer
opt_conv = tf.train.RMSPropOptimizer(learning_rate=learning_rate, decay=0.9, momentum=args.momentum, centered=True, name='RMSProp_conv')
opt_sp_w = tf.train.RMSPropOptimizer(learning_rate=learning_rate * 10, decay=0.9, momentum=args.momentum, centered=True, name='RMSProp_special_w')
opt_sp_b = tf.train.RMSPropOptimizer(learning_rate=learning_rate * 20, decay=0.9, momentum=args.momentum, centered=True, name='RMSProp_special_b')
# Momentum optimizer (original)
#opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
#opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
#opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)
# Minimization of optimizers for specific trainable variables...
op_c_all = opt_conv.minimize(reduced_loss, var_list=[conv_trainable, batch_trainable])
op_spc_w = opt_sp_w.minimize(reduced_loss, var_list=[fc_w_trainable, arm_trainable, ffm_trainable])
op_spc_b = opt_sp_b.minimize(reduced_loss, var_list=[fc_b_trainable])
train_op = tf.group(op_c_all, op_spc_w, op_spc_b)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=1)
# Load variables if the checkpoint is provided.
#if args.restore_from is not None:
# loader = tf.train.Saver(var_list=restore_var)
# load(loader, sess, args.restore_from)
# RESTORE PARTIAL WEIGHTS (which are available)
restored_vars = get_tensors_in_checkpoint_file(file_name=args.restore_from, restore_last_bool=args.not_restore_last)
tensors_to_load = build_tensors_in_checkpoint_file(restored_vars)
loader = tf.train.Saver(var_list=tensors_to_load)
loader.restore(sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
feed_dict = { step_ph : step }
loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict)
if step % args.save_pred_every == 0:
save(saver, sess, args.snapshot_dir, step)
duration = time.time() - start_time
print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
# join threads
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
input_size, # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord)
# image, label = reader.image, reader.label
image_batch, label_batch = reader.dequeue(args.batch_size)
# image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(label, dim=0) # Add one batch dimension.
image_name_list = reader.image_list
x = tf.placeholder(tf.float32, shape=(BATCH_SIZE, h, w, 3))
y = tf.placeholder(tf.float32, shape=(BATCH_SIZE, h, w, 1))
# Create network.
net = DeepLabResNetModel({'data': x},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
label_proc = prepare_label(y,
tf.stack(raw_output.get_shape()[1:3]),
num_classes=args.num_classes,
one_hot=False) # [batch_size, h, w]
raw_gt = tf.reshape(label_proc, [
-1,
])
indices_0 = tf.squeeze(
tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1)
gt_0 = tf.cast(tf.gather(raw_gt, indices_0), tf.int32)
prediction = tf.gather(raw_prediction, indices_0)
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction,
labels=gt_0)
img_mean_t = tf.convert_to_tensor(IMG_MEAN, dtype=tf.float32)
if args.attack == 'fgs':
x_adv = fgs(x, loss, args.eps, img_mean_t, input_size, BATCH_SIZE,
MASK_FLAG, args.targeted)
elif args.attack == 'ifgs':
x_adv = fgs(x, loss, args.beta, img_mean_t, input_size, BATCH_SIZE,
MASK_FLAG, args.targeted)
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output = tf.argmax(raw_output, dimension=3)
pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
# mIoU
pred = tf.reshape(pred, [
-1,
])
gt = tf.reshape(label_batch, [
-1,
])
indices = tf.squeeze(tf.where(tf.less_equal(gt, args.num_classes - 1)),
1) ## ignore all labels >= num_classes
gt = tf.cast(tf.gather(gt, indices), tf.int32)
pred = tf.gather(pred, indices)
mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(
pred, gt, num_classes=args.num_classes)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over evaluation steps.
image_list, label_list = read_labeled_image_list(args.data_dir,
args.data_list)
for step in range(args.num_steps):
X = image_converter(image_list, step)
if args.targeted:
Y = np.zeros((BATCH_SIZE, 321, 321, 1))
else:
Y = label_converter(label_list, step)
if args.attack == 'fgs':
preds, _, X_adv = sess.run([pred, update_op, x_adv],
feed_dict={
x: X,
y: Y
})
elif args.attack == 'ifgs':
X_adv = X
for i in range(args.iter):
preds, _, X_adv, loss_v = sess.run(
[pred, update_op, x_adv, loss], feed_dict={
x: X_adv,
y: Y
})
r = np.clip(X_adv - X, -args.eps, args.eps)
X_adv = X + r
# preds, _ = sess.run([pred, update_op], feed_dict={x: X_adv})
if SAVE_FLAG is not None:
image_saver(X_adv, X, image_name_list[step], args.attack, args.eps,
args.targeted)
if step % 100 == 0:
print('step {:d}'.format(step))
print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
input_size, # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord)
image, label = reader.image, reader.label
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(label, dim=0) # Add one batch dimension.
# Create network.
#net = DeepLabResNetModel({'data': image_batch}, is_training=False, num_classes=args.num_classes)
do_rate = tf.placeholder(dtype=tf.float32, name='dropout_rate')
sizes, padded_input, raw_output = alexfcn(image_batch, NUM_CLASSES, do_rate, reuse=False)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3,])
raw_output = tf.argmax(raw_output, dimension=3)
pred_images = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
# mIoU
pred = tf.reshape(pred_images, [-1,])
gt = tf.reshape(label_batch, [-1,])
weights = tf.cast(tf.less_equal(gt, args.num_classes - 1), tf.int32) # Ignoring all labels greater than or equal to n_classes.
mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred, gt, num_classes=args.num_classes, weights=weights)
# Image summary.
images_summary = tf.py_func(inv_preprocess, [image_batch, 1, IMG_MEAN], tf.uint8)
labels_summary = tf.py_func(decode_labels, [label_batch, 1, args.num_classes], tf.uint8)
preds_summary = tf.py_func(decode_labels, [pred_images, 1, args.num_classes], tf.uint8)
total_summary = tf.summary.image('images',
tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]),
max_outputs=1) # Concatenate row-wise.
summary_writer = tf.summary.FileWriter(args.snapshot_dir,
graph=tf.get_default_graph())
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
summary,io,lo,preds, _ = sess.run([total_summary, image_batch, label_batch,pred, update_op])
if step % 100 == 0:
print('step {:d}'.format(step))
summary_writer.add_summary(summary, step)
print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
coord)
image, label = reader.image, reader.label
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(
label, dim=0) # Add one batch dimension.
h_orig, w_orig = tf.to_float(tf.shape(image_batch)[1]), tf.to_float(
tf.shape(image_batch)[2])
image_batch075 = tf.image.resize_images(
image_batch,
tf.pack([
tf.to_int32(tf.mul(h_orig, 0.75)),
tf.to_int32(tf.mul(w_orig, 0.75))
]))
image_batch05 = tf.image.resize_images(
image_batch,
tf.pack([
tf.to_int32(tf.mul(h_orig, 0.5)),
tf.to_int32(tf.mul(w_orig, 0.5))
]))
# Create network.
with tf.variable_scope('', reuse=False):
net = DeepLabResNetModel({'data': image_batch}, is_training=False)
with tf.variable_scope('', reuse=True):
net075 = DeepLabResNetModel({'data': image_batch075},
is_training=False)
with tf.variable_scope('', reuse=True):
net05 = DeepLabResNetModel({'data': image_batch05}, is_training=False)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output100 = net.layers['fc1_voc12']
raw_output075 = tf.image.resize_images(net075.layers['fc1_voc12'],
tf.shape(raw_output100)[1:3, ])
raw_output05 = tf.image.resize_images(net05.layers['fc1_voc12'],
tf.shape(raw_output100)[1:3, ])
raw_output = tf.reduce_max(tf.stack(
[raw_output100, raw_output075, raw_output05]),
axis=0)
pred = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3, ])
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Get the color palette
palette = voc_colour_map()
f = open(args.data_list, 'r')
image_names = []
for line in f:
image_names.append(line)
# Iterate over training steps.
for step in range(args.num_steps):
preds = sess.run(pred)
preds = np.argmax(preds, axis=3).squeeze().astype(np.uint8)
im = Image.fromarray(preds)
im.putpalette(palette)
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
mask_name = image_names[step].strip("\n").rsplit('/', 1)[1].replace(
'jpg', 'png')
im.save(args.save_dir + "/" + mask_name)
print('The segmentation masks have been saved to {}'.format(args.save_dir))
coord.request_stop()
coord.join(threads)
def main():
args, preds = get_arguments(), []
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.target_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord)
image_orig = reader.image
for rots in range(4):
image = tf.image.rot90(image_orig, k=rots)
image_batch = tf.expand_dims(image, dim=0) # Add one batch dimension.
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
tf.get_variable_scope().reuse_variables()
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
# CRF.
if args.crf:
inv_image = tf.py_func(inv_preprocess, [image_batch, 1, IMG_MEAN],
tf.uint8)
raw_output = tf.py_func(dense_crf,
[tf.nn.softmax(raw_output), inv_image],
tf.float32)
# Rotate to original
raw_output = tf.image.rot90(tf.squeeze(raw_output), k=(4 - rots))
raw_output = tf.expand_dims(raw_output, dim=0)
preds.append(raw_output)
if not args.augment:
break
pred = tf.reduce_mean(tf.concat(preds, axis=0), axis=0)
if args.heatmap < 0:
pred = tf.argmax(tf.expand_dims(pred, dim=0), dimension=3)
pred = tf.cast(tf.expand_dims(pred, dim=3), tf.int32)
else:
pred = tf.expand_dims(pred[:, :, args.heatmap], dim=0)
pred = tf.cast(pred, tf.int32)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
# Iterate over training steps.
for step in tqdm(range(args.num_steps)):
preds, img_path = sess.run([pred, reader.queue[0]])
if args.heatmap < 0:
preds = decode_labels(preds, num_classes=args.num_classes)
im = Image.fromarray(preds[0])
else:
pr = np.zeros((1, preds.shape[1], preds.shape[2], 3))
preds += abs(np.min(preds))
preds *= 255 / np.max(preds)
pr[:, :, :, 0] = preds
pr[:, :, :, 1] = preds
pr[:, :, :, 2] = preds
im = Image.fromarray(pr[0].astype('uint8'))
img_name = os.path.split(img_path)[-1]
im.save(os.path.join(args.save_dir + img_name))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
# remove_huge_images(args.data_list, args.img_path)
num_steps = file_len(args.data_list)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
print(args.img_path, ' ', file_len(args.data_list))
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.img_path,
args.data_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
255,
IMG_MEAN,
coord)
image, label = reader.image, reader.label
title = reader.queue[0]
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(
label, dim=0) # Add one batch dimension.
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
fc1_voc12_layer = net.layers['fc1_voc12']
raw_output_up = tf.image.resize_bilinear(fc1_voc12_layer,
tf.shape(image_batch)[1:3, ])
# uncomment to see only stock segmentation
# raw_output_up = tf.slice(raw_output_up, [0,0,0,0], [-1,-1,-1,7])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Set up TF session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, args.model_weights)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
start_time = time.time()
os.makedirs(args.save_dir, exist_ok=True)
path_parts = args.img_path.split("/")
if path_parts[-1].strip() == "":
path_parts = path_parts[:-1]
if path_parts[0] == "":
path_parts[0] = "/"
bottleneck_dir = os.path.join(*path_parts[:-1],
path_parts[-1] + "_hp_bottlenecks")
os.makedirs(bottleneck_dir, exist_ok=True)
# Perform inference.
for step in range(num_steps):
jpg_name = None
try:
preds, jpg_path, fc1_voc12_val = sess.run(
[pred, title, fc1_voc12_layer])
msk = decode_labels(preds, num_classes=args.num_classes)
im = Image.fromarray(msk[0])
img_o = Image.open(jpg_path)
jpg_path = str(jpg_path)
jpg_name = Path(jpg_path).name.split('.')[0]
img = np.array(im) * 0.9 + np.array(img_o) * 0.7
img[img > 255] = 255
img = Image.fromarray(np.uint8(img))
img.save(os.path.join(args.save_dir, str(jpg_name + '.png')))
img_bgr = cv2.cvtColor(np.array(img_o), cv2.COLOR_BGR2RGB)
cv2.imwrite(
os.path.join(args.save_dir,
"stacked_" + str(jpg_name + '.png')),
np.hstack([img_bgr, im]))
bottleneck_path = os.path.join(bottleneck_dir,
jpg_name + "_hp_bottleneck.h5")
with h5py.File(bottleneck_path, "w") as bottleneck_file:
bottleneck_file.create_dataset("fc1_voc12", data=fc1_voc12_val)
print('Image processed {}.png'.format(jpg_name))
print(
'Wrote human parsing bottleneck to {}'.format(bottleneck_path))
except Exception as e:
print(e)
print('Image failed: ', jpg_name)
total_time = time.time() - start_time
print('The output files have been saved to {}'.format(args.save_dir))
print('It took {} sec on each image.'.format(total_time / num_steps))
def val():
def get_arguments():
"""Parse all the arguments provided from the CLI.
Returns:
A list of parsed arguments.
"""
parser = argparse.ArgumentParser(description="DeepLabLFOV Network")
parser.add_argument(
"--data-dir",
type=str,
default=DATA_DIRECTORY,
help="Path to the directory containing the PASCAL VOC dataset.")
parser.add_argument(
"--data-list",
type=str,
default=VAL_DATA_LIST_PATH,
help="Path to the file listing the images in the dataset.")
parser.add_argument(
"--ignore-label",
type=int,
default=IGNORE_LABEL,
help="The index of the label to ignore during the training.")
parser.add_argument(
"--num-classes",
type=int,
default=NUM_CLASSES,
help="Number of classes to predict (including background).")
parser.add_argument("--num-steps",
type=int,
default=VAL_NUM_STEPS,
help="Number of images in the validation set.")
parser.add_argument("--restore-from",
type=str,
default=SNAPSHOT_DIR,
help="Where restore model parameters from.")
return parser.parse_args()
def load(saver, sess, ckpt_path):
'''Load trained weights.
Args:
saver: TensorFlow saver object.
sess: TensorFlow session.
ckpt_path: path to checkpoint file with parameters.
'''
if os.path.isdir(ckpt_path):
ckpt = tf.train.get_checkpoint_state(ckpt_path)
ckpt_path = ckpt.model_checkpoint_path
saver.restore(sess, ckpt_path)
print("Restored model parameters from {}".format(ckpt_path))
with tf.variable_scope(name_or_scope='', reuse=tf.AUTO_REUSE):
"""Create the model and start the evaluation process."""
args = get_arguments()
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
[321, 321], # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord)
image, label = reader.image, reader.label
image_batch, label_batch = tf.expand_dims(
image, dim=0), tf.expand_dims(label,
dim=0) # Add one batch dimension.
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output = tf.argmax(raw_output, dimension=3)
pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
# mIoU
pred = tf.reshape(pred, [
-1,
])
gt = tf.reshape(label_batch, [
-1,
])
# tensorflow 1.3.0 conflict
# weights = tf.cast(tf.less_equal(gt, args.num_classes - 1), tf.int32) # Ignoring all labels greater than or equal to n_classes.
# mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred, gt, num_classes=args.num_classes, weights=weights)
indices = tf.squeeze(tf.where(tf.less_equal(gt, args.num_classes - 1)),
1) # ignore all labels >= num_classes
gt = tf.cast(tf.gather(gt, indices), tf.int32)
pred = tf.gather(pred, indices)
mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(
pred, gt, num_classes=args.num_classes)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# if args.restore_from is not None:
# if os.path.isdir(args.restore_from):
# ckpt = tf.train.get_checkpoint_state(args.restore_from)
# loader.restore(sess, ckpt.model_checkpoint_path)
# else:
# load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
preds, _ = sess.run([pred, update_op])
if step % 100 == 0:
print('step {:d}'.format(step))
print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the training."""
args = get_arguments()
print(args)
if args.not_restore_last:
try:
shutil.rmtree(args.snapshot_dir)
except Exception as e:
print(e)
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_mask
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
tf.set_random_seed(args.random_seed)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
mode = tf.placeholder(tf.bool, shape=())
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
input_size,
args.random_scale,
args.random_mirror,
args.ignore_label,
IMG_MEAN,
coord)
image_batch_train, label_batch_train = reader.dequeue(args.batch_size)
with tf.name_scope("val_inputs"):
reader = ImageReader(
args.data_dir,
args.val_data_list,
input_size,
args.random_scale,
args.random_mirror,
args.ignore_label,
IMG_MEAN,
coord)
image_batch_val, label_batch_val = reader.dequeue(args.batch_size)
image_batch = tf.cond(mode, lambda: image_batch_train, lambda: image_batch_val)
label_batch = tf.cond(mode, lambda: label_batch_train, lambda: label_batch_val)
# Create network.
net = DeepLabResNetModel({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes)
# For a small batch size, it is better to keep
# the statistics of the BN layers (running means and variances)
# frozen, and to not update the values provided by the pre-trained model.
# If is_training=True, the statistics will be updated during the training.
# Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
# if they are presented in var_list of the optimiser definition.
# Predictions.
raw_output = net.layers['fc1_voc12']
# Which variables to load. Running means and variances are not trainable,
# thus all_variables() should be restored.
restore_var = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name]
fc_trainable = [v for v in all_trainable if 'fc' in v.name]
conv_trainable = [v for v in all_trainable if 'fc' not in v.name] # lr * 1.0
fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0
fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0
assert (len(all_trainable) == len(fc_trainable) + len(conv_trainable))
assert (len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))
# Predictions: ignoring all predictions with labels greater or equal than n_classes
raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes,
one_hot=False) # [batch_size, h, w]
raw_gt = tf.reshape(label_proc, [-1, ])
indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
prediction = tf.gather(raw_prediction, indices)
output_op = tf.cast(tf.argmax(prediction, axis=-1), tf.int32)
correct_pred = tf.equal(output_op, gt)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Pixel-wise softmax loss.
loss = []
accuracy_per_class = []
softmax_weights_per_class = tf.constant(LUNA16_softmax_weights, dtype=tf.float32)
for i in xrange(0, args.num_classes):
curr_class = tf.constant(i, tf.int32)
loss.append(softmax_weights_per_class[i] * tf.losses.sparse_softmax_cross_entropy(logits=prediction, labels=gt,
weights=tf.where(
tf.equal(gt, curr_class),
tf.zeros_like(gt),
tf.ones_like(gt))))
accuracy_per_class.append(
tf.reduce_mean(tf.cast(tf.gather(correct_pred, tf.where(tf.equal(gt, curr_class))), tf.float32)))
l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
reduced_loss = tf.reduce_mean(tf.stack(loss)) + tf.add_n(l2_losses)
# Processed predictions: for visualisation.
raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3, ])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Image summary.
reduced_loss_train = tf.Variable(0, trainable=False, dtype=tf.float32)
accuracy_train = tf.Variable(0, trainable=False, dtype=tf.float32)
reduced_loss_val = tf.Variable(0, trainable=False, dtype=tf.float32)
accuracy_val = tf.Variable(0, trainable=False, dtype=tf.float32)
reduced_loss_train = tf.cond(mode, lambda: tf.assign(reduced_loss_train, reduced_loss), lambda: reduced_loss_train)
accuracy_train = tf.cond(mode, lambda: tf.assign(accuracy_train, accuracy), lambda: accuracy_train)
reduced_loss_val = tf.cond(mode, lambda: reduced_loss_val, lambda: tf.assign(reduced_loss_val, reduced_loss))
accuracy_val = tf.cond(mode, lambda: accuracy_val, lambda: tf.assign(accuracy_val, accuracy))
accuracy_per_class_train = []
accuracy_per_class_val = []
for i in xrange(0, args.num_classes):
temp_train_var = tf.Variable(0, trainable=False, dtype=tf.float32)
temp_val_var = tf.Variable(0, trainable=False, dtype=tf.float32)
accuracy_per_class_train.append(
tf.cond(mode, lambda: tf.assign(temp_train_var, accuracy_per_class[i]), lambda: temp_train_var))
accuracy_per_class_val.append(
tf.cond(mode, lambda: temp_val_var, lambda: tf.assign(temp_val_var, accuracy_per_class[i])))
accuracy_output = tf.cond(mode, lambda: accuracy_train, lambda: accuracy_val)
loss_output = tf.cond(mode, lambda: reduced_loss_train, lambda: reduced_loss_val)
tf.summary.scalar("Loss", loss_output, collections=['all'])
tf.summary.scalar("Accuracy", accuracy_output, collections=['all'])
images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images, IMG_MEAN], tf.uint8)
labels_summary = tf.py_func(decode_labels, [label_batch, args.save_num_images, args.num_classes], tf.uint8)
preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images, args.num_classes], tf.uint8)
counter_no_reset = tf.Variable(tf.zeros([2, args.num_classes]), trainable=False, dtype=tf.float32)
counter = tf.Variable(tf.zeros([2, args.num_classes]), trainable=False, dtype=tf.float32)
counter_no_reset_val = tf.Variable(tf.zeros([2, args.num_classes]), trainable=False, dtype=tf.float32)
counter_val = tf.Variable(tf.zeros([2, args.num_classes]), trainable=False, dtype=tf.float32)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
counter, counter_no_reset = tf.cond(mode, lambda: tf.py_func(update_IoU, [tf.squeeze(pred, axis=-1),
tf.squeeze(label_batch, axis=-1), counter,
counter_no_reset, args.num_classes,
args.batch_size, step_ph,
args.save_pred_every],
[tf.float32, tf.float32]),
lambda: [counter, counter_no_reset])
counter_val, counter_no_reset_val = tf.cond(mode,
lambda: [counter_val, counter_no_reset_val],
lambda: tf.py_func(update_IoU, [tf.squeeze(pred, axis=-1),
tf.squeeze(label_batch, axis=-1),
counter_val, counter_no_reset_val,
args.num_classes, args.batch_size,
step_ph, args.save_pred_every],
[tf.float32, tf.float32]))
eps = tf.constant(1e-10, dtype=tf.float32)
IoU_summary = counter[0] / tf.add(eps, counter[1])
IoU_summary_no_reset = counter_no_reset[0] / tf.add(eps, counter_no_reset[1])
Val_IoU_summary = counter_val[0] / tf.add(eps, counter_val[1])
Val_IoU_summary_no_reset = counter_no_reset_val[0] / tf.add(eps, counter_no_reset_val[1])
mIoU = tf.reduce_mean(IoU_summary)
mIoU_no_reset = tf.reduce_mean(IoU_summary_no_reset)
Val_mIoU = tf.reduce_mean(Val_IoU_summary)
Val_mIoU_no_reset = tf.reduce_mean(Val_IoU_summary_no_reset)
IoU_summary_output_intermed = tf.cond(mode, lambda: IoU_summary, lambda: Val_IoU_summary)
IoU_summary_no_reset_output_intermed = tf.cond(mode, lambda: IoU_summary_no_reset, lambda: Val_IoU_summary_no_reset)
accuracy_per_class_output_intermed = tf.cond(mode, lambda: accuracy_per_class_train, lambda: accuracy_per_class_val)
class_number = tf.placeholder(tf.int32, shape=())
IoU_summary_output = tf.gather(IoU_summary_output_intermed, class_number)
IoU_summary_no_reset_output = tf.gather(IoU_summary_no_reset_output_intermed, class_number)
accuracy_per_class_output = tf.gather(accuracy_per_class_output_intermed, class_number)
tf.summary.scalar("IoU per class", IoU_summary_output, collections=['per_class'])
tf.summary.scalar("IoU (no reset) per class", IoU_summary_no_reset_output, collections=['per_class'])
tf.summary.scalar("Accuracy per class", accuracy_per_class_output, collections=['per_class'])
mIoU_output = tf.cond(mode, lambda: mIoU, lambda: Val_mIoU)
mIoU_no_reset_output = tf.cond(mode, lambda: mIoU_no_reset, lambda: Val_mIoU_no_reset)
tf.summary.scalar("mIoU", mIoU_output, collections=['all'])
tf.summary.scalar("mIoU no reset", mIoU_no_reset_output, collections=['all'])
tf.summary.image('images',
tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]),
max_outputs=args.save_num_images, collections=['all']) # Concatenate row-wise.
summary_writer_train = tf.summary.FileWriter(os.path.join(args.snapshot_dir, 'train_all'),
graph=tf.get_default_graph())
summary_writer_val = tf.summary.FileWriter(os.path.join(args.snapshot_dir, 'val_all'),
graph=tf.get_default_graph())
summary_writer_per_class_val = []
summary_writer_per_class_train = []
for i in xrange(args.num_classes):
summary_writer_per_class_train.append(
tf.summary.FileWriter(os.path.join(args.snapshot_dir, 'train_class_' + str(i)),
graph=tf.get_default_graph()))
summary_writer_per_class_val.append(
tf.summary.FileWriter(os.path.join(args.snapshot_dir, 'val_class_' + str(i)),
graph=tf.get_default_graph()))
# Define loss and optimisation parameters.
base_lr = tf.constant(args.learning_rate)
learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
tf.summary.scalar("learning_rate", learning_rate, collections=['all'])
all_summary = tf.summary.merge_all('all')
per_class_summary = tf.summary.merge_all('per_class')
opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)
grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable)
grads_conv = grads[:len(conv_trainable)]
grads_fc_w = grads[len(conv_trainable): (len(conv_trainable) + len(fc_w_trainable))]
grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):]
train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))
train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)
# Set up tf session and initialize variables.
sess = tf.Session()
init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10)
# Load variables if the checkpoint is provided.
if args.restore_from is not None:
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in xrange(1, args.num_steps + 1):
start_time = time.time()
# mode False -> val, mode True -> train
if step % args.save_pred_every == 0:
feed_dict = {step_ph: step, mode: False, class_number: step % args.num_classes}
acc, loss_value, mI, mINR, _, _, _, summary_v_this_class, summary_v = sess.run(
[accuracy_output, loss_output, mIoU_output, mIoU_no_reset_output, accuracy_per_class_output,
IoU_summary_output, IoU_summary_no_reset_output, per_class_summary, all_summary], feed_dict=feed_dict)
save(saver, sess, args.snapshot_dir, step)
summary_writer_val.add_summary(summary_v, step)
summary_writer_per_class_val[step % args.num_classes].add_summary(summary_v_this_class, step)
duration = time.time() - start_time
print(
'step {:d} \t Val_loss = {:.3f}, Val_acc = {:.3f}, Val_mIoU = {:.6f}, Val_mIoU_no_reset = {:.6f}, ({:.3f} sec/step)'.format(
step, loss_value, acc, mI, mINR, duration))
else:
feed_dict = {step_ph: step, mode: True, class_number: step % args.num_classes}
acc, loss_value, mI, mINR, _, _, _, summary_t_this_class, summary_t, _ = sess.run(
[accuracy_output, loss_output, mIoU_output, mIoU_no_reset_output, accuracy_per_class_output,
IoU_summary_output, IoU_summary_no_reset_output, per_class_summary, all_summary, train_op],
feed_dict=feed_dict)
summary_writer_train.add_summary(summary_t, step)
summary_writer_per_class_train[step % args.num_classes].add_summary(summary_t_this_class, step)
duration = time.time() - start_time
print(
'step {:d} \t loss = {:.3f}, acc = {:.3f}, mIoU = {:.6f}, mIoU_no_reset = {:.6f}, ({:.3f} sec/step)'.format(
step, loss_value, acc, mI, mINR, duration))
coord.request_stop()
# tboard_proc.kill()
coord.join(threads)
def main():
"""Create the model and start the training."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
tf.set_random_seed(args.random_seed)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
input_size,
args.random_scale,
args.random_mirror,
args.ignore_label,
IMG_MEAN,
coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
image_batch075 = tf.image.resize_images(image_batch, [int(h * 0.75), int(w * 0.75)])
image_batch05 = tf.image.resize_images(image_batch, [int(h * 0.5), int(w * 0.5)])
# Create network.
with tf.variable_scope('', reuse=False):
net = DeepLabResNetModel({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes)
with tf.variable_scope('', reuse=True):
net075 = DeepLabResNetModel({'data': image_batch075}, is_training=args.is_training, num_classes=args.num_classes)
with tf.variable_scope('', reuse=True):
net05 = DeepLabResNetModel({'data': image_batch05}, is_training=args.is_training, num_classes=args.num_classes)
# For a small batch size, it is better to keep
# the statistics of the BN layers (running means and variances)
# frozen, and to not update the values provided by the pre-trained model.
# If is_training=True, the statistics will be updated during the training.
# Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
# if they are presented in var_list of the optimiser definition.
# Predictions.
raw_output100 = net.layers['fc1_voc12']
raw_output075 = net075.layers['fc1_voc12']
raw_output05 = net05.layers['fc1_voc12']
raw_output = tf.reduce_max(tf.stack([raw_output100,
tf.image.resize_images(raw_output075, tf.shape(raw_output100)[1:3,]),
tf.image.resize_images(raw_output05, tf.shape(raw_output100)[1:3,])]), axis=0)
# Which variables to load. Running means and variances are not trainable,
# thus all_variables() should be restored.
restore_var = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name]
fc_trainable = [v for v in all_trainable if 'fc' in v.name]
conv_trainable = [v for v in all_trainable if 'fc' not in v.name] # lr * 1.0
fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0
fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0
assert(len(all_trainable) == len(fc_trainable) + len(conv_trainable))
assert(len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))
# Predictions: ignoring all predictions with labels greater or equal than n_classes
raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
raw_prediction100 = tf.reshape(raw_output100, [-1, args.num_classes])
raw_prediction075 = tf.reshape(raw_output075, [-1, args.num_classes])
raw_prediction05 = tf.reshape(raw_output05, [-1, args.num_classes])
label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False) # [batch_size, h, w]
label_proc075 = prepare_label(label_batch, tf.stack(raw_output075.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False)
label_proc05 = prepare_label(label_batch, tf.stack(raw_output05.get_shape()[1:3]), num_classes=args.num_classes, one_hot=False)
raw_gt = tf.reshape(label_proc, [-1,])
raw_gt075 = tf.reshape(label_proc075, [-1,])
raw_gt05 = tf.reshape(label_proc05, [-1,])
indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1)
indices075 = tf.squeeze(tf.where(tf.less_equal(raw_gt075, args.num_classes - 1)), 1)
indices05 = tf.squeeze(tf.where(tf.less_equal(raw_gt05, args.num_classes - 1)), 1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
gt075 = tf.cast(tf.gather(raw_gt075, indices075), tf.int32)
gt05 = tf.cast(tf.gather(raw_gt05, indices05), tf.int32)
prediction = tf.gather(raw_prediction, indices)
prediction100 = tf.gather(raw_prediction100, indices)
prediction075 = tf.gather(raw_prediction075, indices075)
prediction05 = tf.gather(raw_prediction05, indices05)
# Pixel-wise softmax loss.
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
loss100 = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction100, labels=gt)
loss075 = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction075, labels=gt075)
loss05 = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction05, labels=gt05)
l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
reduced_loss = tf.reduce_mean(loss) + tf.reduce_mean(loss100) + tf.reduce_mean(loss075) + tf.reduce_mean(loss05) + tf.add_n(l2_losses)
# Processed predictions: for visualisation.
raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3,])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Image summary.
images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images, IMG_MEAN], tf.uint8)
labels_summary = tf.py_func(decode_labels, [label_batch, args.save_num_images, args.num_classes], tf.uint8)
preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images, args.num_classes], tf.uint8)
total_summary = tf.summary.image('images',
tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]),
max_outputs=args.save_num_images) # Concatenate row-wise.
summary_writer = tf.summary.FileWriter(args.snapshot_dir,
graph=tf.get_default_graph())
# Define loss and optimisation parameters.
base_lr = tf.constant(args.learning_rate)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)
# Define a variable to accumulate gradients.
accum_grads = [tf.Variable(tf.zeros_like(v.initialized_value()),
trainable=False) for v in conv_trainable + fc_w_trainable + fc_b_trainable]
# Define an operation to clear the accumulated gradients for next batch.
zero_op = [v.assign(tf.zeros_like(v)) for v in accum_grads]
# Compute gradients.
grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable)
# Accumulate and normalise the gradients.
accum_grads_op = [accum_grads[i].assign_add(grad / args.grad_update_every) for i, grad in
enumerate(grads)]
grads_conv = accum_grads[:len(conv_trainable)]
grads_fc_w = accum_grads[len(conv_trainable) : (len(conv_trainable) + len(fc_w_trainable))]
grads_fc_b = accum_grads[(len(conv_trainable) + len(fc_w_trainable)):]
# Apply the gradients.
train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))
train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10)
# Load variables if the checkpoint is provided.
if args.restore_from is not None:
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
feed_dict = { step_ph : step }
loss_value = 0
# Clear the accumulated gradients.
sess.run(zero_op, feed_dict=feed_dict)
# Accumulate gradients.
for i in range(args.grad_update_every):
_, l_val = sess.run([accum_grads_op, reduced_loss], feed_dict=feed_dict)
loss_value += l_val
# Normalise the loss.
loss_value /= args.grad_update_every
# Apply gradients.
if step % args.save_pred_every == 0:
images, labels, summary, _ = sess.run([image_batch, label_batch, total_summary, train_op], feed_dict=feed_dict)
summary_writer.add_summary(summary, step)
save(saver, sess, args.snapshot_dir, step)
else:
sess.run(train_op, feed_dict=feed_dict)
duration = time.time() - start_time
print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
None,
False,
## args preprocessing: random_scale, crop, mirror
coord)
image, label = reader.image, reader.label
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(
label, dim=0) # add one batch dimension.
# Create network.
net = DeepLabResNetModel({'data': image_batch})
# Which variables to load.
trainable = tf.trainable_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output = tf.argmax(raw_output, dimension=3)
pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
# mIoU
mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred,
label_batch,
num_classes=21)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.initialize_all_variables()
sess.run(init)
sess.run(tf.initialize_local_variables())
# Load weights.
saver = tf.train.Saver(var_list=trainable)
if args.restore_from is not None:
load(saver, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
#mIoU_value = sess.run([mIoU])
#_ = update_op.eval(session=sess)
preds, _ = sess.run([pred, update_op])
# make the below optional
#img = decode_labels(preds[0, :, :, 0])
#im = Image.fromarray(img)
#im.save(args.save_dir + str(step) + '.png')
if step % 100 == 0:
print('step {:d} \t'.format(step))
print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the training."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(args.data_dir, args.data_list, input_size,
args.random_scale, coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=args.is_training)
# For a small batch size, it is better to keep
# the statistics of the BN layers (running means and variances)
# frozen, and to not update the values provided by the pre-trained model.
# If is_training=True, the statistics will be updated during the training.
# Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
# if they are presented in var_list of the optimiser definition.
# Predictions.
raw_output = net.layers['fc1_voc12']
# Which variables to load. Running means and variances are not trainable,
# thus all_variables() should be restored.
restore_var = tf.global_variables()
trainable = tf.trainable_variables()
prediction = tf.reshape(raw_output, [-1, n_classes])
label_proc = prepare_label(label_batch,
tf.pack(raw_output.get_shape()[1:3]))
gt = tf.reshape(label_proc, [-1, n_classes])
# Pixel-wise softmax loss.
loss = tf.nn.softmax_cross_entropy_with_logits(prediction, gt)
reduced_loss = tf.reduce_mean(loss)
# Processed predictions.
raw_output_up = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Image summary.
images_summary = tf.py_func(inv_preprocess,
[image_batch, args.save_num_images], tf.uint8)
labels_summary = tf.py_func(decode_labels,
[label_batch, args.save_num_images], tf.uint8)
preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images],
tf.uint8)
total_summary = tf.summary.image(
'images',
tf.concat(2, [images_summary, labels_summary, preds_summary]),
max_outputs=args.save_num_images) # Concatenate row-wise.
summary_writer = tf.summary.FileWriter(args.snapshot_dir)
# Define loss and optimisation parameters.
optimiser = tf.train.AdamOptimizer(learning_rate=args.learning_rate)
optim = optimiser.minimize(reduced_loss, var_list=trainable)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=restore_var, max_to_keep=40)
# Load variables if the checkpoint is provided.
if args.restore_from is not None:
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
if step % args.save_pred_every == 0:
loss_value, images, labels, preds, summary, _ = sess.run([
reduced_loss, image_batch, label_batch, pred, total_summary,
optim
])
summary_writer.add_summary(summary, step)
save(saver, sess, args.snapshot_dir, step)
else:
loss_value, _ = sess.run([reduced_loss, optim])
duration = time.time() - start_time
print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(
step, loss_value, duration))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
#image list/ label list
f = open(DATA_ID_PATH, 'r')
maskslist = []
for line in f:
mask = line.strip("\n")
maskslist.append(mask)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord)
image, label = reader.image, reader.label
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(
label, dim=0) # Add one batch dimension.
h_orig, w_orig = tf.to_float(tf.shape(image_batch)[1]), tf.to_float(
tf.shape(image_batch)[2])
image_batch075 = tf.image.resize_images(
image_batch,
tf.stack([
tf.to_int32(tf.multiply(h_orig, 0.75)),
tf.to_int32(tf.multiply(w_orig, 0.75))
]))
image_batch05 = tf.image.resize_images(
image_batch,
tf.stack([
tf.to_int32(tf.multiply(h_orig, 0.5)),
tf.to_int32(tf.multiply(w_orig, 0.5))
]))
# Create network.
with tf.variable_scope('', reuse=False):
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
with tf.variable_scope('', reuse=True):
net075 = DeepLabResNetModel({'data': image_batch075},
is_training=False,
num_classes=args.num_classes)
with tf.variable_scope('', reuse=True):
net05 = DeepLabResNetModel({'data': image_batch05},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output100 = net.layers['fc1_voc12']
raw_output075 = tf.image.resize_images(net075.layers['fc1_voc12'],
tf.shape(raw_output100)[1:3, ])
raw_output05 = tf.image.resize_images(net05.layers['fc1_voc12'],
tf.shape(raw_output100)[1:3, ])
raw_output = tf.reduce_max(tf.stack(
[raw_output100, raw_output075, raw_output05]),
axis=0)
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output = tf.argmax(raw_output, dimension=3)
pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
# # mIoU first convert pred and gt to vector,then compute mIoU
# pred = tf.reshape(pred, [-1, ])
# gt = tf.reshape(label_batch, [-1, ])
# # tensorflow 1.3.0 conflict
# # weights = tf.cast(tf.less_equal(gt, args.num_classes - 1), tf.int32) # Ignoring all labels greater than or equal to n_classes.
# # mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred, gt, num_classes=args.num_classes, weights=weights)
# indices = tf.squeeze(tf.where(tf.less_equal(gt, args.num_classes - 1)), 1) # ignore all labels >= num_classes
# gt = tf.cast(tf.gather(gt, indices), tf.int32)
# pred = tf.gather(pred, indices)
# mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred, gt, num_classes=args.num_classes)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
start = time.time()
for step in range(args.num_steps):
# preds, _ = sess.run([pred, update_op])
once_start = time.time()
preds = sess.run(pred)
once_end = time.time()
print('image %d, net forward cost %d s' %
(step + 1, once_end - once_start))
# save predicted label.png
# msk = decode_labels(preds, num_classes=args.num_classes)
mask = preds
msk = np.array(mask[0, :, :, 0], dtype=np.uint8)
im = Image.fromarray(msk)
# im = Image.fromarray(msk[0])
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
im.save(args.save_dir + maskslist[step] + '.png')
print('The output file has been saved to {}'.format(args.save_dir +
maskslist[step] +
'.png'))
end = time.time()
print('image %d, postprocessing cost %d s' %
(step + 1, end - once_start))
avgfps = (step + 1) / (end - start)
print('frame %d , %s /s' % (step + 1, avgfps))
if step % 100 == 0:
print('step {:d}'.format(step))
# print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord)
image, label = reader.image, reader.label
# Add one batch dimension.
image_batch, label_batch = tf.expand_dims(image,
dim=0), tf.expand_dims(label,
dim=0)
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output = tf.argmax(raw_output, dimension=3)
pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
# mIoU
pred = tf.reshape(pred, [
-1,
])
gt = tf.reshape(label_batch, [
-1,
])
# Ignoring all labels greater than or equal to n_classes.
weights = tf.cast(tf.less_equal(gt, args.num_classes - 1), tf.int32)
mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(
pred, gt, num_classes=args.num_classes, weights=weights)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
preds, _ = sess.run([pred, update_op])
if args.save_dir is not None:
img = decode_labels(preds[0, :, :, 0])
im = Image.fromarray(img)
im.save(args.save_dir + str(step) + '_vis.png')
cv2.imwrite(args.save_dir + str(step) + '.png', preds[0, :, :, 0])
if step % 100 == 0:
print('step {:d}'.format(step))
print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the training."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
tf.set_random_seed(args.random_seed)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(args.data_dir, args.data_list, input_size,
args.random_scale, args.random_mirror,
args.ignore_label, IMG_MEAN, coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
# Create network.
#net = DeepLabResNetModel({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes)
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
# For a small batch size, it is better to keep
# the statistics of the BN layers (running means and variances)
# frozen, and to not update the values provided by the pre-trained model.
# If is_training=True, the statistics will be updated during the training.
# Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
# if they are presented in var_list of the optimiser definition.
# Predictions.
raw_output = net.layers['fc1_voc12']
# Which variables to load. Running means and variances are not trainable,
# thus all_variables() should be restored.
# Restore all variables, or all except the last ones.
prediction = tf.reshape(raw_output, [-1, args.num_classes])
label_proc = prepare_label(label_batch,
tf.stack(raw_output.get_shape()[1:3]),
num_classes=args.num_classes)
gt = tf.reshape(label_proc, [-1, args.num_classes])
# Pixel-wise softmax loss.
loss = tf.nn.softmax_cross_entropy_with_logits(logits=prediction,
labels=gt)
reduced_loss = tf.reduce_mean(loss)
# Processed predictions.
raw_output_up = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Calculate IoU
mean_IoU = tf.metrics.mean_iou(labels=label_batch,
predictions=pred,
num_classes=args.num_classes)
mean_per_class_acc = tf.metrics.mean_per_class_accuracy(
labels=label_batch, predictions=pred, num_classes=args.num_classes)
mean_acc = tf.metrics.accuracy(labels=label_batch, predictions=pred)
# Image summary.
images_summary = tf.py_func(inv_preprocess,
[image_batch, args.save_num_images, IMG_MEAN],
tf.uint8)
labels_summary = tf.py_func(
decode_labels, [label_batch, args.save_num_images, args.num_classes],
tf.uint8)
preds_summary = tf.py_func(decode_labels,
[pred, args.save_num_images, args.num_classes],
tf.uint8)
final_output = tf.concat(
axis=2, values=[images_summary, labels_summary, preds_summary])
#total_summary = tf.summary.image('images',
# tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]),
# max_outputs=args.save_num_images) # Concatenate row-wise.
#loss_summary = tf.summary.scalar('training_loss', reduced_loss)
#overall_summary = tf.summary.merge([total_summary, loss_summary])
#summary_writer = tf.summary.FileWriter(args.snapshot_dir,
# graph=tf.get_default_graph())
# Define loss and optimisation parameters.
#optimiser = tf.train.AdamOptimizer(learning_rate=args.learning_rate)
#optim = optimiser.minimize(reduced_loss, var_list=trainable)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=40)
saver.restore(sess, args.restore_from)
# Load variables if the checkpoint is provided.
#if args.restore_from is not None:
# loader = tf.train.Saver(var_list=restore_var)
# load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
start_time = time.time()
for step in range(25):
loss, IoU, per_class_acc, acc, output = sess.run([
reduced_loss, mean_IoU, mean_per_class_acc, mean_acc, final_output
])
ipdb.set_trace()
print(IoU, per_class_acc, acc)
'''
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
loss, image, overall_output = sess.run([reduced_loss, image_batch, final_output])
#if step % args.save_pred_every == 0:
# loss_value, images, labels, preds, summary, _ = sess.run([reduced_loss, image_batch, label_batch, pred, overall_summary, optim])
# summary_writer.add_summary(summary, step)
# save(saver, sess, args.snapshot_dir, step)
#else:
# loss_value, _, summary = sess.run([reduced_loss, optim, loss_summary])
# summary_writer.add_summary(summary, step)
duration = time.time() - start_time
#print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
'''
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
num_steps = file_len(args.data_list)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.img_path,
args.data_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
255,
IMG_MEAN,
coord)
image, label = reader.image, reader.label
title = reader.queue[0]
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(
label, dim=0) # Add one batch dimension.
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output_up = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Set up TF session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, args.model_weights)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
start_time = time.time()
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
# Perform inference.
for step in range(num_steps):
preds, jpg_path = sess.run([pred, title])
msk = decode_labels(preds, num_classes=args.num_classes)
im = Image.fromarray(msk[0])
img_o = Image.open(jpg_path)
jpg_path = jpg_path.decode()
jpg_path = jpg_path.split('/')[-1].split('.')[0]
img = np.array(im) * 0.9 + np.array(img_o) * 0.7
img[img > 255] = 255
img = Image.fromarray(np.uint8(img))
img.save(args.save_dir + jpg_path + '.png')
print('Image processed {}.png'.format(jpg_path))
total_time = time.time() - start_time
print('The output files have been saved to {}'.format(args.save_dir))
print('It took {} sec on each image.'.format(total_time / num_steps))
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
print(args)
# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
# os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
try:
os.makedirs('eval/imageout')
os.makedirs('eval/imageout_raw')
os.makedirs('eval/mhdout')
except:
pass
dict = {}
with open(args.data_list, 'r') as f, open('eval/output.csv', 'wb') as logfile:
csvwriter = csv.DictWriter(logfile, fieldnames=['File', 'IoU Class 0',
'IoU Class 1', 'mIoU'
])
csvwriter.writeheader()
for line in f:
if re.match(".*\\/(.*)\\.mhd.*", line).group(1) not in dict:
dict[re.match(".*\\/(.*)\\.mhd.*", line).group(1)] = []
dict[re.match(".*\\/(.*)\\.mhd.*", line).group(1)].append(line)
counter_global = np.zeros((2, args.num_classes))
step = 0
for key in dict:
with tempfile.NamedTemporaryFile(mode='w') as tempf, open('eval/' + key + '.csv',
'wb') as logfile_per_file:
csvwriter_per_file = csv.DictWriter(logfile_per_file, fieldnames=['Z Coord', 'IoU Class 0',
'IoU Class 1', 'mIoU',
'Acc Class 0', 'Acc Class 1',
'Total Acc'])
csvwriter_per_file.writeheader()
tempf.writelines(dict[key])
tempf.flush()
prediction_out = np.zeros((len(dict[key]), 512, 512),dtype=np.int16)
with tf.Graph().as_default():
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
tempf.name,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord,
shuffle=False)
image, label = reader.image, reader.label
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(label,
dim=0) # Add one batch dimension.
# Create network.
net = DeepLabResNetModel({'data': image_batch}, is_training=False, num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3, ])
raw_output = tf.argmax(raw_output, dimension=3)
image_output_raw = tf.image.encode_png(tf.cast(tf.transpose(raw_output, (1, 2, 0)), tf.uint8))
pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
image_output = tf.image.encode_png(
tf.squeeze(tf.py_func(decode_labels, [pred, 1, args.num_classes], tf.uint8), axis=0))
# mIoU
pred = tf.reshape(pred, [-1, ])
gt = tf.reshape(label_batch, [-1, ])
# weights = tf.cast(tf.less_equal(gt, args.num_classes - 1),
# tf.int32) # Ignoring all labels greater than or equal to n_classes.
correct_pred = tf.equal(tf.cast(pred, tf.uint8), gt)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
accuracy_per_class = []
for i in xrange(0, args.num_classes):
curr_class = tf.constant(i, tf.uint8)
accuracy_per_class.append(tf.reduce_mean(
tf.cast(tf.gather(correct_pred, tf.where(tf.equal(gt, curr_class))), tf.float32)))
sess = tf.Session()
sess.run(tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()))
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
counter = np.zeros((2, args.num_classes))
acc_per_class = np.zeros(args.num_classes)
for idx, line in enumerate(dict[key]):
colored_label_png, raw_preds_png, preds, labels, acc, acc_per_class[0], acc_per_class[1] = sess.run(
[image_output, image_output_raw, raw_output, label_batch, accuracy, accuracy_per_class[0],accuracy_per_class[1]])
with open('eval/imageout_raw/' + key + "_" + str(idx) + '.png', 'wb') as f:
f.write(raw_preds_png)
with open('eval/imageout/' + key + "_" + str(idx) + '.png', 'wb') as f:
f.write(colored_label_png)
area_intersection, area_union = intersectionAndUnion(preds[0], labels[0, :, :, 0],
args.num_classes)
counter[0] += area_intersection
counter[1] += area_union
counter_global[0] += area_intersection
counter_global[1] += area_union
IoU_per_class = area_intersection / (np.spacing(1) + area_union)
csvwriter_per_file.writerow({'Z Coord': idx, 'IoU Class 0': IoU_per_class[0],
'IoU Class 1': IoU_per_class[1],
'mIoU': np.mean(IoU_per_class),
'Acc Class 0': acc_per_class[0], 'Acc Class 1': acc_per_class[1], 'Total Acc': acc})
step += 1
prediction_out[idx] = preds
IoU_per_class = counter[0] / (np.spacing(1) + counter[1])
csvwriter.writerow({'File': key, 'IoU Class 0': IoU_per_class[0],
'IoU Class 1': IoU_per_class[1],
'mIoU': np.mean(IoU_per_class)
})
print("Writing: " + 'eval/mhdout/' + key + '.mhd')
sitk.WriteImage(sitk.GetImageFromArray(prediction_out),
'eval/mhdout/' + key + '.mhd')
coord.request_stop()
coord.join(threads)
global_IoU_per_class = counter_global[0] / (np.spacing(1) + counter_global[1])
csvwriter.writerow({'File': 'Global', 'IoU Class 0': global_IoU_per_class[0],
'IoU Class 1': global_IoU_per_class[1],
'mIoU': np.mean(global_IoU_per_class)
})
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
print(args)
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_mask
try:
os.makedirs('eval/niiout')
except:
pass
event_end = Event()
queue_proc = Queue()
with open(args.data_list, 'r') as f:
list_of_all_lines = f.readlines()
f.seek(0)
dict = {}
for line in f:
if re.match(".*\\/(.*)\\.nii.*", line).group(1) not in dict:
dict[re.match(".*\\/(.*)\\.nii.*", line).group(1)] = []
dict[re.match(".*\\/(.*)\\.nii.*",
line).group(1)].append(line.rsplit()[0])
with tf.Graph().as_default():
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
(512, 512), # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord,
shuffle=False)
image_batch, _ = reader.dequeue(args.batch_size)
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output = tf.argmax(raw_output, dimension=3)
sess = tf.Session()
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
proc = Process(target=saving_process,
args=(queue_proc, event_end, args.data_dir,
args.post_processing))
proc.start()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
for sublist in [
list_of_all_lines[i:i + args.batch_size]
for i in xrange(0, len(list_of_all_lines), args.batch_size)
]:
preds = sess.run([raw_output])[0]
for i, thing in enumerate(sublist):
regex_match = re.match(".*\\/(.*)\\.nii_([0-9]+).*", thing)
# print(regex_match.group(1) + ' ' + str(regex_match.group(2)))
queue_proc.put(
(regex_match.group(1), int(regex_match.group(2)),
preds[i], len(dict[regex_match.group(1)])))
coord.request_stop()
coord.join(threads)
event_end.set()
proc.join()
def main():
"""Create the model and start the training."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
tf.set_random_seed(args.random_seed)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(args.data_dir, args.data_list, input_size,
args.random_scale, args.random_mirror,
args.ignore_label, IMG_MEAN, coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
image_batch075 = tf.image.resize_images(
image_batch, [int(h * 0.75), int(w * 0.75)])
image_batch05 = tf.image.resize_images(
image_batch, [int(h * 0.5), int(w * 0.5)])
# Create network.
with tf.variable_scope('', reuse=False):
net = DeepLabResNetModel({'data': image_batch},
is_training=args.is_training,
num_classes=args.num_classes)
with tf.variable_scope('', reuse=True):
net075 = DeepLabResNetModel({'data': image_batch075},
is_training=args.is_training,
num_classes=args.num_classes)
with tf.variable_scope('', reuse=True):
net05 = DeepLabResNetModel({'data': image_batch05},
is_training=args.is_training,
num_classes=args.num_classes)
# For a small batch size, it is better to keep
# the statistics of the BN layers (running means and variances)
# frozen, and to not update the values provided by the pre-trained model.
# If is_training=True, the statistics will be updated during the training.
# Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
# if they are presented in var_list of the optimiser definition.
# Predictions.
raw_output100 = net.layers['fc1_voc12']
raw_output075 = net075.layers['fc1_voc12']
raw_output05 = net05.layers['fc1_voc12']
raw_output = tf.reduce_max(tf.stack([
raw_output100,
tf.image.resize_images(raw_output075,
tf.shape(raw_output100)[1:3, ]),
tf.image.resize_images(raw_output05,
tf.shape(raw_output100)[1:3, ])
]),
axis=0)
# Which variables to load. Running means and variances are not trainable,
# thus all_variables() should be restored.
restore_var = [
v for v in tf.global_variables()
if 'fc' not in v.name or not args.not_restore_last
]
all_trainable = [
v for v in tf.trainable_variables()
if 'beta' not in v.name and 'gamma' not in v.name
]
fc_trainable = [v for v in all_trainable if 'fc' in v.name]
conv_trainable = [v for v in all_trainable
if 'fc' not in v.name] # lr * 1.0
fc_w_trainable = [v for v in fc_trainable
if 'weights' in v.name] # lr * 10.0
fc_b_trainable = [v for v in fc_trainable
if 'biases' in v.name] # lr * 20.0
assert (len(all_trainable) == len(fc_trainable) + len(conv_trainable))
assert (len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))
# Predictions: ignoring all predictions with labels greater or equal than n_classes
raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
raw_prediction100 = tf.reshape(raw_output100, [-1, args.num_classes])
raw_prediction075 = tf.reshape(raw_output075, [-1, args.num_classes])
raw_prediction05 = tf.reshape(raw_output05, [-1, args.num_classes])
#pdb.set_trace()
label_proc = prepare_label(label_batch,
tf.stack(raw_output.get_shape()[1:3]),
num_classes=args.num_classes,
one_hot=False) # [batch_size, h, w]
label_proc075 = prepare_label(label_batch,
tf.stack(raw_output075.get_shape()[1:3]),
num_classes=args.num_classes,
one_hot=False)
label_proc05 = prepare_label(label_batch,
tf.stack(raw_output05.get_shape()[1:3]),
num_classes=args.num_classes,
one_hot=False)
raw_gt = tf.reshape(label_proc, [
-1,
])
raw_gt075 = tf.reshape(label_proc075, [
-1,
])
raw_gt05 = tf.reshape(label_proc05, [
-1,
])
indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)),
1)
indices075 = tf.squeeze(
tf.where(tf.less_equal(raw_gt075, args.num_classes - 1)), 1)
indices05 = tf.squeeze(
tf.where(tf.less_equal(raw_gt05, args.num_classes - 1)), 1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
gt075 = tf.cast(tf.gather(raw_gt075, indices075), tf.int32)
gt05 = tf.cast(tf.gather(raw_gt05, indices05), tf.int32)
prediction = tf.gather(raw_prediction, indices)
prediction100 = tf.gather(raw_prediction100, indices)
prediction075 = tf.gather(raw_prediction075, indices075)
prediction05 = tf.gather(raw_prediction05, indices05)
# Pixel-wise softmax loss.
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction,
labels=gt)
loss100 = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=prediction100, labels=gt)
loss075 = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=prediction075, labels=gt075)
loss05 = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=prediction05, labels=gt05)
l2_losses = [
args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables()
if 'weights' in v.name
]
reduced_loss = tf.reduce_mean(loss) + tf.reduce_mean(
loss100) + tf.reduce_mean(loss075) + tf.reduce_mean(loss05) + tf.add_n(
l2_losses)
# Processed predictions: for visualisation.
raw_output_up = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Image summary.
images_summary = tf.py_func(inv_preprocess,
[image_batch, args.save_num_images, IMG_MEAN],
tf.uint8)
labels_summary = tf.py_func(
decode_labels, [label_batch, args.save_num_images, args.num_classes],
tf.uint8)
preds_summary = tf.py_func(decode_labels,
[pred, args.save_num_images, args.num_classes],
tf.uint8)
total_summary = tf.summary.image(
'images',
tf.concat(axis=2,
values=[images_summary, labels_summary, preds_summary]),
max_outputs=args.save_num_images) # Concatenate row-wise.
summary_writer = tf.summary.FileWriter(args.snapshot_dir,
graph=tf.get_default_graph())
print('\n\n')
print('-------------------------------------------------------')
print('--------------- Trainable parameters ------------------')
print('-------------------------------------------------------')
total_params = 0
for v in tf.trainable_variables():
shape = v.get_shape()
params = 1
for dim in shape:
params *= dim.value
print('{:<30s}: {:<20s}\t{:<10s}'.format(v.name, str(shape),
str(params)))
total_params += params
print('total_pamars = {}'.format(total_params))
print('-------------------------------------------------------\n\n')
# Define loss and optimisation parameters.
base_lr = tf.constant(args.learning_rate)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(
base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)
# Define a variable to accumulate gradients.
accum_grads = [
tf.Variable(tf.zeros_like(v.initialized_value()), trainable=False)
for v in conv_trainable + fc_w_trainable + fc_b_trainable
]
# Define an operation to clear the accumulated gradients for next batch.
zero_op = [v.assign(tf.zeros_like(v)) for v in accum_grads]
# Compute gradients.
grads = tf.gradients(reduced_loss,
conv_trainable + fc_w_trainable + fc_b_trainable)
# Accumulate and normalise the gradients.
accum_grads_op = [
accum_grads[i].assign_add(grad / args.grad_update_every)
for i, grad in enumerate(grads)
]
grads_conv = accum_grads[:len(conv_trainable)]
grads_fc_w = accum_grads[len(conv_trainable):(len(conv_trainable) +
len(fc_w_trainable))]
grads_fc_b = accum_grads[(len(conv_trainable) + len(fc_w_trainable)):]
# Apply the gradients.
train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))
train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10)
# Load variables if the checkpoint is provided.
if args.restore_from is not None:
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
feed_dict = {step_ph: step}
loss_value = 0
# Clear the accumulated gradients.
sess.run(zero_op, feed_dict=feed_dict)
# Accumulate gradients.
for i in range(args.grad_update_every):
_, l_val = sess.run([accum_grads_op, reduced_loss],
feed_dict=feed_dict)
loss_value += l_val
# Normalise the loss.
loss_value /= args.grad_update_every
# Apply gradients.
if step % args.save_pred_every == 0:
images, labels, summary, _ = sess.run(
[image_batch, label_batch, total_summary, train_op],
feed_dict=feed_dict)
summary_writer.add_summary(summary, step)
save(saver, sess, args.snapshot_dir, step)
else:
sess.run(train_op, feed_dict=feed_dict)
duration = time.time() - start_time
print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(
step, loss_value, duration))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load validation
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord)
image, label, file, mask = reader.image, reader.label, reader.image_list, reader.label_list
image_batch, label_batch, file_batch, mask_batch = tf.expand_dims(image, dim=0), \
tf.expand_dims(label, dim=0), \
tf.expand_dims(file, dim=0), \
tf.expand_dims(mask, dim=0)# Add one batch dimension.
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
probabilities = tf.nn.softmax(raw_output)
raw_output = tf.argmax(raw_output, dimension=3)
preds = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
# mIoU
# pred = tf.reshape(preds, [-1,])
# gt = tf.reshape(label_batch, [-1,])
# weights = tf.cast(tf.less_equal(gt, args.num_classes - 1), tf.int32) # Ignoring all labels greater than or equal to n_classes.
# mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(pred, gt, num_classes=args.num_classes, weights=weights)
file_name = file_batch
mask_link = mask_batch
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
result_data = {}
total_miuo_score = 0
nb = 0
global OUT_DIR
global IMAGE
global IMAGE_PATH
for step in range(args.num_steps):
predict, probmap, fb, mk = sess.run(
[preds, probabilities, file_name, mask_link])
print('step {:d}'.format(step))
if IS_POD:
IMAGE = fb[0][step].decode('utf8').replace(
"D:\Data\POD/JpegImages/", "").replace(".jpg", "")
IMAGE_PATH = "D:\Data\POD\JpegImages\\" + IMAGE + ".jpg"
labels, bounding_boxs, confidence_score, miuo_score, pixel_score = \
get_bounding_boxs(probmap)
result_data[IMAGE] = {
"labels": labels,
"confidence_score": confidence_score,
"boxes": bounding_boxs,
"pixel_accuracy": pixel_score,
"mIoU": miuo_score
}
# Draw bouding boxes to image
# drw_img(cv2.imread(IMAGE_PATH), labels, bounding_boxs)
OUT_DIR = "mask_pod/"
else:
# print(mk[0][step].decode('utf8'))
IMAGE = mk[0][step].decode('utf8').replace(
"D:\Data\PLAD/MaskImage/", "")
result_data[IMAGE] = probabilities_map_evaluate(
probmap, mk[0][step].decode('utf8'))
OUT_DIR = "mask_plad/"
print(result_data)
# Draw mask image
msk = decode_labels(predict, num_classes=args.num_classes)
im = Image.fromarray(msk[0])
im.save(OUT_DIR + IMAGE + '.png')
# with open('predicted_boxes.json', 'w') as outfile:
# json.dump(result_data, outfile)
# print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord)
image, label = reader.image, reader.label
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(
label, dim=0) # Add one batch dimension.
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output = tf.argmax(raw_output, dimension=3)
pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
output = raw_output
# mIoU
pred = tf.reshape(pred, [
-1,
])
gt = tf.reshape(label_batch, [
-1,
])
indices = tf.squeeze(tf.where(tf.less_equal(gt, args.num_classes - 1)),
1) # ignore all labels >= num_classes
gt = tf.cast(tf.gather(gt, indices), tf.int32)
pred = tf.gather(pred, indices)
mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(
pred, gt, num_classes=args.num_classes)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = False
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
f = open(DATA_LIST_PATH, "r")
for step in range(args.num_steps):
line = f.readline()
name = line[12:23]
image_in, output_value, preds, _ = sess.run(
[image_batch, output, pred, update_op])
##print(image_in[0])
#im1=Image.fromarray(np.uint8(output_value[0]))
#im1.save('./output/'+name+'.png')
if step % 100 == 0:
print('step {:d}'.format(step))
print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
def main(argv=None):
# 1.获取测试集数据
# map()会根据提供的函数对指定序列做映射。FLAGS.input_size.split(',')得到['321','321']
image_height, image_width = map(int, FLAGS.input_size.split(','))
input_size = (image_height, image_width)
# 创建一个线程管理器
coord = tf.train.Coordinator()
with tf.name_scope("create_inputs"):
reader = ImageReader(FLAGS.data_dir,
FLAGS.data_list,
input_size=None,
random_scale=False,
coord=coord)
image, label = reader.image, reader.label
image_batch, label_batch = tf.expand_dims(image,
dim=0), tf.expand_dims(label,
dim=0)
# 2.建立模型
net = DeepLabV3Model(is_training=False, num_classes=FLAGS.num_classes)
# 3.预测结果
raw_output_up = net.preds(image_batch)
#
pred = tf.expand_dims(raw_output_up, dim=3)
# 4.定义一个初始化变量op
init_variable = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_variable)
# 5.
restore_var = tf.global_variables()
loader = tf.train.Saver(var_list=restore_var)
loader.restore(sess, FLAGS.model_weights)
print("Restored model parameters from {}".format(FLAGS.model_weights))
# 6.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# 7.mIoU
predictions = tf.reshape(pred, [
-1,
])
labels = tf.reshape(tf.cast(label_batch, tf.int32), [
-1,
])
mask = labels <= FLAGS.num_classes - 1
predictions = tf.boolean_mask(predictions, mask)
labels = tf.boolean_mask(labels, mask)
# Define the evaluation metric.
mIoU, update_op = tf.contrib.metrics.streaming_mean_iou(
predictions, labels, FLAGS.num_classes)
# 一定要有
sess.run(tf.local_variables_initializer())
# ###
# 8.Iterate over images.
for step in range(FLAGS.num_steps):
# Perform inference.
preds, img, _ = sess.run(
[pred, tf.cast(image_batch, tf.uint8), update_op])
msk = decode_labels(preds, num_classes=FLAGS.num_classes)
im0 = Image.fromarray(msk[0])
im0.save(FLAGS.save_image_dir + 'mask' + str(step) + '.png')
print('The output file has been saved to {}'.format(
FLAGS.save_image_dir + 'mask' + str(step) + '.png'))
# CRF
raw_im = dense_crf(img[0], msk[0])
#
# plt.subplot(1,2,1)
# plt.imshow(msk[0])
# plt.subplot(1, 2, 2)
# plt.imshow(raw_im)
# plt.show()
im = Image.fromarray(raw_im)
im.save(FLAGS.save_image_dir + 'mask_crf' + str(step) + '.png')
print('The output file has been saved to {}'.format(
FLAGS.save_image_dir + 'mask_crf' + str(step) + '.png'))
print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
coord.request_stop()
coord.join(threads)
return None
def main():
"""Create the model and start the training."""
args = get_arguments()
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
tf.set_random_seed(args.random_seed)
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
input_size,
args.random_scale,
args.random_mirror,
args.ignore_label,
IMG_MEAN,
coord)
image_batch, label_batch = reader.dequeue(args.batch_size)
# Create network.
net = DeepLabResNetModelOri50gcnaspp({'data': image_batch}, is_training=args.is_training, num_classes=args.num_classes)
# For a small batch size, it is better to keep
# the statistics of the BN layers (running means and variances)
# frozen, and to not update the values provided by the pre-trained model.
# If is_training=True, the statistics will be updated during the training.
# Note that is_training=False still updates BN parameters gamma (scale) and beta (offset)
# if they are presented in var_list of the optimiser definition.
# Predictions.
raw_output = net.layers['fc1_voc12']
# Which variables to load. Running means and variances are not trainable,
# thus all_variables() should be restored.
restore_var = [v for v in tf.global_variables() if 'fc' not in v.name or not args.not_restore_last]
all_trainable = [v for v in tf.trainable_variables() if 'beta' not in v.name and 'gamma' not in v.name]
fc_trainable = [v for v in all_trainable if 'fc' in v.name]
conv_trainable = [v for v in all_trainable if 'fc' not in v.name] # lr * 1.0
fc_w_trainable = [v for v in fc_trainable if 'weights' in v.name] # lr * 10.0
fc_b_trainable = [v for v in fc_trainable if 'biases' in v.name] # lr * 20.0
assert (len(all_trainable) == len(fc_trainable) + len(conv_trainable))
assert (len(fc_trainable) == len(fc_w_trainable) + len(fc_b_trainable))
# Predictions: ignoring all predictions with labels greater or equal than n_classes
raw_prediction = tf.reshape(raw_output, [-1, args.num_classes])
label_proc = prepare_label(label_batch, tf.stack(raw_output.get_shape()[1:3]), num_classes=args.num_classes,
one_hot=False) # [batch_size, h, w]
raw_gt = tf.reshape(label_proc, [-1, ])
indices = tf.squeeze(tf.where(tf.less_equal(raw_gt, args.num_classes - 1)), 1)
gt = tf.cast(tf.gather(raw_gt, indices), tf.int32)
prediction = tf.gather(raw_prediction, indices)
# Pixel-wise softmax loss.
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=prediction, labels=gt)
l2_losses = [args.weight_decay * tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'weights' in v.name]
reduced_loss = tf.reduce_mean(loss) + tf.add_n(l2_losses)
loss_summary = tf.summary.scalar('loss', reduced_loss)
# Processed predictions: for visualisation.
raw_output_up = tf.image.resize_bilinear(raw_output, tf.shape(image_batch)[1:3, ])
raw_output_up = tf.argmax(raw_output_up, dimension=3)
pred = tf.expand_dims(raw_output_up, dim=3)
# Image summary.
images_summary = tf.py_func(inv_preprocess, [image_batch, args.save_num_images, IMG_MEAN], tf.uint8)
labels_summary = tf.py_func(decode_labels, [label_batch, args.save_num_images, args.num_classes], tf.uint8)
preds_summary = tf.py_func(decode_labels, [pred, args.save_num_images, args.num_classes], tf.uint8)
total_summary = tf.summary.image('images',
tf.concat(axis=2, values=[images_summary, labels_summary, preds_summary]),
max_outputs=args.save_num_images) # Concatenate row-wise.
merged = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(args.snapshot_dir,
graph=tf.get_default_graph())
# Define loss and optimisation parameters.
base_lr = tf.constant(args.learning_rate)
step_ph = tf.placeholder(dtype=tf.float32, shape=())
learning_rate = tf.scalar_mul(base_lr, tf.pow((1 - step_ph / args.num_steps), args.power))
opt_conv = tf.train.MomentumOptimizer(learning_rate, args.momentum)
opt_fc_w = tf.train.MomentumOptimizer(learning_rate * 10.0, args.momentum)
opt_fc_b = tf.train.MomentumOptimizer(learning_rate * 20.0, args.momentum)
grads = tf.gradients(reduced_loss, conv_trainable + fc_w_trainable + fc_b_trainable)
grads_conv = grads[:len(conv_trainable)]
grads_fc_w = grads[len(conv_trainable): (len(conv_trainable) + len(fc_w_trainable))]
grads_fc_b = grads[(len(conv_trainable) + len(fc_w_trainable)):]
train_op_conv = opt_conv.apply_gradients(zip(grads_conv, conv_trainable))
train_op_fc_w = opt_fc_w.apply_gradients(zip(grads_fc_w, fc_w_trainable))
train_op_fc_b = opt_fc_b.apply_gradients(zip(grads_fc_b, fc_b_trainable))
train_op = tf.group(train_op_conv, train_op_fc_w, train_op_fc_b)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
# Saver for storing checkpoints of the model.
saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=10)
# Load variables if the checkpoint is provided.
if args.restore_from is not None:
loader = tf.train.Saver(var_list=restore_var)
load(loader, sess, args.restore_from)
#ckpt = tf.train.get_checkpoint_state('./snapshots50/')
#loader.restore(sess, ckpt.model_checkpoint_path)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
start_time = time.time()
feed_dict = {step_ph: step}
if step % args.save_pred_every == 0:
loss_value, images, labels, preds, summary, _ = sess.run(
[reduced_loss, image_batch, label_batch, pred, merged, train_op], feed_dict=feed_dict)
summary_writer.add_summary(summary, step)
save(saver, sess, args.snapshot_dir, step)
else:
loss_value, _ = sess.run([reduced_loss, train_op], feed_dict=feed_dict)
duration = time.time() - start_time
print('step {:d} \t loss = {:.3f}, ({:.3f} sec/step)'.format(step, loss_value, duration))
coord.request_stop()
coord.join(threads)
def main():
"""Create the model and start the evaluation process."""
args = get_arguments()
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(
args.data_dir,
args.data_list,
None, # No defined input size.
False, # No random scale.
False, # No random mirror.
args.ignore_label,
IMG_MEAN,
coord)
image, label = reader.image, reader.label
image_batch, label_batch = tf.expand_dims(image, dim=0), tf.expand_dims(
label, dim=0) # Add one batch dimension.
# Create network.
net = DeepLabResNetModel({'data': image_batch},
is_training=False,
num_classes=args.num_classes)
# Which variables to load.
restore_var = tf.global_variables()
# Predictions.
raw_output = net.layers['fc1_voc12']
raw_output = tf.image.resize_bilinear(raw_output,
tf.shape(image_batch)[1:3, ])
raw_output = tf.argmax(raw_output, dimension=3)
pred = tf.expand_dims(raw_output, dim=3) # Create 4-d tensor.
decode_pred = tf.py_func(decode_labels, [pred, 1, args.num_classes],
tf.uint8)
# mIoU
pred = tf.reshape(pred, [
-1,
])
gt = tf.reshape(label_batch, [
-1,
])
weights = tf.cast(
tf.less_equal(gt, args.num_classes - 1),
tf.int32) # Ignoring all labels greater than or equal to n_classes.
mIoU, IoU_update_op = tf.metrics.mean_iou(pred,
gt,
num_classes=args.num_classes,
weights=weights)
pc_accuracy, pc_acc_update_op = tf.metrics.mean_per_class_accuracy(
pred, gt, num_classes=args.num_classes, weights=weights)
accuracy, acc_update_op = tf.metrics.accuracy(pred, gt, weights=weights)
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if args.restore_from is not None:
load(loader, sess, args.restore_from)
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# Iterate over training steps.
for step in range(args.num_steps):
preds, _, _, _, pred_img = sess.run([
pred, IoU_update_op, acc_update_op, pc_acc_update_op, decode_pred
])
pred_p = os.path.join(
'output',
reader.image_list[step].split('/')[-1].replace('img', 'pred'))
scipy.misc.imsave(pred_p, pred_img[0])
if step % 100 == 0:
print('step {:d}'.format(step))
print('Mean IoU: {:.3f}'.format(mIoU.eval(session=sess)))
print('Per-Class Accuracy:{:.3f}'.format(pc_accuracy.eval(session=sess)))
print('Accuracy: {:.3f}'.format(accuracy.eval(session=sess)))
coord.request_stop()
coord.join(threads)
