import fcn32_vgg
import utils
from tensorflow.python.framework import ops
os.environ['CUDA_VISIBLE_DEVICES'] = ''
img1 = skimage.io.imread("./test_data/tabby_cat.png")
with tf.Session() as sess:
images = tf.placeholder("float")
feed_dict = {images: img1}
batch_images = tf.expand_dims(images, 0)
vgg_fcn = fcn32_vgg.FCN32VGG()
with tf.name_scope("content_vgg"):
vgg_fcn.build(batch_images, debug=True)
print('Finished building Network.')
init = tf.initialize_all_variables()
sess.run(tf.initialize_all_variables())
print('Running the Network')
tensors = [vgg_fcn.pred, vgg_fcn.pred_up]
down, up = sess.run(tensors, feed_dict=feed_dict)
down_color = utils.color_image(down[0])
up_color = utils.color_image(up[0])
def main(argv=None):
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
image = tf.placeholder(tf.float32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
name="input_image")
annotation = tf.placeholder(tf.int32,
shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
name="annotation")
FM_pl = tf.placeholder(tf.float32, [])
total_acc_pl = tf.placeholder(tf.float32, [])
acc_pl = tf.placeholder(tf.float32, [])
iu_pl = tf.placeholder(tf.float32, [])
fwavacc_pl = tf.placeholder(tf.float32, [])
# is_traing = tf.placeholder('bool')
vgg_fcn = fcn32_vgg.FCN32VGG()
vgg_fcn.build(image,
num_classes=num_classes,
keep_probability=keep_probability,
random_init_fc8=True)
logits = vgg_fcn.upscore
pred_annotation = vgg_fcn.pred_up
# loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
# labels=tf.squeeze(annotation, squeeze_dims=[3]),
# name="entropy")))
# loss_summary = tf.summary.scalar("entropy", loss)
# trainable_var = tf.trainable_variables()
# S_vars = [svar for svar in tf.trainable_variables() if 'weight' in svar.name]
# l2 = tf.add_n([tf.nn.l2_loss(var) for var in S_vars])
# # loss = loss + l2 * FLAGS.weight_decay
# # train_op = tf.train.MomentumOptimizer(FLAGS.learning_rate, 0.9).minimize(loss + l2 * FLAGS.weight_decay)
# train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss + l2 * FLAGS.weight_decay)
# # train_op = train(loss, trainable_var)
""" median-frequency re-weighting """
# class_weights = np.array([
# 0.5501,
# 5.4915
# ])
# loss = tf.reduce_mean((tf.nn.weighted_cross_entropy_with_logits(logits=logits,
# targets=tf.one_hot(tf.squeeze(annotation, squeeze_dims=[3]), depth=num_classes),
# pos_weight=class_weights,
# name="entropy")))
loss = LOSS.loss(
logits,
tf.one_hot(tf.squeeze(annotation, squeeze_dims=[3]),
depth=num_classes))
regularization_loss = tf.add_n(
tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
t_loss = loss + regularization_loss
loss_summary = tf.summary.scalar("entropy", loss)
FM_summary = tf.summary.scalar('FM', FM_pl)
acc_total_summary = tf.summary.scalar("total_acc", total_acc_pl)
acc_summary = tf.summary.scalar("acc", acc_pl)
iu_summary = tf.summary.scalar("iu", iu_pl)
fwavacc_summary = tf.summary.scalar("fwavacc", fwavacc_pl)
train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss)
#train_op = tf.train.MomentumOptimizer(FLAGS.learning_rate, 0.9).minimize(t_loss)
# train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(t_loss)
summary_op = tf.summary.merge_all()
train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir)
print(len(train_records))
print(len(valid_records))
print("Setting up dataset reader")
image_options = {'resize': False, 'resize_size': IMAGE_SIZE}
if FLAGS.mode == 'train':
train_dataset_reader = dataset.BatchDatset(train_records,
image_options)
validation_dataset_reader = dataset.BatchDatset(valid_records,
image_options)
sess = tf.Session(config=config)
saver = tf.train.Saver(max_to_keep=3)
# create two summary writers to show training loss and validation loss in the same graph
# need to create two folders 'train' and 'validation' inside FLAGS.logs_dir
praph_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/graph', sess.graph)
train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train')
validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation')
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
if FLAGS.mode == "train":
for itr in range(1, MAX_ITERATION):
train_images, train_annotations = train_dataset_reader.next_batch(
FLAGS.batch_size)
feed_dict = {
image: train_images,
annotation: train_annotations,
keep_probability: 0.5
}
sess.run(train_op, feed_dict=feed_dict)
if itr % 10 == 0:
train_loss, summary_str = sess.run([loss, loss_summary],
feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
train_writer.add_summary(summary_str, itr)
if itr % 210 == 0:
valid_iamges, valid_annotations = validation_dataset_reader.get_records(
)
val_count = 0
total_loss = 0
hist = np.zeros((num_classes, num_classes))
fm = 0
for i in range(1, 21):
val_images = valid_iamges[val_count:val_count +
val_batch_size]
val_annotations = valid_annotations[val_count:val_count +
val_batch_size]
val_loss, val_pred_dense = sess.run(
[loss, logits],
feed_dict={
image: val_images,
annotation: val_annotations,
keep_probability: 1.0
})
total_loss = total_loss + val_loss
val_count = val_count + val_batch_size
hist += get_hist(val_pred_dense, val_annotations)
fm += get_FM(val_pred_dense, val_annotations)
valid_loss = total_loss / 20
FM = fm / (20 * val_batch_size)
acc_total = np.diag(hist).sum() / hist.sum()
acc = np.diag(hist) / hist.sum(1)
iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) -
np.diag(hist))
freq = hist.sum(1) / hist.sum()
# summary_st = sess.run(summary_op,feed_dict=feed_dict)
summary_sva = sess.run(loss_summary,
feed_dict={loss: valid_loss})
summary_FM = sess.run(FM_summary, feed_dict={FM_pl: FM})
summary_acc_total = sess.run(
acc_total_summary, feed_dict={total_acc_pl: acc_total})
summary_acc = sess.run(acc_summary,
feed_dict={acc_pl: np.nanmean(acc)})
summary_iu = sess.run(iu_summary,
feed_dict={iu_pl: np.nanmean(iu)})
summary_fwavacc = sess.run(
fwavacc_summary,
feed_dict={
fwavacc_pl: (freq[freq > 0] * iu[freq > 0]).sum()
})
print("Step: %d, Valid_loss:%g" % (itr, valid_loss))
print(" >>> Step: %d, f1_score:%g" % (itr, FM))
# overall accuracy
print(" >>> Step: %d, overall accuracy:%g" % (itr, acc_total))
print(" >>> Step: %d, mean accuracy:%g" %
(itr, np.nanmean(acc)))
print(" >>> Step: %d, mean IU:%g" % (itr, np.nanmean(iu)))
print(" >>> Step: %d, fwavacc:%g" %
(itr, (freq[freq > 0] * iu[freq > 0]).sum()))
# validation_writer.add_summary(summary_st, step)
validation_writer.add_summary(summary_sva, itr)
validation_writer.add_summary(summary_FM, itr)
validation_writer.add_summary(summary_acc_total, itr)
validation_writer.add_summary(summary_acc, itr)
validation_writer.add_summary(summary_iu, itr)
validation_writer.add_summary(summary_fwavacc, itr)
saver.save(sess, FLAGS.logs_dir + "model.ckpt", itr)
va_images, va_annotations = validation_dataset_reader.get_random_batch(
20)
pred = sess.run(pred_annotation,
feed_dict={
image: va_images,
annotation: va_annotations,
keep_probability: 1.0
})
va_annotations = np.squeeze(va_annotations, axis=3)
# pred = np.squeeze(pred, axis=3)
pred = pred * 255
va_annotations = va_annotations * 255
for it in range(20):
utils.save_image(va_images[it].astype(np.uint8),
FLAGS.logs_dir,
name="inp_" + str(5 + it))
utils.save_image(va_annotations[it].astype(np.uint8),
FLAGS.logs_dir,
name="gt_" + str(5 + it))
utils.save_image(pred[it].astype(np.uint8),
FLAGS.logs_dir,
name="pred_" + str(5 + it))
elif FLAGS.mode == "visualize":
it = 0
valid_iamge, val_annotation = validation_dataset_reader.get_records()
val_annotation = np.squeeze(val_annotation, axis=3)
val_annotation = val_annotation * 255
for filename in valid_records:
val_image = np.array(misc.imread(filename['image']))
val_image = np.reshape(val_image, (1, 256, 256, 3))
pred = sess.run(pred_annotation,
feed_dict={
image: val_image,
keep_probability: 1.0
})
pred = pred * 255
# pred = sess.run(pred_annotation, feed_dict={image: val_image,
# keep_probability:1.0})
utils.save_image(pred[0].astype(np.uint8),
FLAGS.logs_dir + 'pred01',
name=os.path.splitext(
filename['image'].split("/")[-1])[0])
utils.save_image(val_annotation[it].astype(np.uint8),
FLAGS.logs_dir + 'gt01',
name=os.path.splitext(
filename['annotation'].split("/")[-1])[0])
it = it + 1
#img1 = skimage.io.imread("./test_data/tabby_cat.png")
parser = argparse.ArgumentParser()
parser.add_argument('--npypath',help="path to weights",default="/scratch/gallowaa/")
parser.add_argument('--imgpath',help="path to input image",default="/scratch/gallowaa/")
args = parser.parse_args()
img1 = skimage.io.imread(args.imgpath+"2008_004499_02.png")
with tf.Session() as sess:
images = tf.placeholder("float")
feed_dict = {images: img1}
batch_images = tf.expand_dims(images, 0)
vgg_fcn = fcn32_vgg.FCN32VGG(args.npypath+"vgg16.npy")
with tf.name_scope("content_vgg"):
vgg_fcn.build(batch_images, debug=True)
print('Finished building Network.')
init = tf.initialize_all_variables()
sess.run(tf.initialize_all_variables())
print('Running the Network')
tensors = [vgg_fcn.pred, vgg_fcn.pred_up]
down, up = sess.run(tensors, feed_dict=feed_dict)
down_color = utils.color_image(down[0])
up_color = utils.color_image(up[0])
import numpy as np
import tensorflow as tf
import fcn32_vgg
import utils
from tensorflow.python.framework import ops
img1 = scp.misc.imread("./test_data/tabby_cat.png")
with tf.Session() as sess:
images = tf.placeholder("float")
feed_dict = {images: img1}
batch_images = tf.expand_dims(images, 0) # 第一维是n_examples
vgg_fcn = fcn32_vgg.FCN32VGG() # 获得class
with tf.name_scope("content_vgg"):
vgg_fcn.build(batch_images, debug=True)
print('Finished building Network.')
init = tf.global_variables_initializer()
sess.run(init)
print('Running the Network')
tensors = [vgg_fcn.pred, vgg_fcn.pred_up] # 下采样后和上采样后
down, up = sess.run(tensors, feed_dict=feed_dict)
down_color = utils.color_image(down[0])
up_color = utils.color_image(up[0])
def run_network():
with tf.Session() as sess:
images = tf.placeholder("float")
# batch_images = tf.expand_dims(images, 0)
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
vgg_fcn = fcn32_vgg.FCN32VGG()
with tf.name_scope("content_vgg"):
vgg_fcn.build(images, num_classes=NUM_CLASSES, train=True, debug=True, keep_prob=keep_probability)
print('Finished building Network.')
logging.warning("Score weights are initialized random.")
logging.warning("Do not expect meaningful results.")
print("Build Training network")
annotations = tf.placeholder("float")
# batch_annotations = tf.expand_dims(annotations, 0)
vgg_fcn_loss = loss(vgg_fcn.upscore, annotations, NUM_CLASSES)
tf.summary.scalar("entropy", vgg_fcn_loss)
trainable_vars = tf.trainable_variables()
optimizer = tf.train.AdamOptimizer(1e-6)
grads = optimizer.compute_gradients(vgg_fcn_loss, var_list=trainable_vars)
for grad, var in grads:
if grad is not None:
tf.summary.histogram("gradient/" + var.op.name, grad)
train_step = optimizer.apply_gradients(grads)
pred2color = int(255.0/(NUM_CLASSES-1.0))
CLASS_WE_CARE_ABOUT = 1
tf.summary.image("input_image", images, max_outputs=1)
tf.summary.image("ground_truth", tf.cast(tf.split(annotations, NUM_CLASSES, axis=3)[CLASS_WE_CARE_ABOUT], tf.uint8)*pred2color, max_outputs=1)
tf.summary.image("pred_annotation", tf.cast((tf.split(vgg_fcn.prob_up, NUM_CLASSES, axis=3)[CLASS_WE_CARE_ABOUT])*pred2color, tf.uint8), max_outputs=1)
logging.info("Start Initializing Variables.")
init = tf.global_variables_initializer()
sess.run(init)
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter("./chkpt/", sess.graph)
sum_valid_image = tf.summary.image("valid_input_image", images, max_outputs=20)
sum_valid_gt = tf.summary.image("valid_ground_truth", tf.cast(tf.split(annotations, NUM_CLASSES, axis=3)[1], tf.uint8) * pred2color,
max_outputs=20)
sum_valid_pred = tf.summary.image("valid_pred_annotation",
tf.cast((tf.split(vgg_fcn.prob_up, NUM_CLASSES, axis=3)[CLASS_WE_CARE_ABOUT]) * pred2color, tf.uint8), max_outputs=20)
sum_valid_entropy = tf.summary.scalar("valid_entropy", vgg_fcn_loss)
valid_summary_op = tf.summary.merge([sum_valid_image,sum_valid_gt,sum_valid_pred,sum_valid_entropy])
training_records, validation_records = load_data("E:/videoseg/VanuatuGT")
image_options = {'resize': True, 'resize_size': 244}
DURATION = 30 * 60 * 2
train_dataset_reader = BatchDataset2(training_records, NUM_CLASSES, image_options, fromFrameId=0, uptoFrameId=DURATION)
validation_dataset_reader = BatchDataset2(validation_records, NUM_CLASSES, image_options)
print("Training the Network")
saver = tf.train.Saver()
# saver.restore(sess, "./chkpt/model32.ckpt")
MAX_ITERATION = int(500)
VIDEO_STEPS = int(5)
BATCH_SIZE = 1
for itr in range(1, MAX_ITERATION*VIDEO_STEPS + 2):
train_images, train_annotations = train_dataset_reader.next_batch(BATCH_SIZE)
feed_dict = {images: train_images, annotations: train_annotations, keep_probability: 0.4}
train_step.run(feed_dict=feed_dict)
if itr % 10 == 0:
train_loss, summary_str = sess.run([vgg_fcn_loss, summary_op], feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
summary_writer.add_summary(summary_str, itr)
valid_images, valid_annotations = validation_dataset_reader.next_batch(20)
feed_dict = {images: valid_images, annotations: valid_annotations, keep_probability: 1.0}
valid_loss, valid_summary_str = sess.run([vgg_fcn_loss, valid_summary_op], feed_dict=feed_dict)
print("%s ---> Validation_loss: %g" % (datetime.datetime.now(), valid_loss))
summary_writer.add_summary(valid_summary_str, itr)
if itr % 500 == 0:
save_path = saver.save(sess, "./chkpt/model32.ckpt")
print("Model saved in file: %s" % save_path)
if itr % MAX_ITERATION == 0:
#we've finished training for this portion of the video, lets save out the video and start training on the next part of the video
videoItr = int(itr / MAX_ITERATION)
print("processing the entire video")
cap = cv2.VideoCapture("E:\\vanuatu\\vanuatu35\\640x360\\vanuatu35_%06d.jpg")
frameNum = 0
allFrames = None
while (True):
# Capture frame-by-frame
frames = []
ret = False
for i in range(20):
ret, frame = cap.read()
if not ret:
break
frameNum += 1
frame = cv2.resize(frame, (image_options['resize_size'], image_options['resize_size']))
if frame is not None and frame.any():
frames.append(frame)
if frames:
feed_dict = {images: np.array(frames), keep_probability: 1.0}
prob = sess.run([vgg_fcn.prob_up], feed_dict=feed_dict)
prob = prob[0][:, :, :, 1, None]
if allFrames is None:
allFrames = prob
else:
allFrames = np.vstack((allFrames, prob))
if allFrames.shape[0] > 1000 or not ret:
print("saving frames {} - {}".format(frameNum-allFrames.shape[0], frameNum))
p_path = os.path.join("E:/videoseg/VanuatuProb", "{:d}_{:05d}_P.npy".format(videoItr, frameNum))
np.save(p_path, allFrames)
allFrames = None
if not ret:
break
cap.release()
startFrameId = int(videoItr * 30 * 60)
print("loading the next section of the training data. {} - {}".format(startFrameId,
startFrameId + DURATION))
train_dataset_reader.changeFrameIdRange(fromFrameId=startFrameId, uptoFrameId=startFrameId+DURATION)
label_image = tf.one_hot(label_image, 19)
train_image = tf.cast(train_image, tf.float32) / 255.
image = tf.Print(train_image, [tf.shape(label_image)])
train_image_batch, train_label_batch = tf.train.shuffle_batch(
[train_image, label_image],
batch_size=BATCH_SIZE,
capacity=3 + 3 * BATCH_SIZE,
min_after_dequeue=3)
with tf.device('/cpu:0'):
sess = tf.Session()
images_tf = tf.placeholder(tf.float32, [None, 1024, 2048, 3])
labels_tf = tf.placeholder(tf.float32, [None, 1024, 2048, 19])
vgg_fcn = fcn32_vgg.FCN32VGG('./vgg16.npy')
with tf.name_scope("content_vgg"):
vgg_fcn.build(images_tf,
train=True,
num_classes=19,
random_init_fc8=True,
debug=False)
#head=[]
cross_entropy = -tf.reduce_sum(labels_tf * tf.log(vgg_fcn.softmax),
reduction_indices=[1])
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='xentropy_mean')
#loss_tf = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(vgg_fcn.upscore,
# tf.squeeze(labels_tf, squeeze_dims=[3]),
# name="entropy")))