예제 #1
0
import fcn32_vgg
import utils

from tensorflow.python.framework import ops

os.environ['CUDA_VISIBLE_DEVICES'] = ''

img1 = skimage.io.imread("./test_data/tabby_cat.png")

with tf.Session() as sess:
    images = tf.placeholder("float")
    feed_dict = {images: img1}
    batch_images = tf.expand_dims(images, 0)

    vgg_fcn = fcn32_vgg.FCN32VGG()
    with tf.name_scope("content_vgg"):
        vgg_fcn.build(batch_images, debug=True)

    print('Finished building Network.')

    init = tf.initialize_all_variables()
    sess.run(tf.initialize_all_variables())

    print('Running the Network')
    tensors = [vgg_fcn.pred, vgg_fcn.pred_up]
    down, up = sess.run(tensors, feed_dict=feed_dict)

    down_color = utils.color_image(down[0])
    up_color = utils.color_image(up[0])
예제 #2
0
def main(argv=None):

    keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
    image = tf.placeholder(tf.float32,
                           shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
                           name="input_image")
    annotation = tf.placeholder(tf.int32,
                                shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1],
                                name="annotation")
    FM_pl = tf.placeholder(tf.float32, [])
    total_acc_pl = tf.placeholder(tf.float32, [])
    acc_pl = tf.placeholder(tf.float32, [])
    iu_pl = tf.placeholder(tf.float32, [])
    fwavacc_pl = tf.placeholder(tf.float32, [])
    # is_traing = tf.placeholder('bool')

    vgg_fcn = fcn32_vgg.FCN32VGG()
    vgg_fcn.build(image,
                  num_classes=num_classes,
                  keep_probability=keep_probability,
                  random_init_fc8=True)

    logits = vgg_fcn.upscore
    pred_annotation = vgg_fcn.pred_up
    # loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
    #                                                                       labels=tf.squeeze(annotation, squeeze_dims=[3]),
    #                                                                       name="entropy")))
    # loss_summary = tf.summary.scalar("entropy", loss)

    # trainable_var = tf.trainable_variables()
    # S_vars = [svar for svar in tf.trainable_variables() if 'weight' in svar.name]
    # l2 = tf.add_n([tf.nn.l2_loss(var) for var in S_vars])
    # # loss = loss + l2 * FLAGS.weight_decay
    # # train_op = tf.train.MomentumOptimizer(FLAGS.learning_rate, 0.9).minimize(loss + l2 * FLAGS.weight_decay)
    # train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss + l2 * FLAGS.weight_decay)
    # # train_op = train(loss, trainable_var)
    """ median-frequency re-weighting """
    # class_weights = np.array([
    #     0.5501,
    #     5.4915
    # ])
    # loss = tf.reduce_mean((tf.nn.weighted_cross_entropy_with_logits(logits=logits,
    #                                                                 targets=tf.one_hot(tf.squeeze(annotation, squeeze_dims=[3]), depth=num_classes),
    #                                                                 pos_weight=class_weights,
    #                                                                 name="entropy")))
    loss = LOSS.loss(
        logits,
        tf.one_hot(tf.squeeze(annotation, squeeze_dims=[3]),
                   depth=num_classes))
    regularization_loss = tf.add_n(
        tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
    t_loss = loss + regularization_loss

    loss_summary = tf.summary.scalar("entropy", loss)
    FM_summary = tf.summary.scalar('FM', FM_pl)
    acc_total_summary = tf.summary.scalar("total_acc", total_acc_pl)
    acc_summary = tf.summary.scalar("acc", acc_pl)
    iu_summary = tf.summary.scalar("iu", iu_pl)
    fwavacc_summary = tf.summary.scalar("fwavacc", fwavacc_pl)

    train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss)
    #train_op = tf.train.MomentumOptimizer(FLAGS.learning_rate, 0.9).minimize(t_loss)
    # train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(t_loss)
    summary_op = tf.summary.merge_all()

    train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir)
    print(len(train_records))
    print(len(valid_records))

    print("Setting up dataset reader")
    image_options = {'resize': False, 'resize_size': IMAGE_SIZE}
    if FLAGS.mode == 'train':
        train_dataset_reader = dataset.BatchDatset(train_records,
                                                   image_options)
    validation_dataset_reader = dataset.BatchDatset(valid_records,
                                                    image_options)

    sess = tf.Session(config=config)
    saver = tf.train.Saver(max_to_keep=3)

    # create two summary writers to show training loss and validation loss in the same graph
    # need to create two folders 'train' and 'validation' inside FLAGS.logs_dir
    praph_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/graph', sess.graph)
    train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train')
    validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation')

    sess.run(tf.global_variables_initializer())
    ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
    if ckpt and ckpt.model_checkpoint_path:
        saver.restore(sess, ckpt.model_checkpoint_path)
        print("Model restored...")

    if FLAGS.mode == "train":
        for itr in range(1, MAX_ITERATION):
            train_images, train_annotations = train_dataset_reader.next_batch(
                FLAGS.batch_size)
            feed_dict = {
                image: train_images,
                annotation: train_annotations,
                keep_probability: 0.5
            }

            sess.run(train_op, feed_dict=feed_dict)

            if itr % 10 == 0:
                train_loss, summary_str = sess.run([loss, loss_summary],
                                                   feed_dict=feed_dict)
                print("Step: %d, Train_loss:%g" % (itr, train_loss))
                train_writer.add_summary(summary_str, itr)

            if itr % 210 == 0:
                valid_iamges, valid_annotations = validation_dataset_reader.get_records(
                )
                val_count = 0
                total_loss = 0
                hist = np.zeros((num_classes, num_classes))
                fm = 0
                for i in range(1, 21):
                    val_images = valid_iamges[val_count:val_count +
                                              val_batch_size]
                    val_annotations = valid_annotations[val_count:val_count +
                                                        val_batch_size]
                    val_loss, val_pred_dense = sess.run(
                        [loss, logits],
                        feed_dict={
                            image: val_images,
                            annotation: val_annotations,
                            keep_probability: 1.0
                        })
                    total_loss = total_loss + val_loss
                    val_count = val_count + val_batch_size
                    hist += get_hist(val_pred_dense, val_annotations)
                    fm += get_FM(val_pred_dense, val_annotations)

                valid_loss = total_loss / 20
                FM = fm / (20 * val_batch_size)
                acc_total = np.diag(hist).sum() / hist.sum()
                acc = np.diag(hist) / hist.sum(1)
                iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) -
                                      np.diag(hist))
                freq = hist.sum(1) / hist.sum()

                # summary_st = sess.run(summary_op,feed_dict=feed_dict)

                summary_sva = sess.run(loss_summary,
                                       feed_dict={loss: valid_loss})
                summary_FM = sess.run(FM_summary, feed_dict={FM_pl: FM})
                summary_acc_total = sess.run(
                    acc_total_summary, feed_dict={total_acc_pl: acc_total})
                summary_acc = sess.run(acc_summary,
                                       feed_dict={acc_pl: np.nanmean(acc)})
                summary_iu = sess.run(iu_summary,
                                      feed_dict={iu_pl: np.nanmean(iu)})
                summary_fwavacc = sess.run(
                    fwavacc_summary,
                    feed_dict={
                        fwavacc_pl: (freq[freq > 0] * iu[freq > 0]).sum()
                    })
                print("Step: %d, Valid_loss:%g" % (itr, valid_loss))
                print(" >>> Step: %d, f1_score:%g" % (itr, FM))
                # overall accuracy
                print(" >>> Step: %d, overall accuracy:%g" % (itr, acc_total))
                print(" >>> Step: %d, mean accuracy:%g" %
                      (itr, np.nanmean(acc)))
                print(" >>> Step: %d, mean IU:%g" % (itr, np.nanmean(iu)))
                print(" >>> Step: %d, fwavacc:%g" %
                      (itr, (freq[freq > 0] * iu[freq > 0]).sum()))

                # validation_writer.add_summary(summary_st, step)
                validation_writer.add_summary(summary_sva, itr)
                validation_writer.add_summary(summary_FM, itr)
                validation_writer.add_summary(summary_acc_total, itr)
                validation_writer.add_summary(summary_acc, itr)
                validation_writer.add_summary(summary_iu, itr)
                validation_writer.add_summary(summary_fwavacc, itr)

                saver.save(sess, FLAGS.logs_dir + "model.ckpt", itr)

                va_images, va_annotations = validation_dataset_reader.get_random_batch(
                    20)

                pred = sess.run(pred_annotation,
                                feed_dict={
                                    image: va_images,
                                    annotation: va_annotations,
                                    keep_probability: 1.0
                                })
                va_annotations = np.squeeze(va_annotations, axis=3)
                # pred = np.squeeze(pred, axis=3)
                pred = pred * 255
                va_annotations = va_annotations * 255
                for it in range(20):
                    utils.save_image(va_images[it].astype(np.uint8),
                                     FLAGS.logs_dir,
                                     name="inp_" + str(5 + it))
                    utils.save_image(va_annotations[it].astype(np.uint8),
                                     FLAGS.logs_dir,
                                     name="gt_" + str(5 + it))
                    utils.save_image(pred[it].astype(np.uint8),
                                     FLAGS.logs_dir,
                                     name="pred_" + str(5 + it))

    elif FLAGS.mode == "visualize":
        it = 0
        valid_iamge, val_annotation = validation_dataset_reader.get_records()
        val_annotation = np.squeeze(val_annotation, axis=3)
        val_annotation = val_annotation * 255
        for filename in valid_records:
            val_image = np.array(misc.imread(filename['image']))
            val_image = np.reshape(val_image, (1, 256, 256, 3))
            pred = sess.run(pred_annotation,
                            feed_dict={
                                image: val_image,
                                keep_probability: 1.0
                            })
            pred = pred * 255
            # pred = sess.run(pred_annotation, feed_dict={image: val_image,
            #                                         keep_probability:1.0})
            utils.save_image(pred[0].astype(np.uint8),
                             FLAGS.logs_dir + 'pred01',
                             name=os.path.splitext(
                                 filename['image'].split("/")[-1])[0])
            utils.save_image(val_annotation[it].astype(np.uint8),
                             FLAGS.logs_dir + 'gt01',
                             name=os.path.splitext(
                                 filename['annotation'].split("/")[-1])[0])
            it = it + 1
예제 #3
0
#img1 = skimage.io.imread("./test_data/tabby_cat.png")

parser = argparse.ArgumentParser()
parser.add_argument('--npypath',help="path to weights",default="/scratch/gallowaa/")
parser.add_argument('--imgpath',help="path to input image",default="/scratch/gallowaa/")
args = parser.parse_args()

img1 = skimage.io.imread(args.imgpath+"2008_004499_02.png")

with tf.Session() as sess:
    images = tf.placeholder("float")
    feed_dict = {images: img1}
    batch_images = tf.expand_dims(images, 0)

    vgg_fcn = fcn32_vgg.FCN32VGG(args.npypath+"vgg16.npy")
    with tf.name_scope("content_vgg"):
        vgg_fcn.build(batch_images, debug=True)

    print('Finished building Network.')

    init = tf.initialize_all_variables()
    sess.run(tf.initialize_all_variables())

    print('Running the Network')
    tensors = [vgg_fcn.pred, vgg_fcn.pred_up]
    down, up = sess.run(tensors, feed_dict=feed_dict)

    down_color = utils.color_image(down[0])
    up_color = utils.color_image(up[0])
예제 #4
0
import numpy as np
import tensorflow as tf

import fcn32_vgg
import utils

from tensorflow.python.framework import ops

img1 = scp.misc.imread("./test_data/tabby_cat.png")

with tf.Session() as sess:
    images = tf.placeholder("float")
    feed_dict = {images: img1}
    batch_images = tf.expand_dims(images, 0)  # 第一维是n_examples

    vgg_fcn = fcn32_vgg.FCN32VGG()  # 获得class
    with tf.name_scope("content_vgg"):
        vgg_fcn.build(batch_images, debug=True)

    print('Finished building Network.')

    init = tf.global_variables_initializer()
    sess.run(init)

    print('Running the Network')
    tensors = [vgg_fcn.pred, vgg_fcn.pred_up]  #  下采样后和上采样后
    down, up = sess.run(tensors, feed_dict=feed_dict)

    down_color = utils.color_image(down[0])
    up_color = utils.color_image(up[0])
def run_network():
    with tf.Session() as sess:
        images = tf.placeholder("float")
        # batch_images = tf.expand_dims(images, 0)

        keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")

        vgg_fcn = fcn32_vgg.FCN32VGG()
        with tf.name_scope("content_vgg"):
            vgg_fcn.build(images, num_classes=NUM_CLASSES, train=True, debug=True, keep_prob=keep_probability)

        print('Finished building Network.')

        logging.warning("Score weights are initialized random.")
        logging.warning("Do not expect meaningful results.")


        print("Build Training network")
        annotations = tf.placeholder("float")
        # batch_annotations = tf.expand_dims(annotations, 0)

        vgg_fcn_loss = loss(vgg_fcn.upscore, annotations, NUM_CLASSES)
        tf.summary.scalar("entropy", vgg_fcn_loss)

        trainable_vars = tf.trainable_variables()
        optimizer = tf.train.AdamOptimizer(1e-6)
        grads = optimizer.compute_gradients(vgg_fcn_loss, var_list=trainable_vars)
        for grad, var in grads:
            if grad is not None:
                tf.summary.histogram("gradient/" + var.op.name, grad)
        train_step = optimizer.apply_gradients(grads)


        pred2color = int(255.0/(NUM_CLASSES-1.0))
        CLASS_WE_CARE_ABOUT = 1
        tf.summary.image("input_image", images, max_outputs=1)
        tf.summary.image("ground_truth", tf.cast(tf.split(annotations, NUM_CLASSES, axis=3)[CLASS_WE_CARE_ABOUT], tf.uint8)*pred2color, max_outputs=1)
        tf.summary.image("pred_annotation", tf.cast((tf.split(vgg_fcn.prob_up, NUM_CLASSES, axis=3)[CLASS_WE_CARE_ABOUT])*pred2color, tf.uint8), max_outputs=1)

        logging.info("Start Initializing Variables.")
        init = tf.global_variables_initializer()
        sess.run(init)

        summary_op = tf.summary.merge_all()
        summary_writer = tf.summary.FileWriter("./chkpt/", sess.graph)

        sum_valid_image = tf.summary.image("valid_input_image", images, max_outputs=20)
        sum_valid_gt = tf.summary.image("valid_ground_truth", tf.cast(tf.split(annotations, NUM_CLASSES, axis=3)[1], tf.uint8) * pred2color,
                         max_outputs=20)
        sum_valid_pred = tf.summary.image("valid_pred_annotation",
                         tf.cast((tf.split(vgg_fcn.prob_up, NUM_CLASSES, axis=3)[CLASS_WE_CARE_ABOUT]) * pred2color, tf.uint8), max_outputs=20)
        sum_valid_entropy = tf.summary.scalar("valid_entropy", vgg_fcn_loss)

        valid_summary_op = tf.summary.merge([sum_valid_image,sum_valid_gt,sum_valid_pred,sum_valid_entropy])

        training_records, validation_records = load_data("E:/videoseg/VanuatuGT")

        image_options = {'resize': True, 'resize_size': 244}
        DURATION = 30 * 60 * 2
        train_dataset_reader = BatchDataset2(training_records, NUM_CLASSES, image_options, fromFrameId=0, uptoFrameId=DURATION)
        validation_dataset_reader = BatchDataset2(validation_records, NUM_CLASSES, image_options)

        print("Training the Network")
        saver = tf.train.Saver()
        # saver.restore(sess, "./chkpt/model32.ckpt")
        MAX_ITERATION = int(500)
        VIDEO_STEPS = int(5)
        BATCH_SIZE = 1
        for itr in range(1, MAX_ITERATION*VIDEO_STEPS + 2):
            train_images, train_annotations = train_dataset_reader.next_batch(BATCH_SIZE)
            feed_dict = {images: train_images, annotations: train_annotations, keep_probability: 0.4}

            train_step.run(feed_dict=feed_dict)

            if itr % 10 == 0:
                train_loss, summary_str = sess.run([vgg_fcn_loss, summary_op], feed_dict=feed_dict)
                print("Step: %d, Train_loss:%g" % (itr, train_loss))
                summary_writer.add_summary(summary_str, itr)

                valid_images, valid_annotations = validation_dataset_reader.next_batch(20)
                feed_dict = {images: valid_images, annotations: valid_annotations, keep_probability: 1.0}
                valid_loss, valid_summary_str = sess.run([vgg_fcn_loss, valid_summary_op], feed_dict=feed_dict)
                print("%s ---> Validation_loss: %g" % (datetime.datetime.now(), valid_loss))
                summary_writer.add_summary(valid_summary_str, itr)
            if itr % 500 == 0:
                save_path = saver.save(sess, "./chkpt/model32.ckpt")
                print("Model saved in file: %s" % save_path)

            if itr % MAX_ITERATION == 0:
                #we've finished training for this portion of the video, lets save out the video and start training on the next part of the video
                videoItr = int(itr / MAX_ITERATION)

                print("processing the entire video")
                cap = cv2.VideoCapture("E:\\vanuatu\\vanuatu35\\640x360\\vanuatu35_%06d.jpg")
                frameNum = 0
                allFrames = None
                while (True):
                    # Capture frame-by-frame
                    frames = []
                    ret = False
                    for i in range(20):
                        ret, frame = cap.read()
                        if not ret:
                            break
                        frameNum += 1

                        frame = cv2.resize(frame, (image_options['resize_size'], image_options['resize_size']))

                        if frame is not None and frame.any():
                            frames.append(frame)

                    if frames:
                        feed_dict = {images: np.array(frames), keep_probability: 1.0}
                        prob = sess.run([vgg_fcn.prob_up], feed_dict=feed_dict)
                        prob = prob[0][:, :, :, 1, None]

                        if allFrames is None:
                            allFrames = prob
                        else:
                            allFrames = np.vstack((allFrames, prob))

                    if allFrames.shape[0] > 1000 or not ret:
                        print("saving frames {} - {}".format(frameNum-allFrames.shape[0], frameNum))
                        p_path = os.path.join("E:/videoseg/VanuatuProb", "{:d}_{:05d}_P.npy".format(videoItr, frameNum))
                        np.save(p_path, allFrames)
                        allFrames = None

                    if not ret:
                        break
                cap.release()

                startFrameId = int(videoItr * 30 * 60)
                print("loading the next section of the training data. {} - {}".format(startFrameId,
                                                                                      startFrameId + DURATION))
                train_dataset_reader.changeFrameIdRange(fromFrameId=startFrameId, uptoFrameId=startFrameId+DURATION)
label_image = tf.one_hot(label_image, 19)

train_image = tf.cast(train_image, tf.float32) / 255.
image = tf.Print(train_image, [tf.shape(label_image)])
train_image_batch, train_label_batch = tf.train.shuffle_batch(
    [train_image, label_image],
    batch_size=BATCH_SIZE,
    capacity=3 + 3 * BATCH_SIZE,
    min_after_dequeue=3)
with tf.device('/cpu:0'):

    sess = tf.Session()
    images_tf = tf.placeholder(tf.float32, [None, 1024, 2048, 3])
    labels_tf = tf.placeholder(tf.float32, [None, 1024, 2048, 19])

    vgg_fcn = fcn32_vgg.FCN32VGG('./vgg16.npy')
    with tf.name_scope("content_vgg"):
        vgg_fcn.build(images_tf,
                      train=True,
                      num_classes=19,
                      random_init_fc8=True,
                      debug=False)

    #head=[]
    cross_entropy = -tf.reduce_sum(labels_tf * tf.log(vgg_fcn.softmax),
                                   reduction_indices=[1])
    cross_entropy_mean = tf.reduce_mean(cross_entropy, name='xentropy_mean')
    #loss_tf = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(vgg_fcn.upscore,
    #                                                                     tf.squeeze(labels_tf, squeeze_dims=[3]),
    #                                                                    name="entropy")))