コード例 #1
0
ファイル: train.py プロジェクト: MasazI/tiny_distributed_tf
def main(_):
    if gfile.Exists(TRAIN_DIR):
        gfile.DeleteRecursively(TRAIN_DIR)
    gfile.MakeDirs(TRAIN_DIR)

    # locally
    #train()

    print("ps: %s" % (DFLAGS.task_index))

    ps_hosts = DFLAGS.ps_hosts.split(",")
    worker_hosts = DFLAGS.worker_hosts.split(",")

    # Create a cluster from the parameter server and worker hosts.
    cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts})

    # Create and start a server for the local task.
    server = tf.train.Server(cluster,
                             job_name=FLAGS.job_name,
                             task_index=FLAGS.task_index)

    # training data
    filename_queue = tf.train.string_input_producer(
        ["output/data/airquality.csv"])
    datas, targets = dataset.mini_batch(filename_queue, BATCH_SIZE)

    if DFLAGS.job_name == "ps":
        server.join()
    elif DFLAGS.job_name == "worker":
        # Assigns ops to the local worker by default.
        with tf.device(
                tf.train.replica_device_setter(
                    worker_device="/job:worker/task:%d" % DFLAGS.task_index,
                    cluster=cluster)):

            # step num of global
            global_step = tf.Variable(0, trainable=False)

            # inference
            logits = model.inference(datas)

            debug_value = model.debug(logits)

            # loss graphのoutputとlabelを利用
            loss = model.loss(logits, targets)

            global_step = tf.Variable(0)

            #train_op = tf.train.AdagradOptimizer(0.0001).minimize(
            #    loss, global_step=global_step)
            train_op = op.train(loss, global_step)

            saver = tf.train.Saver()
            summary_op = tf.merge_all_summaries()
            init_op = tf.initialize_all_variables()

        # Create a "supervisor", which oversees the training process.
        sv = tf.train.Supervisor(is_chief=(FLAGS.task_index == 0),
                                 logdir="/tmp/train_logs",
                                 init_op=init_op,
                                 init_feed_dict=None,
                                 summary_op=summary_op,
                                 saver=saver,
                                 global_step=global_step,
                                 save_model_secs=60)

        # The supervisor takes care of session initialization, restoring from
        # a checkpoint, and closing when done or an error occurs.
        with sv.managed_session(server.target) as sess:
            # Loop until the supervisor shuts down or 1000000 steps have completed.
            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(sess=sess, coord=coord)

            step = 0
            while not sv.should_stop() and step < 1000000:
                # Run a training step asynchronously.
                # See `tf.train.SyncReplicasOptimizer` for additional details on how to
                # perform *synchronous* training.
                start_time = time.time()
                _, loss_value, predict_value, targets_eval, step = sess.run(
                    [train_op, loss, debug_value, targets, global_step])
                #_, step = sess.run([train_op, global_step])
                duration = time.time() - start_time

                if step % 100 == 0:
                    # mini batch size
                    num_examples_per_step = BATCH_SIZE

                    # examples num per sec
                    examples_per_sec = num_examples_per_step / duration

                    # duration per batch
                    sec_per_batch = float(duration)

                    # time, step num, loss, exampeles num per sec, time per batch
                    format_str = '$s: step %d, loss = %.2f (%.1f examples/sec; %.3f sec/batch)'
                    print str(datetime.now()) + ': step' + str(
                        step) + ', loss= ' + str(loss_value) + ' ' + str(
                            examples_per_sec) + ' examples/sec; ' + str(
                                sec_per_batch) + ' sec/batch'
                    print "predict: ", predict_value
                    print "targets: ", targets_eval

            coord.request_stop()
            coord.join(threads)
            sess.close()

        # Ask for all the services to stop.
        sv.stop()
コード例 #2
0
def train():
    '''
    Train CNN_tiny for a number of steps.
    '''
    with tf.Graph().as_default():
        # globalなstep数
        global_step = tf.Variable(0, trainable=False)

        # 教師データ
        #images, labels = data_inputs.distorted_inputs(TF_RECORDS)
        # 教師データ
        mnist = np.load('./data/mnist_sequence1_sample_5distortions5x5.npz')
        trX = mnist['X_train']
        trY = mnist['y_train']
        # X_valid = mnist_cluttered['X_valid']
        # y_valid = mnist_cluttered['y_valid']
        teX = mnist['X_test']
        teY = mnist['y_test']
        trX = trX.reshape(-1, 40, 40, 1)
        teX = teX.reshape(-1, 40, 40, 1)
        
        # % turn from dense to one hot representation
        trY = dense_to_one_hot(trY, n_classes=10)
        # Y_valid = dense_to_one_hot(y_valid, n_classes=10)
        teY = dense_to_one_hot(teY, n_classes=10)

        print("the number of train data: %d" % (len(trX)))

        # create mini_batch
        #datas, targets = trX.(trX, trY, BATCH_SIZE)

        images = tf.placeholder(tf.float32, [None, 40, 40, 1])
        labels = tf.placeholder(tf.float32, [None, 10])
        keep_conv = tf.placeholder("float")
        keep_hidden = tf.placeholder("float")

        # graphのoutput
        logits = model.inference(images, keep_conv, keep_hidden)

        # loss graphのoutputとlabelを利用
        #loss = model.loss(logits, labels)
        
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, labels))
        predict_op = tf.argmax(logits, 1)

        # 学習オペレーション
        train_op = op.train(loss, global_step)

        # saver
        saver = tf.train.Saver(tf.all_variables())

        # サマリー
        summary_op = tf.merge_all_summaries()

        # 初期化オペレーション
        init_op = tf.initialize_all_variables()

        # Session
        sess = tf.Session(config=tf.ConfigProto(log_device_placement=LOG_DEVICE_PLACEMENT))
        sess.run(init_op)

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        # サマリーのライターを設定
        summary_writer = tf.train.SummaryWriter(TRAIN_DIR, graph_def=sess.graph_def)
 
        # max_stepまで繰り返し学習
        for step in xrange(MAX_STEPS):
            start_time = time.time()
            previous_time = start_time
            index = 0
            for start, end in zip(range(0, len(trX), BATCH_SIZE), range(BATCH_SIZE, len(trX), BATCH_SIZE)):
                _, loss_value = sess.run([train_op, loss], feed_dict={images: trX[start:end], labels: trY[start:end], keep_conv: 0.8, keep_hidden: 0.5})
                if index % 10 == 0:
                    end_time = time.time()
                    duration = end_time - previous_time
                    num_examples_per_step = BATCH_SIZE * 10 * (step+1)
                    examples_per_sec = num_examples_per_step / duration
                    print("%s: %d[epoch]: %d[iteration]: train loss %f: %d[examples/step]: %f[examples/sec]: %f[sec/iteration]" % (datetime.now(), step, index, loss_value, num_examples_per_step, examples_per_sec, duration))
                    index += 1
                    assert not np.isnan(loss_value), 'Model diverged with loss = NaN'

                    test_indices = np.arange(len(teX)) # Get A Test Batch
                    np.random.shuffle(test_indices)
                    test_indices = test_indices[0:5]
                    print "="*20
                    print teY[test_indices]
                    predict, cost_value = sess.run([predict_op, loss], feed_dict={images: teX[test_indices],
                                                                     labels: teY[test_indices], keep_conv: 1.0, keep_hidden: 1.0})
                    print predict
                    print("test loss: %f" % (cost_value))
                    print "="*20
                    previous_time = end_time

                index += 1
                assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
                
                # 1000回ごと
                if index % 100 == 0:
                    pass
                    summary_str = sess.run(summary_op, feed_dict={images: teX[test_indices],
                                                                     labels: teY[test_indices], keep_conv: 1.0, keep_hidden: 1.0})
                    # サマリーに書き込む
                    summary_writer.add_summary(summary_str, step)
            if step % 1 == 0 or (step * 1) == MAX_STEPS:
                checkpoint_path = TRAIN_DIR + '/model.ckpt'
                saver.save(sess, checkpoint_path, global_step=step)

        coord.request_stop()
        coord.join(threads)
        sess.close()
コード例 #3
0
def train():
    '''
    Train CNN_tiny for a number of steps.
    '''
    with tf.Graph().as_default():
        # globalなstep数
        global_step = tf.Variable(0, trainable=False)

        # 教師データ
        filename_queue = tf.train.string_input_producer(
            ["data/airquality.csv"])
        datas, targets = load.mini_batch(filename_queue, BATCH_SIZE)

        # placeholder
        x = tf.placeholder(tf.float32, shape=[None, 5])
        y = tf.placeholder(tf.float32, shape=[None, 1])

        # graphのoutput
        logits = model.inference(x)

        debug_value = model.debug(logits)

        # loss graphのoutputとlabelを利用
        loss = model.loss(logits, y)

        # 学習オペレーション
        train_op = op.train(loss, global_step)

        # saver
        saver = tf.train.Saver(tf.all_variables())

        # サマリー
        summary_op = tf.merge_all_summaries()

        # 初期化オペレーション
        init_op = tf.initialize_all_variables()

        # Session
        sess = tf.Session(config=tf.ConfigProto(
            log_device_placement=LOG_DEVICE_PLACEMENT))
        sess.run(init_op)

        print("settion start.")

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        # サマリーのライターを設定
        summary_writer = tf.train.SummaryWriter(TRAIN_DIR,
                                                graph_def=sess.graph_def)

        # model名
        model_name = '/model%s.ckpt' % (tdatetime.strftime('%Y%m%d%H%M%S'))

        # max_stepまで繰り返し学習
        for step in xrange(MAX_STEPS):
            start_time = time.time()
            a, b = sess.run([datas, targets])
            _, loss_value, predict_value = sess.run(
                [train_op, loss, debug_value], feed_dict={
                    x: a,
                    y: b
                })

            duration = time.time() - start_time

            assert not np.isnan(loss_value), 'Model diverged with loss = NaN'

            # 100回ごと
            if step % 100 == 0:
                # stepごとの事例数 = mini batch size
                num_examples_per_step = BATCH_SIZE

                # 1秒ごとの事例数
                examples_per_sec = num_examples_per_step / duration

                # バッチごとの時間
                sec_per_batch = float(duration)

                # time, step数, loss, 1秒で実行できた事例数, バッチあたりの時間
                format_str = '$s: step %d, loss = %.2f (%.1f examples/sec; %.3f sec/batch)'
                print str(datetime.now()) + ': step' + str(
                    step) + ', loss= ' + str(loss_value) + ' ' + str(
                        examples_per_sec) + ' examples/sec; ' + str(
                            sec_per_batch) + ' sec/batch'

                print "x", a
                print "ground truth:", b
                print "predict: ", predict_value

            # 100回ごと
            if step % 100 == 0:
                pass
                #summary_str = sess.run(summary_op)
                # サマリーに書き込む
                #summary_writer.add_summary(summary_str, step)

            if step % 1000 == 0 or (step * 1) == MAX_STEPS:
                checkpoint_path = TRAIN_DIR + model_name
                saver.save(sess, checkpoint_path, global_step=step)

        coord.request_stop()
        coord.join(threads)
        sess.close()
コード例 #4
0
def train():
    '''
    Train CNN_tiny for a number of steps.
    '''
    with tf.Graph().as_default():
        # globalなstep数
        global_step = tf.Variable(0, trainable=False)

        # 教師データ
        image_input = ImageInput('./data/101Caltech_shuffles.txt')
        print("the number of train data: %d" % (len(image_input.image_paths)))

        images = tf.placeholder(tf.float32, [None, 224, 224, 3])
        labels = tf.placeholder(tf.float32, [None, 101])
        keep_conv = tf.placeholder(tf.float32)
        keep_hidden = tf.placeholder(tf.float32)

        # graphのoutput
        logits, transform_result = model.inference(images, keep_conv,
                                                   keep_hidden)

        # loss graphのoutputとlabelを利用
        # loss = model.loss(logits, labels)

        loss = tf.reduce_mean(
            tf.nn.softmax_cross_entropy_with_logits(logits, labels))
        predict_op = tf.argmax(logits, 1)

        # 学習オペレーション
        train_op = op.train(loss, global_step)

        # サマリー
        summary_op = tf.merge_all_summaries()

        # 初期化オペレーション
        init_op = tf.initialize_all_variables()

        # Session
        sess = tf.Session(config=tf.ConfigProto(
            log_device_placement=LOG_DEVICE_PLACEMENT))

        # saver
        #saver = tf.train.Saver(tf.all_variables())

        sess.run(init_op)
        # pretrainと全体を分けて保存
        pretrain_params = {}
        train_params = {}
        for variable in tf.trainable_variables():
            variable_name = variable.name
            #print("parameter: %s" %(variable_name))
            scope, name = variable_name.split("/")
            target, _ = name.split(":")
            if variable_name.find('spatial_transformer') < 0:
                print("pretrain parameter: %s" % (variable_name))
                pretrain_params[variable_name] = variable
            print("train parameter: %s" % (variable_name))
            train_params[variable_name] = variable
        saver_cnn = tf.train.Saver(pretrain_params)
        saver_transformers = tf.train.Saver(train_params)

        # pretrained_model
        if FLAGS.fine_tune:
            ckpt = tf.train.get_checkpoint_state(PRETRAIN_DIR)
            if ckpt and ckpt.model_checkpoint_path:
                print("Pretrained Model Loading.")
                saver_cnn.restore(sess, ckpt.model_checkpoint_path)
                print("Pretrained Model Restored.")
            else:
                print("No Pretrained Model.")

        # pretrained model from another type models.
        #    # saver
        #    print type(tf.all_variables())
        #    for variable in tf.trainable_variables():
        #        variable_name = variable.name
        #        variable_value = variable.eval(sess)
        #        if variable_name.find('softmax_linear') < 0 and variable_name.find('spatial_transformer') < 0:
        #            print("trained parameter: %s" %(variable_name))
        #            scope, name = variable_name.split("/")
        #            target, _ = name.split(":")
        #            with tf.variable_scope(scope, reuse=True):
        #                sess.run(tf.get_variable(target).assign(variable_value))
        #    trained_model = FLAGS.trained_model
        #    print trained_model

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        # サマリーのライターを設定
        summary_writer = tf.train.SummaryWriter(TRAIN_DIR,
                                                graph_def=sess.graph_def)

        # max_stepまで繰り返し学習
        for step in xrange(MAX_STEPS):
            start_time = time.time()
            previous_time = start_time
            index = 0

            batches = image_input.get_batches(FLAGS.batch_size)
            for batch in batches:
                train = batch[0]
                label = batch[1]
                _, loss_value = sess.run([train_op, loss],
                                         feed_dict={
                                             images: train,
                                             labels: label,
                                             keep_conv: 0.8,
                                             keep_hidden: 0.5
                                         })
                if index % 10 == 0:
                    end_time = time.time()
                    duration = end_time - previous_time
                    num_examples_per_step = BATCH_SIZE * 10
                    examples_per_sec = num_examples_per_step / duration
                    print(
                        "%s: %d[epoch]: %d[iteration]: train loss %f: %d[examples/iteration]: %f[examples/sec]: %f[sec/iteration]"
                        % (datetime.now(), step, index, loss_value,
                           num_examples_per_step, examples_per_sec, duration))
                    index += 1
                    assert not np.isnan(
                        loss_value), 'Model diverged with loss = NaN'

                    # test_indices = np.arange(len(teX)) # Get A Test Batch
                    # np.random.shuffle(test_indices)
                    # test_indices = test_indices[0:5]
                    print "=" * 20
                    testx = train[0:2]
                    #print testx
                    testy = label[0:2]
                    print np.argmax(testy[0])
                    print np.argmax(testy[1])
                    output_vec, predict, cost_value = sess.run(
                        [logits, predict_op, loss],
                        feed_dict={
                            images: testx,
                            labels: testy,
                            keep_conv: 1.0,
                            keep_hidden: 1.0
                        })
                    print predict
                    print("test loss: %f" % (cost_value))
                    print "=" * 20
                    previous_time = end_time

                index += 1
                assert not np.isnan(
                    loss_value), 'Model diverged with loss = NaN'

                # 100回ごと
                if index % 100 == 0:
                    pass
                    summary_str = sess.run(summary_op,
                                           feed_dict={
                                               images: train,
                                               labels: label,
                                               keep_conv: 0.8,
                                               keep_hidden: 0.5
                                           })
                    # サマリーに書き込む
                    summary_writer.add_summary(summary_str, step)

            if step % 1 == 0 or (step * 1) == MAX_STEPS:
                pretrain_checkpoint_path = PRETRAIN_DIR + '/model.ckpt'
                train_checkpoint_path = TRAIN_DIR + '/model.ckpt'
                saver_cnn.save(sess,
                               pretrain_checkpoint_path,
                               global_step=step)
                saver_transformers.save(sess,
                                        train_checkpoint_path,
                                        global_step=step)
        coord.request_stop()
        coord.join(threads)
        sess.close()
コード例 #5
0
def train():
    '''
    Train CNN_tiny for a number of steps.
    '''
    with tf.Graph().as_default():
        # globalなstep数
        global_step = tf.Variable(0, trainable=False)

        # 教師データ
        filename_queue = tf.train.string_input_producer(["data/airquality.csv"])
        datas, targets = load.mini_batch(filename_queue, BATCH_SIZE)

        # placeholder
        x = tf.placeholder(tf.float32, shape=[None, 5])
        y = tf.placeholder(tf.float32, shape=[None, 1])

        # graphのoutput
        logits = model.inference(x)

        debug_value = model.debug(logits)

        # loss graphのoutputとlabelを利用
        loss = model.loss(logits, y)

        # 学習オペレーション
        train_op = op.train(loss, global_step)

        # saver
        saver = tf.train.Saver(tf.all_variables())

        # サマリー
        summary_op = tf.merge_all_summaries()

        # 初期化オペレーション
        init_op = tf.initialize_all_variables()

        # Session
        sess = tf.Session(config=tf.ConfigProto(log_device_placement=LOG_DEVICE_PLACEMENT))
        sess.run(init_op)

        print("settion start.")

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        # サマリーのライターを設定
        summary_writer = tf.train.SummaryWriter(TRAIN_DIR, graph_def=sess.graph_def)

        # model名
        model_name = '/model%s.ckpt' % (tdatetime.strftime('%Y%m%d%H%M%S'))
   
        # max_stepまで繰り返し学習
        for step in xrange(MAX_STEPS):
            start_time = time.time()
            a, b = sess.run([datas, targets])
            _, loss_value, predict_value = sess.run([train_op, loss, debug_value], feed_dict={x: a, y: b})

            duration = time.time() - start_time

            assert not np.isnan(loss_value), 'Model diverged with loss = NaN'

            # 100回ごと
            if step % 100 == 0:
                # stepごとの事例数 = mini batch size
                num_examples_per_step = BATCH_SIZE

                # 1秒ごとの事例数
                examples_per_sec = num_examples_per_step / duration
                
                # バッチごとの時間
                sec_per_batch = float(duration)

                # time, step数, loss, 1秒で実行できた事例数, バッチあたりの時間
                format_str = '$s: step %d, loss = %.2f (%.1f examples/sec; %.3f sec/batch)'
                print str(datetime.now()) + ': step' + str(step) + ', loss= '+ str(loss_value) + ' ' + str(examples_per_sec) + ' examples/sec; ' + str(sec_per_batch) + ' sec/batch'

                print "x", a
                print "ground truth:", b
                print "predict: ", predict_value



            # 100回ごと
            if step % 100 == 0:
                pass
                #summary_str = sess.run(summary_op)
                # サマリーに書き込む
                #summary_writer.add_summary(summary_str, step)

            if step % 1000 == 0 or (step * 1) == MAX_STEPS:
                checkpoint_path = TRAIN_DIR + model_name
                saver.save(sess, checkpoint_path, global_step=step)

        coord.request_stop()
        coord.join(threads)
        sess.close()
コード例 #6
0
def train():
    '''
    Train CNN_tiny for a number of steps.
    '''
    with tf.Graph().as_default():
        # globalなstep数
        global_step = tf.Variable(0, trainable=False)

        # 教師データ
        images, labels = data_inputs.distorted_inputs(TF_RECORDS)

        # graphのoutput
        logits = model.inference(images)

        # loss graphのoutputとlabelを利用
        loss = model.loss(logits, labels)

        # 学習オペレーション
        train_op = op.train(loss, global_step)

        # saver
        saver = tf.train.Saver(tf.all_variables())

        # サマリー
        summary_op = tf.merge_all_summaries()

        # 初期化オペレーション
        init_op = tf.initialize_all_variables()

        # Session
        sess = tf.Session(config=tf.ConfigProto(
            log_device_placement=LOG_DEVICE_PLACEMENT))
        sess.run(init_op)

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        # サマリーのライターを設定
        summary_writer = tf.train.SummaryWriter(TRAIN_DIR,
                                                graph_def=sess.graph_def)

        # model名
        model_name = '/model%s.ckpt' % (tdatetime.strftime('%Y%m%d%H%M%S'))

        # max_stepまで繰り返し学習
        for step in xrange(MAX_STEPS):
            start_time = time.time()
            _, loss_value = sess.run([train_op, loss])
            duration = time.time() - start_time

            assert not np.isnan(loss_value), 'Model diverged with loss = NaN'

            # 100回ごと
            if step % 100 == 0:
                # stepごとの事例数 = mini batch size
                num_examples_per_step = BATCH_SIZE

                # 1秒ごとの事例数
                examples_per_sec = num_examples_per_step / duration

                # バッチごとの時間
                sec_per_batch = float(duration)

                # time, step数, loss, 1秒で実行できた事例数, バッチあたりの時間
                format_str = '$s: step %d, loss = %.2f (%.1f examples/sec; %.3f sec/batch)'
                print str(datetime.now()) + ': step' + str(
                    step) + ', loss= ' + str(loss_value) + ' ' + str(
                        examples_per_sec) + ' examples/sec; ' + str(
                            sec_per_batch) + ' sec/batch'

            # 100回ごと
            if step % 100 == 0:
                pass
                summary_str = sess.run(summary_op)
                # サマリーに書き込む
                summary_writer.add_summary(summary_str, step)

            if step % 1000 == 0 or (step * 1) == MAX_STEPS:
                checkpoint_path = TRAIN_DIR + model_name
                saver.save(sess, checkpoint_path, global_step=step)

        coord.request_stop()
        coord.join(threads)
        sess.close()
コード例 #7
0
def train():
    '''
    Train
    '''
    with tf.Graph().as_default():
        # globalなstep数
        global_step = tf.Variable(0, trainable=False)

        # NYU Dataset V2 original size(480 x 640 x 3) -> crop -> (460 x 620 x 3)
        image_input = ImageInput('./data/nyu_depth_v2_labeled.mat')
        print("the number of train data: %d" % (len(image_input.images)))

        images = tf.placeholder(tf.float32, [None, FLAGS.crop_size_height, FLAGS.crop_size_width, FLAGS.image_depth])
        depths = tf.placeholder(tf.float32, [None, 1, 55, 74])
        invalid_depths = tf.placeholder(tf.float32, [None, 1, 55, 74])
        keep_conv = tf.placeholder(tf.float32)
        keep_hidden = tf.placeholder(tf.float32)

        # graphのoutput
        if FLAGS.refine_train:
            print("refine train.")
            logits = model.inference_refine(images, keep_conv, keep_hidden)
        else:
            print("coarse train.")
            logits = model.inference(images, keep_conv, keep_hidden)

        # loss graphのoutputとlabelを利用
        loss = model.loss(logits, depths, invalid_depths)
        # 学習オペレーション
        train_op = op.train(loss, global_step)

        # サマリー
        summary_op = tf.merge_all_summaries()

        # 初期化オペレーション
        init_op = tf.initialize_all_variables()

        # Session
        sess = tf.Session(config=tf.ConfigProto(log_device_placement=LOG_DEVICE_PLACEMENT))

        # saver
        #saver = tf.train.Saver(tf.all_variables())

        sess.run(init_op)    

        # coarseとrefineを分けて保存
        coarse_params = {}
        refine_params = {}

        if FLAGS.refine_train:
            for variable in tf.all_variables():
                variable_name = variable.name
                print("parameter: %s" % (variable_name))
                if variable_name.find("/") < 0 or variable_name.count("/") != 1:
                    print("ignore.")
                    continue
                scope, name = variable_name.split("/")
                target, _ = name.split(":")
                if variable_name.find('coarse') >= 0:
                    print("coarse parameter: %s" % (variable_name))
                    coarse_params[variable_name] = variable
                if variable_name.find('fine') >= 0:
                    print("refine parameter: %s" % (variable_name))
                    refine_params[variable_name] = variable
        else:
            for variable in tf.trainable_variables():
                variable_name = variable.name
                print("parameter: %s" %(variable_name))
                if variable_name.find("/") < 0 or variable_name.count("/") != 1:
                    print("ignore.")
                    continue
                scope, name = variable_name.split("/")
                target, _ = name.split(":")
                if variable_name.find('coarse') >= 0:
                    print("coarse parameter: %s" %(variable_name))
                    coarse_params[variable_name] = variable
                if variable_name.find('fine') >= 0:
                    print("refine parameter: %s" %(variable_name))
                    refine_params[variable_name] = variable

        # define saver
        saver_coarse = tf.train.Saver(coarse_params)
        saver_refine = tf.train.Saver(refine_params)

        # fine tune
        if FLAGS.fine_tune:
            # load coarse paramteters
            coarse_ckpt = tf.train.get_checkpoint_state(COARSE_DIR)
            if coarse_ckpt and coarse_ckpt.model_checkpoint_path:
                print("Pretrained coarse Model Loading.")
                saver_coarse.restore(sess, coarse_ckpt.model_checkpoint_path)
                print("Pretrained coarse Model Restored.")
            else:
                print("No Pretrained coarse Model.")

            # load refine parameters
            refine_ckpt = tf.train.get_checkpoint_state(REFINE_DIR)
            if refine_ckpt and refine_ckpt.model_checkpoint_path:
                print("Pretrained refine Model Loading.")
                saver_refine.restore(sess, refine_ckpt.model_checkpoint_path)
                print("Pretrained refine Model Restored.")
            else:
                print("No Pretrained refine Model.")

        # TODO train coarse or refine (change trainable)
        #if not FLAGS.coarse_train:
        #    for val in coarse_params:
        #        print val
        #if not FLAGS.refine_train:
        #    for val in coarse_params:
        #        print val

        # train refine
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        # debug
        # サマリーのライターを設定
        #summary_writer = tf.train.SummaryWriter(TRAIN_DIR, graph_def=sess.graph_def)
        #batches = image_input.get_batches(FLAGS.batch_size)a
        #d = np.asarray(batches[0][0])
        #print d.shape
        #a = np.asarray(batches[0][1])
        #print a.shape
        #logits_val, logits_fine_val, loss_value = sess.run([logits, logits_fine, loss], feed_dict={images: batches[0][0], depths: batches[0][1], invalid_depths: batches[0][2], keep_conv: 1.0, keep_hidden: 1.0})
        #print len(logits_val[0])
        #print len(logits_fine_val[0])
        #print loss_value

        # max_stepまで繰り返し学習
        for step in xrange(MAX_STEPS):
            start_time = time.time()
            previous_time = start_time
            index = 0

            batches = image_input.get_batches(FLAGS.batch_size)
            vals = image_input.get_validation()
            for batch in batches:
                train = batch[0]
                depth = batch[1]
                ignore_depth = batch[2]
                _, loss_value = sess.run([train_op, loss], feed_dict={images: train, depths: depth, invalid_depths: ignore_depth, keep_conv: 0.8, keep_hidden: 0.5})
                if index % 10 == 0:
                    end_time = time.time()
                    duration = end_time - previous_time
                    num_examples_per_step = BATCH_SIZE * 10
                    examples_per_sec = num_examples_per_step / duration
                    print("%s: %d[epoch]: %d[iteration]: train loss %f: %d[examples/iteration]: %f[examples/sec]: %f[sec/iteration]" % (datetime.now(), step, index, loss_value, num_examples_per_step, examples_per_sec, duration))
                    assert not np.isnan(loss_value), 'Model diverged with loss = NaN'

                if index % 50 == 0:
                    output_vec, cost_value = sess.run([logits, loss], feed_dict={images: vals[0], depths: vals[1], invalid_depths: vals[2], keep_conv: 1.0, keep_hidden: 1.0})
                    print("%s: %d[epoch]: %d[iteration]: validation loss: %f" % (datetime.now(), step, index, cost_value))
                    if index % 100 == 0:
                        output_dir = "predicts_%05d_%08d" % (step, index)
                        print("predicts output: %s" % output_dir)
                        data_feed_inputs_nyu.output_predict(output_vec, output_dir)

                previous_time = end_time
                index += 1
                
        #        if index % 100 == 0:
        #            pass
        #            summary_str = sess.run(summary_op, feed_dict={images: train, labels: label, keep_conv: 0.8, keep_hidden: 0.5})
        #            # サマリーに書き込む
        #            summary_writer.add_summary(summary_str, step)
        #    
            if step % 5 == 0 or (step * 1) == MAX_STEPS:
                if FLAGS.refine_train:
                    refine_checkpoint_path = REFINE_DIR + '/model.ckpt'
                    saver_refine.save(sess, refine_checkpoint_path, global_step=step)
                else:
                    coarse_checkpoint_path = COARSE_DIR + '/model.ckpt'
                    saver_coarse.save(sess, coarse_checkpoint_path, global_step=step)

        coord.request_stop()
        coord.join(threads)
        sess.close()
コード例 #8
0
def train():
    '''
    Train CNN_tiny for a number of steps.
    '''
    with tf.Graph().as_default():
        # globalなstep数
        global_step = tf.Variable(0, trainable=False)

        # 教師データ
        #images, labels = data_inputs.distorted_inputs(TF_RECORDS)
        # 教師データ
        mnist = np.load('./data/mnist_sequence1_sample_5distortions5x5.npz')
        trX = mnist['X_train']
        trY = mnist['y_train']
        # X_valid = mnist_cluttered['X_valid']
        # y_valid = mnist_cluttered['y_valid']
        teX = mnist['X_test']
        teY = mnist['y_test']
        trX = trX.reshape(-1, 40, 40, 1)
        teX = teX.reshape(-1, 40, 40, 1)

        # % turn from dense to one hot representation
        trY = dense_to_one_hot(trY, n_classes=10)
        trY = trY.reshape(-1, 10)
        # Y_valid = dense_to_one_hot(y_valid, n_classes=10)
        teY = dense_to_one_hot(teY, n_classes=10)
        teY = teY.reshape(-1, 10)

        print("the number of train data: %d" % (len(trX)))

        # create mini_batch
        #datas, targets = trX.(trX, trY, BATCH_SIZE)

        images = tf.placeholder(tf.float32, [None, 40, 40, 1])
        labels = tf.placeholder(tf.float32, [None, 10])
        keep_conv = tf.placeholder(tf.float32)
        keep_hidden = tf.placeholder(tf.float32)

        # graphのoutput
        logits = model.inference(images, keep_conv, keep_hidden)

        # loss graphのoutputとlabelを利用
        #loss = model.loss(logits, labels)

        loss = tf.reduce_mean(
            tf.nn.softmax_cross_entropy_with_logits(logits, labels))
        predict_op = tf.argmax(logits, 1)

        # 学習オペレーション
        train_op = op.train(loss, global_step)

        # saver
        saver = tf.train.Saver(tf.all_variables())

        # サマリー
        summary_op = tf.merge_all_summaries()

        # 初期化オペレーション
        init_op = tf.initialize_all_variables()

        # Session
        sess = tf.Session(config=tf.ConfigProto(
            log_device_placement=LOG_DEVICE_PLACEMENT))
        sess.run(init_op)

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        # サマリーのライターを設定
        summary_writer = tf.train.SummaryWriter(TRAIN_DIR,
                                                graph_def=sess.graph_def)

        # max_stepまで繰り返し学習
        for step in xrange(MAX_STEPS):
            start_time = time.time()
            previous_time = start_time
            index = 0
            for start, end in zip(range(0, len(trX), BATCH_SIZE),
                                  range(BATCH_SIZE, len(trX), BATCH_SIZE)):
                _, loss_value = sess.run(
                    [train_op, loss],
                    feed_dict={
                        images: trX[start:end],
                        labels: trY[start:end],
                        keep_conv: 0.8,
                        keep_hidden: 0.5
                    })
                if index % 10 == 0:
                    end_time = time.time()
                    duration = end_time - previous_time
                    num_examples_per_step = BATCH_SIZE * 10 * (step + 1)
                    examples_per_sec = num_examples_per_step / duration
                    print(
                        "%s: %d[epoch]: %d[iteration]: train loss %f: %d[examples/step]: %f[examples/sec]: %f[sec/iteration]"
                        % (datetime.now(), step, index, loss_value,
                           num_examples_per_step, examples_per_sec, duration))
                    index += 1
                    assert not np.isnan(
                        loss_value), 'Model diverged with loss = NaN'

                    test_indices = np.arange(len(teX))  # Get A Test Batch
                    np.random.shuffle(test_indices)
                    test_indices = test_indices[0:5]
                    print "=" * 20
                    print teY[test_indices]
                    predict, cost_value = sess.run(
                        [predict_op, loss],
                        feed_dict={
                            images: teX[test_indices],
                            labels: teY[test_indices],
                            keep_conv: 1.0,
                            keep_hidden: 1.0
                        })
                    print predict
                    print("test loss: %f" % (cost_value))
                    print "=" * 20
                    previous_time = end_time

                index += 1
                assert not np.isnan(
                    loss_value), 'Model diverged with loss = NaN'

                # 1000回ごと
                if index % 100 == 0:
                    pass
                    summary_str = sess.run(summary_op,
                                           feed_dict={
                                               images: trX[start:end],
                                               labels: trY[start:end],
                                               keep_conv: 0.8,
                                               keep_hidden: 0.5
                                           })
                    # サマリーに書き込む
                    summary_writer.add_summary(summary_str, step)
            if step % 1 == 0 or (step * 1) == MAX_STEPS:
                checkpoint_path = TRAIN_DIR + '/model.ckpt'
                saver.save(sess, checkpoint_path, global_step=step)
        coord.request_stop()
        coord.join(threads)
        sess.close()
コード例 #9
0
def train():
    '''
    Train CNN_tiny for a number of steps.
    '''
    with tf.Graph().as_default():
        # globalなstep数
        global_step = tf.Variable(0, trainable=False)

        # 教師データ
        image_input = ImageInput('./data/101Caltech_shuffles.txt')
        print("the number of train data: %d" % (len(image_input.image_paths)))

        images = tf.placeholder(tf.float32, [None, 224, 224, 3])
        labels = tf.placeholder(tf.float32, [None, 101])
        keep_conv = tf.placeholder(tf.float32)
        keep_hidden = tf.placeholder(tf.float32)

        # graphのoutput
        logits, transform_result = model.inference(images, keep_conv, keep_hidden)

        # loss graphのoutputとlabelを利用
        # loss = model.loss(logits, labels)
        
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, labels))
        predict_op = tf.argmax(logits, 1)

        # 学習オペレーション
        train_op = op.train(loss, global_step)

        # サマリー
        summary_op = tf.merge_all_summaries()

        # 初期化オペレーション
        init_op = tf.initialize_all_variables()

        # Session
        sess = tf.Session(config=tf.ConfigProto(log_device_placement=LOG_DEVICE_PLACEMENT))

        # saver
        #saver = tf.train.Saver(tf.all_variables())

        sess.run(init_op)    
        # pretrainと全体を分けて保存
        pretrain_params = {}
        train_params = {}
        for variable in tf.trainable_variables():
            variable_name = variable.name
            #print("parameter: %s" %(variable_name))
            scope, name = variable_name.split("/")
            target, _ = name.split(":")
            if variable_name.find('spatial_transformer') <  0:
                print("pretrain parameter: %s" %(variable_name))
                pretrain_params[variable_name] = variable
            print("train parameter: %s" %(variable_name))
            train_params[variable_name] = variable
        saver_cnn = tf.train.Saver(pretrain_params)
        saver_transformers = tf.train.Saver(train_params)

        # pretrained_model
        if FLAGS.fine_tune:
            ckpt = tf.train.get_checkpoint_state(PRETRAIN_DIR)
            if ckpt and ckpt.model_checkpoint_path:
                print("Pretrained Model Loading.")
                saver_cnn.restore(sess, ckpt.model_checkpoint_path)
                print("Pretrained Model Restored.")
            else:
                print("No Pretrained Model.")       

        # pretrained model from another type models.
        #    # saver
        #    print type(tf.all_variables())
        #    for variable in tf.trainable_variables():
        #        variable_name = variable.name
        #        variable_value = variable.eval(sess)
        #        if variable_name.find('softmax_linear') < 0 and variable_name.find('spatial_transformer') < 0:
        #            print("trained parameter: %s" %(variable_name))
        #            scope, name = variable_name.split("/")
        #            target, _ = name.split(":")
        #            with tf.variable_scope(scope, reuse=True):
        #                sess.run(tf.get_variable(target).assign(variable_value))
        #    trained_model = FLAGS.trained_model
        #    print trained_model

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        # サマリーのライターを設定
        summary_writer = tf.train.SummaryWriter(TRAIN_DIR, graph_def=sess.graph_def)
 
        # max_stepまで繰り返し学習
        for step in xrange(MAX_STEPS):
            start_time = time.time()
            previous_time = start_time
            index = 0

            batches = image_input.get_batches(FLAGS.batch_size)
            for batch in batches:
                train = batch[0]
                label = batch[1]
                _, loss_value = sess.run([train_op, loss], feed_dict={images: train, labels: label, keep_conv: 0.8, keep_hidden: 0.5})
                if index % 10 == 0:
                    end_time = time.time()
                    duration = end_time - previous_time
                    num_examples_per_step = BATCH_SIZE * 10
                    examples_per_sec = num_examples_per_step / duration
                    print("%s: %d[epoch]: %d[iteration]: train loss %f: %d[examples/iteration]: %f[examples/sec]: %f[sec/iteration]" % (datetime.now(), step, index, loss_value, num_examples_per_step, examples_per_sec, duration))
                    index += 1
                    assert not np.isnan(loss_value), 'Model diverged with loss = NaN'

                    # test_indices = np.arange(len(teX)) # Get A Test Batch
                    # np.random.shuffle(test_indices)
                    # test_indices = test_indices[0:5]
                    print "="*20
                    testx = train[0:2]
                    #print testx
                    testy = label[0:2]
                    print np.argmax(testy[0])
                    print np.argmax(testy[1])
                    output_vec, predict, cost_value = sess.run([logits, predict_op, loss], feed_dict={images: testx, labels: testy, keep_conv: 1.0, keep_hidden: 1.0})
                    print predict
                    print("test loss: %f" % (cost_value))
                    print "="*20
                    previous_time = end_time

                index += 1
                assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
                
                # 100回ごと
                if index % 100 == 0:
                    pass
                    summary_str = sess.run(summary_op, feed_dict={images: train, labels: label, keep_conv: 0.8, keep_hidden: 0.5})
                    # サマリーに書き込む
                    summary_writer.add_summary(summary_str, step)
            
            if step % 1 == 0 or (step * 1) == MAX_STEPS:
                pretrain_checkpoint_path = PRETRAIN_DIR + '/model.ckpt'
                train_checkpoint_path = TRAIN_DIR + '/model.ckpt'
                saver_cnn.save(sess, pretrain_checkpoint_path, global_step=step)
                saver_transformers.save(sess, train_checkpoint_path, global_step=step)
        coord.request_stop()
        coord.join(threads)
        sess.close()
コード例 #10
0
def train():
    '''
    Train CNN_tiny for a number of steps.
    '''
    with tf.Graph().as_default():
        # globalなstep数
        global_step = tf.Variable(0, trainable=False)

        # 教師データ
        images, labels = data_inputs.distorted_inputs(TF_RECORDS)

        # graphのoutput
        logits = model.inference(images)

        # loss graphのoutputとlabelを利用
        loss = model.loss(logits, labels)

        # 学習オペレーション
        train_op = op.train(loss, global_step)

        # saver
        saver = tf.train.Saver(tf.all_variables())

        # サマリー
        summary_op = tf.merge_all_summaries()

        # 初期化オペレーション
        init_op = tf.initialize_all_variables()

        # Session
        sess = tf.Session(config=tf.ConfigProto(log_device_placement=LOG_DEVICE_PLACEMENT))
        sess.run(init_op)

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        # サマリーのライターを設定
        summary_writer = tf.train.SummaryWriter(TRAIN_DIR, graph_def=sess.graph_def)

        # model名
        model_name = '/model%s.ckpt' % (tdatetime.strftime('%Y%m%d%H%M%S'))
   
        # max_stepまで繰り返し学習
        for step in xrange(MAX_STEPS):
            start_time = time.time()
            _, loss_value = sess.run([train_op, loss])
            duration = time.time() - start_time

            assert not np.isnan(loss_value), 'Model diverged with loss = NaN'

            # 100回ごと
            if step % 100 == 0:
                # stepごとの事例数 = mini batch size
                num_examples_per_step = BATCH_SIZE

                # 1秒ごとの事例数
                examples_per_sec = num_examples_per_step / duration
                
                # バッチごとの時間
                sec_per_batch = float(duration)

                # time, step数, loss, 1秒で実行できた事例数, バッチあたりの時間
                format_str = '$s: step %d, loss = %.2f (%.1f examples/sec; %.3f sec/batch)'
                print str(datetime.now()) + ': step' + str(step) + ', loss= '+ str(loss_value) + ' ' + str(examples_per_sec) + ' examples/sec; ' + str(sec_per_batch) + ' sec/batch'

            # 100回ごと
            if step % 100 == 0:
                pass
                summary_str = sess.run(summary_op)
                # サマリーに書き込む
                summary_writer.add_summary(summary_str, step)

            if step % 1000 == 0 or (step * 1) == MAX_STEPS:
                checkpoint_path = TRAIN_DIR + model_name
                saver.save(sess, checkpoint_path, global_step=step)

        coord.request_stop()
        coord.join(threads)
        sess.close()
コード例 #11
0
def train():
    '''
    Train
    '''
    with tf.Graph().as_default():
        # globalなstep数
        global_step = tf.Variable(0, trainable=False)

        # NYU Dataset V2 original size(480 x 640 x 3) -> crop -> (460 x 620 x 3)
        image_input = ImageInput('./data/nyu_depth_v2_labeled.mat')
        print("the number of train data: %d" % (len(image_input.images)))

        images = tf.placeholder(tf.float32, [
            None, FLAGS.crop_size_height, FLAGS.crop_size_width,
            FLAGS.image_depth
        ])
        depths = tf.placeholder(tf.float32, [None, 1, 55, 74])
        invalid_depths = tf.placeholder(tf.float32, [None, 1, 55, 74])
        keep_conv = tf.placeholder(tf.float32)
        keep_hidden = tf.placeholder(tf.float32)

        # graphのoutput
        if FLAGS.refine_train:
            print("refine train.")
            logits = model.inference_refine(images, keep_conv, keep_hidden)
        else:
            print("coarse train.")
            logits = model.inference(images, keep_conv, keep_hidden)

        # loss graphのoutputとlabelを利用
        loss = model.loss(logits, depths, invalid_depths)
        # 学習オペレーション
        train_op = op.train(loss, global_step)

        # サマリー
        summary_op = tf.merge_all_summaries()

        # 初期化オペレーション
        init_op = tf.initialize_all_variables()

        # Session
        sess = tf.Session(config=tf.ConfigProto(
            log_device_placement=LOG_DEVICE_PLACEMENT))

        # saver
        #saver = tf.train.Saver(tf.all_variables())

        sess.run(init_op)

        # coarseとrefineを分けて保存
        coarse_params = {}
        refine_params = {}

        if FLAGS.refine_train:
            for variable in tf.all_variables():
                variable_name = variable.name
                print("parameter: %s" % (variable_name))
                if variable_name.find("/") < 0 or variable_name.count(
                        "/") != 1:
                    print("ignore.")
                    continue
                scope, name = variable_name.split("/")
                target, _ = name.split(":")
                if variable_name.find('coarse') >= 0:
                    print("coarse parameter: %s" % (variable_name))
                    coarse_params[variable_name] = variable
                if variable_name.find('fine') >= 0:
                    print("refine parameter: %s" % (variable_name))
                    refine_params[variable_name] = variable
        else:
            for variable in tf.trainable_variables():
                variable_name = variable.name
                print("parameter: %s" % (variable_name))
                if variable_name.find("/") < 0 or variable_name.count(
                        "/") != 1:
                    print("ignore.")
                    continue
                scope, name = variable_name.split("/")
                target, _ = name.split(":")
                if variable_name.find('coarse') >= 0:
                    print("coarse parameter: %s" % (variable_name))
                    coarse_params[variable_name] = variable
                if variable_name.find('fine') >= 0:
                    print("refine parameter: %s" % (variable_name))
                    refine_params[variable_name] = variable

        # define saver
        saver_coarse = tf.train.Saver(coarse_params)
        saver_refine = tf.train.Saver(refine_params)

        # fine tune
        if FLAGS.fine_tune:
            # load coarse paramteters
            coarse_ckpt = tf.train.get_checkpoint_state(COARSE_DIR)
            if coarse_ckpt and coarse_ckpt.model_checkpoint_path:
                print("Pretrained coarse Model Loading.")
                saver_coarse.restore(sess, coarse_ckpt.model_checkpoint_path)
                print("Pretrained coarse Model Restored.")
            else:
                print("No Pretrained coarse Model.")

            # load refine parameters
            refine_ckpt = tf.train.get_checkpoint_state(REFINE_DIR)
            if refine_ckpt and refine_ckpt.model_checkpoint_path:
                print("Pretrained refine Model Loading.")
                saver_refine.restore(sess, refine_ckpt.model_checkpoint_path)
                print("Pretrained refine Model Restored.")
            else:
                print("No Pretrained refine Model.")

        # TODO train coarse or refine (change trainable)
        #if not FLAGS.coarse_train:
        #    for val in coarse_params:
        #        print val
        #if not FLAGS.refine_train:
        #    for val in coarse_params:
        #        print val

        # train refine
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        # debug
        # サマリーのライターを設定
        #summary_writer = tf.train.SummaryWriter(TRAIN_DIR, graph_def=sess.graph_def)
        #batches = image_input.get_batches(FLAGS.batch_size)a
        #d = np.asarray(batches[0][0])
        #print d.shape
        #a = np.asarray(batches[0][1])
        #print a.shape
        #logits_val, logits_fine_val, loss_value = sess.run([logits, logits_fine, loss], feed_dict={images: batches[0][0], depths: batches[0][1], invalid_depths: batches[0][2], keep_conv: 1.0, keep_hidden: 1.0})
        #print len(logits_val[0])
        #print len(logits_fine_val[0])
        #print loss_value

        # max_stepまで繰り返し学習
        for step in xrange(MAX_STEPS):
            start_time = time.time()
            previous_time = start_time
            index = 0

            batches = image_input.get_batches(FLAGS.batch_size)
            vals = image_input.get_validation()
            for batch in batches:
                train = batch[0]
                depth = batch[1]
                ignore_depth = batch[2]
                _, loss_value = sess.run(
                    [train_op, loss],
                    feed_dict={
                        images: train,
                        depths: depth,
                        invalid_depths: ignore_depth,
                        keep_conv: 0.8,
                        keep_hidden: 0.5
                    })
                if index % 10 == 0:
                    end_time = time.time()
                    duration = end_time - previous_time
                    num_examples_per_step = BATCH_SIZE * 10
                    examples_per_sec = num_examples_per_step / duration
                    print(
                        "%s: %d[epoch]: %d[iteration]: train loss %f: %d[examples/iteration]: %f[examples/sec]: %f[sec/iteration]"
                        % (datetime.now(), step, index, loss_value,
                           num_examples_per_step, examples_per_sec, duration))
                    assert not np.isnan(
                        loss_value), 'Model diverged with loss = NaN'

                if index % 50 == 0:
                    output_vec, cost_value = sess.run(
                        [logits, loss],
                        feed_dict={
                            images: vals[0],
                            depths: vals[1],
                            invalid_depths: vals[2],
                            keep_conv: 1.0,
                            keep_hidden: 1.0
                        })
                    print("%s: %d[epoch]: %d[iteration]: validation loss: %f" %
                          (datetime.now(), step, index, cost_value))
                    if index % 100 == 0:
                        output_dir = "predicts_%05d_%08d" % (step, index)
                        print("predicts output: %s" % output_dir)
                        data_feed_inputs_nyu.output_predict(
                            output_vec, output_dir)

                previous_time = end_time
                index += 1

        #        if index % 100 == 0:
        #            pass
        #            summary_str = sess.run(summary_op, feed_dict={images: train, labels: label, keep_conv: 0.8, keep_hidden: 0.5})
        #            # サマリーに書き込む
        #            summary_writer.add_summary(summary_str, step)
        #
            if step % 5 == 0 or (step * 1) == MAX_STEPS:
                if FLAGS.refine_train:
                    refine_checkpoint_path = REFINE_DIR + '/model.ckpt'
                    saver_refine.save(sess,
                                      refine_checkpoint_path,
                                      global_step=step)
                else:
                    coarse_checkpoint_path = COARSE_DIR + '/model.ckpt'
                    saver_coarse.save(sess,
                                      coarse_checkpoint_path,
                                      global_step=step)

        coord.request_stop()
        coord.join(threads)
        sess.close()