def test(FLAGS): max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size train_dir = FLAGS.log_dir # /tmp3/first350/TensorFlow/Logs test_dir = FLAGS.test_dir # /tmp3/first350/SegNet-Tutorial/CamVid/train.txt test_ckpt = FLAGS.testing image_w = FLAGS.image_w image_h = FLAGS.image_h image_c = FLAGS.image_c # testing should set BATCH_SIZE = 1 batch_size = 1 image_filenames, label_filenames = get_filename_list(test_dir) test_data_node = tf.placeholder( tf.float32, shape=[batch_size, image_h, image_w, image_c]) test_labels_node = tf.placeholder(tf.int64, shape=[batch_size, 360, 480, 1]) phase_train = tf.placeholder(tf.bool, name='phase_train') loss, logits = inference(test_data_node, test_labels_node, batch_size, phase_train) pred = tf.argmax(logits, dimension=3) # get moving avg variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) with tf.Session() as sess: # Load checkpoint saver.restore(sess, test_ckpt) images, labels = get_all_test_data(image_filenames, label_filenames) threads = tf.train.start_queue_runners(sess=sess) hist = np.zeros((NUM_CLASSES, NUM_CLASSES)) for image_batch, label_batch in zip(images, labels): feed_dict = { test_data_node: image_batch, test_labels_node: label_batch, phase_train: False } dense_prediction, im = sess.run([logits, pred], feed_dict=feed_dict) # output_image to verify if (FLAGS.save_image): writeImage(im[0], 'testing_image.png') hist += get_hist(dense_prediction, label_batch) acc_total = np.diag(hist).sum() / hist.sum() iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist)) print("acc: ", acc_total) print("mean IU: ", np.nanmean(iu))
def test_model(two): global max_steps, batch_size, image_w, image_h, image_c, image_dir, log_dir count=0 #test_dir = FLAGS.test_dir # /tmp3/first350/SegNet-Tutorial/CamVid/train.txt test_ckpt = '/home/shantams/deep-learning/tf/logs/Test/run1/model.ckpt-9000' # testing should set BATCH_SIZE = 1 batch_size = 1 image_filenames, label_filenames = get_filename_list(image_dir) test_data_node = tf.placeholder( tf.float32, shape=[batch_size, image_h, image_w, image_c]) test_labels_node = tf.placeholder(tf.int64, shape=[batch_size, 240,320,1]) phase_train = tf.placeholder(tf.bool, name='phase_train') loss, logits = inference(test_data_node, test_labels_node, phase_train) pred = tf.argmax(logits, axis=3) # get moving avg variable_averages = tf.train.ExponentialMovingAverage( MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.0001) with tf.Session() as sess: # Load checkpoint saver.restore(sess, test_ckpt) images, labels = get_all_test_data(image_filenames, label_filenames) threads = tf.train.start_queue_runners(sess=sess) hist = np.zeros((3, 3)) #number of classes, number of classes for image_batch, label_batch in zip(images, labels): #print image_batch.shape feed_dict = { test_data_node: image_batch, test_labels_node: label_batch, phase_train: False } dense_prediction, im = sess.run([logits, pred], feed_dict=feed_dict) # output_image to verify if (True): #writeImage(im[0], 'testing_image.png') writeImage(im[0], str(image_filenames[count]).split('/')[-1]) hist += get_hist(dense_prediction, label_batch) count+=1 acc_total = np.diag(hist).sum() / hist.sum() iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist)) print("acc: ", acc_total) print("mean IU: ", np.nanmean(iu))
def test(): max_steps = config.max_steps batch_size = config.batch_size test_dir = config.test_dir test_ckpt = config.testing image_w = config.image_w image_h = config.image_h image_c = config.image_c # testing should set BATCH_SIZE = 1 batch_size = 1 image_filenames, label_filenames = get_filename_list(test_dir) test_data_node = tf.placeholder( tf.float32, shape=[batch_size, image_h, image_w, image_c]) test_labels_node = tf.placeholder(tf.int64, shape=[batch_size, 360, 480, 1]) phase_train = tf.placeholder(tf.bool, name='phase_train') loss, logits = inference(test_data_node, test_labels_node, batch_size, phase_train) pred = tf.argmax(logits, axis=3) # get moving avg variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.0001) with tf.Session() as sess: # Load checkpoint saver.restore(sess, test_ckpt) images, labels = get_all_test_data(image_filenames, label_filenames) threads = tf.train.start_queue_runners(sess=sess) hist = np.zeros((NUM_CLASSES, NUM_CLASSES)) for image_batch, label_batch in zip(images, labels): feed_dict = { test_data_node: image_batch, test_labels_node: label_batch, phase_train: False } dense_prediction, im = sess.run([logits, pred], feed_dict=feed_dict) # output_image to verify if (config.save_image): writeImage(im[0], '%s/testing_image.png' % config.log_dir) print('Testing image saved.') hist += get_hist(dense_prediction, label_batch) # count+=1 acc_total = np.diag(hist).sum() / hist.sum() iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist)) print("acc: ", acc_total) print("mean IU: ", np.nanmean(iu))
def training(FLAGS, is_finetune=False): max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size train_dir = FLAGS.log_dir # /tmp3/first350/TensorFlow/Logs image_dir = FLAGS.image_dir # /tmp3/first350/SegNet-Tutorial/CamVid/train.txt val_dir = FLAGS.val_dir # /tmp3/first350/SegNet-Tutorial/CamVid/val.txt finetune_ckpt = FLAGS.finetune image_w = FLAGS.image_w image_h = FLAGS.image_h image_c = FLAGS.image_c # should be changed if your model stored by different convention startstep = 0 if not is_finetune else int(FLAGS.finetune.split('-')[-1]) image_filenames, label_filenames = get_filename_list(image_dir) val_image_filenames, val_label_filenames = get_filename_list(val_dir) with tf.Graph().as_default(): train_data_node = tf.placeholder( tf.float32, shape=[batch_size, image_h, image_w, image_c]) train_labels_node = tf.placeholder( tf.int64, shape=[batch_size, image_h, image_w, 1]) phase_train = tf.placeholder(tf.bool, name='phase_train') global_step = tf.Variable(0, trainable=False) # For CamVid images, labels = CamVidInputs(image_filenames, label_filenames, batch_size) val_images, val_labels = CamVidInputs(val_image_filenames, val_label_filenames, batch_size) # Build a Graph that computes the logits predictions from the inference model. loss, eval_prediction = inference(train_data_node, train_labels_node, batch_size, phase_train) # Build a Graph that trains the model with one batch of examples and updates the model parameters. train_op = train(loss, global_step) saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.0001) with tf.Session() as sess: # Build an initialization operation to run below. if (is_finetune == True): saver.restore(sess, finetune_ckpt) else: init = tf.global_variables_initializer() sess.run(init) # Start the queue runners. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Summery placeholders summary_writer = tf.summary.FileWriter(train_dir, sess.graph) average_pl = tf.placeholder(tf.float32) acc_pl = tf.placeholder(tf.float32) iu_pl = tf.placeholder(tf.float32) average_summary = tf.summary.scalar("test_average_loss", average_pl) acc_summary = tf.summary.scalar("test_accuracy", acc_pl) iu_summary = tf.summary.scalar("Mean_IU", iu_pl) for step in range(startstep, startstep + max_steps): image_batch, label_batch = sess.run([images, labels]) # since we still use mini-batches in validation, still set bn-layer phase_train = True feed_dict = { train_data_node: image_batch, train_labels_node: label_batch, phase_train: True } start_time = time.time() _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) duration = time.time() - start_time assert not np.isnan( loss_value), 'Model diverged with loss = NaN' if step % 10 == 0: num_examples_per_step = batch_size examples_per_sec = num_examples_per_step / duration sec_per_batch = float(duration) format_str = ( '%s: step %d, loss = %.2f (%.1f examples/sec; %.3f ' 'sec/batch)') print(format_str % (datetime.now(), step, loss_value, examples_per_sec, sec_per_batch)) # eval current training batch pre-class accuracy pred = sess.run(eval_prediction, feed_dict=feed_dict) per_class_acc(pred, label_batch) if step % 100 == 0: print("start validating.....") total_val_loss = 0.0 hist = np.zeros((NUM_CLASSES, NUM_CLASSES)) for test_step in range(int(TEST_ITER)): val_images_batch, val_labels_batch = sess.run( [val_images, val_labels]) _val_loss, _val_pred = sess.run( [loss, eval_prediction], feed_dict={ train_data_node: val_images_batch, train_labels_node: val_labels_batch, phase_train: True }) total_val_loss += _val_loss hist += get_hist(_val_pred, val_labels_batch) print("val loss: ", total_val_loss / TEST_ITER) acc_total = np.diag(hist).sum() / hist.sum() iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist)) test_summary_str = sess.run( average_summary, feed_dict={average_pl: total_val_loss / TEST_ITER}) acc_summary_str = sess.run(acc_summary, feed_dict={acc_pl: acc_total}) iu_summary_str = sess.run( iu_summary, feed_dict={iu_pl: np.nanmean(iu)}) print_hist_summery(hist) print(" end validating.... ") summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) summary_writer.add_summary(test_summary_str, step) summary_writer.add_summary(acc_summary_str, step) summary_writer.add_summary(iu_summary_str, step) # Save the model checkpoint periodically. if step % 1000 == 0 or (step + 1) == max_steps: checkpoint_path = os.path.join(train_dir, 'model.ckpt') saver.save(sess, checkpoint_path, global_step=step) coord.request_stop() coord.join(threads)
def training(FLAGS, is_finetune=False): max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size train_dir = FLAGS.log_dir # /tmp3/first350/TensorFlow/Logs image_dir = FLAGS.image_dir # /tmp3/first350/SegNet-Tutorial/CamVid/train.txt val_dir = FLAGS.val_dir # /tmp3/first350/SegNet-Tutorial/CamVid/val.txt finetune_ckpt = FLAGS.finetune image_w = FLAGS.image_w image_h = FLAGS.image_h image_c = FLAGS.image_c # should be changed if your model stored by different convention startstep = 0 if not is_finetune else int(FLAGS.finetune.split('-')[-1]) image_filenames, label_filenames = get_filename_list(image_dir) val_image_filenames, val_label_filenames = get_filename_list(val_dir) with tf.Graph().as_default(): train_data_node = tf.placeholder( tf.float32, shape=[batch_size, image_h, image_w, image_c]) train_labels_node = tf.placeholder(tf.int64, shape=[batch_size, image_h, image_w, 1]) phase_train = tf.placeholder(tf.bool, name='phase_train') global_step = tf.Variable(0, trainable=False) # For CamVid images, labels = CamVidInputs(image_filenames, label_filenames, batch_size) val_images, val_labels = CamVidInputs(val_image_filenames, val_label_filenames, batch_size) # Build a Graph that computes the logits predictions from the inference model. loss, eval_prediction = inference(train_data_node, train_labels_node, batch_size, phase_train) # Build a Graph that trains the model with one batch of examples and updates the model parameters. train_op = train(loss, global_step) saver = tf.train.Saver(tf.global_variables()) summary_op = tf.summary.merge_all() # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.0001) with tf.Session() as sess: # Build an initialization operation to run below. if (is_finetune == True): saver.restore(sess, finetune_ckpt ) else: init = tf.global_variables_initializer() sess.run(init) # Start the queue runners. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Summery placeholders summary_writer = tf.summary.FileWriter(train_dir, sess.graph) average_pl = tf.placeholder(tf.float32) acc_pl = tf.placeholder(tf.float32) iu_pl = tf.placeholder(tf.float32) average_summary = tf.summary.scalar("test_average_loss", average_pl) acc_summary = tf.summary.scalar("test_accuracy", acc_pl) iu_summary = tf.summary.scalar("Mean_IU", iu_pl) for step in range(startstep, startstep + max_steps): image_batch ,label_batch = sess.run([images, labels]) # since we still use mini-batches in validation, still set bn-layer phase_train = True feed_dict = { train_data_node: image_batch, train_labels_node: label_batch, phase_train: True } start_time = time.time() _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) duration = time.time() - start_time assert not np.isnan(loss_value), 'Model diverged with loss = NaN' if step % 10 == 0: num_examples_per_step = batch_size examples_per_sec = num_examples_per_step / duration sec_per_batch = float(duration) format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f ' 'sec/batch)') print (format_str % (datetime.now(), step, loss_value, examples_per_sec, sec_per_batch)) # eval current training batch pre-class accuracy pred = sess.run(eval_prediction, feed_dict=feed_dict) per_class_acc(pred, label_batch) if step % 100 == 0: print("start validating.....") total_val_loss = 0.0 hist = np.zeros((NUM_CLASSES, NUM_CLASSES)) for test_step in range(int(TEST_ITER)): val_images_batch, val_labels_batch = sess.run([val_images, val_labels]) _val_loss, _val_pred = sess.run([loss, eval_prediction], feed_dict={ train_data_node: val_images_batch, train_labels_node: val_labels_batch, phase_train: True }) total_val_loss += _val_loss hist += get_hist(_val_pred, val_labels_batch) print("val loss: ", total_val_loss / TEST_ITER) acc_total = np.diag(hist).sum() / hist.sum() iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist)) test_summary_str = sess.run(average_summary, feed_dict={average_pl: total_val_loss / TEST_ITER}) acc_summary_str = sess.run(acc_summary, feed_dict={acc_pl: acc_total}) iu_summary_str = sess.run(iu_summary, feed_dict={iu_pl: np.nanmean(iu)}) print_hist_summery(hist) print(" end validating.... ") summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) summary_writer.add_summary(test_summary_str, step) summary_writer.add_summary(acc_summary_str, step) summary_writer.add_summary(iu_summary_str, step) # Save the model checkpoint periodically. if step % 1000 == 0 or (step + 1) == max_steps: checkpoint_path = os.path.join(train_dir, 'model.ckpt') saver.save(sess, checkpoint_path, global_step=step) coord.request_stop() coord.join(threads)
def test(FLAGS): max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size train_dir = FLAGS.log_dir # /tmp3/first350/TensorFlow/Logs test_dir = FLAGS.test_dir # /tmp3/first350/SegNet-Tutorial/CamVid/train.txt test_ckpt = FLAGS.testing image_w = FLAGS.image_w image_h = FLAGS.image_h image_c = FLAGS.image_c # testing should set BATCH_SIZE = 1 batch_size = 1 image_filenames, label_filenames = get_filename_list(test_dir) test_data_node = tf.placeholder( tf.float32, shape=[batch_size, image_h, image_w, image_c]) test_labels_node = tf.placeholder(tf.int64, shape=[batch_size, 360, 480, 1]) phase_train = tf.placeholder(tf.bool, name='phase_train') loss, logits = inference(test_data_node, test_labels_node, batch_size, phase_train) pred = tf.argmax(logits, axis=3) # get moving avg variable_averages = tf.train.ExponentialMovingAverage( MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.0001) with tf.Session() as sess: # Load checkpoint saver.restore(sess, test_ckpt ) images, labels = get_all_test_data(image_filenames, label_filenames) threads = tf.train.start_queue_runners(sess=sess) hist = np.zeros((NUM_CLASSES, NUM_CLASSES)) for image_batch, label_batch in zip(images, labels): feed_dict = { test_data_node: image_batch, test_labels_node: label_batch, phase_train: False } dense_prediction, im = sess.run([logits, pred], feed_dict=feed_dict) # output_image to verify if (FLAGS.save_image): writeImage(im[0], 'testing_image.png') # writeImage(im[0], 'out_image/'+str(image_filenames[count]).split('/')[-1]) hist += get_hist(dense_prediction, label_batch) # count+=1 acc_total = np.diag(hist).sum() / hist.sum() iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist)) print("acc: ", acc_total) print("mean IU: ", np.nanmean(iu))
def test(FLAGS): max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size train_dir = FLAGS.log_dir # /tmp3/first350/TensorFlow/Logs test_dir = FLAGS.test_dir # /tmp3/first350/SegNet-Tutorial/CamVid/train.txt test_ckpt = FLAGS.testing image_w = FLAGS.image_w image_h = FLAGS.image_h image_c = FLAGS.image_c # testing should set BATCH_SIZE = 1 batch_size = 1 image_filenames, label_filenames = get_filename_list(test_dir) test_data_node = tf.placeholder( tf.float32, shape=[batch_size, image_h, image_w, image_c]) test_labels_node = tf.placeholder(tf.int64, shape=[batch_size, 360, 480, 1]) phase_train = tf.placeholder(tf.bool, name='phase_train') loss, logits = inference(test_data_node, test_labels_node, batch_size, phase_train) pred = tf.argmax(logits, axis=3) # get moving avg variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) #gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.05) #config=tf.ConfigProto(gpu_options=gpu_options) count = 0 with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: # Load checkpoint saver.restore(sess, test_ckpt) images, labels = get_all_test_data(image_filenames, label_filenames) threads = tf.train.start_queue_runners(sess=sess) hist = np.zeros((NUM_CLASSES, NUM_CLASSES)) for image_batch, label_batch in zip(images, labels): #image_batch = cv2.resize(image_batch, (480, 360)) #print(image_batch.shape) #label_batch = image_batch feed_dict = { test_data_node: image_batch, test_labels_node: label_batch, phase_train: False } dense_prediction, im = sess.run([logits, pred], feed_dict=feed_dict) # output_image to verify if (FLAGS.save_image): #writeImage(im[0], 'testing_image0.png') #writeImage(im[1], 'testing_image1.png') #writeImage(im[2], 'testing_image2.png') writeImage( im[0], 'out_image/' + str(image_filenames[count]).split('/')[-1]) hist += get_hist(dense_prediction, label_batch) count += 1
def test(FLAGS): iterations = FLAGS.iterations batch_size = FLAGS.batch_size train_dir = FLAGS.log_dir # /tmp3/first350/TensorFlow/Logs test_image_list = FLAGS.test_image_list # /tmp3/first350/SegNet-Tutorial/CamVid/train.txt test_ckpt = FLAGS.ckpt_for_test image_w = FLAGS.image_w image_h = FLAGS.image_h image_c = FLAGS.image_c # test should set batch_size = 1 batch_size = 1 image_filenames, label_filenames = get_filename_list(test_image_list) test_data_node = tf.placeholder( tf.float32, shape=[batch_size, image_h, image_w, image_c]) test_labels_node = tf.placeholder(tf.int64, shape=[batch_size, image_h, image_w, 1]) phase_train = tf.placeholder(tf.bool, name='phase_train') loss, logits = inference(FLAGS, test_data_node, test_labels_node, batch_size, phase_train) pred = tf.argmax(logits, axis=3) # get moving avg variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.0001) with tf.Session() as sess: # Load checkpoint saver.restore(sess, test_ckpt) images, labels = get_all_test_data(image_filenames, label_filenames) threads = tf.train.start_queue_runners(sess=sess) hist = np.zeros((FLAGS.num_classes, FLAGS.num_classes)) i = -1 for image_batch, label_batch in zip(images, labels): i += 1 feed_dict = { test_data_node: image_batch, test_labels_node: label_batch, phase_train: False } dense_prediction, im = sess.run([logits, pred], feed_dict=feed_dict) # output_image to verify if (FLAGS.save_image): writeImage( FLAGS, im[0], str(image_filenames[i]).rstrip('.png') + '_hanteikekka.png') hist += get_hist(dense_prediction, label_batch) # count+=1 acc_total = np.diag(hist).sum() / hist.sum() iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist)) print("acc: ", acc_total) print("mean IU: ", np.nanmean(iu))
def test(FLAGS): max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size train_dir = FLAGS.log_dir # /tmp3/first350/TensorFlow/Logs test_dir = FLAGS.test_dir # /tmp3/first350/SegNet-Tutorial/CamVid/train.txt test_ckpt = FLAGS.testing image_w = FLAGS.image_w image_h = FLAGS.image_h image_c = FLAGS.image_c # testing should set BATCH_SIZE = 1 batch_size = 1 image_filenames, label_filenames = get_filename_list(test_dir) test_data_node = tf.placeholder( tf.float32, shape=[batch_size, image_h, image_w, image_c], name = 'input') test_labels_node = tf.placeholder(tf.int64, shape=[batch_size, 90, 120, 1]) phase_train = tf.placeholder(tf.bool, name='phase_train') loss, logits = inference(test_data_node, test_labels_node, batch_size, phase_train) logits = tf.identity(logits, name = 'output') pred = tf.argmax(logits, axis=3) # get moving avg variable_averages = tf.train.ExponentialMovingAverage( MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.0001) with tf.Session() as sess: # Load checkpoint count = 0 saver.restore(sess, test_ckpt ) images, labels = get_all_test_data(image_filenames, label_filenames) threads = tf.train.start_queue_runners(sess=sess) hist = np.zeros((NUM_CLASSES, NUM_CLASSES)) for image_batch, label_batch in zip(images, labels): feed_dict = { test_data_node: image_batch, test_labels_node: label_batch, phase_train: False } dense_prediction, im = sess.run([logits, pred], feed_dict=feed_dict) # output_image to verify if (FLAGS.save_image): writeImage(im[0], 'testing_image.png') writeImage(im[0], 'out_image/'+str(image_filenames[count]).split('/')[-1]) hist += get_hist(dense_prediction, label_batch) count+=1 acc_total = np.diag(hist).sum() / hist.sum() iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist)) print("acc: ", acc_total) print("mean IU: ", np.nanmean(iu)) graph_def = sess.graph.as_graph_def() for node in graph_def.node: p#rint(node) if node.op == 'RefSwitch': node.op = 'Switch' for index in range(len(node.input)): if 'moving_' in node.input[index]: node.input[index] = node.input[index] + '/read' elif node.op == 'AssignSub': node.op = 'Sub' if 'use_locking' in node.attr: del node.attr['use_locking'] elif node.op == 'AssignAdd': node.op = 'Add' if 'use_locking' in node.attr: del node.attr['use_locking'] constant_graph = graph_util.convert_variables_to_constants(sess, graph_def, ["output"]) graph_io.write_graph(constant_graph, './', "FoosballBallTrackerHeatmap.pb", as_text=False)
def training(FLAGS, is_finetune=False): max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size train_dir = FLAGS.log_dir # /tmp3/first350/TensorFlow/Logs train_list = FLAGS.train_dir # /tmp3/first350/SegNet-Tutorial/CamVid/train.txt val_dir = FLAGS.val_dir # /tmp3/first350/SegNet-Tutorial/CamVid/val.txt finetune_ckpt = FLAGS.finetune not_restore_last = FLAGS.not_restore_last image_w = FLAGS.image_w image_h = FLAGS.image_h image_c = FLAGS.image_c datadir = FLAGS.datadir gpu_frac = FLAGS.gpu_usage dataset = FLAGS.dataset lr = FLAGS.learning_rate max_runtime = FLAGS.max_runtime use_weights = FLAGS.use_weights print("DEBUG 558: " + str(use_weights)) max_time_seconds = 3600 * max_runtime TEST_ITER = int(get_dataset_params(dataset)["num_val"] / batch_size) EPOCH_ITER = int(get_dataset_params(dataset)["num_train"] / batch_size) EPOCHS_UNTIL_VAL = 1 PATIENCE = FLAGS.patience print(TEST_ITER) print('batchsize training: ' + str(batch_size)) print('max_steps training: ' + str(max_steps)) print('lr training: ' + str(lr)) print('Epochs until save ' + str(EPOCHS_UNTIL_VAL)) # should be changed if your model stored by different convention startstep = 0 if not is_finetune else 0 # int((FLAGS.finetune.split('-')[-1]).split('.')[0]) image_filenames, label_filenames = get_filename_list(train_list) val_image_filenames, val_label_filenames = get_filename_list(val_dir) with tf.Graph().as_default(): train_data_node = tf.placeholder( tf.float32, shape=[batch_size, image_h, image_w, image_c]) train_labels_node = tf.placeholder( tf.int64, shape=[batch_size, image_h, image_w, 1]) phase_train = tf.placeholder(tf.bool, name='phase_train') global_step = tf.Variable(0, trainable=False) # select weights # class weight calculation used in segnet # weights for dataset de_top14 # "detop15", "eutop25", "worldtiny2k", "kaggle_dstl", "vaihingen", # "detop15_nores", "eutop25_nores", "worldtiny2k_nores" if not use_weights: print('dont use weights') use_weights = tf.constant([1.0 for i in range(FLAGS.num_class)]) else: print('use weights') if dataset == "detop15": use_weights = np.array([ 0.975644, 1.025603, 0.601745, 6.600600, 1.328684, 0.454776 ]) elif dataset == "eutop25": use_weights = np.array([ 0.970664, 1.031165, 0.790741, 5.320133, 1.384649, 0.718765 ]) elif dataset == "worldtiny2k": use_weights = np.array([ 0.879195, 1.439660, 0.683112, 4.628286, 1.159291, 0.322113 ]) elif dataset == "eutop25_nores": use_weights = np.array( [0.400486, 1.000000, 0.766842, 5.159342, 1.342801]) elif dataset == "detop15_nores": use_weights = np.array( [0.303529, 1.000000, 0.604396, 5.941638, 1.305352]) elif dataset == "worldtiny2k_nores": use_weights = np.array( [0.203351, 1.241845, 0.589249, 3.992340, 1.000000]) elif dataset == "vaihingen": use_weights = np.array([ 0.808506, 0.855016, 1.086051, 0.926584, 18.435326, 26.644663 ]) elif dataset == "kaggle_dstl": use_weights = np.array([ 0.014317, 0.227888, 2.175962, 1.000000, 0.300450, 0.081639, 0.046646, 1.740426, 8.405148, 749.202109, 73.475000 ]) else: print('Error: No weights for dataset ' + dataset + ' could be found.') # early stop variables last_val_loss_tf = tf.Variable(10000.0, name="last_loss") steps_total_tf = tf.Variable(0, name="steps_total") val_increased_t_tf = tf.Variable(0, name="loss_increased_t") # For Inputs images, labels = OSMInputs(image_filenames, label_filenames, batch_size, datadir, dataset) val_images, val_labels = OSMInputs(val_image_filenames, val_label_filenames, batch_size, datadir, dataset) # Build a Graph that computes the logits predictions from the inference model. loss, eval_prediction = inference(FLAGS.num_class, train_data_node, train_labels_node, batch_size, phase_train, use_weights) # Build a Graph that trains the model with one batch of examples and updates the model parameters. train_op = train(loss, global_step, lr) # print(vars_to_restore) # print([v.name for v in restore_var]) # thanks to https://stackoverflow.com/a/50216949/8862202 # v.name[:-2] to transform 'conv1_1_3x3_s2/weights:0' to 'conv1_1_3x3_s2/weights' saver = tf.train.Saver(tf.global_variables(), max_to_keep=10) summary_op = tf.summary.merge_all() glob_start_time = time.time() # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.0001) # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=gpu_frac) with tf.Session() as sess: # init always (necessary for early stop vars) init = tf.global_variables_initializer() sess.run(init) # Build an initialization operation to run below. if is_finetune: ckpt = tf.train.get_checkpoint_state(finetune_ckpt) ckpt_path = ckpt.model_checkpoint_path # restore only vars previously (in the last save) defined vars_to_restore = get_tensors_in_checkpoint_file( file_name=ckpt.model_checkpoint_path) # print(vars_to_restore) # print([v.name for v in tf.global_variables()]) vars_to_restore = [ v for v in tf.global_variables() if (not 'conv_classifier' in v.name or not FLAGS. not_restore_last) and v.name[:-2] in vars_to_restore ] # print(vars_to_restore) loader = tf.train.Saver(vars_to_restore, max_to_keep=10) # saver = tf.train.Saver(tf.global_variables(), max_to_keep=10) loader.restore(sess, ckpt_path) # saver.restore(sess, finetune_ckpt ) print("Restored model parameters from {}".format(ckpt_path)) else: init = tf.global_variables_initializer() sess.run(init) # debug, check early stop vars print(sess.run(last_val_loss_tf)) print(sess.run(steps_total_tf)) print(sess.run(val_increased_t_tf)) # Start the queue runners. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # Summery placeholders summary_writer = tf.summary.FileWriter(train_dir, sess.graph) average_pl = tf.placeholder(tf.float32) acc_pl = tf.placeholder(tf.float32) iu_pl = tf.placeholder(tf.float32) average_summary = tf.summary.scalar("test_average_loss", average_pl) acc_summary = tf.summary.scalar("test_accuracy", acc_pl) iu_summary = tf.summary.scalar("Mean_IU", iu_pl) last_val_loss = sess.run(last_val_loss_tf) val_not_imp_t = sess.run(val_increased_t_tf) total_steps = sess.run(steps_total_tf) for step in trange(startstep + total_steps, startstep + max_steps + total_steps, desc='training', leave=True): image_batch, label_batch = sess.run([images, labels]) # since we still use mini-batches in validation, still set bn-layer phase_train = True feed_dict = { train_data_node: image_batch, train_labels_node: label_batch, phase_train: True } start_time = time.time() _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) duration = time.time() - start_time # assert not np.isnan(loss_value), 'Model diverged with loss = NaN' if step % 100 == 0: num_examples_per_step = batch_size examples_per_sec = num_examples_per_step / duration sec_per_batch = float(duration) # time update elapsed = time.time() - glob_start_time remaining = max_time_seconds - elapsed format_str = ( '%s: step %d, loss = %.2f (%.1f examples/sec; %.3f ' 'sec/batch), %.1f seconds until stop') print(format_str % (datetime.now(), step, loss_value, examples_per_sec, sec_per_batch, remaining)) # eval current training batch pre-class accuracy pred = sess.run(eval_prediction, feed_dict=feed_dict) per_class_acc(pred, label_batch, dataset=dataset) # generate image and send it to event file argmax_t = tf.argmax(pred, axis=3) argmax = sess.run(argmax_t) sat = image_batch[0][:][:][:] sat = np.expand_dims(sat, axis=0) im = predToLabelledImg(argmax[0]) gt = predToLabelledImg(label_batch[0]) # concat images to a single 4D vector to get all images in single line in tensorboard sat_pred_gt = np.concatenate((sat, im, gt), 0) sat_pred_gt_summary = tf.summary.image( 'sat_pred_gt', tf.convert_to_tensor(sat_pred_gt)) sat_pred_gt_summary = sess.run(sat_pred_gt_summary) # img_sum_pred = tf.summary.image('pred img', tf.convert_to_tensor(im)) # img_sum_pred = sess.run(img_sum_pred) # img_sum_gt = tf.summary.image('gt img', tf.convert_to_tensor(gt)) # img_sum_sat = tf.summary.image('satellite img', tf.convert_to_tensor(sat)) # img_sum_gt = sess.run(img_sum_gt) # img_sum_sat = sess.run(img_sum_sat) # debug line # writeImage(argmax[0], str(step)+"_labelled.png", "osm") #summary_writer.add_summary(img_sum_pred, step) # summary_writer.add_summary(img_sum_gt, step) summary_writer.add_summary(sat_pred_gt_summary, step) if step % EPOCH_ITER * EPOCHS_UNTIL_VAL == 0: print("start validating.....") total_val_loss = 0.0 hist = np.zeros((FLAGS.num_class, FLAGS.num_class)) for test_step in range(int(TEST_ITER)): val_images_batch, val_labels_batch = sess.run( [val_images, val_labels]) _val_loss, _val_pred = sess.run( [loss, eval_prediction], feed_dict={ train_data_node: val_images_batch, train_labels_node: val_labels_batch, phase_train: True }) total_val_loss += _val_loss hist += get_hist(_val_pred, val_labels_batch) total_val_loss = total_val_loss / TEST_ITER print("val loss: {:.3f} , last val loss: {:.3f}".format( total_val_loss, last_val_loss)) acc_total = np.diag(hist).sum() / hist.sum() iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist)) test_summary_str = sess.run( average_summary, feed_dict={average_pl: total_val_loss}) acc_summary_str = sess.run(acc_summary, feed_dict={acc_pl: acc_total}) iu_summary_str = sess.run( iu_summary, feed_dict={iu_pl: np.nanmean(iu)}) print_hist_summery(hist, dataset=dataset) print(" end validating.... ") if total_val_loss > last_val_loss: val_not_imp_t = val_not_imp_t + 1 if val_not_imp_t >= PATIENCE: print( "Terminated Training, Best Model (at step %d) saved %d validations ago" % (best_model_step, PATIENCE)) f = open("./FINISHED_SEGNET", "w+") f.close() break else: val_not_imp_t = 0 best_model_step = step print("Loss not since improved %d times" % val_not_imp_t) last_val_loss = total_val_loss # update early stop tensors steps_assign = tf.assign(steps_total_tf, step) last_val_assign = tf.assign(last_val_loss_tf, last_val_loss) increased_assign = tf.assign(val_increased_t_tf, val_not_imp_t) print(sess.run(steps_assign)) print(sess.run(last_val_assign)) print(sess.run(increased_assign)) summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) summary_writer.add_summary(test_summary_str, step) summary_writer.add_summary(acc_summary_str, step) summary_writer.add_summary(iu_summary_str, step) # Save the model checkpoint periodically. checkpoint_path = os.path.join(train_dir, 'model.ckpt') saver.save(sess, checkpoint_path, global_step=step) print("Checkpoint created at %s" % checkpoint_path) # check if max run time is already over elapsed = time.time() - glob_start_time if (elapsed + 300) > max_time_seconds: print("Training stopped: max run time elapsed") os.remove("./RUNNING_SEGNET") break coord.request_stop() coord.join(threads)
def test(FLAGS): max_steps = FLAGS.max_steps batch_size = FLAGS.batch_size train_dir = FLAGS.log_dir # /tmp3/first350/TensorFlow/Logs test_dir = FLAGS.test_dir # /tmp3/first350/SegNet-Tutorial/CamVid/train.txt test_ckpt = FLAGS.testing image_w = FLAGS.image_w image_h = FLAGS.image_h image_c = FLAGS.image_c datadir = FLAGS.datadir outdir = FLAGS.output_dir dataset = FLAGS.dataset use_weights = FLAGS.use_weights # testing should set BATCH_SIZE = 1 batch_size = 1 # select weights # class weight calculation used in segnet # weights for dataset de_top14 # "detop15", "eutop25", "worldtiny2k", "kaggle_dstl", "vaihingen", # "detop15_nores", "eutop25_nores", "worldtiny2k_nores" if not use_weights: print('dont use weights during evaluation') use_weights = tf.constant([1.0 for i in range(FLAGS.num_class)]) image_filenames, label_filenames = get_filename_list(test_dir) test_data_node = tf.placeholder( tf.float32, shape=[batch_size, image_h, image_w, image_c]) test_labels_node = tf.placeholder(tf.int64, shape=[batch_size, image_h, image_w, 1]) phase_train = tf.placeholder(tf.bool, name='phase_train') loss, logits = inference(FLAGS.num_class, test_data_node, test_labels_node, batch_size, phase_train, use_weights) pred = tf.argmax(logits, axis=3) # get moving avg variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.0001) with tf.Session() as sess: ckpt = tf.train.get_checkpoint_state(test_ckpt) # Load checkpoint vars_to_restore = get_tensors_in_checkpoint_file( file_name=ckpt.model_checkpoint_path) # print(vars_to_restore) # print([v.name for v in tf.global_variables()]) vars_to_restore = [ v for v in tf.global_variables() if (not 'conv_classifier' in v.name or not FLAGS.not_restore_last) and v.name[:-2] in vars_to_restore ] # saver = tf.train.Saver(variables_to_restore) saver = tf.train.Saver(vars_to_restore) saver.restore(sess, ckpt.model_checkpoint_path) images, labels = get_all_test_data(image_filenames, label_filenames, datadir) threads = tf.train.start_queue_runners(sess=sess) hist = np.zeros((FLAGS.num_class, FLAGS.num_class)) eval_file = open(FLAGS.dataset, 'wt+') for image_batch, label_batch in tqdm(zip(images, labels), desc="Testing", leave=True, total=len(images)): feed_dict = { test_data_node: image_batch, test_labels_node: label_batch, phase_train: False } dense_prediction, im = sess.run([logits, pred], feed_dict=feed_dict) # output_image to verify if (FLAGS.save_image): if not os.path.exists(outdir): os.makedirs(outdir) writeImage(im[0], outdir + '/testing_image.png', dataset=dataset) # writeImage(im[0], 'out_image/'+str(image_filenames[count]).split('/')[-1]) # calc iou and acc values for each image (for significance test) lhist = get_hist(dense_prediction, label_batch) lacc = np.diag(lhist).sum() / lhist.sum() liu = np.diag(lhist) / (lhist.sum(1) + lhist.sum(0) - np.diag(lhist)) acc_line = "acc" iou_line = "iou" for cls in range(lhist.shape[0]): iou = liu[cls] if float(lhist.sum(1)[cls]) == 0: local_acc = 0.0 else: local_acc = np.diag(lhist)[cls] / float(lhist.sum(1)[cls]) acc_line = acc_line + ", " + str(local_acc) iou_line = iou_line + ", " + str(iou) eval_file.write(acc_line + "\n") eval_file.write(iou_line + "\n") eval_file.flush() hist += lhist # count+=1 acc_total = np.diag(hist).sum() / hist.sum() iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist)) eval_file.close() print("acc: ", acc_total) print("mean IU: ", np.nanmean(iu)) cl_name = get_dataset_classnames(dataset) for cls in range(hist.shape[0]): iou = iu[cls] if float(hist.sum(1)[cls]) == 0: acc = 0.0 else: acc = np.diag(hist)[cls] / float(hist.sum(1)[cls]) print(" class %s accuracy = %f, IoU = %f" % (cl_name[cls].ljust(12), acc, iou))