def confusion_matrix(y_pred, y_true, labels=None): """ Computes the confusion matrix of given vectors containing actual observations and predicted observations. Parameters ---------- pred : 1-d or 2-d tensor variable actual : 1-d or 2-d tensor variable labels : array, shape = [n_classes], optional List of labels to index the matrix. This may be used to reorder or select a subset of labels. If none is given, those that appear at least once in ``y_true`` or ``y_pred`` are used in sorted order. """ from tensorflow.contrib.metrics import confusion_matrix if y_true.get_shape().ndims == 2: y_true = argmax(y_true, -1) elif y_true.get_shape().ndims != 1: raise ValueError('actual must be 1-d or 2-d tensor variable') if y_pred.get_shape().ndims == 2: y_pred = argmax(y_pred, -1) elif y_pred.get_shape().ndims != 1: raise ValueError('pred must be 1-d or 2-d tensor variable') return confusion_matrix( y_pred, y_true, num_classes=None if labels is None else len(labels))
def test_setup(self): # Load reader with tf.name_scope("create_inputs"): reader = ImageReader(self.conf.data_dir, self.conf.valid_data_list, None, False, False, self.conf.ignore_label, IMG_MEAN, self.coord) image, label = reader.image, reader.label # [h, w, 3 or 1] # Add one batch dimension [1, h, w, 3 or 1] self.image_batch, self.label_batch = tf.expand_dims( image, dim=0), tf.expand_dims(label, dim=0) # Create network if self.conf.encoder_name not in ['res101', 'res50', 'deeplab']: print('encoder_name ERROR!') print("Please input: res101, res50, or deeplab") sys.exit(-1) elif self.conf.encoder_name == 'deeplab': net = Deeplab_v2(self.image_batch, self.conf.num_classes, False) else: net = ResNet_segmentation(self.image_batch, self.conf.num_classes, False, self.conf.encoder_name) pass # predictions raw_output = net.outputs raw_output = tf.image.resize_bilinear( raw_output, tf.shape(self.image_batch)[1:3, ]) raw_output = tf.argmax(raw_output, axis=3) pred = tf.expand_dims(raw_output, dim=3) self.pred = tf.reshape(pred, [ -1, ]) # labels gt = tf.reshape(self.label_batch, [ -1, ]) # Ignoring all labels greater than or equal to n_classes. temp = tf.less_equal(gt, self.conf.num_classes - 1) weights = tf.cast(temp, tf.int32) # fix for tf 1.3.0 gt = tf.where(temp, gt, tf.cast(temp, tf.uint8)) # Pixel accuracy self.accu, self.accu_update_op = tcm.streaming_accuracy( self.pred, gt, weights=weights) # mIoU self.mIoU, self.mIou_update_op = tcm.streaming_mean_iou( self.pred, gt, self.conf.num_classes, weights) # confusion matrix self.confusion_matrix = tcm.confusion_matrix( self.pred, gt, num_classes=self.conf.num_classes, weights=weights) # Loader for loading the checkpoint self.loader = tf.train.Saver(var_list=tf.global_variables()) pass
def evaluate(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, labels = image_processing.inputs(dataset, BATCH_SIZE) print("image processing is done.") # Number of classes in the Dataset label set plus 1. # Label 0 is reserved tf.get_variable_scope()for an (unused) background class. num_classes = dataset.num_classes() # Build a Graph that computes the logits predictions from the # inference model. logits, _, _ = inception.inference(images, num_classes) # sparse_labels = tf.reshape(labels, [BATCH_SIZE, 1]) # indices = tf.reshape(tf.range(BATCH_SIZE), [BATCH_SIZE, 1]) # concated = tf.concat(1, [indices, sparse_labels]) # num_classes = logits[0].get_shape()[-1].value # dense_labels = tf.sparse_to_dense(concated,[BATCH_SIZE, num_classes],1, 0) #argmax返回沿着某个维度最大值的位置# confusion_matrix_op = metrics.confusion_matrix(labels, tf.argmax(logits, axis=1)) # false_positive_op = metrics.streaming_false_positives(logits, dense_labels) # false_negative_op = metrics.streaming_false_negatives(logits, dense_labels) # Calculate predictions. '''predictions:预测的结果,预测矩阵大小为样本数×标注的label类的个数的二维矩阵。targets:实际的标签,大小为样本数。k:每个样本的预测结果的前k个最大的数里面是否包含targets 预测中的标签,一般都是取1,即取预测最大概率的索引与标签对比。name:名字。 ''' accuracy = tf.nn.in_top_k(logits, labels, 1) # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage(inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.summary.merge_all() graph_def = tf.get_default_graph().as_graph_def() summary_writer = tf.summary.FileWriter(FLAGS.eval_logs, graph_def=graph_def) while True: # _eval_once(saver, summary_writer, accuracy, summary_op, confusion_matrix_op, logits, labels, dense_labels) _eval_once(saver, summary_writer, accuracy, summary_op, confusion_matrix_op,labels,logits) if FLAGS.run_once: break time.sleep(FLAGS.eval_interval_secs)
def evaluate(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, labels = image_processing.inputs(dataset, BATCH_SIZE) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class. num_classes = dataset.num_classes() # Build a Graph that computes the logits predictions from the # inference model. logits, _, _ = inception.inference(images, num_classes) sparse_labels = tf.reshape(labels, [BATCH_SIZE, 1]) indices = tf.reshape(tf.range(BATCH_SIZE), [BATCH_SIZE, 1]) concated = tf.concat(1, [indices, sparse_labels]) num_classes = logits[0].get_shape()[-1].value dense_labels = tf.sparse_to_dense(concated, [BATCH_SIZE, num_classes], 1, 0) confusion_matrix_op = metrics.confusion_matrix( labels, tf.argmax(logits, axis=1)) # false_positive_op = metrics.streaming_false_positives(logits, dense_labels) # false_negative_op = metrics.streaming_false_negatives(logits, dense_labels) # Calculate predictions. accuracy = tf.nn.in_top_k(logits, labels, 1) # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.summary.merge_all() graph_def = tf.get_default_graph().as_graph_def() summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, graph_def=graph_def) while True: # _eval_once(saver, summary_writer, accuracy, summary_op, confusion_matrix_op, logits, labels, dense_labels) _eval_once(saver, summary_writer, accuracy, summary_op, confusion_matrix_op) if FLAGS.run_once: break time.sleep(FLAGS.eval_interval_secs)
def evaluate(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, labels = image_processing.inputs(dataset, BATCH_SIZE) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class. num_classes = dataset.num_classes() # Build a Graph that computes the logits predictions from the # inference model. logits, _, _ = inception.inference(images, num_classes) sparse_labels = tf.reshape(labels, [BATCH_SIZE, 1]) indices = tf.reshape(tf.range(BATCH_SIZE), [BATCH_SIZE, 1]) concated = tf.concat(1, [indices, sparse_labels]) num_classes = logits[0].get_shape()[-1].value dense_labels = tf.sparse_to_dense(concated, [BATCH_SIZE, num_classes], 1, 0) confusion_matrix_op = metrics.confusion_matrix(labels, tf.argmax(logits, axis=1)) # false_positive_op = metrics.streaming_false_positives(logits, dense_labels) # false_negative_op = metrics.streaming_false_negatives(logits, dense_labels) # Calculate predictions. accuracy = tf.nn.in_top_k(logits, labels, 1) # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.summary.merge_all() graph_def = tf.get_default_graph().as_graph_def() summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, graph_def=graph_def) while True: # _eval_once(saver, summary_writer, accuracy, summary_op, confusion_matrix_op, logits, labels, dense_labels) _eval_once(saver, summary_writer, accuracy, summary_op, confusion_matrix_op) if FLAGS.run_once: break time.sleep(FLAGS.eval_interval_secs)
def miou(y_true, y_pred, n_classes=FLAGS.nb_classes): """ Calculate mIOU :param y_true: labels of one batch, shape=(batch_size, H, W, n_classes) (one-hot format) :param y_pred: predicts of one batch, shape=(batch_size, H, W, n_classes) :param n_classes: Number of categories :return: Average of the IOU for each category """ class_label = K.reshape(y_true, shape=[-1]) class_pred = K.argmax(y_pred, -1) class_pred = K.reshape(class_pred, shape=[-1]) confusion_metrics = confusion_matrix(labels=class_label, predictions=class_pred, num_classes=n_classes) denominator = tf.cast(n_classes, dtype=tf.float32) avg_iou = 0. for class_index in range(0, n_classes): sum_of_one_class = K.sum(confusion_metrics[class_index]) denominator = tf.cond( tf.equal(sum_of_one_class, 0), lambda: tf.cast(denominator - 1, dtype=tf.float32), lambda: tf.cast(denominator, dtype=tf.float32)) # | X ∩ Y | intersection = confusion_metrics[class_index, class_index] # | X | + | Y | union = K.sum(confusion_metrics[class_index, :]) + K.sum( confusion_metrics[:, class_index]) - intersection iou_one_class = tf.cond( tf.equal(intersection, 0), lambda: 0., lambda: tf.cast(intersection / union, dtype=tf.float32)) avg_iou += iou_one_class return tf.cond(tf.equal(denominator, 0), lambda: tf.cast(0, dtype=tf.float32), lambda: tf.cast(avg_iou / denominator, dtype=tf.float32))
def test(self): # 图上下文 with tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions( allow_growth=True))) as sess: # 改变数据大小 shape = tf.shape(self.images_tensor) h, w = (tf.maximum(self.conf.input_size, shape[1]), tf.maximum(self.conf.input_size, shape[2])) img = tf.image.resize_nearest_neighbor(self.images_tensor, [h, w]) # logits logits = self.net.fit({"data": img}) # 预测 raw_output_up = tf.image.resize_bilinear(logits, size=[h, w], align_corners=True) raw_output_up = tf.image.crop_to_bounding_box( raw_output_up, 0, 0, shape[1], shape[2]) raw_output_up = tf.argmax(raw_output_up, dimension=3) prediction = tf.expand_dims(raw_output_up, dim=3) # 评估 pred = tf.reshape(prediction, [ -1, ]) gt = tf.reshape(self.labels_tensor, [ -1, ]) temp = tf.less_equal(gt, self.conf.num_classes - 1) weights = tf.cast(temp, tf.int32) gt = tf.where(temp, gt, tf.cast(temp, tf.uint8)) acc, acc_update_op = tcm.streaming_accuracy(pred, gt, weights=weights) # confusion matrix confusion_matrix_tensor = tcm.confusion_matrix( pred, gt, num_classes=self.conf.num_classes, weights=weights) # 启动初始化 sess.run([ tf.global_variables_initializer(), tf.local_variables_initializer() ]) # 保存 saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=100) # 模型加载 ckpt = tf.train.get_checkpoint_state(self.conf.checkpoints_path) if ckpt and ckpt.model_checkpoint_path: print('test from {}'.format(ckpt.model_checkpoint_path)) saver.restore(sess, ckpt.model_checkpoint_path) else: raise ("请先训练模型..., Train.py first") coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord, sess=sess) # test confusion_matrix = np.zeros( (self.conf.num_classes, self.conf.num_classes), dtype=np.int) for i in range(self.conf.test_num_steps // self.conf.batch_size): start = time.time() pred, _, c_matrix = sess.run( [prediction, acc_update_op, confusion_matrix_tensor]) confusion_matrix += c_matrix _diff_time = time.time() - start print('{}: cost {:.0f}ms'.format(i, _diff_time * 1000)) # 总体 self.compute_IoU_per_class(confusion_matrix) print("Pascal VOC 2012 validation dataset pixel accuracy: " + str(sess.run(acc))) coord.request_stop() coord.join(threads) pass
def __init__(self, data, conf): self.conf = conf # model self.weight_decay = self.conf.weight_decay self.is_training = self.conf.is_training # data self.data = data self.num_classes = self.data.num_classes self.image_height = self.data.image_height self.image_width = self.data.image_width self.image_channel = self.data.image_channel self.label_channel = self.data.label_channel self.label_channel = self.data.label_channel self.batch_size = self.data.batch_size # 学习 self.learning_rate_tensor = tf.convert_to_tensor( self.conf.learning_rate) self.global_step = tf.get_variable( 'global_step', [], initializer=tf.constant_initializer(0), trainable=False, dtype=tf.int32) # add summary tf.summary.scalar('learning_rate', self.learning_rate_tensor) # inputs self.images_tensor, self.labels_tensor = self.data.get_next_data() # output logits = self._network(self.images_tensor) self.prediction = tf.cast( tf.expand_dims(tf.argmax(logits, axis=3), dim=3), tf.uint8) # 评估 pred = tf.reshape(self.prediction, [ -1, ]) gt = tf.reshape(self.labels_tensor, [ -1, ]) temp = tf.less_equal(gt, self.num_classes - 1) weights = tf.cast(temp, tf.int32) gt = tf.where(temp, gt, tf.cast(temp, tf.uint8)) self.acc, self.acc_update_op = tcm.streaming_accuracy(pred, gt, weights=weights) # confusion matrix self.confusion_matrix = tcm.confusion_matrix( pred, gt, num_classes=self.num_classes, weights=weights) # loss self.loss = self._loss(logits) # save moving average variables_averages_op = tf.train.ExponentialMovingAverage( self.conf.moving_average_decay, self.global_step).apply(tf.trainable_variables()) # 优化 with tf.control_dependencies([variables_averages_op]): self.train_op = self._get_train_op() self.summary_op = tf.summary.merge_all() pass
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) x = tf.placeholder(tf.float32, [None, 784]) W = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) y = tf.nn.softmax(tf.matmul(x, W) + b) y_ = tf.placeholder(tf.float32, [None, 10]) # cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1])) cross_entropy = losses.sum_of_squares(y, y_) train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy) init = tf.initialize_all_variables() sess = tf.Session() sess.run(init) for i in range(1000): batch_xs, batch_ys = mnist.train.next_batch(100) sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) cm = metrics.confusion_matrix(tf.argmax(y, 1), tf.argmax(y_, 1)) C = sess.run(cm, feed_dict={x: mnist.test.images, y_: mnist.test.labels}) print(C) # correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) # accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # print(sess.run(accura cy, feed_dict={x: mnist.test.images, y_: mnist.test.labels}))
def main(_): sess = tf.InteractiveSession() with tf.variable_scope('inputs') as scope: validation_x, validation_labels = data.validation_examples() validation_x_var = tf.Variable(validation_x, validate_shape=False, trainable=False, dtype=tf.float32, name='validation_images') validation_labels_var = tf.Variable(validation_labels, validate_shape=False, trainable=False, dtype=tf.int64, name='validation_labels') training_x, training_labels = data.training_examples( batch_size=FLAGS.batch_size) is_val = tf.placeholder(tf.bool, name='is_val') dropout = tf.placeholder(tf.float32) x = tf.cond(is_val, lambda: validation_x_var, lambda: training_x, name='images') labels = tf.cond(is_val, lambda: validation_labels_var, lambda: training_labels, name='labels') y = nn.inference(x, data.IMAGE_SIZE, data.IMAGE_CHANNELS, data.NUM_CLASSES, options={ 'dropout': dropout, 'regularization': FLAGS.regularization, 'is_val': is_val }) with tf.variable_scope('outputs') as scope: y_max = tf.argmax(y, 1, name="prediction") corrects = tf.equal(y_max, labels, name="corrects") accuracy = tf.reduce_mean(tf.cast(corrects, tf.float32), name="accuracy") tf.summary.scalar('accuracy', accuracy) # note difference between r12 and master confusion = confusion_matrix(labels, y_max, num_classes=data.NUM_CLASSES, name="confusion") y_softmax = tf.nn.softmax(y, name="y_softmax") with tf.variable_scope('loss') as scope: cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=labels, logits=y) cross_entropy_loss = tf.reduce_mean(cross_entropy) tf.add_to_collection('losses', cross_entropy_loss) losses = tf.get_collection('losses') loss = tf.add_n(losses) tf.summary.scalar('cross_entropy', cross_entropy_loss) tf.summary.scalar('total', loss) global_step = tf.Variable(0, name='global_step', trainable=False) with tf.variable_scope('training') as scope: optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.rate) train = optimizer.minimize(loss, global_step=global_step) merged = tf.summary.merge_all() saver = tf.train.Saver(tf.trainable_variables() + [global_step]) tf.train.start_queue_runners(sess=sess) sess.run(tf.global_variables_initializer()) if tf.gfile.Exists(FLAGS.logdir): checkpoint = tf.train.latest_checkpoint(FLAGS.logdir) if (checkpoint): print('Restoring from %s' % checkpoint) saver.restore(sess=sess, save_path=checkpoint) else: tf.gfile.MakeDirs(FLAGS.logdir) if FLAGS.predict != '': print(' ', end='\t') for c in data.CLASSES: print('%7s' % c, end='\t') print() files = tf.matching_files(FLAGS.predict).eval() for f in files: image = data.single_example(f) image_out = sess.run(image) y_out = sess.run(y_softmax, feed_dict={ dropout: 0.0, is_val: False, x: [image_out] }) print('%18s' % os.path.basename(f).decode("utf-8"), end='\t') for i, c in enumerate(data.CLASSES): print('%7.2f' % (y_out[0][i] * 100.0), end='\t') print() sys.exit() if tf.train.global_step(sess, global_step) == 0: training_logger = tf.summary.FileWriter( os.path.join(FLAGS.logdir, 'training'), sess.graph) else: training_logger = tf.summary.FileWriter( os.path.join(FLAGS.logdir, 'training')) validation_logger = tf.summary.FileWriter( os.path.join(FLAGS.logdir, 'validation')) images_val, labels_val = sess.run([validation_x, validation_labels]) for st in range(FLAGS.steps + 1): step = tf.train.global_step(sess, global_step) if step % 10 == 0: summary_output, training_accuracy = \ sess.run([merged, accuracy], feed_dict={ dropout: 0.0, is_val: False }) training_logger.add_summary(summary_output, step) summary_output, validation_accuracy, y_out, labels_out, y_max_out, confusion_out = \ sess.run([merged, accuracy, y_softmax, labels, y_max, confusion], feed_dict={ dropout: 0.0, is_val: True }) validation_logger.add_summary(summary_output, step) print('[Step %4d] Training: %6.2f%% Validation: %6.2f%%' % (step, training_accuracy * 100, validation_accuracy * 100)) saver.save(sess, os.path.join(FLAGS.logdir, "model.ckpt"), global_step=step) jsondata = { "confusion": confusion_out.tolist(), "correct": round(validation_accuracy * len(labels_val)), "tops": [[[] for i in range(data.NUM_CLASSES)] for j in range(data.NUM_CLASSES)], "actuals": confusion_out.sum(1).tolist(), "predictions": confusion_out.sum(0).tolist(), "total": len(labels_val) } if step % 50 == 0: for i in range(len(labels_val)): jsondata['tops'][int(labels_out[i])][int( y_max_out[i])].append({ "idx": i, "prob": float(y_out[i][int(y_max_out[i])]) }) for i in range(data.NUM_CLASSES): for j in range(data.NUM_CLASSES): jsondata['tops'][i][j] = sorted(jsondata['tops'][i][j], key=itemgetter('prob'), reverse=True) with open(os.path.join("confusion", data.NAME, "summary.json"), 'w') as outfile: json.dump(jsondata, outfile) sess.run(train, feed_dict={dropout: FLAGS.dropout, is_val: False}) training_logger.close() validation_logger.close()