def evaluate_model(config): """ Train the model using the passed in config """ ########################################################### # Generate the model ########################################################### outputs = create_generator(config, input_utils.get_data_shape(config.dataset)) ########################################################### # Setup the evaluation metrics and summaries ########################################################### # Generate the canvases that lead to the final output image summaries = [] summaries.extend(layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS)) with tf.name_scope('canvases'): for step, canvas in enumerate(outputs): canvas = input_utils.reshape_images(canvas, config.dataset) tiled_images = image_utils.tile_images(canvas) summaries.append(tf.summary.image('step{0}'.format(step), tiled_images)) summary_op = tf.summary.merge(summaries, name='summaries') ########################################################### # Begin evaluation ########################################################### checkpoint_path = FLAGS.checkpoint_path if tf.gfile.IsDirectory(checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) eval_ops = tf.group(*outputs) hooks = [ training.SummaryAtEndHook(FLAGS.log_dir, summary_op), training.StopAfterNEvalsHook(FLAGS.count)] training.evaluate_once(checkpoint_path, hooks=hooks, eval_ops=eval_ops)
def evaluate(train_dir, eval_dir, config, dataset_fn, num_batches, master=''): """Evaluate the model repeatedly.""" tf.gfile.MakeDirs(eval_dir) _trial_summary( config.hparams, config.eval_examples_path or config.tfds_name, eval_dir) with tf.Graph().as_default(): model = config.model model.build(config.hparams, config.data_converter.output_depth, is_training=False) eval_op = model.eval( **_get_input_tensors(dataset_fn().take(num_batches), config)) hooks = [ contrib_training.StopAfterNEvalsHook(num_batches), contrib_training.SummaryAtEndHook(eval_dir) ] contrib_training.evaluate_repeatedly( train_dir, eval_ops=eval_op, hooks=hooks, eval_interval_secs=60, master=master)
def run_eval(build_graph_fn, train_dir, eval_dir, num_batches, timeout_secs=300): """Runs the training loop. Args: build_graph_fn: A function that builds the graph ops. train_dir: The path to the directory where checkpoints will be loaded from for evaluation. eval_dir: The path to the directory where the evaluation summary events will be written to. num_batches: The number of full batches to use for each evaluation step. timeout_secs: The number of seconds after which to stop waiting for a new checkpoint. Raises: ValueError: If `num_batches` is less than or equal to 0. """ if num_batches <= 0: raise ValueError( '`num_batches` must be greater than 0. Check that the batch size is ' 'no larger than the number of records in the eval set.') with tf.Graph().as_default(): build_graph_fn() global_step = tf.train.get_or_create_global_step() loss = tf.get_collection('loss')[0] perplexity = tf.get_collection('metrics/perplexity')[0] accuracy = tf.get_collection('metrics/accuracy')[0] eval_ops = tf.get_collection('eval_ops') logging_dict = { 'Global Step': global_step, 'Loss': loss, 'Perplexity': perplexity, 'Accuracy': accuracy } hooks = [ EvalLoggingTensorHook(logging_dict, every_n_iter=num_batches), contrib_training.StopAfterNEvalsHook(num_batches), contrib_training.SummaryAtEndHook(eval_dir), ] contrib_training.evaluate_repeatedly(train_dir, eval_ops=eval_ops, hooks=hooks, eval_interval_secs=60, timeout=timeout_secs)
def Evaluacion(build_graph_fn, train_dir, eval_dir, num_batches, timeout_secs=300): tf.compat.v1.logging.set_verbosity('INFO') if num_batches <= 0: raise ValueError( '`num_batches` must be greater than 0. Check that the batch size is ' 'no larger than the number of records in the eval set.') with tf.Graph().as_default(): # Creamos un modelo igual que el del entrenamiento build_graph_fn() # Define the summaries to write: global_step = tf.train.get_or_create_global_step() loss = tf.get_collection('loss')[0] perplexity = tf.get_collection('metrics/perplexity')[0] accuracy = tf.get_collection('metrics/accuracy')[0] eval_ops = tf.get_collection('eval_ops') logging_dict = { 'Global Step': global_step, 'Loss': loss, 'Perplexity': perplexity, 'Accuracy': accuracy } hooks = [ EvalLoggingTensorHook(logging_dict, every_n_iter=num_batches), contrib_training.StopAfterNEvalsHook(num_batches), contrib_training.SummaryAtEndHook(eval_dir), ] # names_to_values = contrib_training.evaluate_once( # checkpoint_path=train_dir, # eval_ops=eval_ops, # final_ops=logging_dict, # hooks=hooks, # config=None) # for name in names_to_values: # print('Metric %s has value %f.' % (name, names_to_values[name])) contrib_training.evaluate_repeatedly(train_dir, eval_ops=eval_ops, hooks=hooks, eval_interval_secs=2, timeout=timeout_secs)
def main(unused_argv): tf.logging.set_verbosity(tf.logging.INFO) dataset = data_generator.Dataset( dataset_name=FLAGS.dataset, split_name=FLAGS.eval_split, dataset_dir=FLAGS.dataset_dir, batch_size=FLAGS.eval_batch_size, crop_size=[int(sz) for sz in FLAGS.eval_crop_size], min_resize_value=FLAGS.min_resize_value, max_resize_value=FLAGS.max_resize_value, resize_factor=FLAGS.resize_factor, model_variant=FLAGS.model_variant, num_readers=2, is_training=False, should_shuffle=False, should_repeat=False) tf.gfile.MakeDirs(FLAGS.eval_logdir) tf.logging.info('Evaluating on %s set', FLAGS.eval_split) with tf.Graph().as_default(): samples = dataset.get_one_shot_iterator().get_next() model_options = common.ModelOptions( model_name=FLAGS.model_name, outputs_to_num_classes={ common.OUTPUT_TYPE: dataset.num_of_classes }, crop_size=[int(sz) for sz in FLAGS.eval_crop_size], atrous_rates=FLAGS.atrous_rates, output_stride=FLAGS.output_stride) # Set shape in order for tf.contrib.tfprof.model_analyzer to work properly. samples[common.IMAGE].set_shape([ FLAGS.eval_batch_size, int(FLAGS.eval_crop_size[0]), int(FLAGS.eval_crop_size[1]), 3 ]) if tuple(FLAGS.eval_scales) == (1.0, ): tf.logging.info('Performing single-scale test.') predictions = model.predict_labels( samples[common.IMAGE], model_options, image_pyramid=FLAGS.image_pyramid) else: tf.logging.info('Performing multi-scale test.') if FLAGS.quantize_delay_step >= 0: raise ValueError( 'Quantize mode is not supported with multi-scale test.') predictions = model.predict_labels_multi_scale( samples[common.IMAGE], model_options=model_options, eval_scales=FLAGS.eval_scales, add_flipped_images=FLAGS.add_flipped_images) predictions = predictions[common.OUTPUT_TYPE] predictions = tf.reshape(predictions, shape=[-1]) labels = tf.reshape(samples[common.LABEL], shape=[-1]) weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label)) # Set ignore_label regions to label 0, because metrics.mean_iou requires # range of labels = [0, dataset.num_classes). Note the ignore_label regions # are not evaluated since the corresponding regions contain weights = 0. labels = tf.where(tf.equal(labels, dataset.ignore_label), tf.zeros_like(labels), labels) predictions_tag = 'miou' for eval_scale in FLAGS.eval_scales: predictions_tag += '_' + str(eval_scale) if FLAGS.add_flipped_images: predictions_tag += '_flipped' # Define the evaluation metric. metric_map = {} num_classes = dataset.num_of_classes metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou( labels=labels, predictions=predictions, num_classes=num_classes, weights=weights) # IoU for each class. one_hot_predictions = tf.one_hot(predictions, num_classes) one_hot_predictions = tf.reshape(one_hot_predictions, [-1, num_classes]) one_hot_labels = tf.one_hot(labels, num_classes) one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes]) for c in range(num_classes): predictions_tag_c = '%s_class_%d' % (predictions_tag, c) tp, tp_op = tf.metrics.true_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fp, fp_op = tf.metrics.false_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fn, fn_op = tf.metrics.false_negatives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op) iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp), tf.constant(np.NaN)) metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op) (metrics_to_values, metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map) summary_ops = [] for metric_name, metric_value in six.iteritems(metrics_to_values): op = tf.summary.scalar(metric_name, metric_value) op = tf.Print(op, [metric_value], metric_name) summary_ops.append(op) summary_op = tf.summary.merge(summary_ops) summary_hook = contrib_training.SummaryAtEndHook( log_dir=FLAGS.eval_logdir, summary_op=summary_op) hooks = [summary_hook] num_eval_iters = None if FLAGS.max_number_of_evaluations > 0: num_eval_iters = FLAGS.max_number_of_evaluations if FLAGS.quantize_delay_step >= 0: contrib_quantize.create_eval_graph() contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer. TRAINABLE_VARS_PARAMS_STAT_OPTIONS) contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS) contrib_training.evaluate_repeatedly( checkpoint_dir=FLAGS.checkpoint_dir, master=FLAGS.master, eval_ops=list(metrics_to_updates.values()), max_number_of_evaluations=num_eval_iters, hooks=hooks, eval_interval_secs=FLAGS.eval_interval_secs)
def main(unused_argv): tf.logging.set_verbosity(tf.logging.INFO) dataset = data_generator.Dataset( dataset_name=FLAGS.dataset, split_name=FLAGS.eval_split, dataset_dir=FLAGS.dataset_dir, batch_size=FLAGS.eval_batch_size, crop_size=[int(sz) for sz in FLAGS.eval_crop_size], min_resize_value=FLAGS.min_resize_value, max_resize_value=FLAGS.max_resize_value, resize_factor=FLAGS.resize_factor, model_variant=FLAGS.model_variant, num_readers=2, is_training=False, should_shuffle=False, should_repeat=False, with_cls=True, cls_only=False, output_valid=True) tf.gfile.MakeDirs(FLAGS.eval_logdir) tf.logging.info('Evaluating on %s set', FLAGS.eval_split) with tf.Graph().as_default(): samples = dataset.get_one_shot_iterator().get_next() model_options = common.ModelOptions( outputs_to_num_classes={ common.OUTPUT_TYPE: dataset.num_of_classes }, crop_size=[int(sz) for sz in FLAGS.eval_crop_size], atrous_rates=FLAGS.atrous_rates, output_stride=FLAGS.output_stride) # Set shape in order for tf.contrib.tfprof.model_analyzer to work properly. samples[common.IMAGE].set_shape([ FLAGS.eval_batch_size, int(FLAGS.eval_crop_size[0]), int(FLAGS.eval_crop_size[1]), 3 ]) if tuple(FLAGS.eval_scales) == (1.0, ): tf.logging.info('Performing single-scale test.') predictions = model.predict_labels( samples[common.IMAGE], model_options, image_pyramid=FLAGS.image_pyramid) else: tf.logging.info('Performing multi-scale test.') raise NotImplementedError('Multi-scale is not supported yet!') metric_map = {} ## Extract cls logits if FLAGS.weakly: _, end_points = feature_extractor.extract_features( samples[common.IMAGE], output_stride=model_options.output_stride, multi_grid=model_options.multi_grid, model_variant=model_options.model_variant, depth_multiplier=model_options.depth_multiplier, divisible_by=model_options.divisible_by, reuse=tf.AUTO_REUSE, is_training=False, preprocessed_images_dtype=model_options. preprocessed_images_dtype, global_pool=True, num_classes=dataset.num_of_classes - 1) # ResNet beta version has an additional suffix in FLAGS.model_variant, but # it shares the same variable names with original version. Add a special # handling here for beta version ResNet. logits = end_points['{}/logits'.format( FLAGS.model_variant).replace('_beta', '')] logits = tf.reshape(logits, [-1, dataset.num_of_classes - 1]) cls_pred = tf.sigmoid(logits) # Multi-label classification evaluation cls_label = samples['cls_label'] cls_pred = tf.cast(tf.greater_equal(cls_pred, 0.5), tf.int32) ## For classification metric_map['eval/cls_overall'] = tf.metrics.accuracy( labels=cls_label, predictions=cls_pred) metric_map['eval/cls_precision'] = tf.metrics.precision( labels=cls_label, predictions=cls_pred) metric_map['eval/cls_recall'] = tf.metrics.recall( labels=cls_label, predictions=cls_pred) ## For segmentation branch eval predictions = predictions[common.OUTPUT_TYPE] predictions = tf.reshape(predictions, shape=[-1]) labels = tf.reshape(samples[common.LABEL], shape=[-1]) weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label)) # Set ignore_label regions to label 0, because metrics.mean_iou requires # range of labels = [0, dataset.num_classes). Note the ignore_label regions # are not evaluated since the corresponding regions contain weights = 0. labels = tf.where(tf.equal(labels, dataset.ignore_label), tf.zeros_like(labels), labels) predictions_tag = 'miou' # Define the evaluation metric. num_classes = dataset.num_of_classes ## For segmentation metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou( labels=labels, predictions=predictions, num_classes=num_classes, weights=weights) # IoU for each class. one_hot_predictions = tf.one_hot(predictions, num_classes) one_hot_predictions = tf.reshape(one_hot_predictions, [-1, num_classes]) one_hot_labels = tf.one_hot(labels, num_classes) one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes]) for c in range(num_classes): predictions_tag_c = '%s_class_%d' % (predictions_tag, c) tp, tp_op = tf.metrics.true_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fp, fp_op = tf.metrics.false_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fn, fn_op = tf.metrics.false_negatives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op) iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp), tf.constant(np.NaN)) metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op) (metrics_to_values, metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map) summary_ops = [] for metric_name, metric_value in six.iteritems(metrics_to_values): op = tf.summary.scalar(metric_name, metric_value) op = tf.Print(op, [metric_value], metric_name) summary_ops.append(op) summary_op = tf.summary.merge(summary_ops) summary_hook = contrib_training.SummaryAtEndHook( log_dir=FLAGS.eval_logdir, summary_op=summary_op) hooks = [summary_hook] num_eval_iters = None if FLAGS.max_number_of_evaluations > 0: num_eval_iters = FLAGS.max_number_of_evaluations if FLAGS.quantize_delay_step >= 0: contrib_quantize.create_eval_graph() contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer. TRAINABLE_VARS_PARAMS_STAT_OPTIONS) contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS) contrib_training.evaluate_repeatedly( checkpoint_dir=FLAGS.checkpoint_dir, master=FLAGS.master, eval_ops=list(metrics_to_updates.values()), max_number_of_evaluations=num_eval_iters, hooks=hooks, eval_interval_secs=FLAGS.eval_interval_secs)
def evaluate_model(config): """ Train the model using the passed in config """ ########################################################### # Create the input pipeline ########################################################### with tf.name_scope('input_pipeline'): dataset = input_utils.get_dataset(config.datadir, config.dataset, config.datasubset, num_folds=config.fold_count, fold=config.fold, holdout=True) init_op, init_feed_dict, image = input_utils.get_data( config.dataset, dataset, config.batch_size, num_epochs=config.num_epochs, num_readers=config.num_readers) images = tf.train.batch([image], config.batch_size, num_threads=config.num_preprocessing_threads, capacity=5 * config.batch_size) ########################################################### # Generate the model ########################################################### outputs = create_model(config, images, dataset) ########################################################### # Setup the evaluation metrics and summaries ########################################################### summaries = [] metrics_map = {} for loss in tf.losses.get_losses(): metrics_map[loss.op.name] = metrics.streaming_mean(loss) for metric in tf.get_collection(graph_utils.GraphKeys.METRICS): metrics_map[metric.op.name] = metrics.streaming_mean(metric) total_loss = tf.losses.get_total_loss() metrics_map[total_loss.op.name] = metrics.streaming_mean(total_loss) names_to_values, names_to_updates = metrics.aggregate_metric_map( metrics_map) # Create summaries of the metrics and print them to the screen for name, value in names_to_values.iteritems(): summary = tf.summary.scalar(name, value, collections=[]) summaries.append(tf.Print(summary, [value], name)) summaries.extend(layers.summarize_collection(tf.GraphKeys.MODEL_VARIABLES)) summaries.extend(layers.summarize_collection( graph_utils.GraphKeys.METRICS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS)) images = input_utils.reshape_images(images, config.dataset) tiled_images = image_utils.tile_images(images) summaries.append(tf.summary.image('input_batch', tiled_images)) # Generate the canvases that lead to the final output image with tf.name_scope('canvases'): for step, canvas in enumerate(outputs): canvas = input_utils.reshape_images(canvas, config.dataset) tiled_images = image_utils.tile_images(canvas) summaries.append( tf.summary.image('step{0}'.format(step), tiled_images)) summary_op = tf.summary.merge(summaries, name='summaries') ########################################################### # Begin evaluation ########################################################### checkpoint_path = FLAGS.checkpoint_path eval_ops = tf.group(*names_to_updates.values()) hooks = [ training.SummaryAtEndHook(log_dir=FLAGS.log_dir, summary_op=summary_op), training.StopAfterNEvalsHook( math.ceil(dataset.num_samples / float(config.batch_size))) ] eval_kwargs = {} eval_fn = training.evaluate_repeatedly if FLAGS.once: if tf.gfile.IsDirectory(checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) eval_fn = training.evaluate_once else: assert tf.gfile.IsDirectory(checkpoint_path), ( 'checkpoint path must be a directory when using loop evaluation') eval_fn(checkpoint_path, hooks=hooks, eval_ops=eval_ops, **eval_kwargs)
def main(unused_argv): tf.logging.set_verbosity(tf.logging.INFO) dataset = data_generator.Dataset( # 获取验证集图片数据 dataset_name=FLAGS.dataset, # 数据集名称 cityscapes 默认为 pascal_voc_seg split_name=FLAGS.eval_split, # 指定带有val的tfrecorder数据集 默认为“val” dataset_dir=FLAGS.dataset_dir, # 数据集目录 tfrecoder文件的数据集目录 batch_size=FLAGS.eval_batch_size, # 每个batch包含的image数量 默认为1 crop_size=[int(sz) for sz in FLAGS.eval_crop_size], # 评估时crop_size 默认为513,513 min_resize_value=FLAGS.min_resize_value, # 默认为None max_resize_value=FLAGS.max_resize_value, # 默认为None resize_factor=FLAGS.resize_factor, # 默认为None model_variant=FLAGS.model_variant, # 模型的变体 本次训练为 xception_65 num_readers=2, # 并行读取图片的数量 is_training=False, # 不训练 should_shuffle=False, # 不将输入的数据随机打乱 should_repeat=False) # 不一直重复 tf.gfile.MakeDirs(FLAGS.eval_logdir) # 创建评估目录 tf.logging.info('Evaluating on %s set', FLAGS.eval_split) with tf.Graph().as_default(): samples = dataset.get_one_shot_iterator().get_next() # 获取一次迭代的验证集数据 ''' samples: {'image_name': <tf.Tensor 'IteratorGetNext:2' shape=(?,) dtype=string>, 'width': <tf.Tensor 'IteratorGetNext:5' shape=(?,) dtype=int64>, 'image': <tf.Tensor 'IteratorGetNext:1' shape=(?, 1024, 2048, 3) dtype=float32>, 'height': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=int64>, 'label': <tf.Tensor 'IteratorGetNext:3' shape=(?, 1024, 2048, 1) dtype=int32>, 'original_image': <tf.Tensor 'IteratorGetNext:4' shape=(?, ?, ?, 3) dtype=uint8>} ''' model_options = common.ModelOptions( # 模型参数 outputs_to_num_classes={ common.OUTPUT_TYPE: dataset.num_of_classes }, # {semantic: 19} crop_size=[int(sz) for sz in FLAGS.eval_crop_size], # 1024,2048 atrous_rates=FLAGS.atrous_rates, # 6,12,18 output_stride=FLAGS.output_stride) # 16 # Set shape in order for tf.contrib.tfprof.model_analyzer to work properly. samples[common.IMAGE].set_shape( # 设置形状 [ FLAGS.eval_batch_size, # 默认为1 int(FLAGS.eval_crop_size[0]), int(FLAGS.eval_crop_size[1]), 3 ]) if tuple(FLAGS.eval_scales) == (1.0, ): # 默认 评估尺度为1 tf.logging.info('Performing single-scale test.') predictions = model.predict_labels( samples[common.IMAGE], model_options, # 进行每个像素点预测 image_pyramid=FLAGS.image_pyramid) ''' predictions: {'semantic': <tf.Tensor 'ArgMax:0' shape=(1, 1024, 2048) dtype=int64>, 'semantic_prob': <tf.Tensor 'Softmax:0' shape=(1, 1024, 2048, 19) dtype=float32>} ''' else: tf.logging.info('Performing multi-scale test.') if FLAGS.quantize_delay_step >= 0: raise ValueError( 'Quantize mode is not supported with multi-scale test.') predictions = model.predict_labels_multi_scale( samples[common.IMAGE], model_options=model_options, eval_scales=FLAGS.eval_scales, add_flipped_images=FLAGS.add_flipped_images) predictions = predictions[common.OUTPUT_TYPE] predictions = tf.reshape(predictions, shape=[-1]) # 预测标签 labels = tf.reshape(samples[common.LABEL], shape=[-1]) # 真实标签 weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label)) # 各标签权重 # Set ignore_label regions to label 0, because metrics.mean_iou requires # range of labels = [0, dataset.num_classes). Note the ignore_label regions # are not evaluated since the corresponding regions contain weights = 0. labels = tf.where(tf.equal(labels, dataset.ignore_label), tf.zeros_like(labels), labels) predictions_tag = 'miou' # MIoU predictions_tag1 = 'accuracy_pixel' # 像素精度 for eval_scale in FLAGS.eval_scales: # 默认为单尺度[1.0] predictions_tag += '_' + str(eval_scale) predictions_tag1 += '_' + str(eval_scale) if FLAGS.add_flipped_images: # 默认为False 不设置左右翻转来评估模型 predictions_tag += '_flipped' predictions_tag1 += '_flipped' # Define the evaluation metric. metric_map = {} num_classes = dataset.num_of_classes # 19 metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou( labels=labels, predictions=predictions, num_classes=num_classes, weights=weights) ''' metric_map: {'eval/miou_1.0_overall': (<tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>, <tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>)} ''' metric_map['eval/%s_overall_accuracy_' % predictions_tag] = tf.metrics.accuracy( labels=labels, predictions=predictions, weights=weights) # IoU for each class. ''' tf.one_hot(indices, depth, on_value=None, off_value=None, axis=None, dtype=None, name=None) Returns a one-hot tensor. ndices表示输入的多个数值,通常是矩阵形式;depth表示输出的尺寸。 ''' one_hot_predictions = tf.one_hot(predictions, num_classes) one_hot_predictions = tf.reshape(one_hot_predictions, [-1, num_classes]) # 预测输出的one_hot one_hot_labels = tf.one_hot(labels, num_classes) one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes]) # 真实label的one_hot for c in range(num_classes): predictions_tag_c = '%s_class_%d' % (predictions_tag, c ) # miou_1.0_class_c predictions_tag_c1 = '%s_class_%d' % (predictions_tag1, c) tp, tp_op = tf.metrics.true_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fp, fp_op = tf.metrics.false_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fn, fn_op = tf.metrics.false_negatives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) tn, tn_op = tf.metrics.true_negatives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op) iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp), tf.constant(np.NaN)) ap = tf.where(tf.greater(tp + fn, 0.0), (tp + tn) / (tp + tn + fn + fp), tf.constant(np.NaN)) metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op) metric_map['eval/%s' % predictions_tag_c1] = (ap, tp_fp_fn_op) (metrics_to_values, metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map) ''' (metrics_to_values, metrics_to_updates): ({'eval/miou_1.0_class_5': <tf.Tensor 'Select_6:0' shape=() dtype=float32>, 'eval/miou_1.0_class_18': <tf.Tensor 'Select_19:0' shape=() dtype=float32>, 'eval/miou_1.0_class_13': <tf.Tensor 'Select_14:0' shape=() dtype=float32>, 'eval/miou_1.0_class_1': <tf.Tensor 'Select_2:0' shape=() dtype=float32>, 'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>, 'eval/miou_1.0_class_17': <tf.Tensor 'Select_18:0' shape=() dtype=float32>, 'eval/miou_1.0_class_8': <tf.Tensor 'Select_9:0' shape=() dtype=float32>, 'eval/miou_1.0_class_2': <tf.Tensor 'Select_3:0' shape=() dtype=float32>, 'eval/miou_1.0_class_0': <tf.Tensor 'Select_1:0' shape=() dtype=float32>, 'eval/miou_1.0_class_3': <tf.Tensor 'Select_4:0' shape=() dtype=float32>, 'eval/miou_1.0_class_14': <tf.Tensor 'Select_15:0' shape=() dtype=float32>, 'eval/miou_1.0_class_11': <tf.Tensor 'Select_12:0' shape=() dtype=float32>, 'eval/miou_1.0_class_6': <tf.Tensor 'Select_7:0' shape=() dtype=float32>, 'eval/miou_1.0_class_15': <tf.Tensor 'Select_16:0' shape=() dtype=float32>, 'eval/miou_1.0_class_4': <tf.Tensor 'Select_5:0' shape=() dtype=float32>, 'eval/miou_1.0_class_9': <tf.Tensor 'Select_10:0' shape=() dtype=float32>, 'eval/miou_1.0_class_16': <tf.Tensor 'Select_17:0' shape=() dtype=float32>, 'eval/miou_1.0_class_7': <tf.Tensor 'Select_8:0' shape=() dtype=float32>, 'eval/miou_1.0_class_10': <tf.Tensor 'Select_11:0' shape=() dtype=float32>, 'eval/miou_1.0_class_12': <tf.Tensor 'Select_13:0' shape=() dtype=float32>}, {'eval/miou_1.0_class_5': <tf.Operation 'group_deps_5' type=NoOp>, 'eval/miou_1.0_class_18': <tf.Operation 'group_deps_18' type=NoOp>, 'eval/miou_1.0_class_13': <tf.Operation 'group_deps_13' type=NoOp>, 'eval/miou_1.0_class_1': <tf.Operation 'group_deps_1' type=NoOp>, 'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>, 'eval/miou_1.0_class_17': <tf.Operation 'group_deps_17' type=NoOp>, 'eval/miou_1.0_class_8': <tf.Operation 'group_deps_8' type=NoOp>, 'eval/miou_1.0_class_2': <tf.Operation 'group_deps_2' type=NoOp>, 'eval/miou_1.0_class_0': <tf.Operation 'group_deps' type=NoOp>, 'eval/miou_1.0_class_3': <tf.Operation 'group_deps_3' type=NoOp>, 'eval/miou_1.0_class_14': <tf.Operation 'group_deps_14' type=NoOp>, 'eval/miou_1.0_class_11': <tf.Operation 'group_deps_11' type=NoOp>, 'eval/miou_1.0_class_6': <tf.Operation 'group_deps_6' type=NoOp>, 'eval/miou_1.0_class_15': <tf.Operation 'group_deps_15' type=NoOp>, 'eval/miou_1.0_class_4': <tf.Operation 'group_deps_4' type=NoOp>, 'eval/miou_1.0_class_9': <tf.Operation 'group_deps_9' type=NoOp>, 'eval/miou_1.0_class_16': <tf.Operation 'group_deps_16' type=NoOp>, 'eval/miou_1.0_class_7': <tf.Operation 'group_deps_7' type=NoOp>, 'eval/miou_1.0_class_10': <tf.Operation 'group_deps_10' type=NoOp>, 'eval/miou_1.0_class_12': <tf.Operation 'group_deps_12' type=NoOp>}) ''' ''' tf.Print(input, data, message=None, first_n=None, summarize=None, name=None) 最低要求两个输入,input和data,input是需要打印的变量的名字,data要求是一个list,里面包含要打印的内容。 ''' summary_ops = [] for metric_name, metric_value in six.iteritems(metrics_to_values): op = tf.summary.scalar(metric_name, metric_value) # 显示标量信息 op = tf.Print(op, [metric_value], metric_name) summary_ops.append(op) summary_op = tf.summary.merge(summary_ops) summary_hook = contrib_training.SummaryAtEndHook( log_dir=FLAGS.eval_logdir, summary_op=summary_op) hooks = [summary_hook] num_eval_iters = None if FLAGS.max_number_of_evaluations > 0: # 为0 暂不考虑 num_eval_iters = FLAGS.max_number_of_evaluations if FLAGS.quantize_delay_step >= 0: # -1 暂不考虑 contrib_quantize.create_eval_graph() contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer. TRAINABLE_VARS_PARAMS_STAT_OPTIONS) contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS) contrib_training.evaluate_repeatedly( checkpoint_dir=FLAGS.checkpoint_dir, master=FLAGS.master, eval_ops=list(metrics_to_updates.values()), max_number_of_evaluations=num_eval_iters, hooks=hooks, eval_interval_secs=FLAGS.eval_interval_secs)
def evaluate_model(config): """ Train the model using the passed in config """ ########################################################### # Create the input pipeline ########################################################### with tf.name_scope('input_pipeline'): dataset = input_utils.get_dataset(config.datadir, config.dataset, config.datasubset) init_op, init_feed_dict, image, label = input_utils.get_data( config.dataset, dataset, config.batch_size, num_epochs=config.num_epochs, num_readers=config.num_readers) images, labels = tf.train.batch( [image, label], config.batch_size, num_threads=config.num_preprocessing_threads, capacity=5 * config.batch_size) ########################################################### # Generate the model ########################################################### outputs = create_model(config, images, dataset) tfprof.model_analyzer.print_model_analysis(tf.get_default_graph()) ########################################################### # Setup the evaluation metrics and summaries ########################################################### summaries = [] metrics_map = {} for metric in tf.get_collection(graph_utils.GraphKeys.METRICS): metrics_map[metric.op.name] = metrics.streaming_mean(metric) predictions = tf.argmax(outputs, 1) metrics_map['accuracy'] = metrics.streaming_accuracy(predictions, labels) metrics_map['recall_5'] = metrics.streaming_sparse_recall_at_k( outputs, tf.expand_dims(labels, 1), 5) names_to_values, names_to_updates = metrics.aggregate_metric_map( metrics_map) # Create summaries of the metrics and print them to the screen for name, value in names_to_values.iteritems(): summary = tf.summary.scalar(name, value, collections=[]) summaries.append(tf.Print(summary, [value], name)) summaries.extend(layers.summarize_collection( graph_utils.GraphKeys.METRICS)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.QUANTIZED_VARIABLES)) summaries.extend( layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS)) tiled_images = image_utils.tile_images(images) summaries.append(tf.summary.image('input_batch', tiled_images)) summary_op = tf.summary.merge(summaries, name='summaries') ########################################################### # Begin evaluation ########################################################### checkpoint_path = FLAGS.checkpoint_path eval_ops = tf.group(*names_to_updates.values()) scaffold = tf.train.Scaffold(init_op, init_feed_dict) hooks = [ training.SummaryAtEndHook(FLAGS.log_dir, summary_op), training.StopAfterNEvalsHook( math.ceil(dataset.num_samples / float(config.batch_size))) ] eval_kwargs = {} eval_fn = training.evaluate_repeatedly if FLAGS.once: if tf.gfile.IsDirectory(checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) eval_fn = training.evaluate_once else: assert tf.gfile.IsDirectory(checkpoint_path), ( 'checkpoint path must be a directory when using loop evaluation') # On Tensorflow master fd87896 fixes this, but for now just set a very large number eval_kwargs['max_number_of_evaluations'] = sys.maxint eval_fn(checkpoint_path, scaffold=scaffold, hooks=hooks, eval_ops=eval_ops, **eval_kwargs)