def evaluate(train_dir, eval_dir, config, dataset_fn, num_batches, master=''): """Evaluate the model repeatedly.""" tf.gfile.MakeDirs(eval_dir) _trial_summary( config.hparams, config.eval_examples_path or config.tfds_name, eval_dir) with tf.Graph().as_default(): model = config.model model.build(config.hparams, config.data_converter.output_depth, is_training=False) eval_op = model.eval( **_get_input_tensors(dataset_fn().take(num_batches), config)) hooks = [ contrib_training.StopAfterNEvalsHook(num_batches), contrib_training.SummaryAtEndHook(eval_dir) ] contrib_training.evaluate_repeatedly( train_dir, eval_ops=eval_op, hooks=hooks, eval_interval_secs=60, master=master)
def main(_): FLAGS.properties_file = "properties_test.json" properties = get_properties(FLAGS) logdir = setup_logdir(FLAGS, properties) FLAGS.running_mode = "test" model = get_model(FLAGS, properties) with tf.variable_scope('model', reuse=True): batch = model.data_handler.get_batch(FLAGS.batch_size, FLAGS) noise = tf.random_uniform([FLAGS.batch_size, FLAGS.z_dim], minval=-1.0, maxval=1.0, dtype=tf.float32, name='z0') generated_data = model.get_generated_data(noise, batch[1:]) model.data_handler.display_fake_data(generated_data, batch[1], FLAGS.batch_size) session_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) id_to_enzyme_class_dict = properties["class_mapping"] evaluate_repeatedly(checkpoint_dir=logdir, hooks=[ BlastHook(id_to_enzyme_class_dict, every_n_steps=1, output_dir=logdir, n_examples=FLAGS.batch_size, running_mode=FLAGS.running_mode), tf.contrib.training.SummaryAtEndHook(logdir), tf.contrib.training.StopAfterNEvalsHook(1) ], eval_ops=generated_data, config=session_config)
def main(_): FLAGS.properties_file = "properties_test.json" properties = get_properties(FLAGS) logdir = setup_logdir(FLAGS, properties) FLAGS.running_mode = "test" path = os.path.join(FLAGS.data_dir, FLAGS.dataset.replace("\\", os.sep), "d_test.csv") # data_to_test = pd.read_csv(path, header=None)[:FLAGS.batch_size] # data_to_test = from_amino_acid_to_id(data_to_test, 0) model = get_model(FLAGS, properties) path = os.path.join(FLAGS.data_dir, FLAGS.dataset.replace("\\", os.sep), "properties_test_original.json") with open(path) as json_data_file: properties = json.load(json_data_file) with tf.variable_scope('model', reuse=True): real_x, labels = model.data_handler.get_batch(FLAGS.batch_size, FLAGS) real_x, labels = model.data_handler.prepare_real_data(real_x, labels) # # data = model.data_handler.get_embedded_seqs(data_to_test) # d_scores, _ = model.get_discriminator_result(data, batch[1:], reuse=True) # printing_d_scores = tf.py_func( # lambda vals, scores: print_protein_seq(vals, properties["class_mapping"], d_scores=scores), # [tf.squeeze(data_to_test), d_scores], tf.string) noise = np.random.normal(size=[FLAGS.batch_size, FLAGS.z_dim], loc=0.0, scale=1.0) z = noise generated_data = model.get_generated_data( tf.convert_to_tensor(z, dtype=tf.float32), labels) d_fake_scores, _ = model.get_discriminator_result(generated_data, labels, reuse=True) generated_data_ids = model.data_handler.convert_to_acid_ids( generated_data) printing_sequeces = tf.py_func( lambda vals, labels: print_protein_seq( vals, properties["class_mapping"], labels=labels), [tf.squeeze(generated_data_ids), labels[0]], tf.string) session_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) evaluate_repeatedly( checkpoint_dir=logdir, hooks=[ # BlastHook(id_to_enzyme_class_dict, every_n_steps=1, output_dir=logdir, n_examples=FLAGS.batch_size, # running_mode=FLAGS.running_mode), tf.contrib.training.SummaryAtEndHook(logdir), tf.contrib.training.StopAfterNEvalsHook(1) ], eval_ops=[printing_sequeces], max_number_of_evaluations=1, config=session_config)
def main(_): FLAGS.properties_file = "properties_test.json" properties = get_properties(FLAGS) logdir = setup_logdir(FLAGS, properties) FLAGS.running_mode = "test" path = os.path.join(FLAGS.data_dir, FLAGS.dataset.replace("\\", os.sep), "d_test.csv") data_to_test = pd.read_csv(path, header=None)[:FLAGS.batch_size] data_to_test = from_amino_acid_to_id(data_to_test, 0) model = get_model(FLAGS, properties) with tf.variable_scope('model', reuse=True): batch = model.data_handler.get_batch(FLAGS.batch_size, FLAGS) data = model.data_handler.get_embedded_seqs(data_to_test) d_scores, _ = model.get_discriminator_result(data, batch[1:], reuse=True) printing_d_scores = tf.py_func( lambda vals, scores: print_protein_seq( vals, properties["class_mapping"], d_scores=scores), [tf.squeeze(data_to_test), d_scores], tf.string) noise1 = np.random.uniform(size=[FLAGS.z_dim], low=-1.0, high=1.0) noise2 = np.random.uniform(size=[FLAGS.z_dim], low=-1.0, high=1.0) z = np.stack([ slerp(ratio, noise1, noise2) for ratio in np.linspace(0, 1, FLAGS.batch_size) ]) generated_data = model.get_generated_data( tf.convert_to_tensor(z, dtype=tf.float32), batch[1:]) d_fake_scores, _ = model.get_discriminator_result(generated_data, batch[1:], reuse=True) generated_data_ids = model.data_handler.convert_to_acid_ids( generated_data) printing_sequeces = tf.py_func( lambda vals, scores: print_protein_seq( vals, properties["class_mapping"], d_scores=scores), [tf.squeeze(generated_data_ids), d_fake_scores], tf.string) session_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) evaluate_repeatedly( checkpoint_dir=logdir, hooks=[ # BlastHook(id_to_enzyme_class_dict, every_n_steps=1, output_dir=logdir, n_examples=FLAGS.batch_size, # running_mode=FLAGS.running_mode), tf.contrib.training.SummaryAtEndHook(logdir), tf.contrib.training.StopAfterNEvalsHook(1) ], eval_ops=[printing_d_scores, printing_sequeces], max_number_of_evaluations=1, config=session_config)
def run_eval(build_graph_fn, train_dir, eval_dir, num_batches, timeout_secs=300): """Runs the training loop. Args: build_graph_fn: A function that builds the graph ops. train_dir: The path to the directory where checkpoints will be loaded from for evaluation. eval_dir: The path to the directory where the evaluation summary events will be written to. num_batches: The number of full batches to use for each evaluation step. timeout_secs: The number of seconds after which to stop waiting for a new checkpoint. Raises: ValueError: If `num_batches` is less than or equal to 0. """ if num_batches <= 0: raise ValueError( '`num_batches` must be greater than 0. Check that the batch size is ' 'no larger than the number of records in the eval set.') with tf.Graph().as_default(): build_graph_fn() global_step = tf.train.get_or_create_global_step() loss = tf.get_collection('loss')[0] perplexity = tf.get_collection('metrics/perplexity')[0] accuracy = tf.get_collection('metrics/accuracy')[0] eval_ops = tf.get_collection('eval_ops') logging_dict = { 'Global Step': global_step, 'Loss': loss, 'Perplexity': perplexity, 'Accuracy': accuracy } hooks = [ EvalLoggingTensorHook(logging_dict, every_n_iter=num_batches), contrib_training.StopAfterNEvalsHook(num_batches), contrib_training.SummaryAtEndHook(eval_dir), ] contrib_training.evaluate_repeatedly(train_dir, eval_ops=eval_ops, hooks=hooks, eval_interval_secs=60, timeout=timeout_secs)
def Evaluacion(build_graph_fn, train_dir, eval_dir, num_batches, timeout_secs=300): tf.compat.v1.logging.set_verbosity('INFO') if num_batches <= 0: raise ValueError( '`num_batches` must be greater than 0. Check that the batch size is ' 'no larger than the number of records in the eval set.') with tf.Graph().as_default(): # Creamos un modelo igual que el del entrenamiento build_graph_fn() # Define the summaries to write: global_step = tf.train.get_or_create_global_step() loss = tf.get_collection('loss')[0] perplexity = tf.get_collection('metrics/perplexity')[0] accuracy = tf.get_collection('metrics/accuracy')[0] eval_ops = tf.get_collection('eval_ops') logging_dict = { 'Global Step': global_step, 'Loss': loss, 'Perplexity': perplexity, 'Accuracy': accuracy } hooks = [ EvalLoggingTensorHook(logging_dict, every_n_iter=num_batches), contrib_training.StopAfterNEvalsHook(num_batches), contrib_training.SummaryAtEndHook(eval_dir), ] # names_to_values = contrib_training.evaluate_once( # checkpoint_path=train_dir, # eval_ops=eval_ops, # final_ops=logging_dict, # hooks=hooks, # config=None) # for name in names_to_values: # print('Metric %s has value %f.' % (name, names_to_values[name])) contrib_training.evaluate_repeatedly(train_dir, eval_ops=eval_ops, hooks=hooks, eval_interval_secs=2, timeout=timeout_secs)
def main(unused_argv): tf.logging.set_verbosity(tf.logging.INFO) dataset = data_generator.Dataset( dataset_name=FLAGS.dataset, split_name=FLAGS.eval_split, dataset_dir=FLAGS.dataset_dir, batch_size=FLAGS.eval_batch_size, crop_size=[int(sz) for sz in FLAGS.eval_crop_size], min_resize_value=FLAGS.min_resize_value, max_resize_value=FLAGS.max_resize_value, resize_factor=FLAGS.resize_factor, model_variant=FLAGS.model_variant, num_readers=2, is_training=False, should_shuffle=False, should_repeat=False) tf.gfile.MakeDirs(FLAGS.eval_logdir) tf.logging.info('Evaluating on %s set', FLAGS.eval_split) with tf.Graph().as_default(): samples = dataset.get_one_shot_iterator().get_next() model_options = common.ModelOptions( model_name=FLAGS.model_name, outputs_to_num_classes={ common.OUTPUT_TYPE: dataset.num_of_classes }, crop_size=[int(sz) for sz in FLAGS.eval_crop_size], atrous_rates=FLAGS.atrous_rates, output_stride=FLAGS.output_stride) # Set shape in order for tf.contrib.tfprof.model_analyzer to work properly. samples[common.IMAGE].set_shape([ FLAGS.eval_batch_size, int(FLAGS.eval_crop_size[0]), int(FLAGS.eval_crop_size[1]), 3 ]) if tuple(FLAGS.eval_scales) == (1.0, ): tf.logging.info('Performing single-scale test.') predictions = model.predict_labels( samples[common.IMAGE], model_options, image_pyramid=FLAGS.image_pyramid) else: tf.logging.info('Performing multi-scale test.') if FLAGS.quantize_delay_step >= 0: raise ValueError( 'Quantize mode is not supported with multi-scale test.') predictions = model.predict_labels_multi_scale( samples[common.IMAGE], model_options=model_options, eval_scales=FLAGS.eval_scales, add_flipped_images=FLAGS.add_flipped_images) predictions = predictions[common.OUTPUT_TYPE] predictions = tf.reshape(predictions, shape=[-1]) labels = tf.reshape(samples[common.LABEL], shape=[-1]) weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label)) # Set ignore_label regions to label 0, because metrics.mean_iou requires # range of labels = [0, dataset.num_classes). Note the ignore_label regions # are not evaluated since the corresponding regions contain weights = 0. labels = tf.where(tf.equal(labels, dataset.ignore_label), tf.zeros_like(labels), labels) predictions_tag = 'miou' for eval_scale in FLAGS.eval_scales: predictions_tag += '_' + str(eval_scale) if FLAGS.add_flipped_images: predictions_tag += '_flipped' # Define the evaluation metric. metric_map = {} num_classes = dataset.num_of_classes metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou( labels=labels, predictions=predictions, num_classes=num_classes, weights=weights) # IoU for each class. one_hot_predictions = tf.one_hot(predictions, num_classes) one_hot_predictions = tf.reshape(one_hot_predictions, [-1, num_classes]) one_hot_labels = tf.one_hot(labels, num_classes) one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes]) for c in range(num_classes): predictions_tag_c = '%s_class_%d' % (predictions_tag, c) tp, tp_op = tf.metrics.true_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fp, fp_op = tf.metrics.false_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fn, fn_op = tf.metrics.false_negatives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op) iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp), tf.constant(np.NaN)) metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op) (metrics_to_values, metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map) summary_ops = [] for metric_name, metric_value in six.iteritems(metrics_to_values): op = tf.summary.scalar(metric_name, metric_value) op = tf.Print(op, [metric_value], metric_name) summary_ops.append(op) summary_op = tf.summary.merge(summary_ops) summary_hook = contrib_training.SummaryAtEndHook( log_dir=FLAGS.eval_logdir, summary_op=summary_op) hooks = [summary_hook] num_eval_iters = None if FLAGS.max_number_of_evaluations > 0: num_eval_iters = FLAGS.max_number_of_evaluations if FLAGS.quantize_delay_step >= 0: contrib_quantize.create_eval_graph() contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer. TRAINABLE_VARS_PARAMS_STAT_OPTIONS) contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS) contrib_training.evaluate_repeatedly( checkpoint_dir=FLAGS.checkpoint_dir, master=FLAGS.master, eval_ops=list(metrics_to_updates.values()), max_number_of_evaluations=num_eval_iters, hooks=hooks, eval_interval_secs=FLAGS.eval_interval_secs)
def main(unused_argv): tf.logging.set_verbosity(tf.logging.INFO) dataset = data_generator.Dataset( dataset_name=FLAGS.dataset, split_name=FLAGS.eval_split, dataset_dir=FLAGS.dataset_dir, batch_size=FLAGS.eval_batch_size, crop_size=[int(sz) for sz in FLAGS.eval_crop_size], min_resize_value=FLAGS.min_resize_value, max_resize_value=FLAGS.max_resize_value, resize_factor=FLAGS.resize_factor, model_variant=FLAGS.model_variant, num_readers=2, is_training=False, should_shuffle=False, should_repeat=False, with_cls=True, cls_only=False, output_valid=True) tf.gfile.MakeDirs(FLAGS.eval_logdir) tf.logging.info('Evaluating on %s set', FLAGS.eval_split) with tf.Graph().as_default(): samples = dataset.get_one_shot_iterator().get_next() model_options = common.ModelOptions( outputs_to_num_classes={ common.OUTPUT_TYPE: dataset.num_of_classes }, crop_size=[int(sz) for sz in FLAGS.eval_crop_size], atrous_rates=FLAGS.atrous_rates, output_stride=FLAGS.output_stride) # Set shape in order for tf.contrib.tfprof.model_analyzer to work properly. samples[common.IMAGE].set_shape([ FLAGS.eval_batch_size, int(FLAGS.eval_crop_size[0]), int(FLAGS.eval_crop_size[1]), 3 ]) if tuple(FLAGS.eval_scales) == (1.0, ): tf.logging.info('Performing single-scale test.') predictions = model.predict_labels( samples[common.IMAGE], model_options, image_pyramid=FLAGS.image_pyramid) else: tf.logging.info('Performing multi-scale test.') raise NotImplementedError('Multi-scale is not supported yet!') metric_map = {} ## Extract cls logits if FLAGS.weakly: _, end_points = feature_extractor.extract_features( samples[common.IMAGE], output_stride=model_options.output_stride, multi_grid=model_options.multi_grid, model_variant=model_options.model_variant, depth_multiplier=model_options.depth_multiplier, divisible_by=model_options.divisible_by, reuse=tf.AUTO_REUSE, is_training=False, preprocessed_images_dtype=model_options. preprocessed_images_dtype, global_pool=True, num_classes=dataset.num_of_classes - 1) # ResNet beta version has an additional suffix in FLAGS.model_variant, but # it shares the same variable names with original version. Add a special # handling here for beta version ResNet. logits = end_points['{}/logits'.format( FLAGS.model_variant).replace('_beta', '')] logits = tf.reshape(logits, [-1, dataset.num_of_classes - 1]) cls_pred = tf.sigmoid(logits) # Multi-label classification evaluation cls_label = samples['cls_label'] cls_pred = tf.cast(tf.greater_equal(cls_pred, 0.5), tf.int32) ## For classification metric_map['eval/cls_overall'] = tf.metrics.accuracy( labels=cls_label, predictions=cls_pred) metric_map['eval/cls_precision'] = tf.metrics.precision( labels=cls_label, predictions=cls_pred) metric_map['eval/cls_recall'] = tf.metrics.recall( labels=cls_label, predictions=cls_pred) ## For segmentation branch eval predictions = predictions[common.OUTPUT_TYPE] predictions = tf.reshape(predictions, shape=[-1]) labels = tf.reshape(samples[common.LABEL], shape=[-1]) weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label)) # Set ignore_label regions to label 0, because metrics.mean_iou requires # range of labels = [0, dataset.num_classes). Note the ignore_label regions # are not evaluated since the corresponding regions contain weights = 0. labels = tf.where(tf.equal(labels, dataset.ignore_label), tf.zeros_like(labels), labels) predictions_tag = 'miou' # Define the evaluation metric. num_classes = dataset.num_of_classes ## For segmentation metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou( labels=labels, predictions=predictions, num_classes=num_classes, weights=weights) # IoU for each class. one_hot_predictions = tf.one_hot(predictions, num_classes) one_hot_predictions = tf.reshape(one_hot_predictions, [-1, num_classes]) one_hot_labels = tf.one_hot(labels, num_classes) one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes]) for c in range(num_classes): predictions_tag_c = '%s_class_%d' % (predictions_tag, c) tp, tp_op = tf.metrics.true_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fp, fp_op = tf.metrics.false_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fn, fn_op = tf.metrics.false_negatives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op) iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp), tf.constant(np.NaN)) metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op) (metrics_to_values, metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map) summary_ops = [] for metric_name, metric_value in six.iteritems(metrics_to_values): op = tf.summary.scalar(metric_name, metric_value) op = tf.Print(op, [metric_value], metric_name) summary_ops.append(op) summary_op = tf.summary.merge(summary_ops) summary_hook = contrib_training.SummaryAtEndHook( log_dir=FLAGS.eval_logdir, summary_op=summary_op) hooks = [summary_hook] num_eval_iters = None if FLAGS.max_number_of_evaluations > 0: num_eval_iters = FLAGS.max_number_of_evaluations if FLAGS.quantize_delay_step >= 0: contrib_quantize.create_eval_graph() contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer. TRAINABLE_VARS_PARAMS_STAT_OPTIONS) contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS) contrib_training.evaluate_repeatedly( checkpoint_dir=FLAGS.checkpoint_dir, master=FLAGS.master, eval_ops=list(metrics_to_updates.values()), max_number_of_evaluations=num_eval_iters, hooks=hooks, eval_interval_secs=FLAGS.eval_interval_secs)
def main(unused_argv): tf.logging.set_verbosity(tf.logging.INFO) dataset = data_generator.Dataset( # 获取验证集图片数据 dataset_name=FLAGS.dataset, # 数据集名称 cityscapes 默认为 pascal_voc_seg split_name=FLAGS.eval_split, # 指定带有val的tfrecorder数据集 默认为“val” dataset_dir=FLAGS.dataset_dir, # 数据集目录 tfrecoder文件的数据集目录 batch_size=FLAGS.eval_batch_size, # 每个batch包含的image数量 默认为1 crop_size=[int(sz) for sz in FLAGS.eval_crop_size], # 评估时crop_size 默认为513,513 min_resize_value=FLAGS.min_resize_value, # 默认为None max_resize_value=FLAGS.max_resize_value, # 默认为None resize_factor=FLAGS.resize_factor, # 默认为None model_variant=FLAGS.model_variant, # 模型的变体 本次训练为 xception_65 num_readers=2, # 并行读取图片的数量 is_training=False, # 不训练 should_shuffle=False, # 不将输入的数据随机打乱 should_repeat=False) # 不一直重复 tf.gfile.MakeDirs(FLAGS.eval_logdir) # 创建评估目录 tf.logging.info('Evaluating on %s set', FLAGS.eval_split) with tf.Graph().as_default(): samples = dataset.get_one_shot_iterator().get_next() # 获取一次迭代的验证集数据 ''' samples: {'image_name': <tf.Tensor 'IteratorGetNext:2' shape=(?,) dtype=string>, 'width': <tf.Tensor 'IteratorGetNext:5' shape=(?,) dtype=int64>, 'image': <tf.Tensor 'IteratorGetNext:1' shape=(?, 1024, 2048, 3) dtype=float32>, 'height': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=int64>, 'label': <tf.Tensor 'IteratorGetNext:3' shape=(?, 1024, 2048, 1) dtype=int32>, 'original_image': <tf.Tensor 'IteratorGetNext:4' shape=(?, ?, ?, 3) dtype=uint8>} ''' model_options = common.ModelOptions( # 模型参数 outputs_to_num_classes={ common.OUTPUT_TYPE: dataset.num_of_classes }, # {semantic: 19} crop_size=[int(sz) for sz in FLAGS.eval_crop_size], # 1024,2048 atrous_rates=FLAGS.atrous_rates, # 6,12,18 output_stride=FLAGS.output_stride) # 16 # Set shape in order for tf.contrib.tfprof.model_analyzer to work properly. samples[common.IMAGE].set_shape( # 设置形状 [ FLAGS.eval_batch_size, # 默认为1 int(FLAGS.eval_crop_size[0]), int(FLAGS.eval_crop_size[1]), 3 ]) if tuple(FLAGS.eval_scales) == (1.0, ): # 默认 评估尺度为1 tf.logging.info('Performing single-scale test.') predictions = model.predict_labels( samples[common.IMAGE], model_options, # 进行每个像素点预测 image_pyramid=FLAGS.image_pyramid) ''' predictions: {'semantic': <tf.Tensor 'ArgMax:0' shape=(1, 1024, 2048) dtype=int64>, 'semantic_prob': <tf.Tensor 'Softmax:0' shape=(1, 1024, 2048, 19) dtype=float32>} ''' else: tf.logging.info('Performing multi-scale test.') if FLAGS.quantize_delay_step >= 0: raise ValueError( 'Quantize mode is not supported with multi-scale test.') predictions = model.predict_labels_multi_scale( samples[common.IMAGE], model_options=model_options, eval_scales=FLAGS.eval_scales, add_flipped_images=FLAGS.add_flipped_images) predictions = predictions[common.OUTPUT_TYPE] predictions = tf.reshape(predictions, shape=[-1]) # 预测标签 labels = tf.reshape(samples[common.LABEL], shape=[-1]) # 真实标签 weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label)) # 各标签权重 # Set ignore_label regions to label 0, because metrics.mean_iou requires # range of labels = [0, dataset.num_classes). Note the ignore_label regions # are not evaluated since the corresponding regions contain weights = 0. labels = tf.where(tf.equal(labels, dataset.ignore_label), tf.zeros_like(labels), labels) predictions_tag = 'miou' # MIoU predictions_tag1 = 'accuracy_pixel' # 像素精度 for eval_scale in FLAGS.eval_scales: # 默认为单尺度[1.0] predictions_tag += '_' + str(eval_scale) predictions_tag1 += '_' + str(eval_scale) if FLAGS.add_flipped_images: # 默认为False 不设置左右翻转来评估模型 predictions_tag += '_flipped' predictions_tag1 += '_flipped' # Define the evaluation metric. metric_map = {} num_classes = dataset.num_of_classes # 19 metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou( labels=labels, predictions=predictions, num_classes=num_classes, weights=weights) ''' metric_map: {'eval/miou_1.0_overall': (<tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>, <tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>)} ''' metric_map['eval/%s_overall_accuracy_' % predictions_tag] = tf.metrics.accuracy( labels=labels, predictions=predictions, weights=weights) # IoU for each class. ''' tf.one_hot(indices, depth, on_value=None, off_value=None, axis=None, dtype=None, name=None) Returns a one-hot tensor. ndices表示输入的多个数值,通常是矩阵形式;depth表示输出的尺寸。 ''' one_hot_predictions = tf.one_hot(predictions, num_classes) one_hot_predictions = tf.reshape(one_hot_predictions, [-1, num_classes]) # 预测输出的one_hot one_hot_labels = tf.one_hot(labels, num_classes) one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes]) # 真实label的one_hot for c in range(num_classes): predictions_tag_c = '%s_class_%d' % (predictions_tag, c ) # miou_1.0_class_c predictions_tag_c1 = '%s_class_%d' % (predictions_tag1, c) tp, tp_op = tf.metrics.true_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fp, fp_op = tf.metrics.false_positives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) fn, fn_op = tf.metrics.false_negatives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) tn, tn_op = tf.metrics.true_negatives( labels=one_hot_labels[:, c], predictions=one_hot_predictions[:, c], weights=weights) tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op) iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp), tf.constant(np.NaN)) ap = tf.where(tf.greater(tp + fn, 0.0), (tp + tn) / (tp + tn + fn + fp), tf.constant(np.NaN)) metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op) metric_map['eval/%s' % predictions_tag_c1] = (ap, tp_fp_fn_op) (metrics_to_values, metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map) ''' (metrics_to_values, metrics_to_updates): ({'eval/miou_1.0_class_5': <tf.Tensor 'Select_6:0' shape=() dtype=float32>, 'eval/miou_1.0_class_18': <tf.Tensor 'Select_19:0' shape=() dtype=float32>, 'eval/miou_1.0_class_13': <tf.Tensor 'Select_14:0' shape=() dtype=float32>, 'eval/miou_1.0_class_1': <tf.Tensor 'Select_2:0' shape=() dtype=float32>, 'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>, 'eval/miou_1.0_class_17': <tf.Tensor 'Select_18:0' shape=() dtype=float32>, 'eval/miou_1.0_class_8': <tf.Tensor 'Select_9:0' shape=() dtype=float32>, 'eval/miou_1.0_class_2': <tf.Tensor 'Select_3:0' shape=() dtype=float32>, 'eval/miou_1.0_class_0': <tf.Tensor 'Select_1:0' shape=() dtype=float32>, 'eval/miou_1.0_class_3': <tf.Tensor 'Select_4:0' shape=() dtype=float32>, 'eval/miou_1.0_class_14': <tf.Tensor 'Select_15:0' shape=() dtype=float32>, 'eval/miou_1.0_class_11': <tf.Tensor 'Select_12:0' shape=() dtype=float32>, 'eval/miou_1.0_class_6': <tf.Tensor 'Select_7:0' shape=() dtype=float32>, 'eval/miou_1.0_class_15': <tf.Tensor 'Select_16:0' shape=() dtype=float32>, 'eval/miou_1.0_class_4': <tf.Tensor 'Select_5:0' shape=() dtype=float32>, 'eval/miou_1.0_class_9': <tf.Tensor 'Select_10:0' shape=() dtype=float32>, 'eval/miou_1.0_class_16': <tf.Tensor 'Select_17:0' shape=() dtype=float32>, 'eval/miou_1.0_class_7': <tf.Tensor 'Select_8:0' shape=() dtype=float32>, 'eval/miou_1.0_class_10': <tf.Tensor 'Select_11:0' shape=() dtype=float32>, 'eval/miou_1.0_class_12': <tf.Tensor 'Select_13:0' shape=() dtype=float32>}, {'eval/miou_1.0_class_5': <tf.Operation 'group_deps_5' type=NoOp>, 'eval/miou_1.0_class_18': <tf.Operation 'group_deps_18' type=NoOp>, 'eval/miou_1.0_class_13': <tf.Operation 'group_deps_13' type=NoOp>, 'eval/miou_1.0_class_1': <tf.Operation 'group_deps_1' type=NoOp>, 'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>, 'eval/miou_1.0_class_17': <tf.Operation 'group_deps_17' type=NoOp>, 'eval/miou_1.0_class_8': <tf.Operation 'group_deps_8' type=NoOp>, 'eval/miou_1.0_class_2': <tf.Operation 'group_deps_2' type=NoOp>, 'eval/miou_1.0_class_0': <tf.Operation 'group_deps' type=NoOp>, 'eval/miou_1.0_class_3': <tf.Operation 'group_deps_3' type=NoOp>, 'eval/miou_1.0_class_14': <tf.Operation 'group_deps_14' type=NoOp>, 'eval/miou_1.0_class_11': <tf.Operation 'group_deps_11' type=NoOp>, 'eval/miou_1.0_class_6': <tf.Operation 'group_deps_6' type=NoOp>, 'eval/miou_1.0_class_15': <tf.Operation 'group_deps_15' type=NoOp>, 'eval/miou_1.0_class_4': <tf.Operation 'group_deps_4' type=NoOp>, 'eval/miou_1.0_class_9': <tf.Operation 'group_deps_9' type=NoOp>, 'eval/miou_1.0_class_16': <tf.Operation 'group_deps_16' type=NoOp>, 'eval/miou_1.0_class_7': <tf.Operation 'group_deps_7' type=NoOp>, 'eval/miou_1.0_class_10': <tf.Operation 'group_deps_10' type=NoOp>, 'eval/miou_1.0_class_12': <tf.Operation 'group_deps_12' type=NoOp>}) ''' ''' tf.Print(input, data, message=None, first_n=None, summarize=None, name=None) 最低要求两个输入,input和data,input是需要打印的变量的名字,data要求是一个list,里面包含要打印的内容。 ''' summary_ops = [] for metric_name, metric_value in six.iteritems(metrics_to_values): op = tf.summary.scalar(metric_name, metric_value) # 显示标量信息 op = tf.Print(op, [metric_value], metric_name) summary_ops.append(op) summary_op = tf.summary.merge(summary_ops) summary_hook = contrib_training.SummaryAtEndHook( log_dir=FLAGS.eval_logdir, summary_op=summary_op) hooks = [summary_hook] num_eval_iters = None if FLAGS.max_number_of_evaluations > 0: # 为0 暂不考虑 num_eval_iters = FLAGS.max_number_of_evaluations if FLAGS.quantize_delay_step >= 0: # -1 暂不考虑 contrib_quantize.create_eval_graph() contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer. TRAINABLE_VARS_PARAMS_STAT_OPTIONS) contrib_tfprof.model_analyzer.print_model_analysis( tf.get_default_graph(), tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS) contrib_training.evaluate_repeatedly( checkpoint_dir=FLAGS.checkpoint_dir, master=FLAGS.master, eval_ops=list(metrics_to_updates.values()), max_number_of_evaluations=num_eval_iters, hooks=hooks, eval_interval_secs=FLAGS.eval_interval_secs)