def after_run(self, run_context, run_values): results = run_values.results global_step = results.get('global_step') if self._draw_images: self._timer.update_last_triggered_step(global_step) prediction_dict = results.get('prediction_dict') if prediction_dict is not None: summaries = image_vis_summaries(prediction_dict, with_rcnn=self._with_rcnn) for summary in summaries: self._summary_writer.add_summary(summary, global_step) self._next_step = global_step + 1
def after_run(self, run_context, run_values): results = run_values.results global_step = results.get('global_step') if self._draw_images: self._timer.update_last_triggered_step(global_step) prediction_dict = results.get('prediction_dict') if prediction_dict is not None: summaries = image_vis_summaries( prediction_dict, config=self._config, image_visualization_mode=self._image_visualization_mode, image=results.get('image'), gt_bboxes=results.get('gt_bboxes')) if self._summary_writer is not None: for summary in summaries: self._summary_writer.add_summary(summary, global_step) self._next_step = global_step + 1
def evaluate_once( config, writer, saver, ops, checkpoint, class_labels, metrics_scope="metrics", image_vis=None, files_per_class=None, files_to_visualize=None, ): """Run the evaluation once. Create a new session with the previously-built graph, run it through the dataset, calculate the evaluation metrics and write the corresponding summaries. Args: config: Config object for the model. writer: Summary writers. saver: Saver object to restore checkpoint parameters. ops (dict): All the operations needed to successfully run the model. Expects the following keys: ``init_op``, ``metric_ops``, ``pred_objects``, ``pred_objects_classes``, ``pred_objects_scores``, ``train_objects``, ``losses``, ``train_image``. checkpoint (dict): Checkpoint-related data. Expects the following keys: ``global_step``, ``file``. """ # Output of the detector, per batch. output_per_batch = { "bboxes": [], # Bounding boxes detected. "classes": [], # Class associated to each bounding box. "scores": [], # Score for each detection. "gt_bboxes": [], # Ground-truth bounding boxes for the batch. "gt_classes": [], # Ground-truth classes for each bounding box. } with tf.Session() as sess: sess.run(ops["init_op"]) saver.restore(sess, checkpoint["file"]) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) total_evaluated = 0 start_time = time.time() try: track_start = start_time track_count = 0 while not coord.should_stop(): fetches = { "metric_ops": ops["metric_ops"], "bboxes": ops["pred_objects"], "classes": ops["pred_objects_classes"], "scores": ops["pred_objects_scores"], "gt_bboxes": ops["train_objects"], "losses": ops["losses"], "filename": ops["filename"], } if image_vis is not None: fetches["prediction_dict"] = ops["prediction_dict"] fetches["train_image"] = ops["train_image"] batch_fetched = sess.run(fetches) output_per_batch["bboxes"].append(batch_fetched.get("bboxes")) output_per_batch["classes"].append(batch_fetched["classes"]) output_per_batch["scores"].append(batch_fetched["scores"]) batch_gt_objects = batch_fetched["gt_bboxes"] output_per_batch["gt_bboxes"].append(batch_gt_objects[:, :4]) batch_gt_classes = batch_gt_objects[:, 4] output_per_batch["gt_classes"].append(batch_gt_classes) val_losses = batch_fetched["losses"] if image_vis is not None: filename = batch_fetched["filename"].decode("utf-8") visualize_file = False for gt_class in batch_gt_classes: cls_files = files_to_visualize.get(gt_class, set()) if len(cls_files) < files_per_class: files_to_visualize.setdefault(gt_class, set()).add(filename) visualize_file = True break elif filename in cls_files: visualize_file = True break if visualize_file: image_summaries = image_vis_summaries( batch_fetched["prediction_dict"], config=config.model, extra_tag=filename, image_visualization_mode=image_vis, image=batch_fetched["train_image"], gt_bboxes=batch_fetched["gt_bboxes"], ) for image_summary in image_summaries: writer.add_summary(image_summary, checkpoint["global_step"]) total_evaluated += 1 track_count += 1 track_end = time.time() if track_end - track_start > 20.0: click.echo( "{} processed in {:.2f}s (global {:.2f} images/s, " "period {:.2f} images/s)".format( total_evaluated, track_end - start_time, total_evaluated / (track_end - start_time), track_count / (track_end - track_start), )) track_count = 0 track_start = track_end except tf.errors.OutOfRangeError: # Save final evaluation stats into summary under the checkpoint's # global step. ap_per_class, ar_per_class = calculate_metrics( output_per_batch, config.model.network.num_classes) map_at_50 = np.mean(ap_per_class[:, 0]) map_at_75 = np.mean(ap_per_class[:, 5]) map_at_range = np.mean(ap_per_class) mar_at_range = np.mean(ar_per_class) tf.logging.info("Finished evaluation at step {}.".format( checkpoint["global_step"])) tf.logging.info("Evaluated {} images.".format(total_evaluated)) tf.logging.info( "Average Precision (AP) @ [0.50] = {:.3f}".format(map_at_50)) tf.logging.info( "Average Precision (AP) @ [0.75] = {:.3f}".format(map_at_75)) tf.logging.info( "Average Precision (AP) @ [0.50:0.95] = {:.3f}".format( map_at_range)) tf.logging.info( "Average Recall (AR) @ [0.50:0.95] = {:.3f}".format( mar_at_range)) for idx, val in enumerate(ap_per_class[:, 0]): class_label = ("{} ({})".format(class_labels[idx], idx) if class_labels else idx) tf.logging.debug( "Average Precision (AP) @ [0.50] for {} = {:.3f}".format( class_label, val)) summary = [ tf.Summary.Value(tag="{}/[email protected]".format(metrics_scope), simple_value=map_at_50), tf.Summary.Value(tag="{}/[email protected]".format(metrics_scope), simple_value=map_at_75), tf.Summary.Value( tag="{}/AP@[0.50:0.95]".format(metrics_scope), simple_value=map_at_range, ), tf.Summary.Value( tag="{}/AR@[0.50:0.95]".format(metrics_scope), simple_value=mar_at_range, ), tf.Summary.Value( tag="{}/total_evaluated".format(metrics_scope), simple_value=total_evaluated, ), tf.Summary.Value( tag="{}/evaluation_time".format(metrics_scope), simple_value=time.time() - start_time, ), ] for loss_name, loss_value in val_losses.items(): tf.logging.debug("{} loss = {:.4f}".format( loss_name, loss_value)) summary.append( tf.Summary.Value(tag=loss_name, simple_value=loss_value)) writer.add_summary(tf.Summary(value=summary), checkpoint["global_step"]) finally: coord.request_stop() # Wait for all threads to stop. coord.join(threads)
def evaluate_once(config, writer, saver, ops, checkpoint, metrics_scope='metrics', image_vis=None, files_per_class=None, files_to_visualize=None): """Run the evaluation once. Create a new session with the previously-built graph, run it through the dataset, calculate the evaluation metrics and write the corresponding summaries. Args: config: Config object for the model. writer: Summary writers. saver: Saver object to restore checkpoint parameters. ops (dict): All the operations needed to successfully run the model. Expects the following keys: ``init_op``, ``metric_ops``, ``pred_objects``, ``pred_objects_classes``, ``pred_objects_scores``, ``train_objects``, ``losses``, ``train_image``. checkpoint (dict): Checkpoint-related data. Expects the following keys: ``global_step``, ``file``. """ # Output of the detector, per batch. output_per_batch = { 'bboxes': [], # Bounding boxes detected. 'classes': [], # Class associated to each bounding box. 'scores': [], # Score for each detection. 'gt_bboxes': [], # Ground-truth bounding boxes for the batch. 'gt_classes': [], # Ground-truth classes for each bounding box. } with tf.Session() as sess: sess.run(ops['init_op']) saver.restore(sess, checkpoint['file']) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) try: while not coord.should_stop(): fetches = { 'metric_ops': ops['metric_ops'], 'bboxes': ops['pred_objects'], 'classes': ops['pred_objects_classes'], 'scores': ops['pred_objects_scores'], 'gt_bboxes': ops['train_objects'] } if image_vis is not None: fetches['prediction_dict'] = ops['prediction_dict'] fetches['filename'] = ops['filename'] fetches['train_image'] = ops['train_image'] batch_fetched = sess.run(fetches) output_per_batch['bboxes'].append(batch_fetched.get('bboxes')) output_per_batch['classes'].append(batch_fetched['classes']) output_per_batch['scores'].append(batch_fetched['scores']) batch_gt_objects = batch_fetched['gt_bboxes'] output_per_batch['gt_bboxes'].append(batch_gt_objects[:, :4]) batch_gt_classes = batch_gt_objects[:, 4] output_per_batch['gt_classes'].append(batch_gt_classes) val_losses = sess.run(ops['losses']) if image_vis is not None: filename = batch_fetched['filename'].decode('utf-8') visualize_file = False for gt_class in batch_gt_classes: cls_files = files_to_visualize.get( gt_class, set() ) if len(cls_files) < files_per_class: files_to_visualize.setdefault( gt_class, set() ).add(filename) visualize_file = True break elif filename in cls_files: visualize_file = True break if visualize_file: image_summaries = image_vis_summaries( batch_fetched['prediction_dict'], with_rcnn=config.model.network.with_rcnn, extra_tag=filename, image_visualization_mode=image_vis, image=batch_fetched['train_image'], gt_bboxes=batch_fetched['gt_bboxes'] ) for image_summary in image_summaries: writer.add_summary( image_summary, checkpoint['global_step'] ) except tf.errors.OutOfRangeError: # Save final evaluation stats into summary under the checkpoint's # global step. map_0_5, per_class_0_5 = calculate_map( output_per_batch, config.model.network.num_classes, 0.5 ) tf.logging.info('Finished evaluation at step {}.'.format( checkpoint['global_step'])) tf.logging.info('[email protected] = {:.2f}'.format(map_0_5)) # TODO: Find a way to generate these summaries automatically, or # less manually. summary = [ tf.Summary.Value( tag='{}/[email protected]'.format(metrics_scope), simple_value=map_0_5 ), ] for idx, val in enumerate(per_class_0_5): tf.logging.debug('[email protected] for {} = {:.2f}'.format(idx, val)) summary.append(tf.Summary.Value( tag='{}/[email protected]/{}'.format(metrics_scope, idx), simple_value=val )) for loss_name, loss_value in val_losses.items(): tf.logging.debug('{} loss = {:.4f}'.format( loss_name, loss_value)) summary.append(tf.Summary.Value( tag=loss_name, simple_value=loss_value )) total_bboxes_per_batch = [ len(bboxes) for bboxes in output_per_batch['bboxes'] ] summary.append(tf.Summary.Value( tag='{}/avg_bboxes'.format(metrics_scope), simple_value=np.mean(total_bboxes_per_batch) )) writer.add_summary( tf.Summary(value=summary), checkpoint['global_step'] ) finally: coord.request_stop() # Wait for all threads to stop. coord.join(threads)
def evaluate_once(config, writer, saver, ops, checkpoint, metrics_scope='metrics', image_vis=None, files_per_class=None, files_to_visualize=None, iou_threshold=0.5, min_probability=0.5): """Run the evaluation once. Create a new session with the previously-built graph, run it through the dataset, calculate the evaluation metrics and write the corresponding summaries. Args: config: Config object for the model. writer: Summary writers. saver: Saver object to restore checkpoint parameters. ops (dict): All the operations needed to successfully run the model. Expects the following keys: ``init_op``, ``metric_ops``, ``pred_objects``, ``pred_objects_classes``, ``pred_objects_scores``, ``train_objects``, ``losses``, ``train_image``. checkpoint (dict): Checkpoint-related data. Expects the following keys: ``global_step``, ``file``. """ # Output of the detector, per batch. output_per_batch = { 'bboxes': [], # Bounding boxes detected. 'classes': [], # Class associated to each bounding box. 'scores': [], # Score for each detection. 'gt_bboxes': [], # Ground-truth bounding boxes for the batch. 'gt_classes': [], # Ground-truth classes for each bounding box. } with tf.Session() as sess: sess.run(ops['init_op']) saver.restore(sess, checkpoint['file']) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) total_evaluated = 0 start_time = time.time() try: track_start = start_time track_count = 0 period_detected_count = 0 period_detected_max = 0 while not coord.should_stop(): fetches = { 'metric_ops': ops['metric_ops'], 'bboxes': ops['pred_objects'], 'classes': ops['pred_objects_classes'], 'scores': ops['pred_objects_scores'], 'gt_bboxes': ops['train_objects'], 'losses': ops['losses'], 'filename': ops['filename'], } if image_vis is not None: fetches['prediction_dict'] = ops['prediction_dict'] fetches['train_image'] = ops['train_image'] batch_fetched = sess.run(fetches) output_per_batch['bboxes'].append(batch_fetched.get('bboxes')) output_per_batch['classes'].append(batch_fetched['classes']) output_per_batch['scores'].append(batch_fetched['scores']) batch_gt_objects = batch_fetched['gt_bboxes'] output_per_batch['gt_bboxes'].append(batch_gt_objects[:, :4]) batch_gt_classes = batch_gt_objects[:, 4] output_per_batch['gt_classes'].append(batch_gt_classes) val_losses = batch_fetched['losses'] num_detected = len(batch_fetched['scores']) period_detected_count += num_detected period_detected_max = max(period_detected_max, num_detected) if period_detected_max == len(batch_fetched['scores']): period_max_img = batch_fetched['filename'] if image_vis is not None: filename = batch_fetched['filename'].decode('utf-8') visualize_file = False for gt_class in batch_gt_classes: cls_files = files_to_visualize.get( gt_class, set() ) if len(cls_files) < files_per_class: files_to_visualize.setdefault( gt_class, set() ).add(filename) visualize_file = True break elif filename in cls_files: visualize_file = True break if visualize_file: image_summaries = image_vis_summaries( batch_fetched['prediction_dict'], with_rcnn=config.model.network.with_rcnn, extra_tag=filename, image_visualization_mode=image_vis, image=batch_fetched['train_image'], gt_bboxes=batch_fetched['gt_bboxes'] ) for image_summary in image_summaries: writer.add_summary( image_summary, checkpoint['global_step'] ) total_evaluated += 1 track_count += 1 track_end = time.time() if track_end - track_start > 20.: click.echo( '{} processed in {:.2f}s (global {:.2f} images/s, ' 'period {:.2f} images/s, avg dets {:.2f}, ' 'max dets {} on {})'.format( total_evaluated, track_end - start_time, total_evaluated / (track_end - start_time), track_count / (track_end - track_start), period_detected_count / track_count, period_detected_max, period_max_img )) track_count = 0 track_start = track_end period_detected_count = 0 period_detected_max = 0 except tf.errors.OutOfRangeError: # Save final evaluation stats into summary under the checkpoint's # global step. map_at_iou, per_class_at_iou = calculate_map( output_per_batch, config.model.network.num_classes, iou_threshold ) tf.logging.info('Finished evaluation at step {}.'.format( checkpoint['global_step'])) tf.logging.info('Evaluated {} images.'.format(total_evaluated)) tf.logging.info('mAP@{}@{} = {:.2f}'.format( iou_threshold, min_probability, map_at_iou)) # TODO: Find a way to generate these summaries automatically, or # less manually. summary = [ tf.Summary.Value( tag='{}/mAP@{}@{}'.format( metrics_scope, iou_threshold, min_probability ), simple_value=map_at_iou ), tf.Summary.Value( tag='{}/total_evaluated'.format(metrics_scope), simple_value=total_evaluated ), tf.Summary.Value( tag='{}/evaluation_time'.format(metrics_scope), simple_value=time.time() - start_time ), ] for idx, val in enumerate(per_class_at_iou): tf.logging.debug('AP@{}@{} for {} = {:.2f}'.format( iou_threshold, min_probability, idx, val)) summary.append(tf.Summary.Value( tag='{}/AP@{}@{}/{}'.format( metrics_scope, iou_threshold, min_probability, idx ), simple_value=val )) for loss_name, loss_value in val_losses.items(): tf.logging.debug('{} loss = {:.4f}'.format( loss_name, loss_value)) summary.append(tf.Summary.Value( tag=loss_name, simple_value=loss_value )) total_bboxes_per_batch = [ len(bboxes) for bboxes in output_per_batch['bboxes'] ] summary.append(tf.Summary.Value( tag='{}/avg_bboxes'.format(metrics_scope), simple_value=np.mean(total_bboxes_per_batch) )) writer.add_summary( tf.Summary(value=summary), checkpoint['global_step'] ) finally: coord.request_stop() # Wait for all threads to stop. coord.join(threads)