def do_evaluate(pred_config, output_file): num_gpu = cfg.TRAIN.NUM_GPUS graph_funcs = MultiTowerOfflinePredictor(pred_config, list( range(num_gpu))).get_predictors() for dataset in cfg.DATA.VAL: logger.info("Evaluating {} ...".format(dataset)) dataflows = [ get_eval_dataflow(dataset, shard=k, num_shards=num_gpu) for k in range(num_gpu) ] all_results = multithread_predict_dataflow(dataflows, graph_funcs) output = output_file + '-' + dataset DetectionDataset().eval_or_save_inference_results( all_results, dataset, output) for dataset in cfg.DATA.TEST: logger.info("Evaluating {} ...".format(dataset)) dataflows = [ get_eval_dataflow(dataset, shard=k, num_shards=num_gpu) for k in range(num_gpu) ] all_results = multithread_predict_dataflow(dataflows, graph_funcs) output = output_file + '-' + dataset DetectionDataset().eval_or_save_inference_results( all_results, dataset, output)
def do_evaluate(pred_config, output_file): num_tower = max(cfg.TRAIN.NUM_GPUS, 1) graph_funcs = MultiTowerOfflinePredictor( pred_config, list(range(num_tower))).get_predictors() for dataset in cfg.DATA.VAL: logger.info("Evaluating {} ...".format(dataset)) dataflows = [ get_eval_dataflow(dataset, shard=k, num_shards=num_tower) for k in range(num_tower)] all_results = multithread_predict_dataflow(dataflows, graph_funcs) output = output_file + '-' + dataset DatasetRegistry.get(dataset).eval_inference_results(all_results, output)
def compute_validation_metrics(self) -> Any: if self.trainer_type == "replicated": all_results = multithread_predict_dataflow( self.dataflows, self.predictors ) # type: ignore else: filenames = [ os.path.join( self._output_dir, "outputs{}-part{}.json".format(self.trainer.global_step, rank) ) for rank in range(hvd.local_size()) ] if self.machine_rank == 0: local_results = predict_dataflow(self.dataflow, self.predictor) fname = filenames[hvd.local_rank()] with open(fname, "w") as f: json.dump(local_results, f) self.barrier.eval() if hvd.rank() > 0: return all_results = [] for fname in filenames: with open(fname, "r") as f: obj = json.load(f) all_results.extend(obj) output_file = os.path.join( self._output_dir, "{}-outputs{}-{}.json".format( self._eval_dataset, self.trainer.global_step, time.time() ), ) metrics = DatasetRegistry.get(self._eval_dataset).eval_inference_results( # type: ignore all_results, output_file ) # If there are no detections, the metrics result is totally empty, instead of containing # zeroes. Ensure that the main evaluation metric has some value. metrics.setdefault("mAP(bbox)/IoU=0.5:0.95", 0) return metrics
def _eval(self): logdir = args.logdir if cfg.TRAINER == 'replicated': all_results = multithread_predict_dataflow(self.dataflows, self.predictors) else: filenames = [ os.path.join( logdir, 'outputs{}-part{}.json'.format(self.global_step, rank)) for rank in range(hvd.local_size()) ] if self._horovod_run_eval: local_results = predict_dataflow(self.dataflow, self.predictor) fname = filenames[hvd.local_rank()] with open(fname, 'w') as f: json.dump(local_results, f) self.barrier.eval() if hvd.rank() > 0: return all_results = [] for fname in filenames: with open(fname, 'r') as f: obj = json.load(f) all_results.extend(obj) os.unlink(fname) output_file = os.path.join( logdir, '{}-outputs{}.json'.format(self._eval_dataset, self.global_step)) scores = DetectionDataset().eval_or_save_inference_results( all_results, self._eval_dataset, output_file) for k, v in scores.items(): self.trainer.monitors.put_scalar(k, v)