def get_evaluator(cfg, dataset_name, output_folder): evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type in ["coco", "coco_panoptic_seg"]: # D2 is in the process of reducing the use of cfg. dataset_evaluators = COCOEvaluator( dataset_name, output_dir=output_folder, kpt_oks_sigmas=cfg.TEST.KEYPOINT_OKS_SIGMAS, max_dets_per_image=cfg.TEST.DETECTIONS_PER_IMAGE, ) elif evaluator_type in ["rotated_coco"]: dataset_evaluators = DatasetEvaluators( [RotatedCOCOEvaluator(dataset_name, cfg, True, output_folder)]) elif evaluator_type in ["lvis"]: dataset_evaluators = LVISEvaluator( dataset_name, output_dir=output_folder, max_dets_per_image=cfg.TEST.DETECTIONS_PER_IMAGE, ) else: dataset_evaluators = D2Trainer.build_evaluator( cfg, dataset_name, output_folder) if not isinstance(dataset_evaluators, DatasetEvaluators): dataset_evaluators = DatasetEvaluators([dataset_evaluators]) return dataset_evaluators
def build_evaluator(cls, cfg, dataset_name, output_folder=None): #It is not implemented by default. #output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") #output_folder = os.path.join(cfg.OUTPUT_DIR) if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluators = [COCOEvaluator(dataset_name, cfg, True, output_folder)] return DatasetEvaluators(evaluators)
def build_evaluator( cls, cfg: CfgNode, dataset_name: str, output_folder: Optional[str] = None, embedder: Embedder = None, ) -> DatasetEvaluators: if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluators = [] # Note: we currently use COCO evaluator for both COCO and LVIS datasets # to have compatible metrics. LVIS bbox evaluator could also be used # with an adapter to properly handle filtered / mapped categories # evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type # if evaluator_type == "coco": # evaluators.append(COCOEvaluator(dataset_name, output_dir=output_folder)) # elif evaluator_type == "lvis": # evaluators.append(LVISEvaluator(dataset_name, output_dir=output_folder)) evaluators.append( Detectron2COCOEvaluatorAdapter(dataset_name, output_dir=output_folder)) if cfg.MODEL.DENSEPOSE_ON: storage = build_densepose_evaluator_storage(cfg, output_folder) evaluators.append( DensePoseCOCOEvaluator( dataset_name, True, output_folder, evaluator_type=cfg.DENSEPOSE_EVALUATION.TYPE, min_iou_threshold=cfg.DENSEPOSE_EVALUATION. MIN_IOU_THRESHOLD, storage=storage, embedder=embedder, )) return DatasetEvaluators(evaluators)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if cfg.DATASETS.NAME == 'flickr30k': evaluator_list.append( FLICKR30KEvaluatorKAC( dataset_name, distributed=True, output_dir=output_folder, )) else: raise NotImplementedError ## TODO ## the result can be passed out as dict return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] metadata = MetadataCatalog.get(dataset_name) # Note(): Check original meta classes, also tells us if we have a meta dataset # Ugly but easier to support than other options classes_to_eval = [] if hasattr(metadata, 'classes_to_eval'): classes_to_eval = metadata.classes_to_eval print(f'Using meta-dataset with classes {classes_to_eval}') evaluator_type = metadata.evaluator_type if evaluator_type == "coco": evaluator_list.append( COCOEvaluator(dataset_name, cfg, True, output_folder, classes_to_eval=classes_to_eval)) if evaluator_type == "pascal_voc": return PascalVOCDetectionEvaluator(dataset_name) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type)) if len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type in ["sem_seg", "coco_panoptic_seg"]: evaluator_list.append( SemSegEvaluator( dataset_name, distributed=True, num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES, ignore_label=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, output_dir=output_folder, )) if evaluator_type in ["coco", "coco_panoptic_seg"]: evaluator_list.append( COCOEvaluator(dataset_name, cfg, True, output_folder)) if evaluator_type == "coco_panoptic_seg": evaluator_list.append( COCOPanopticEvaluator(dataset_name, output_folder)) elif evaluator_type == "lvis": return LVISEvaluator(dataset_name, cfg, True, output_folder) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type)) elif len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg: CfgNode, dataset_name): output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluators = [COCOEvaluator(dataset_name, cfg, True, output_folder)] if cfg.MODEL.DENSEPOSE_ON: evaluators.append( DensePoseCOCOEvaluator(dataset_name, True, output_folder)) return DatasetEvaluators(evaluators)
def evaluate(cfg=None, trainer=None, pdmClasses=None, set="_test", threshold=0.7): if cfg is None: cfg = get_cfg(find_outputn()) dataset = CustomConfig.dataset if pdmClasses is None: pdmClasses = CustomConfig.pdmClasses cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, "model_final.pth") cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = threshold if trainer is None: trainer = CustomTrainer(cfg) trainer.resume_or_load(resume=False) evaluator = DatasetEvaluators([ COCOEvaluator( dataset + set, ("bbox", "segm"), False, output_dir=cfg.OUTPUT_DIR + "/coco_eval_test", ), PDM_Evaluator(dataset + set, pdmClasses), ]) test_loader = build_detection_test_loader(cfg, dataset + set) print( f"{cfg.OUTPUT_DIR.split('/')[-1]}={inference_on_dataset(trainer.model, test_loader, evaluator)}" )
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type == "coco": evaluator_list.append( COCOEvaluator(dataset_name, cfg, True, output_folder)) if evaluator_type == "pascal_voc": return PascalVOCDetectionEvaluator(dataset_name) if evaluator_type == "lvis": return LVISEvaluator(dataset_name, cfg, True, output_folder) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type)) if len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def do_test(cfg, model): dataset_name = cfg.DATASETS.TEST[0] evaluator = DatasetEvaluators( [COCOEvaluator(dataset_name, cfg, True, cfg.OUTPUT_DIR)]) data_loader = build_detection_test_loader(cfg, dataset_name) results = inference_on_dataset(model, data_loader, evaluator) return results
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type == "lvis": return LVISEvaluator(dataset_name, cfg, True, output_folder) if evaluator_type == "coco": return COCOEvaluator(dataset_name, cfg, True, output_folder) if evaluator_type == "cityscapes": assert ( torch.cuda.device_count() >= comm.get_rank() ), "CityscapesEvaluator currently do not work with multiple machines." return CityscapesEvaluator(dataset_name) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type)) if len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type in ["cityscapes_panoptic_seg", "coco_panoptic_seg"]: evaluator_list.append(COCOPanopticEvaluator(dataset_name, output_folder)) if evaluator_type == "cityscapes_panoptic_seg": assert ( torch.cuda.device_count() >= comm.get_rank() ), "CityscapesEvaluator currently do not work with multiple machines." evaluator_list.append(CityscapesSemSegEvaluator(dataset_name)) evaluator_list.append(CityscapesInstanceEvaluator(dataset_name)) if evaluator_type == "coco_panoptic_seg": # Evaluate bbox and segm. cfg.defrost() cfg.MODEL.MASK_ON = True cfg.MODEL.KEYPOINT_ON = False cfg.freeze() evaluator_list.append(COCOEvaluator(dataset_name, cfg, True, output_folder)) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type ) ) elif len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type == "sem_seg": return SemSegEvaluator( dataset_name, distributed=True, num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES, ignore_label=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, output_dir=output_folder, ) if evaluator_type == "cityscapes_sem_seg": assert ( torch.cuda.device_count() >= comm.get_rank() ), "CityscapesEvaluator currently do not work with multiple machines." return CityscapesSemSegEvaluator(dataset_name) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type)) if len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluators_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type in ["sem_seg", "isprs_panoptic_seg"]: evaluators_list.append( ISPRSSemSegEvaluator( dataset_name, distributed=True, output_dir=output_folder, num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES)) if evaluator_type in [ "isprs_instance", "isprs_panoptic_seg", "isprs_rpn" ]: if cfg.ISPRS.LABEL.BOXMODE == "ROTATED": evaluators_list.append( RotatedCOCOEvaluatorWithMask(dataset_name, cfg, True, output_folder)) else: evaluators_list.append( COCOEvaluator(dataset_name, cfg, True, output_folder)) if evaluator_type == "isprs_panoptic_seg": evaluators_list.append( COCOPanopticEvaluator(dataset_name, output_folder)) return DatasetEvaluators(evaluators_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if "dota_val_mini" in dataset_name: evaluator_type = 'rot' if "dota_test" in dataset_name: evaluator_type = 'hw' if 'hw' in evaluator_type: evaluator_list.append(DotaVOCDetectionEvaluator( dataset_name, 'hw')) if 'rot' in evaluator_type: evaluator_list.append( DotaVOCDetectionEvaluator(dataset_name, 'rot')) if 'hbb' in evaluator_type: evaluator_list.append( DotaVOCDetectionEvaluator(dataset_name, 'hbb')) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type)) if len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type in ["sem_seg", "coco_panoptic_seg"]: evaluator_list.append( SemSegEvaluator( dataset_name, distributed=True, num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES, ignore_label=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, output_dir=output_folder)) if evaluator_type in ["coco", "coco_panoptic_seg"]: evaluator_list.append( COCOEvaluator(dataset_name, cfg, True, output_folder)) if evaluator_type == "coco_panoptic_seg": evaluator_list.append( COCOPanopticEvaluatorWith2ChPNG( dataset_name, output_folder, gen_png=cfg.MODEL.SOGNET.GEN_PNG)) elif evaluator_type == "cityscapes": assert ( torch.cuda.device_count() >= comm.get_rank() ), "CityscapesEvaluator currently do not work with multiple machines." return CityscapesEvaluator(dataset_name) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type)) elif len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type == "sem_seg": return SemSegEvaluator( dataset_name, distributed=True, output_dir=output_folder, ) if evaluator_type == "cityscapes_sem_seg": return CityscapesSemSegEvaluator(dataset_name) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type)) if len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") if 'sku110' in dataset_name: return VOCDetectionEvaluator(cfg, dataset_name) evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type in ["sem_seg", "coco_panoptic_seg"]: evaluator_list.append( SemSegEvaluator( dataset_name, distributed=True, num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES, ignore_label=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, output_dir=output_folder, ) ) if evaluator_type in ["coco", "coco_panoptic_seg"]: evaluator_list.append(COCOEvaluator(dataset_name, cfg, True, output_folder)) if evaluator_type == "coco_panoptic_seg": evaluator_list.append(COCOPanopticEvaluator(dataset_name, output_folder)) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type ) ) elif len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") pdmClasses = CustomConfig.pdmClasses return DatasetEvaluators([ COCOEvaluator(dataset_name, ("bbox", "segm"), True, output_folder), PDM_Evaluator(dataset_name, pdmClasses), ])
def build_evaluator(cls, cfg, dataset_name): output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluators = [COCOEvaluator(dataset_name, cfg, True, output_folder)] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if cfg.MODEL.DENSEPOSE_ON and cfg.MODEL.ROI_DENSEPOSE_HEAD.RCNN_HEAD_ON: evaluators.append( DensePoseCOCOEvaluator(dataset_name, True, output_folder)) # if evaluator_type in ["coco", "coco_panoptic_seg"]: # evaluators.append(COCOEvaluator(dataset_name, cfg, True, output_folder)) return DatasetEvaluators(evaluators)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_list.append( COCOEvaluator(dataset_name, cfg, True, output_folder)) if len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type in ["coco"]: evaluator_list.append( PartiallySupervisedEvaluator(dataset_name, cfg, True, output_folder)) elif len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type in ["sem_seg", "coco_panoptic_seg"]: evaluator_list.append( SemSegEvaluator( dataset_name, distributed=True, output_dir=output_folder, ) ) if evaluator_type in ["coco", "coco_panoptic_seg"]: evaluator_list.append( COCOEvaluator(dataset_name, output_dir=output_folder)) if evaluator_type == "coco_panoptic_seg": evaluator_list.append( COCOPanopticEvaluator(dataset_name, output_folder)) if evaluator_type == "cityscapes_instance": assert ( torch.cuda.device_count() >= comm.get_rank() ), "CityscapesEvaluator currently " \ "do not work with multiple machines." return CityscapesInstanceEvaluator(dataset_name) if evaluator_type == "cityscapes_sem_seg": assert ( torch.cuda.device_count() >= comm.get_rank() ), "CityscapesEvaluator currently " \ "do not work with multiple machines." return CityscapesSemSegEvaluator(dataset_name) elif evaluator_type == "pascal_voc": return PascalVOCDetectionEvaluator(dataset_name) elif evaluator_type == "lvis": return LVISEvaluator(dataset_name, output_dir=output_folder) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type ) ) elif len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): if output_folder is None: os.makedirs(cfg.OUTPUT_DIR + f"/eval/{dataset_name}", exist_ok=True) return DatasetEvaluators([ COCOEvaluatorWithRecall(dataset_name, cfg, False, output_dir=cfg.OUTPUT_DIR + f"/eval/{dataset_name}"), SickTreesEvaluator() ])
def build_evaluator(cls, cfg_arg: CfgNode, dataset_name): output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") eval_img = cfg.TEST.TEST_IMAGES[0] if dataset_name != 'kodak_test': evaluators = [COCOEvaluator(dataset_name, cfg_arg, True)] else: evaluators = [] if cfg.MODEL.RECONSTRUCT_HEADS_ON and dataset_name == 'kodak_test': evaluators.append( ReconstructionEvaluator(dataset_name, output_folder, eval_img=eval_img)) return DatasetEvaluators(evaluators)
def build_evaluator(cls, cfg, dataset_name): """ Builds evaluators for post-training mAP report. Args: cfg(CfgNode): a detectron2 CfgNode dataset_name(str): registered dataset name Returns: detectron2 DatasetEvaluators object """ output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluators = [COCOEvaluator(dataset_name, cfg, True, output_folder)] return DatasetEvaluators(evaluators)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, 'inference') evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type in ['sem_seg', 'coco_panoptic_seg']: evaluator_list.append( SemSegEvaluator( dataset_name, distributed=True, num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES, ignore_label=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, output_dir=output_folder, )) if evaluator_type in ['coco', 'coco_panoptic_seg', 'indiscapes']: evaluator_list.append( COCOEvaluator(dataset_name, cfg, True, output_folder)) evaluator_list.append(HDEvaluator(dataset_name)) if evaluator_type == 'coco_panoptic_seg': evaluator_list.append( COCOPanopticEvaluator(dataset_name, output_folder)) if evaluator_type == 'cityscapes_instance': assert ( torch.cuda.device_count() >= comm.get_rank() ), 'CityscapesEvaluator currently do not work with multiple machines.' return CityscapesInstanceEvaluator(dataset_name) if evaluator_type == 'cityscapes_sem_seg': assert ( torch.cuda.device_count() >= comm.get_rank() ), 'CityscapesEvaluator currently do not work with multiple machines.' return CityscapesSemSegEvaluator(dataset_name) elif evaluator_type == 'pascal_voc': return PascalVOCDetectionEvaluator(dataset_name) elif evaluator_type == 'lvis': return LVISEvaluator(dataset_name, cfg, True, output_folder) if len(evaluator_list) == 0: raise NotImplementedError( 'no Evaluator for the dataset {} with the type {}'.format( dataset_name, evaluator_type)) elif len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type == "coco": return COCOEvaluator(dataset_name, cfg, True, output_folder) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type)) if len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type if evaluator_type in ["sem_seg", "coco_panoptic_seg"]: evaluator_list.append( SemSegEvaluator( dataset_name, distributed=True, num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES, ignore_label=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, output_dir=output_folder, ) ) if evaluator_type in ["coco", "coco_panoptic_seg", "d2sa", "coco_amodal_cls", "kins", "sailvos", "cocoa"] \ and cfg.TEST.EVAL_AMODAL_TYPE == "NORMAL": evaluator_list.append(COCOEvaluator(dataset_name, cfg, True, output_folder)) if evaluator_type in ["d2sa", "coco_amodal_cls", "kins", "sailvos", "cocoa"] \ and cfg.TEST.EVAL_AMODAL_TYPE == "AMODAL_VISIBLE": evaluator_list.append(AmodalVisibleEvaluator(dataset_name, cfg, True, output_folder)) # if evaluator_type == "coco_amodal": # evaluator_list.append(COCOAmodalEvaluator(dataset_name, cfg, True, output_folder)) if evaluator_type == "coco_panoptic_seg": evaluator_list.append(COCOPanopticEvaluator(dataset_name, output_folder)) elif evaluator_type == "cityscapes": assert ( torch.cuda.device_count() >= comm.get_rank() ), "CityscapesEvaluator currently do not work with multiple machines." return CityscapesEvaluator(dataset_name) elif evaluator_type == "pascal_voc": return PascalVOCDetectionEvaluator(dataset_name) elif evaluator_type == "lvis": return LVISEvaluator(dataset_name, cfg, True, output_folder) if len(evaluator_list) == 0: raise NotImplementedError( "no Evaluator for the dataset {} with the type {}".format( dataset_name, evaluator_type ) ) elif len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)
def get_evaluator(cfg, dataset_name, output_folder=None): """ Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here. """ if output_folder is None: output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") evaluator_list = [] evaluator_list.append(COCOEvaluator(dataset_name, cfg, True, output_folder)) if len(evaluator_list) == 1: return evaluator_list[0] return DatasetEvaluators(evaluator_list)