def build_evaluator(cls, cfg, dataset_name, output_folder=None):
"""
Create evaluator(s) for a given dataset.
This uses the special metadata "evaluator_type" associated with each builtin dataset.
For your own dataset, you can simply create an evaluator manually in your
script and do not have to worry about the hacky if-else logic here.
"""
if output_folder is None:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference")
evaluator_list = []
metadata = MetadataCatalog.get(dataset_name)
# Note(): Check original meta classes, also tells us if we have a meta dataset
# Ugly but easier to support than other options
classes_to_eval = []
if hasattr(metadata, 'classes_to_eval'):
classes_to_eval = metadata.classes_to_eval
print(f'Using meta-dataset with classes {classes_to_eval}')
evaluator_type = metadata.evaluator_type
if evaluator_type == "coco":
evaluator_list.append(
COCOEvaluator(dataset_name,
cfg,
True,
output_folder,
classes_to_eval=classes_to_eval))
if evaluator_type == "pascal_voc":
return PascalVOCDetectionEvaluator(dataset_name)
if len(evaluator_list) == 0:
raise NotImplementedError(
"no Evaluator for the dataset {} with the type {}".format(
dataset_name, evaluator_type))
if len(evaluator_list) == 1:
return evaluator_list[0]
return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None):
"""
Create evaluator(s) for a given dataset.
This uses the special metadata "evaluator_type" associated with each builtin dataset.
For your own dataset, you can simply create an evaluator manually in your
script and do not have to worry about the hacky if-else logic here.
"""
if output_folder is None:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference")
evaluator_list = []
evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type
if evaluator_type == "coco":
evaluator_list.append(
COCOEvaluator(dataset_name, cfg, True, output_folder))
if evaluator_type == "pascal_voc":
return PascalVOCDetectionEvaluator(dataset_name)
if evaluator_type == "lvis":
return LVISEvaluator(dataset_name, cfg, True, output_folder)
if len(evaluator_list) == 0:
raise NotImplementedError(
"no Evaluator for the dataset {} with the type {}".format(
dataset_name, evaluator_type))
if len(evaluator_list) == 1:
return evaluator_list[0]
return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None):
if output_folder is None:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference")
if "voc" in dataset_name:
return PascalVOCDetectionEvaluator(dataset_name)
else:
return COCOEvaluator(dataset_name, cfg, True, output_folder)
def build_evaluator(cls,
cfg,
dataset_name,
output_folder=None,
AP_method='norm'):
"""
Create evaluator(s) for a given dataset.
This uses the special metadata "evaluator_type" associated with each builtin dataset.
For your own dataset, you can simply create an evaluator manually in your
script and do not have to worry about the hacky if-else logic here.
"""
if output_folder is None:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference")
evaluator_list = []
evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type
if evaluator_type in ["sem_seg", "coco_panoptic_seg"]:
evaluator_list.append(
SemSegEvaluator(
dataset_name,
distributed=True,
num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES,
ignore_label=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE,
output_dir=output_folder,
))
if evaluator_type in ["coco", "coco_panoptic_seg"]:
evaluator_list.append(
COCOEvaluator(dataset_name,
cfg,
True,
output_folder,
AP_method=AP_method))
if evaluator_type == "coco_panoptic_seg":
evaluator_list.append(
COCOPanopticEvaluator(dataset_name, output_folder))
if evaluator_type == "cityscapes_instance":
assert (
torch.cuda.device_count() >= comm.get_rank()
), "CityscapesEvaluator currently do not work with multiple machines."
return CityscapesInstanceEvaluator(dataset_name)
if evaluator_type == "cityscapes_sem_seg":
assert (
torch.cuda.device_count() >= comm.get_rank()
), "CityscapesEvaluator currently do not work with multiple machines."
return CityscapesSemSegEvaluator(dataset_name)
elif evaluator_type == "pascal_voc":
return PascalVOCDetectionEvaluator(dataset_name)
elif evaluator_type == "lvis":
return LVISEvaluator(dataset_name, cfg, True, output_folder)
if len(evaluator_list) == 0:
raise NotImplementedError(
"no Evaluator for the dataset {} with the type {}".format(
dataset_name, evaluator_type))
elif len(evaluator_list) == 1:
return evaluator_list[0]
return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None):
if output_folder is None:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference")
evaluator_list = []
evaluator_type = d2.data.MetadataCatalog.get(dataset_name).evaluator_type
if evaluator_type == "pascal_voc":
return PascalVOCDetectionEvaluator(dataset_name)
elif evaluator_type == "coco":
return COCOEvaluator(dataset_name, cfg, True, output_folder)
elif evaluator_type == "lvis":
return LVISEvaluator(dataset_name, cfg, True, output_folder)
def build_evaluator(cls, cfg, dataset_name, output_folder=None):
if output_folder is None:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference")
if "coco" in dataset_name:
return COCOEvaluator(dataset_name, cfg, True, output_folder)
elif "cityscapes" in dataset_name:
#assert (
#torch.cuda.device_count() >= comm.get_rank()
#), "CityscapesEvaluator currently do not work with multiple machines."
return CityscapesInstanceEvaluator(dataset_name)
else:
assert "voc" in dataset_name
return PascalVOCDetectionEvaluator(dataset_name)
def build_evaluator(cls, cfg, dataset_name, output_folder=None):
"""Create evaluator(s) for a given dataset.
Modified from D2's example `tools/train_net.py`"""
if output_folder is None:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference")
evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type
if evaluator_type == "coco":
return COCOEvaluator(dataset_name, output_dir=output_folder)
if evaluator_type == "pascal_voc":
return PascalVOCDetectionEvaluator(dataset_name)
raise NotImplementedError(
"No evaluator implementation for the dataset {} with the type {}".format(dataset_name, evaluator_type)
)
def build_evaluator(cls, cfg, dataset_name, output_folder=None):
"""
Create evaluator(s) for a given dataset.
This uses the special metadata "evaluator_type" associated with each builtin dataset.
For your own dataset, you can simply create an evaluator manually in your
script and do not have to worry about the hacky if-else logic here.
"""
if output_folder is None:
model_name = osp.basename(cfg.MODEL.WEIGHTS).split(".")[0]
output_folder = osp.join(
cfg.OUTPUT_DIR,
"inference_{}/{}".format(model_name, dataset_name))
evaluator_list = []
evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type
if evaluator_type in ["sem_seg", "coco_panoptic_seg"]:
evaluator_list.append(
SemSegEvaluator(
dataset_name,
distributed=True,
num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES,
ignore_label=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE,
output_dir=output_folder,
))
if evaluator_type in ["coco", "coco_panoptic_seg", "coco_bop"]:
evaluator_list.append(
MyCOCOEvaluator(dataset_name, cfg, True, output_folder))
# evaluator_list.append(COCOEvaluator(dataset_name, cfg, True, output_folder))
if evaluator_type == "coco_panoptic_seg":
evaluator_list.append(
COCOPanopticEvaluator(dataset_name, output_folder))
if evaluator_type == "pascal_voc":
return PascalVOCDetectionEvaluator(dataset_name)
if evaluator_type == "lvis":
return LVISEvaluator(dataset_name, cfg, True, output_folder)
if evaluator_type == "text":
return TextEvaluator(dataset_name, cfg, True, output_folder)
if len(evaluator_list) == 0:
raise NotImplementedError(
"no Evaluator for the dataset {} with the type {}".format(
dataset_name, evaluator_type))
if len(evaluator_list) == 1:
return evaluator_list[0]
return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None):
if output_folder is None:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference")
evaluator_list = []
evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type
if evaluator_type in ["sem_seg", "coco_panoptic_seg"]:
evaluator_list.append(
SemSegEvaluator(
dataset_name,
distributed=True,
num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES,
ignore_label=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE,
output_dir=output_folder,
))
if evaluator_type in ["coco", "coco_panoptic_seg"]:
evaluator_list.append(
COCOEvaluator(dataset_name, cfg, True, output_folder))
if evaluator_type == "coco_panoptic_seg":
evaluator_list.append(
COCOPanopticEvaluator(dataset_name, output_folder))
if evaluator_type == "cityscapes_instance":
assert (
torch.cuda.device_count() >= comm.get_rank()
), "CityscapesEvaluator currently do not work with multiple machines."
return CityscapesInstanceEvaluator(dataset_name)
if evaluator_type == "cityscapes_sem_seg":
assert (
torch.cuda.device_count() >= comm.get_rank()
), "CityscapesEvaluator currently do not work with multiple machines."
return CityscapesSemSegEvaluator(dataset_name)
elif evaluator_type == "pascal_voc":
return PascalVOCDetectionEvaluator(dataset_name)
elif evaluator_type == "lvis":
return LVISEvaluator(dataset_name, cfg, True, output_folder)
if len(evaluator_list) == 0:
raise NotImplementedError(
"no Evaluator for the dataset {} with the type {}".format(
dataset_name, evaluator_type))
elif len(evaluator_list) == 1:
return evaluator_list[0]
return DatasetEvaluators(evaluator_list)
def build_evaluator(cls, cfg, dataset_name, output_folder=None):
if output_folder is None:
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference")
evaluator_list = []
evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type
if evaluator_type == "coco":
evaluator_list.append(COCOEvaluator(
dataset_name, output_dir=output_folder))
elif evaluator_type == "pascal_voc":
return PascalVOCDetectionEvaluator(dataset_name)
if len(evaluator_list) == 0:
raise NotImplementedError(
"no Evaluator for the dataset {} with the type {}".format(
dataset_name, evaluator_type
)
)
elif len(evaluator_list) == 1:
return evaluator_list[0]
return DatasetEvaluators(evaluator_list)
# Fixing the image for PASCAL evaluation:
for image in images:
image["image_id"] = os.path.splitext(os.path.split(image["image_id"])[-1])[0]
# Evaluating the prediction:
# Setting up the COCO evaluation:
coco_evaluator = COCOEvaluator(dataset_name, cfg, distributed=True, output_dir=output_folder_path)
# Setting up the PASCAL VOC evaluation:
my_dataset.dirname = os.path.join(output_folder_path, "PascalVOCAnnotations")
my_dataset.split = 'test'
my_dataset.year = 2012
dataset.to_pascal(my_dataset.dirname) # Converting the dataset to PASCAL VOC
pascal_evaluator = PascalVOCDetectionEvaluator(dataset_name)
# COCO evaluating:
coco_evaluator.reset()
coco_evaluator.process(images, outputs_list)
coco_results = coco_evaluator.evaluate()
# Dumping the COCO evaluation results:
coco_results_file_path = os.path.join(output_folder_path, "coco_eval_results.json")
with open(coco_results_file_path, "w") as coco_results_file:
json.dump(coco_results, coco_results_file, indent=2)
print("COCO EVALUATION FINISHED")
# PASCAL VOC evaluating:
pascal_evaluator.reset()
def build_evaluator(cls, cfg, dataset_name):
output_folder = os.path.join(cfg.OUTPUT_DIR, "inference")
evaluators = [PascalVOCDetectionEvaluator(dataset_name)
] # FIXME: evaluator需要再改
return DatasetEvaluators(evaluators)
model_zoo.get_config_file("COCO-Detection/retinanet_R_101_FPN_3x.yaml"))
cfg_source.DATASETS.TRAIN = ("city_trainS", )
cfg_source.DATALOADER.NUM_WORKERS = 0
cfg_source.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(
"COCO-Detection/retinanet_R_101_FPN_3x.yaml")
cfg_source.SOLVER.IMS_PER_BATCH = 2
cfg_source.SOLVER.BASE_LR = 0.001
cfg_source.SOLVER.MAX_ITER = 70000
cfg_source.SOLVER.STEPS = (50000, )
cfg_source.INPUT.MIN_SIZE_TRAIN = (600, )
cfg_source.INPUT.MIN_SIZE_TEST = 0
os.makedirs(cfg_source.OUTPUT_DIR, exist_ok=True)
cfg_source.MODEL.RETINANET.NUM_CLASSES = 8
model = build_model(cfg_source)
cfg_target = get_cfg()
cfg_target.DATASETS.TRAIN = ("city_trainT", )
cfg_target.INPUT.MIN_SIZE_TRAIN = (600, )
cfg_target.DATALOADER.NUM_WORKERS = 0
cfg_target.SOLVER.IMS_PER_BATCH = 2
do_train(cfg_source, cfg_target, model, False)
from detectron2.evaluation import COCOEvaluator, inference_on_dataset, PascalVOCDetectionEvaluator
from detectron2.data import build_detection_test_loader
evaluator = PascalVOCDetectionEvaluator("city_testT")
val_loader = build_detection_test_loader(cfg_source, "city_testT")
res = inference_on_dataset(model, val_loader, evaluator)
print(res)
def build_evaluator(cls, cfg, dataset_name, output_folder=None):
"""
Create VOC evaluator(s) for a given dataset.
"""
return PascalVOCDetectionEvaluator(dataset_name)
def get_evaluator(self, cfg, dataset_name, output_folder=None):
"""Create evaluator(s) for a given dataset.
This uses the special metadata "evaluator_type" associated with
each builtin dataset. For your own dataset, you can simply
create an evaluator manually in your script and do not have to
worry about the hacky if-else logic here.
"""
if output_folder is None:
output_folder = osp.join(cfg.OUTPUT_DIR, "inference")
evaluator_list = []
evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type
if evaluator_type in ["sem_seg", "coco_panoptic_seg"]:
evaluator_list.append(
SemSegEvaluator(
dataset_name,
distributed=True,
num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES,
ignore_label=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE,
output_dir=output_folder,
))
if evaluator_type in ["coco", "coco_panoptic_seg"]:
evaluator_list.append(
COCOEvaluator(dataset_name, cfg, True, output_folder))
if evaluator_type == "coco_panoptic_seg":
evaluator_list.append(
COCOPanopticEvaluator(dataset_name, output_folder))
if evaluator_type == "cityscapes_instance":
assert (
torch.cuda.device_count() >= self.global_rank
), "CityscapesEvaluator currently do not work with multiple machines."
return CityscapesInstanceEvaluator(dataset_name)
if evaluator_type == "cityscapes_sem_seg":
assert (
torch.cuda.device_count() >= self.global_rank
), "CityscapesEvaluator currently do not work with multiple machines."
return CityscapesSemSegEvaluator(dataset_name)
if evaluator_type == "pascal_voc":
return PascalVOCDetectionEvaluator(dataset_name)
if evaluator_type == "lvis":
return LVISEvaluator(dataset_name, cfg, True, output_folder)
_distributed = self.world_size > 1
dataset_meta = MetadataCatalog.get(cfg.DATASETS.TRAIN[0])
train_obj_names = dataset_meta.objs
if evaluator_type == "bop":
if cfg.VAL.get("USE_BOP", False):
return GDRN_Evaluator(cfg,
dataset_name,
distributed=_distributed,
output_dir=output_folder,
train_objs=train_obj_names)
else:
return GDRN_EvaluatorCustom(cfg,
dataset_name,
distributed=_distributed,
output_dir=output_folder,
train_objs=train_obj_names)
if len(evaluator_list) == 0:
raise NotImplementedError(
"no Evaluator for the dataset {} with the type {}".format(
dataset_name, evaluator_type))
if len(evaluator_list) == 1:
return evaluator_list[0]
return DatasetEvaluators(evaluator_list)
