def __init__(self, dataset_name, cfg, distributed, output_dir=None, *, use_fast_impl=True): self._predictions = [] self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._use_fast_impl = use_fast_impl self._cpu_device = torch.device("cpu") self._logger = logging.getLogger('MyCocoEvaluator') self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__(self, dataset_name, cfg, distributed, output_dir=None): self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._cpu_device = torch.device("cpu") # this is hard-coded so that logging is normal self._logger = logging.getLogger("detectron2.evaluation.coco_evaluation") self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): self._logger.warning( f"json_file was not found in MetaDataCatalog for '{dataset_name}'" ) cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset if "BIT" in cfg.MODEL.DANCE.MASK_IN: self.input_format = "bit" else: self.input_format = "rle"
def __init__(self, dataset_name, cfg, distributed, output_dir=None): self._distributed = distributed self._output_dir = output_dir self._cpu_device = torch.device("cpu") self._logger = logging.getLogger("detectron2") self._metadata = MetadataCatalog.get(dataset_name) self.internal_dataset_mapping = copy.deepcopy( self._metadata.thing_dataset_id_to_contiguous_id) if -1 in self.internal_dataset_mapping.keys(): self.internal_dataset_mapping = dict([ (k, v - 1) for k, v in self.internal_dataset_mapping.items() ]) if not hasattr(self._metadata, "json_file"): self._logger.warning( f"json_file was not found in MetaDataCatalog for '{dataset_name}'." " Trying to convert it to COCO format ...") cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file)
def __init__(self, dataset_name, cfg, distributed, output_dir=None, result_df=None, output_file=None, *, use_fast_impl=True): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. cfg (CfgNode): config instance distributed (True): if True, will collect results from all ranks and run evaluation in the main process. Otherwise, will only evaluate the results in the current process. output_dir (str): optional, an output directory to dump all results predicted on the dataset. The dump contains two files: 1. "instance_predictions.pth" a file in torch serialization format that contains all the raw original predictions. 2. "coco_instances_results.json" a json file in COCO's result format. use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP. Although the results should be very close to the official implementation in COCO API, it is still recommended to compute results with the official API for use in papers. """ self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._use_fast_impl = use_fast_impl self._cpu_device = torch.device("cpu") self._logger = logging.getLogger(__name__) self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): self._logger.info( f"'{dataset_name}' is not registered by `register_coco_instances`." " Therefore trying to convert it to COCO format ..." ) cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset #Changed made by Johan to store all results self.result_df = None if len(result_df) == 0: self.result_df = result_df self._output_file = output_file
def __init__(self, dataset_name, cfg, distributed, output_dir=None): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. cfg (CfgNode): config instance distributed (True): if True, will collect results from all ranks for evaluation. Otherwise, will evaluate the results in the current process. output_dir (str): optional, an output directory to dump all results predicted on the dataset. The dump contains two files: 1. "instance_predictions.pth" a file in torch serialization format that contains all the raw original predictions. 2. "coco_instances_results.json" a json file in COCO's result format. """ self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._cpu_device = torch.device("cpu") self._logger = logging.getLogger(__name__) self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): self._logger.warning( f"json_file was not found in MetaDataCatalog for '{dataset_name}'" ) cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._fashion_api = Fashion(json_file) self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = ("annotations" in self._fashion_api.dataset \ and "annotations2" in self._fashion_api.dataset) self._view_error = cfg.TEST.VIEW_ERROR if self._view_error: self._error_output_path = self._output_dir + '/error_result' os.makedirs(self._error_output_path, exist_ok=True) with open(os.path.join(self._error_output_path, 'info.txt'), 'w') as f: s = "time: " + time.strftime("%Y-%m-%d-%H:%M", time.localtime()) + "\n" s += f"weight: {cfg.MODEL.WEIGHTS}\n" s += f"dataset: {cfg.DATASETS.TEST}\n" f.write(s)
def __init__(self, dataset_name, cfg, distributed, output_dir=None): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. cfg (CfgNode): config instance distributed (True): if True, will collect results from all ranks and run evaluation in the main process. Otherwise, will evaluate the results in the current process. output_dir (str): optional, an output directory to dump all results predicted on the dataset. The dump contains two files: 1. "instance_predictions.pth" a file in torch serialization format that contains all the raw original predictions. 2. "coco_instances_results.json" a json file in COCO's result format. """ self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._cpu_device = torch.device("cpu") self._logger = logging.getLogger(__name__) self._metadata = MetadataCatalog.get(dataset_name) has_json = True if not hasattr(self._metadata, "json_file"): self._logger.warning( f"json_file was not found in MetaDataCatalog for '{dataset_name}'." " Trying to convert it to COCO format ...") cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path try: convert_to_coco_json(dataset_name, cache_path) except KeyError: self._logger.warning( f"Failed to convert {dataset_name} to COCO format. Do inference only." ) has_json = False if has_json: json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset else: self._coco_api = None self._do_evaluation = False self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
def __init__(self, dataset_name, cfg, distributed, output_dir=None): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation; "matlab_file": the original matlab annotation files, Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. cfg (CfgNode): config instance distributed (True): if True, will collect results from all ranks and run evaluation in the main process. Otherwise, will evaluate the results in the current process. output_dir (str): optional, an output directory to dump all results predicted on the dataset. The dump contains two files: 1. "instance_predictions.pth" a file in torch serialization format that contains all the raw original predictions. 2. "coco_instances_results.json" a json file in COCO's result format. 3. "hico_interaction_results.mat" a matlab file used for HICO-DET official evaluation. """ self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._cpu_device = torch.device("cpu") self._matlab = cfg.TEST.MATLAB self._hico_official_matlab_path = cfg.TEST.HICO_OFFICIAL_MATLAB_PATH self._hico_official_anno_file = cfg.TEST.HICO_OFFICIAL_ANNO_FILE self._hico_official_bbox_file = cfg.TEST.HICO_OFFICIAL_BBOX_FILE self.is_hoi_prediction = cfg.MODEL.ROI_HEADS.IS_HOI_PRED self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): logger.warning( f"json_file was not found in MetaDataCatalog for '{dataset_name}'." " Trying to convert it to COCO format ...") cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__(self, dataset_name, cfg, distributed, output_dir=None, *, use_fast_impl=True): self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._cpu_device = torch.device("cpu") self._logger = logging.getLogger('detectron2.evaluation.coco_evaluation') self._metadata = copy.deepcopy(MetadataCatalog.get(dataset_name)) if cfg.DATASETS.UNSEEN_LABEL_SET != '': thing_classes = self._metadata.thing_classes del self._metadata.thing_classes thing_classes.append('unknown') self._metadata.thing_classes = thing_classes #['unknown'] unknown_colors = self._metadata.thing_colors[-1] thing_colors = self._metadata.thing_colors thing_colors.append(unknown_colors) del self._metadata.thing_colors self._metadata.thing_colors = thing_colors # unknown_colors conv_id = self._metadata.thing_dataset_id_to_contiguous_id del self._metadata.thing_dataset_id_to_contiguous_id conv_id[-1] = -1 self._metadata.thing_dataset_id_to_contiguous_id = conv_id #{-1:-1} if not hasattr(self._metadata, "json_file"): self._logger.info( f"'{dataset_name}' is not registered by `register_coco_instances`." " Therefore trying to convert it to COCO format ..." ) cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset if cfg.DATASETS.UNSEEN_LABEL_SET != '': with open(cfg.DATASETS.UNSEEN_LABEL_SET, 'r') as f: self.unknown_label_list = [e.replace('\n', '') for e in f.readlines()] else: self.unknown_label_list = None
def convert_dataset_to_coco_json(output_dir, registered_dataset_name): output_coco_json = os.path.join( output_dir, f'{registered_dataset_name}_coco_format.json') convert_to_coco_json(registered_dataset_name, output_file=output_coco_json, allow_cached=False) # Save old metadata metadata = DatasetCatalog.get(registered_dataset_name) # Remove the current one DatasetCatalog.remove(registered_dataset_name) # Register again # if your dataset is in COCO format, this cell can be replaced by the following three lines: register_coco_instances(registered_dataset_name, {}, output_coco_json, os.path.abspath('./')) return output_coco_json
def to_json(): # /Users/mac7/Desktop/MS_Project/csvtojson5/detectron2/detectron2/data/catalog.py """ 1 : DatasetCatalog.register(name, func): Args: name (str): the name that identifies a dataset, e.g. "coco_2014_train". func (callable): a callable which takes no arguments and returns a list of dicts. Call the registered function and return its results. 2 : Args: name (str): the name that identifies a dataset, e.g. "coco_2014_train". Returns: list[dict]: dataset annotations. A class that supports simple attribute setter/getter. It is intended for storing metadata of a dataset and make it accessible globally. Examples: .. code-block: : python somewhere when you load the data: MetadataCatalog.get("mydataset").thing_classes = ["person", "dog"] somewhere when you print statistics or visualize: classes = MetadataCatalog.get("mydataset").thing_classes. 3 : Convert it to the coco. . . . """ THING_CLASSES = [ 'Past', 'Gorgonia', 'SeaRods', 'Antillo', 'Fish', 'Ssid', 'Orb', 'Other_Coral', 'Apalm', 'Galaxaura' ] # 1 DatasetCatalog.register('coral', _get_coral_dicts) # 2 Metadata(name='test', thing_classes=['first']) MetadataCatalog.get('coral').set(thing_classes=THING_CLASSES) # 3 convert_to_coco_json('coral', output_file=OP, allow_cached=False) # output_folder -> output_file
def __init__(self, dataset_name, cfg, distributed, output_dir=None): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. cfg (CfgNode): config instance distributed (True): if True, will collect results from all ranks for evaluation. Otherwise, will evaluate the results in the current process. output_dir (str): optional, an output directory to dump results. """ self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._cpu_device = torch.device("cpu") self._logger = logging.getLogger(__name__) self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): self._logger.warning(f"json_file was not found in MetaDataCatalog for '{dataset_name}'") cache_path = convert_to_coco_json(dataset_name, output_dir) self._metadata.json_file = cache_path json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__(self, dataset_name, cfg, distributed, output_dir=None): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. cfg (CfgNode): config instance distributed (True): if True, will collect results from all ranks for evaluation. Otherwise, will evaluate the results in the current process. output_dir (str): optional, an output directory to dump results. """ self._cfg = cfg self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._cpu_device = torch.device("cpu") self._logger = logging.getLogger(__name__) self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): self._logger.warning( f"json_file was not found in MetaDataCatalog for '{dataset_name}'" ) cache_path = convert_to_coco_json(dataset_name, output_dir) self._metadata.json_file = cache_path json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) if dataset_name.endswith("visible"): self._logger.info("using visible segmentation") if dataset_name.find('d2sa') != -1: for key, _ in self._coco_api.anns.items(): self._coco_api.anns[key][ "segmentation"] = self._coco_api.anns[key][ "visible_mask"] elif dataset_name.find("kins") != -1: for key, _ in self._coco_api.anns.items(): self._coco_api.anns[key][ "segmentation"] = self._coco_api.anns[key][ "inmodal_seg"] # if dataset_name.startswith("sailvos"): # self._coco_api.dataset['categories'] = [c for c in self._coco_api.dataset['categories'] if c["id"] not in sailvos_ignore] # for i in range(self._coco_api.getCatIds()[-1]): # if i in sailvos_ignore: # self._coco_api.cats.pop(i) self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__( self, dataset_name, # coco_2017_val for onenet tasks=None, # cfg distributed=True, output_dir=None, *, use_fast_impl=True, kpt_oks_sigmas=(), ): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. tasks (tuple[str]): tasks that can be evaluated under the given configuration. A task is one of "bbox", "segm", "keypoints". By default, will infer this automatically from predictions. distributed (True): if True, will collect results from all ranks and run evaluation in the main process. Otherwise, will only evaluate the results in the current process. output_dir (str): optional, an output directory to dump all results predicted on the dataset. The dump contains two files: 1. "instances_predictions.pth" a file in torch serialization format that contains all the raw original predictions. 2. "coco_instances_results.json" a json file in COCO's result format. use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP. Although the results should be very close to the official implementation in COCO API, it is still recommended to compute results with the official API for use in papers. The faster implementation also uses more RAM. kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS. See http://cocodataset.org/#keypoints-eval When empty, it will use the defaults in COCO. Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS. """ self._logger = logging.getLogger(__name__) self._distributed = distributed self._output_dir = output_dir self._use_fast_impl = use_fast_impl if tasks is not None and isinstance(tasks, CfgNode): kpt_oks_sigmas = ( tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas ) self._logger.warn( "COCO Evaluator instantiated using config, this is deprecated behavior." " Please pass in explicit arguments instead." ) self._tasks = None # Infering it from predictions should be better else: self._tasks = tasks self._cpu_device = torch.device("cpu") # 获取元数据, 一般有下面属性 """ json_file: json_file路径名 image_root: 图片所在路径(前部) evaluator_type:"coco" "thing_dataset_id_to_contiguous_id": 类别id映射, "thing_classes":类别名称, "thing_colors":类别对应色, """ self._metadata = MetadataCatalog.get(dataset_name) # 有 # json_file: instances_val2017.json if not hasattr(self._metadata, "json_file"): self._logger.info( f"'{dataset_name}' is not registered by `register_coco_instances`." " Therefore trying to convert it to COCO format ..." ) # 需要转化 cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) # 创建val_coco_api with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). # instances_val2017.json 是有annotatons 的 self._do_evaluation = "annotations" in self._coco_api.dataset if self._do_evaluation: self._kpt_oks_sigmas = kpt_oks_sigmas
def __init__(self, dataset_name, cfg, distributed, output_dir=None, fake=False): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. cfg (CfgNode): config instance distributed (True): if True, will collect results from all ranks and run evaluation in the main process. Otherwise, will evaluate the results in the current process. output_dir (str): optional, an output directory to dump all results predicted on the dataset. The dump contains two files: 1. "instance_predictions.pth" a file in torch serialization format that contains all the raw original predictions. 2. "coco_instances_results.json" a json file in COCO's result format. """ self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._cpu_device = torch.device("cpu") self._logger = logging.getLogger(__name__) if fake: mask = eval_make_mask() DatasetCatalog.register("test_dataset", lambda: eval_make_dataset_dicts(mask)) MetadataCatalog.get("test_dataset").set( thing_classes=["test_label"]) # Dump to json. json_dict = convert_to_coco_dict("test_dataset") with tempfile.TemporaryDirectory() as tmpdir: json_file_name = "test.json" with open(json_file_name, "w") as f: json.dump(json_dict, f) # Load from json. dicts = load_coco_json(json_file_name, "") json_file = json_file_name else: self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): self._logger.warning( f"json_file was not found in MetaDataCatalog for '{dataset_name}'." " Trying to convert it to COCO format ...") cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset self.evaluator_name = "CocoEvaluator" self.reset() self.fake = fake
def __init__(self, dataset_name, cfg, distributed, output_dir=None, classes_to_eval=None): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. cfg (CfgNode): config instance distributed (True): if True, will collect results from all ranks for evaluation. Otherwise, will evaluate the results in the current process. output_dir (str): optional, an output directory to dump all results predicted on the dataset. The dump contains two files: 1. "instance_predictions.pth" a file in torch serialization format that contains all the raw original predictions. 2. "coco_instances_results.json" a json file in COCO's result format. classes_to_eval (list): List of indices with the COCO classes to be used when performing evaluation. Otherwise, if None, will use the default 80 COCO classes. """ # Avoid default mutable argument and convert to list self.classes_to_eval = classes_to_eval if classes_to_eval else [] self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._cpu_device = torch.device("cpu") self._logger = logging.getLogger(__name__) self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): self._logger.warning( f"json_file was not found in MetaDataCatalog for '{dataset_name}'." " Trying to convert it to COCO format ...") cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset self._dataset_name = dataset_name # TODO(): Can be replaced with check if classes_to_eval != None. self._is_few_shot = "_all" in dataset_name or "_base" in dataset_name \ or "_novel" in dataset_name
def __init__(self, dataset_name, cfg, distributed, output_dir=None): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. cfg (CfgNode): config instance distributed (True): if True, will collect results from all ranks and run evaluation in the main process. Otherwise, will evaluate the results in the current process. output_dir (str): optional, an output directory to dump all results predicted on the dataset. The dump contains two files: 1. "instance_predictions.pth" a file in torch serialization format that contains all the raw original predictions. 2. "coco_instances_results.json" a json file in COCO's result format. """ self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._cpu_device = torch.device("cpu") self._logger = logging.getLogger(__name__) self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): self._logger.warning( f"json_file was not found in MetaDataCatalog for '{dataset_name}'." " Trying to convert it to COCO format ...") cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset # for visualization self._visresult = cfg.TEST.VISUAL_OUTPUT self._id2anno = defaultdict(list) for anno in self._coco_api.dataset["annotations"]: ann = copy.deepcopy(anno) ann['bbox_mode'] = BoxMode.XYWH_ABS ann['category_id'] -= 1 self._id2anno[ann['image_id']].append(ann) # score thresh if cfg.MODEL.META_ARCHITECTURE == "GeneralizedRCNN": self._score_thresh = cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST elif cfg.MODEL.META_ARCHITECTURE == "OneStageDetector": if 'FCOS' in cfg.MODEL.PROPOSAL_GENERATOR.NAME: self._score_thresh = cfg.MODEL.FCOS.INFERENCE_TH_TEST elif 'ATSS' in cfg.MODEL.PROPOSAL_GENERATOR.NAME: self._score_thresh = cfg.MODEL.ATSS.INFERENCE_TH else: self._logger.warning( f"score thresh is not implement for {cfg.MODEL.META_ARCHITECTURE}" ) self._score_thresh = 0.35
def __init__(self, dataset_name, cfg, distributed, output_dir=None): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. cfg (CfgNode): config instance distributed (True): if True, will collect results from all ranks for evaluation. Otherwise, will evaluate the results in the current process. output_dir (str): optional, an output directory to dump results. """ self._distributed = distributed self._output_dir = output_dir self._dataset_name = dataset_name self._cpu_device = torch.device("cpu") self._logger = logging.getLogger(__name__) self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): self._logger.warning( f"json_file was not found in MetaDataCatalog for '{dataset_name}'" ) cache_path = convert_to_coco_json(dataset_name, output_dir) self._metadata.json_file = cache_path self._is_splits = "all" in dataset_name or "base" in dataset_name \ or "novel" in dataset_name self._base_classes = [ 8, 10, 11, 13, 14, 15, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 65, 70, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90, ] self._novel_classes = [ 1, 2, 3, 4, 5, 6, 7, 9, 16, 17, 18, 19, 20, 21, 44, 62, 63, 64, 67, 72 ] json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__(self, dataset_name, cfg, distributed, output_dir=None, in_res=None, out_res=None, resize=None): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. cfg (CfgNode): config instance distributed (True): if True, will collect results from all ranks and run evaluation in the main process. Otherwise, will evaluate the results in the current process. output_dir (str): optional, an output directory to dump all results predicted on the dataset. The dump contains two files: 1. "instance_predictions.pth" a file in torch serialization format that contains all the raw original predictions. 2. "coco_instances_results.json" a json file in COCO's result format. """ self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._output_dir = output_dir self._cpu_device = torch.device("cpu") self._logger = logging.getLogger(__name__) self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): self._logger.info( f"'{dataset_name}' is not registered by `register_coco_instances`." " Therefore trying to convert it to COCO format ..." ) cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) if in_res and out_res and in_res < out_res: for i in range(len(self._coco_api.dataset['annotations'])): image_w = [img for img in self._coco_api.dataset['images'] if img['id'] == self._coco_api.dataset[ 'annotations'][i]['image_id']][0]['width'] pad = int(image_w * (out_res / in_res - 1) / 2) # As the first two coords are a point, then we add the pad value and divide by resize # But because the last two coords are width and height, we only divide by resize, as padding does not affect them bbox_coords = ((np.array(self._coco_api.dataset['annotations'][i]['bbox'][:2]) + pad) / (out_res / in_res)).tolist() bbox_coords += (np.array(self._coco_api.dataset['annotations'][i]['bbox'][2:]) / (out_res / in_res)).tolist() self._coco_api.dataset['annotations'][i]['bbox'] = bbox_coords self._coco_api.dataset['annotations'][i]['segmentation'] = ((np.array(self._coco_api.dataset['annotations'][i]['segmentation']) + pad) / (out_res / in_res)).tolist() if resize: for i in range(len(self._coco_api.dataset['annotations'])): self._coco_api.dataset['annotations'][i]['bbox'] = (np.array(self._coco_api.dataset['annotations'][i]['bbox']) * resize).tolist() if 'segmentation' in self._coco_api.dataset['annotations'][i]: self._coco_api.dataset['annotations'][i]['segmentation'] = (np.array(self._coco_api.dataset['annotations'][i]['segmentation']) * resize).tolist() self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__(self, dataset_name, cfg, distributed, output_dir=None): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. cfg (CfgNode): config instance distributed (True): if True, will collect results from all ranks for evaluation. Otherwise, will evaluate the results in the current process. output_dir (str): optional, an output directory to dump all results predicted on the dataset. The dump contains two files: 1. "instance_predictions.pth" a file in torch serialization format that contains all the raw original predictions. 2. "coco_instances_results.json" a json file in COCO's result format. """ self._tasks = self._tasks_from_config(cfg) self._distributed = distributed self._logger = logging.getLogger(__name__) self._cpu_device = torch.device("cpu") self._output_dir = output_dir self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): self._logger.warning(f"json_file was not found in MetaDataCatalog for '{dataset_name}'") cache_path = os.path.join(output_dir, f"{dataset_name}_vg_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) self._classes = ['__background__'] self._class_to_ind = {} self._class_to_ind[self._classes[0]] = 0 with open(os.path.join('evaluation/objects_vocab.txt')) as f: count = 1 for object in f.readlines(): names = [n.lower().strip() for n in object.split(',')] self._classes.append(names[0]) for n in names: self._class_to_ind[n] = count count += 1 # Load attributes self._attributes = ['__no_attribute__'] self._attribute_to_ind = {} self._attribute_to_ind[self._attributes[0]] = 0 with open(os.path.join('evaluation/attributes_vocab.txt')) as f: count = 1 for att in f.readlines(): names = [n.lower().strip() for n in att.split(',')] self._attributes.append(names[0]) for n in names: self._attribute_to_ind[n] = count count += 1 self.roidb, self.image_index = self.gt_roidb(self._coco_api)
def __init__( self, dataset_name, tasks, distributed, output_dir=None, *, use_fast_impl=True, kpt_oks_sigmas=(), ): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. tasks (tuple[str]): tasks that can be evaluated under the given configuration. A task is one of "bbox", "segm", "keypoints". DEPRECATED pass cfgNode here to generate tasks from config distributed (True): if True, will collect results from all ranks and run evaluation in the main process. Otherwise, will only evaluate the results in the current process. output_dir (str): optional, an output directory to dump all results predicted on the dataset. The dump contains two files: 1. "instance_predictions.pth" a file in torch serialization format that contains all the raw original predictions. 2. "coco_instances_results.json" a json file in COCO's result format. use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP. Although the results should be very close to the official implementation in COCO API, it is still recommended to compute results with the official API for use in papers. kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS. See http://cocodataset.org/#keypoints-eval When empty, it will use the defaults in COCO. Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS. """ self._logger = logging.getLogger(__name__) if isinstance(tasks, CfgNode): kpt_oks_sigmas = (tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas) self._tasks = self._tasks_from_config(tasks) self._logger.warn( "COCO Evaluator instantiated using config, this is deprecated behavior." " Please pass tasks in directly") else: self._tasks = tasks self._distributed = distributed self._output_dir = output_dir self._use_fast_impl = use_fast_impl self._cpu_device = torch.device("cpu") self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): self._logger.info( f"'{dataset_name}' is not registered by `register_coco_instances`." " Therefore trying to convert it to COCO format ...") cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) self._kpt_oks_sigmas = kpt_oks_sigmas # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__( self, dataset_name, tasks=None, distributed=True, output_dir=None, *, max_dets_per_image=None, use_fast_impl=True, kpt_oks_sigmas=(), ): """ Args: dataset_name (str): name of the dataset to be evaluated. It must have either the following corresponding metadata: "json_file": the path to the COCO format annotation Or it must be in detectron2's standard dataset format so it can be converted to COCO format automatically. tasks (tuple[str]): tasks that can be evaluated under the given configuration. A task is one of "bbox", "segm", "keypoints". By default, will infer this automatically from predictions. distributed (True): if True, will collect results from all ranks and run evaluation in the main process. Otherwise, will only evaluate the results in the current process. output_dir (str): optional, an output directory to dump all results predicted on the dataset. The dump contains two files: 1. "instances_predictions.pth" a file that can be loaded with `torch.load` and contains all the results in the format they are produced by the model. 2. "coco_instances_results.json" a json file in COCO's result format. max_dets_per_image (int): limit on the maximum number of detections per image. By default in COCO, this limit is to 100, but this can be customized to be greater, as is needed in evaluation metrics AP fixed and AP pool (see https://arxiv.org/pdf/2102.01066.pdf) This doesn't affect keypoint evaluation. use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP. Although the results should be very close to the official implementation in COCO API, it is still recommended to compute results with the official API for use in papers. The faster implementation also uses more RAM. kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS. See http://cocodataset.org/#keypoints-eval When empty, it will use the defaults in COCO. Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS. """ self._logger = logging.getLogger(__name__) self._distributed = distributed self._output_dir = output_dir self._use_fast_impl = use_fast_impl # COCOeval requires the limit on the number of detections per image (maxDets) to be a list # with at least 3 elements. The default maxDets in COCOeval is [1, 10, 100], in which the # 3rd element (100) is used as the limit on the number of detections per image when # evaluating AP. COCOEvaluator expects an integer for max_dets_per_image, so for COCOeval, # we reformat max_dets_per_image into [1, 10, max_dets_per_image], based on the defaults. if max_dets_per_image is None: max_dets_per_image = [1, 10, 100] else: max_dets_per_image = [1, 10, max_dets_per_image] self._max_dets_per_image = max_dets_per_image if tasks is not None and isinstance(tasks, CfgNode): kpt_oks_sigmas = ( tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas ) self._logger.warn( "COCO Evaluator instantiated using config, this is deprecated behavior." " Please pass in explicit arguments instead." ) self._tasks = None # Infering it from predictions should be better else: self._tasks = tasks self._cpu_device = torch.device("cpu") self._metadata = MetadataCatalog.get(dataset_name) if not hasattr(self._metadata, "json_file"): if output_dir is None: raise ValueError( "output_dir must be provided to COCOEvaluator " "for datasets not in COCO format." ) self._logger.info(f"Trying to convert '{dataset_name}' to COCO format ...") cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") self._metadata.json_file = cache_path convert_to_coco_json(dataset_name, cache_path) json_file = PathManager.get_local_path(self._metadata.json_file) with contextlib.redirect_stdout(io.StringIO()): self._coco_api = COCO(json_file) # Test set json files do not contain annotations (evaluation must be # performed using the COCO evaluation server). self._do_evaluation = "annotations" in self._coco_api.dataset if self._do_evaluation: self._kpt_oks_sigmas = kpt_oks_sigmas