def __init__(self,
dataset_name,
cfg,
distributed,
output_dir=None,
*,
use_fast_impl=True):
self._predictions = []
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._use_fast_impl = use_fast_impl
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger('MyCocoEvaluator')
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
cache_path = os.path.join(output_dir,
f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__(self, dataset_name, cfg, distributed, output_dir=None):
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
# this is hard-coded so that logging is normal
self._logger = logging.getLogger("detectron2.evaluation.coco_evaluation")
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
self._logger.warning(
f"json_file was not found in MetaDataCatalog for '{dataset_name}'"
)
cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
if "BIT" in cfg.MODEL.DANCE.MASK_IN:
self.input_format = "bit"
else:
self.input_format = "rle"
def __init__(self, dataset_name, cfg, distributed, output_dir=None):
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger("detectron2")
self._metadata = MetadataCatalog.get(dataset_name)
self.internal_dataset_mapping = copy.deepcopy(
self._metadata.thing_dataset_id_to_contiguous_id)
if -1 in self.internal_dataset_mapping.keys():
self.internal_dataset_mapping = dict([
(k, v - 1) for k, v in self.internal_dataset_mapping.items()
])
if not hasattr(self._metadata, "json_file"):
self._logger.warning(
f"json_file was not found in MetaDataCatalog for '{dataset_name}'."
" Trying to convert it to COCO format ...")
cache_path = os.path.join(output_dir,
f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
def __init__(self, dataset_name, cfg, distributed, output_dir=None, result_df=None, output_file=None, *, use_fast_impl=True):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
cfg (CfgNode): config instance
distributed (True): if True, will collect results from all ranks and run evaluation
in the main process.
Otherwise, will only evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instance_predictions.pth" a file in torch serialization
format that contains all the raw original predictions.
2. "coco_instances_results.json" a json file in COCO's result
format.
use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP.
Although the results should be very close to the official implementation in COCO
API, it is still recommended to compute results with the official API for use in
papers.
"""
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._use_fast_impl = use_fast_impl
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger(__name__)
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
self._logger.info(
f"'{dataset_name}' is not registered by `register_coco_instances`."
" Therefore trying to convert it to COCO format ..."
)
cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
#Changed made by Johan to store all results
self.result_df = None
if len(result_df) == 0:
self.result_df = result_df
self._output_file = output_file
def __init__(self, dataset_name, cfg, distributed, output_dir=None):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
cfg (CfgNode): config instance
distributed (True): if True, will collect results from all ranks for evaluation.
Otherwise, will evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instance_predictions.pth" a file in torch serialization
format that contains all the raw original predictions.
2. "coco_instances_results.json" a json file in COCO's result
format.
"""
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger(__name__)
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
self._logger.warning(
f"json_file was not found in MetaDataCatalog for '{dataset_name}'"
)
cache_path = os.path.join(output_dir,
f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._fashion_api = Fashion(json_file)
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = ("annotations" in self._fashion_api.dataset \
and "annotations2" in self._fashion_api.dataset)
self._view_error = cfg.TEST.VIEW_ERROR
if self._view_error:
self._error_output_path = self._output_dir + '/error_result'
os.makedirs(self._error_output_path, exist_ok=True)
with open(os.path.join(self._error_output_path, 'info.txt'),
'w') as f:
s = "time: " + time.strftime("%Y-%m-%d-%H:%M",
time.localtime()) + "\n"
s += f"weight: {cfg.MODEL.WEIGHTS}\n"
s += f"dataset: {cfg.DATASETS.TEST}\n"
f.write(s)
def __init__(self, dataset_name, cfg, distributed, output_dir=None):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
cfg (CfgNode): config instance
distributed (True): if True, will collect results from all ranks and run evaluation
in the main process.
Otherwise, will evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instance_predictions.pth" a file in torch serialization
format that contains all the raw original predictions.
2. "coco_instances_results.json" a json file in COCO's result
format.
"""
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger(__name__)
self._metadata = MetadataCatalog.get(dataset_name)
has_json = True
if not hasattr(self._metadata, "json_file"):
self._logger.warning(
f"json_file was not found in MetaDataCatalog for '{dataset_name}'."
" Trying to convert it to COCO format ...")
cache_path = os.path.join(output_dir,
f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
try:
convert_to_coco_json(dataset_name, cache_path)
except KeyError:
self._logger.warning(
f"Failed to convert {dataset_name} to COCO format. Do inference only."
)
has_json = False
if has_json:
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
else:
self._coco_api = None
self._do_evaluation = False
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
def __init__(self, dataset_name, cfg, distributed, output_dir=None):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation;
"matlab_file": the original matlab annotation files,
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
cfg (CfgNode): config instance
distributed (True): if True, will collect results from all ranks and run evaluation
in the main process. Otherwise, will evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instance_predictions.pth" a file in torch serialization
format that contains all the raw original predictions.
2. "coco_instances_results.json" a json file in COCO's result format.
3. "hico_interaction_results.mat" a matlab file
used for HICO-DET official evaluation.
"""
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
self._matlab = cfg.TEST.MATLAB
self._hico_official_matlab_path = cfg.TEST.HICO_OFFICIAL_MATLAB_PATH
self._hico_official_anno_file = cfg.TEST.HICO_OFFICIAL_ANNO_FILE
self._hico_official_bbox_file = cfg.TEST.HICO_OFFICIAL_BBOX_FILE
self.is_hoi_prediction = cfg.MODEL.ROI_HEADS.IS_HOI_PRED
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
logger.warning(
f"json_file was not found in MetaDataCatalog for '{dataset_name}'."
" Trying to convert it to COCO format ...")
cache_path = os.path.join(output_dir,
f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__(self, dataset_name, cfg, distributed, output_dir=None, *, use_fast_impl=True):
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger('detectron2.evaluation.coco_evaluation')
self._metadata = copy.deepcopy(MetadataCatalog.get(dataset_name))
if cfg.DATASETS.UNSEEN_LABEL_SET != '':
thing_classes = self._metadata.thing_classes
del self._metadata.thing_classes
thing_classes.append('unknown')
self._metadata.thing_classes = thing_classes #['unknown']
unknown_colors = self._metadata.thing_colors[-1]
thing_colors = self._metadata.thing_colors
thing_colors.append(unknown_colors)
del self._metadata.thing_colors
self._metadata.thing_colors = thing_colors # unknown_colors
conv_id = self._metadata.thing_dataset_id_to_contiguous_id
del self._metadata.thing_dataset_id_to_contiguous_id
conv_id[-1] = -1
self._metadata.thing_dataset_id_to_contiguous_id = conv_id #{-1:-1}
if not hasattr(self._metadata, "json_file"):
self._logger.info(
f"'{dataset_name}' is not registered by `register_coco_instances`."
" Therefore trying to convert it to COCO format ..."
)
cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
if cfg.DATASETS.UNSEEN_LABEL_SET != '':
with open(cfg.DATASETS.UNSEEN_LABEL_SET, 'r') as f:
self.unknown_label_list = [e.replace('\n', '') for e in f.readlines()]
else:
self.unknown_label_list = None
def convert_dataset_to_coco_json(output_dir, registered_dataset_name):
output_coco_json = os.path.join(
output_dir, f'{registered_dataset_name}_coco_format.json')
convert_to_coco_json(registered_dataset_name,
output_file=output_coco_json,
allow_cached=False)
# Save old metadata
metadata = DatasetCatalog.get(registered_dataset_name)
# Remove the current one
DatasetCatalog.remove(registered_dataset_name)
# Register again
# if your dataset is in COCO format, this cell can be replaced by the following three lines:
register_coco_instances(registered_dataset_name, {}, output_coco_json,
os.path.abspath('./'))
return output_coco_json
def to_json():
# /Users/mac7/Desktop/MS_Project/csvtojson5/detectron2/detectron2/data/catalog.py
"""
1 :
DatasetCatalog.register(name, func):
Args:
name (str): the name that identifies a dataset, e.g. "coco_2014_train".
func (callable): a callable which takes no arguments and returns a list of dicts.
Call the registered function and return its results.
2 :
Args:
name (str): the name that identifies a dataset, e.g. "coco_2014_train".
Returns:
list[dict]: dataset annotations.
A class that supports simple attribute setter/getter.
It is intended for storing metadata of a dataset and make it accessible globally.
Examples:
.. code-block: : python
somewhere when you load the data:
MetadataCatalog.get("mydataset").thing_classes = ["person", "dog"]
somewhere when you print statistics or visualize:
classes = MetadataCatalog.get("mydataset").thing_classes.
3 :
Convert it to the coco.
.
.
.
"""
THING_CLASSES = [
'Past', 'Gorgonia', 'SeaRods', 'Antillo', 'Fish', 'Ssid', 'Orb',
'Other_Coral', 'Apalm', 'Galaxaura'
]
# 1
DatasetCatalog.register('coral', _get_coral_dicts)
# 2 Metadata(name='test', thing_classes=['first'])
MetadataCatalog.get('coral').set(thing_classes=THING_CLASSES)
# 3
convert_to_coco_json('coral', output_file=OP,
allow_cached=False) # output_folder -> output_file
def __init__(self, dataset_name, cfg, distributed, output_dir=None):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
cfg (CfgNode): config instance
distributed (True): if True, will collect results from all ranks for evaluation.
Otherwise, will evaluate the results in the current process.
output_dir (str): optional, an output directory to dump results.
"""
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger(__name__)
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
self._logger.warning(f"json_file was not found in MetaDataCatalog for '{dataset_name}'")
cache_path = convert_to_coco_json(dataset_name, output_dir)
self._metadata.json_file = cache_path
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__(self, dataset_name, cfg, distributed, output_dir=None):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
cfg (CfgNode): config instance
distributed (True): if True, will collect results from all ranks for evaluation.
Otherwise, will evaluate the results in the current process.
output_dir (str): optional, an output directory to dump results.
"""
self._cfg = cfg
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger(__name__)
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
self._logger.warning(
f"json_file was not found in MetaDataCatalog for '{dataset_name}'"
)
cache_path = convert_to_coco_json(dataset_name, output_dir)
self._metadata.json_file = cache_path
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
if dataset_name.endswith("visible"):
self._logger.info("using visible segmentation")
if dataset_name.find('d2sa') != -1:
for key, _ in self._coco_api.anns.items():
self._coco_api.anns[key][
"segmentation"] = self._coco_api.anns[key][
"visible_mask"]
elif dataset_name.find("kins") != -1:
for key, _ in self._coco_api.anns.items():
self._coco_api.anns[key][
"segmentation"] = self._coco_api.anns[key][
"inmodal_seg"]
# if dataset_name.startswith("sailvos"):
# self._coco_api.dataset['categories'] = [c for c in self._coco_api.dataset['categories'] if c["id"] not in sailvos_ignore]
# for i in range(self._coco_api.getCatIds()[-1]):
# if i in sailvos_ignore:
# self._coco_api.cats.pop(i)
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__(
self,
dataset_name, # coco_2017_val for onenet
tasks=None, # cfg
distributed=True,
output_dir=None,
*,
use_fast_impl=True,
kpt_oks_sigmas=(),
):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
tasks (tuple[str]): tasks that can be evaluated under the given
configuration. A task is one of "bbox", "segm", "keypoints".
By default, will infer this automatically from predictions.
distributed (True): if True, will collect results from all ranks and run evaluation
in the main process.
Otherwise, will only evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instances_predictions.pth" a file in torch serialization
format that contains all the raw original predictions.
2. "coco_instances_results.json" a json file in COCO's result
format.
use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP.
Although the results should be very close to the official implementation in COCO
API, it is still recommended to compute results with the official API for use in
papers. The faster implementation also uses more RAM.
kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS.
See http://cocodataset.org/#keypoints-eval
When empty, it will use the defaults in COCO.
Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS.
"""
self._logger = logging.getLogger(__name__)
self._distributed = distributed
self._output_dir = output_dir
self._use_fast_impl = use_fast_impl
if tasks is not None and isinstance(tasks, CfgNode):
kpt_oks_sigmas = (
tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas
)
self._logger.warn(
"COCO Evaluator instantiated using config, this is deprecated behavior."
" Please pass in explicit arguments instead."
)
self._tasks = None # Infering it from predictions should be better
else:
self._tasks = tasks
self._cpu_device = torch.device("cpu")
# 获取元数据, 一般有下面属性
"""
json_file: json_file路径名
image_root: 图片所在路径(前部)
evaluator_type:"coco"
"thing_dataset_id_to_contiguous_id": 类别id映射,
"thing_classes":类别名称,
"thing_colors":类别对应色，
"""
self._metadata = MetadataCatalog.get(dataset_name)
# 有
# json_file: instances_val2017.json
if not hasattr(self._metadata, "json_file"):
self._logger.info(
f"'{dataset_name}' is not registered by `register_coco_instances`."
" Therefore trying to convert it to COCO format ..."
)
# 需要转化
cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
# 创建val_coco_api
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
# instances_val2017.json 是有annotatons 的
self._do_evaluation = "annotations" in self._coco_api.dataset
if self._do_evaluation:
self._kpt_oks_sigmas = kpt_oks_sigmas
def __init__(self,
dataset_name,
cfg,
distributed,
output_dir=None,
fake=False):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
cfg (CfgNode): config instance
distributed (True): if True, will collect results from all ranks and run evaluation
in the main process.
Otherwise, will evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instance_predictions.pth" a file in torch serialization
format that contains all the raw original predictions.
2. "coco_instances_results.json" a json file in COCO's result
format.
"""
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger(__name__)
if fake:
mask = eval_make_mask()
DatasetCatalog.register("test_dataset",
lambda: eval_make_dataset_dicts(mask))
MetadataCatalog.get("test_dataset").set(
thing_classes=["test_label"])
# Dump to json.
json_dict = convert_to_coco_dict("test_dataset")
with tempfile.TemporaryDirectory() as tmpdir:
json_file_name = "test.json"
with open(json_file_name, "w") as f:
json.dump(json_dict, f)
# Load from json.
dicts = load_coco_json(json_file_name, "")
json_file = json_file_name
else:
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
self._logger.warning(
f"json_file was not found in MetaDataCatalog for '{dataset_name}'."
" Trying to convert it to COCO format ...")
cache_path = os.path.join(output_dir,
f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
self.evaluator_name = "CocoEvaluator"
self.reset()
self.fake = fake
def __init__(self,
dataset_name,
cfg,
distributed,
output_dir=None,
classes_to_eval=None):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
cfg (CfgNode): config instance
distributed (True): if True, will collect results from all ranks for evaluation.
Otherwise, will evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instance_predictions.pth" a file in torch serialization
format that contains all the raw original predictions.
2. "coco_instances_results.json" a json file in COCO's result
format.
classes_to_eval (list): List of indices with the COCO classes to be used
when performing evaluation.
Otherwise, if None, will use the default 80 COCO classes.
"""
# Avoid default mutable argument and convert to list
self.classes_to_eval = classes_to_eval if classes_to_eval else []
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger(__name__)
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
self._logger.warning(
f"json_file was not found in MetaDataCatalog for '{dataset_name}'."
" Trying to convert it to COCO format ...")
cache_path = os.path.join(output_dir,
f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
self._dataset_name = dataset_name
# TODO(): Can be replaced with check if classes_to_eval != None.
self._is_few_shot = "_all" in dataset_name or "_base" in dataset_name \
or "_novel" in dataset_name
def __init__(self, dataset_name, cfg, distributed, output_dir=None):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
cfg (CfgNode): config instance
distributed (True): if True, will collect results from all ranks and run evaluation
in the main process.
Otherwise, will evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instance_predictions.pth" a file in torch serialization
format that contains all the raw original predictions.
2. "coco_instances_results.json" a json file in COCO's result
format.
"""
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger(__name__)
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
self._logger.warning(
f"json_file was not found in MetaDataCatalog for '{dataset_name}'."
" Trying to convert it to COCO format ...")
cache_path = os.path.join(output_dir,
f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
# for visualization
self._visresult = cfg.TEST.VISUAL_OUTPUT
self._id2anno = defaultdict(list)
for anno in self._coco_api.dataset["annotations"]:
ann = copy.deepcopy(anno)
ann['bbox_mode'] = BoxMode.XYWH_ABS
ann['category_id'] -= 1
self._id2anno[ann['image_id']].append(ann)
# score thresh
if cfg.MODEL.META_ARCHITECTURE == "GeneralizedRCNN":
self._score_thresh = cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST
elif cfg.MODEL.META_ARCHITECTURE == "OneStageDetector":
if 'FCOS' in cfg.MODEL.PROPOSAL_GENERATOR.NAME:
self._score_thresh = cfg.MODEL.FCOS.INFERENCE_TH_TEST
elif 'ATSS' in cfg.MODEL.PROPOSAL_GENERATOR.NAME:
self._score_thresh = cfg.MODEL.ATSS.INFERENCE_TH
else:
self._logger.warning(
f"score thresh is not implement for {cfg.MODEL.META_ARCHITECTURE}"
)
self._score_thresh = 0.35
def __init__(self, dataset_name, cfg, distributed, output_dir=None):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
cfg (CfgNode): config instance
distributed (True):
if True, will collect results from all ranks for evaluation.
Otherwise, will evaluate the results in the current process.
output_dir (str): optional, an output directory to dump results.
"""
self._distributed = distributed
self._output_dir = output_dir
self._dataset_name = dataset_name
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger(__name__)
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
self._logger.warning(
f"json_file was not found in MetaDataCatalog for '{dataset_name}'"
)
cache_path = convert_to_coco_json(dataset_name, output_dir)
self._metadata.json_file = cache_path
self._is_splits = "all" in dataset_name or "base" in dataset_name \
or "novel" in dataset_name
self._base_classes = [
8,
10,
11,
13,
14,
15,
22,
23,
24,
25,
27,
28,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59,
60,
61,
65,
70,
73,
74,
75,
76,
77,
78,
79,
80,
81,
82,
84,
85,
86,
87,
88,
89,
90,
]
self._novel_classes = [
1, 2, 3, 4, 5, 6, 7, 9, 16, 17, 18, 19, 20, 21, 44, 62, 63, 64, 67,
72
]
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__(self, dataset_name, cfg, distributed, output_dir=None, in_res=None, out_res=None, resize=None):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
cfg (CfgNode): config instance
distributed (True): if True, will collect results from all ranks and run evaluation
in the main process.
Otherwise, will evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instance_predictions.pth" a file in torch serialization
format that contains all the raw original predictions.
2. "coco_instances_results.json" a json file in COCO's result
format.
"""
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._output_dir = output_dir
self._cpu_device = torch.device("cpu")
self._logger = logging.getLogger(__name__)
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
self._logger.info(
f"'{dataset_name}' is not registered by `register_coco_instances`."
" Therefore trying to convert it to COCO format ..."
)
cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
if in_res and out_res and in_res < out_res:
for i in range(len(self._coco_api.dataset['annotations'])):
image_w = [img for img in self._coco_api.dataset['images'] if img['id'] == self._coco_api.dataset[
'annotations'][i]['image_id']][0]['width']
pad = int(image_w * (out_res / in_res - 1) / 2)
# As the first two coords are a point, then we add the pad value and divide by resize
# But because the last two coords are width and height, we only divide by resize, as padding does not affect them
bbox_coords = ((np.array(self._coco_api.dataset['annotations'][i]['bbox'][:2]) + pad) / (out_res / in_res)).tolist()
bbox_coords += (np.array(self._coco_api.dataset['annotations'][i]['bbox'][2:]) / (out_res / in_res)).tolist()
self._coco_api.dataset['annotations'][i]['bbox'] = bbox_coords
self._coco_api.dataset['annotations'][i]['segmentation'] = ((np.array(self._coco_api.dataset['annotations'][i]['segmentation']) + pad) / (out_res / in_res)).tolist()
if resize:
for i in range(len(self._coco_api.dataset['annotations'])):
self._coco_api.dataset['annotations'][i]['bbox'] = (np.array(self._coco_api.dataset['annotations'][i]['bbox']) * resize).tolist()
if 'segmentation' in self._coco_api.dataset['annotations'][i]:
self._coco_api.dataset['annotations'][i]['segmentation'] = (np.array(self._coco_api.dataset['annotations'][i]['segmentation']) * resize).tolist()
self._kpt_oks_sigmas = cfg.TEST.KEYPOINT_OKS_SIGMAS
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__(self, dataset_name, cfg, distributed, output_dir=None):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
cfg (CfgNode): config instance
distributed (True): if True, will collect results from all ranks for evaluation.
Otherwise, will evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instance_predictions.pth" a file in torch serialization
format that contains all the raw original predictions.
2. "coco_instances_results.json" a json file in COCO's result
format.
"""
self._tasks = self._tasks_from_config(cfg)
self._distributed = distributed
self._logger = logging.getLogger(__name__)
self._cpu_device = torch.device("cpu")
self._output_dir = output_dir
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
self._logger.warning(f"json_file was not found in MetaDataCatalog for '{dataset_name}'")
cache_path = os.path.join(output_dir, f"{dataset_name}_vg_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
self._classes = ['__background__']
self._class_to_ind = {}
self._class_to_ind[self._classes[0]] = 0
with open(os.path.join('evaluation/objects_vocab.txt')) as f:
count = 1
for object in f.readlines():
names = [n.lower().strip() for n in object.split(',')]
self._classes.append(names[0])
for n in names:
self._class_to_ind[n] = count
count += 1
# Load attributes
self._attributes = ['__no_attribute__']
self._attribute_to_ind = {}
self._attribute_to_ind[self._attributes[0]] = 0
with open(os.path.join('evaluation/attributes_vocab.txt')) as f:
count = 1
for att in f.readlines():
names = [n.lower().strip() for n in att.split(',')]
self._attributes.append(names[0])
for n in names:
self._attribute_to_ind[n] = count
count += 1
self.roidb, self.image_index = self.gt_roidb(self._coco_api)
def __init__(
self,
dataset_name,
tasks,
distributed,
output_dir=None,
*,
use_fast_impl=True,
kpt_oks_sigmas=(),
):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
tasks (tuple[str]): tasks that can be evaluated under the given configuration.
A task is one of "bbox", "segm", "keypoints".
DEPRECATED pass cfgNode here to generate tasks from config
distributed (True): if True, will collect results from all ranks and run evaluation
in the main process.
Otherwise, will only evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instance_predictions.pth" a file in torch serialization
format that contains all the raw original predictions.
2. "coco_instances_results.json" a json file in COCO's result
format.
use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP.
Although the results should be very close to the official implementation in COCO
API, it is still recommended to compute results with the official API for use in
papers.
kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS.
See http://cocodataset.org/#keypoints-eval
When empty, it will use the defaults in COCO.
Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS.
"""
self._logger = logging.getLogger(__name__)
if isinstance(tasks, CfgNode):
kpt_oks_sigmas = (tasks.TEST.KEYPOINT_OKS_SIGMAS
if not kpt_oks_sigmas else kpt_oks_sigmas)
self._tasks = self._tasks_from_config(tasks)
self._logger.warn(
"COCO Evaluator instantiated using config, this is deprecated behavior."
" Please pass tasks in directly")
else:
self._tasks = tasks
self._distributed = distributed
self._output_dir = output_dir
self._use_fast_impl = use_fast_impl
self._cpu_device = torch.device("cpu")
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
self._logger.info(
f"'{dataset_name}' is not registered by `register_coco_instances`."
" Therefore trying to convert it to COCO format ...")
cache_path = os.path.join(output_dir,
f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
self._kpt_oks_sigmas = kpt_oks_sigmas
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
def __init__(
self,
dataset_name,
tasks=None,
distributed=True,
output_dir=None,
*,
max_dets_per_image=None,
use_fast_impl=True,
kpt_oks_sigmas=(),
):
"""
Args:
dataset_name (str): name of the dataset to be evaluated.
It must have either the following corresponding metadata:
"json_file": the path to the COCO format annotation
Or it must be in detectron2's standard dataset format
so it can be converted to COCO format automatically.
tasks (tuple[str]): tasks that can be evaluated under the given
configuration. A task is one of "bbox", "segm", "keypoints".
By default, will infer this automatically from predictions.
distributed (True): if True, will collect results from all ranks and run evaluation
in the main process.
Otherwise, will only evaluate the results in the current process.
output_dir (str): optional, an output directory to dump all
results predicted on the dataset. The dump contains two files:
1. "instances_predictions.pth" a file that can be loaded with `torch.load` and
contains all the results in the format they are produced by the model.
2. "coco_instances_results.json" a json file in COCO's result format.
max_dets_per_image (int): limit on the maximum number of detections per image.
By default in COCO, this limit is to 100, but this can be customized
to be greater, as is needed in evaluation metrics AP fixed and AP pool
(see https://arxiv.org/pdf/2102.01066.pdf)
This doesn't affect keypoint evaluation.
use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP.
Although the results should be very close to the official implementation in COCO
API, it is still recommended to compute results with the official API for use in
papers. The faster implementation also uses more RAM.
kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS.
See http://cocodataset.org/#keypoints-eval
When empty, it will use the defaults in COCO.
Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS.
"""
self._logger = logging.getLogger(__name__)
self._distributed = distributed
self._output_dir = output_dir
self._use_fast_impl = use_fast_impl
# COCOeval requires the limit on the number of detections per image (maxDets) to be a list
# with at least 3 elements. The default maxDets in COCOeval is [1, 10, 100], in which the
# 3rd element (100) is used as the limit on the number of detections per image when
# evaluating AP. COCOEvaluator expects an integer for max_dets_per_image, so for COCOeval,
# we reformat max_dets_per_image into [1, 10, max_dets_per_image], based on the defaults.
if max_dets_per_image is None:
max_dets_per_image = [1, 10, 100]
else:
max_dets_per_image = [1, 10, max_dets_per_image]
self._max_dets_per_image = max_dets_per_image
if tasks is not None and isinstance(tasks, CfgNode):
kpt_oks_sigmas = (
tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas
)
self._logger.warn(
"COCO Evaluator instantiated using config, this is deprecated behavior."
" Please pass in explicit arguments instead."
)
self._tasks = None # Infering it from predictions should be better
else:
self._tasks = tasks
self._cpu_device = torch.device("cpu")
self._metadata = MetadataCatalog.get(dataset_name)
if not hasattr(self._metadata, "json_file"):
if output_dir is None:
raise ValueError(
"output_dir must be provided to COCOEvaluator "
"for datasets not in COCO format."
)
self._logger.info(f"Trying to convert '{dataset_name}' to COCO format ...")
cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json")
self._metadata.json_file = cache_path
convert_to_coco_json(dataset_name, cache_path)
json_file = PathManager.get_local_path(self._metadata.json_file)
with contextlib.redirect_stdout(io.StringIO()):
self._coco_api = COCO(json_file)
# Test set json files do not contain annotations (evaluation must be
# performed using the COCO evaluation server).
self._do_evaluation = "annotations" in self._coco_api.dataset
if self._do_evaluation:
self._kpt_oks_sigmas = kpt_oks_sigmas