def _eval_predictions(self, tasks):
"""
Evaluate self._predictions on the given tasks.
Fill self._results with the metrics of the tasks.
"""
logger.info("Preparing results for CrowdHuman format ...")
self._coco_results = self._predictions
if self._output_dir:
file_path = os.path.join(self._output_dir,
"coco_instances_results.json")
logger.info("Saving results to {}".format(file_path))
with megfile.smart_open(file_path, "w") as f:
for db in self._coco_results:
line = json.dumps(db) + '\n'
f.write(line)
logger.info("Evaluating predictions ...")
for task in sorted(tasks):
coco_eval = (
_evaluate_predictions_on_crowdhuman(self._metadata.json_file,
file_path)
if len(self._coco_results) > 0 else
None # cocoapi does not handle empty results very well
)
res, small_table = self._derive_coco_results(coco_eval, task)
self._results[task] = res
return small_table
def evaluate(self):
if self._distributed:
comm.synchronize()
self._predictions = comm.gather(self._predictions, dst=0)
self._predictions = list(itertools.chain(*self._predictions))
if not comm.is_main_process():
return {}
if len(self._predictions) == 0:
logger.warning(
"[COCOEvaluator] Did not receive valid predictions.")
return {}
if self._output_dir:
ensure_dir(self._output_dir)
file_path = os.path.join(self._output_dir,
"instances_predictions.pth")
with megfile.smart_open(file_path, "wb") as f:
torch.save(self._predictions, f)
self._results = OrderedDict()
if "instances" in self._predictions[0]:
self._eval_predictions(set(self._tasks))
if self._dump:
_dump_to_markdown(self._dump_infos)
# Copy so the caller can do whatever with results
return copy.deepcopy(self._results)
def process(self, inputs, outputs):
"""
Args:
inputs: the inputs to a model.
It is a list of dicts. Each dict corresponds to an image and
contains keys like "height", "width", "file_name".
outputs: the outputs of a model. It is either list of semantic segmentation predictions
(Tensor [H, W]) or list of dicts with key "sem_seg" that contains semantic
segmentation prediction in the same format.
"""
for input, output in zip(inputs, outputs):
output = output["sem_seg"].argmax(dim=0).to(self._cpu_device)
pred = np.array(output, dtype=np.int)
with megfile.smart_open(
self.input_file_to_gt_file[input["file_name"]], "rb") as f:
gt = np.array(Image.open(f), dtype=np.int)
gt[gt == self._ignore_label] = self._num_classes
self._conf_matrix += np.bincount(
self._N * pred.reshape(-1) + gt.reshape(-1),
minlength=self._N**2).reshape(self._N, self._N)
self._predictions.extend(
self.encode_json_sem_seg(pred, input["file_name"]))
def save(self, name: str, tag_checkpoint: bool = True, **kwargs: dict):
"""
Dump model and checkpointables to a file.
Args:
name (str): name of the file.
kwargs (dict): extra arbitrary data to save.
"""
if not self.save_dir or not self.save_to_disk:
return
data = {}
data["model"] = self.model.state_dict()
for key, obj in self.checkpointables.items():
data[key] = obj.state_dict()
data.update(kwargs)
basename = "{}.pth".format(name)
save_file = os.path.join(self.save_dir, basename)
assert os.path.basename(save_file) == basename, basename
logger.info("Saving checkpoint to {}".format(save_file))
with megfile.smart_open(save_file, "wb") as f:
torch.save(data, f)
if tag_checkpoint:
self.tag_last_checkpoint(basename)
def _load_semantic_annotations(self, image_dir, gt_dir):
"""
Args:
image_dir (str): path to the raw dataset. e.g., "~/cityscapes/leftImg8bit/train".
gt_dir (str): path to the raw annotations. e.g., "~/cityscapes/gtFine/train".
Returns:
list[dict]: a list of dict, each has "file_name" and
"sem_seg_file_name".
"""
ret = []
for image_file in glob.glob(os.path.join(image_dir, "**/*.png")):
suffix = "leftImg8bit.png"
assert image_file.endswith(suffix)
prefix = image_dir
label_file = (gt_dir + image_file[len(prefix):-len(suffix)] +
"gtFine_labelTrainIds.png")
assert os.path.isfile(
label_file
), "Please generate labelTrainIds.png with cityscapesscripts/preparation/createTrainIdLabelImgs.py" # noqa
json_file = gt_dir + image_file[
len(prefix):-len(suffix)] + "gtFine_polygons.json"
with megfile.smart_open(json_file, "r") as f:
jsonobj = json.load(f)
ret.append({
"file_name": image_file,
"sem_seg_file_name": label_file,
"height": jsonobj["imgHeight"],
"width": jsonobj["imgWidth"],
})
return ret
def convert_to_coco_json(dataset_name, output_file, allow_cached=True):
"""
Converts dataset into COCO format and saves it to a json file.
dataset_name must be registered in DatasetCatalog and in cvpods's standard format.
Args:
dataset_name:
reference from the config file to the catalogs
must be registered in DatasetCatalog and in cvpods's standard format
output_file: path of json file that will be saved to
allow_cached: if json file is already present then skip conversion
"""
# TODO: The dataset or the conversion script *may* change,
# a checksum would be useful for validating the cached data
ensure_dir(os.path.dirname(output_file))
with file_lock(output_file):
if megfile.smart_exists(output_file) and allow_cached:
logger.info(
f"Cached annotations in COCO format already exist: {output_file}"
)
else:
logger.info(
f"Converting dataset annotations in '{dataset_name}' to COCO format ...)"
)
coco_dict = convert_to_coco_dict(dataset_name)
with megfile.smart_open(output_file, "w") as json_file:
logger.info(
f"Caching annotations in COCO format: {output_file}")
json.dump(coco_dict, json_file)
def _load_annotations(self):
"""
Load Pascal VOC detection annotations to cvpods format.
Args:
dirname: Contain "Annotations", "ImageSets", "JPEGImages"
split (str): one of "train", "test", "val", "trainval"
"""
dirname = self.image_root
split = self.split
with megfile.smart_open(
megfile.smart_path_join(dirname, "ImageSets", "Main",
split + ".txt")) as f:
fileids = np.loadtxt(f, dtype=np.str)
dicts = []
for fileid in fileids:
anno_file = os.path.join(dirname, "Annotations", fileid + ".xml")
jpeg_file = os.path.join(dirname, "JPEGImages", fileid + ".jpg")
tree = ET.parse(anno_file)
r = {
"file_name": jpeg_file,
"image_id": fileid,
"height": int(tree.findall("./size/height")[0].text),
"width": int(tree.findall("./size/width")[0].text),
}
instances = []
for obj in tree.findall("object"):
cls = obj.find("name").text
# We include "difficult" samples in training.
# Based on limited experiments, they don't hurt accuracy.
# difficult = int(obj.find("difficult").text)
# if difficult == 1:
# continue
bbox = obj.find("bndbox")
bbox = [
float(bbox.find(x).text)
for x in ["xmin", "ymin", "xmax", "ymax"]
]
# Original annotations are integers in the range [1, W or H]
# Assuming they mean 1-based pixel indices (inclusive),
# a box with annotation (xmin=1, xmax=W) covers the whole image.
# In coordinate space this is represented by (xmin=0, xmax=W)
bbox[0] -= 1.0
bbox[1] -= 1.0
instances.append({
"category_id": CLASS_NAMES.index(cls),
"bbox": bbox,
"bbox_mode": BoxMode.XYXY_ABS
})
r["annotations"] = instances
dicts.append(r)
return dicts
def __getitem__(self, index):
"""Load data, apply transforms, converto to Instances.
"""
dataset_dict = copy.deepcopy(self.dataset_dicts[index])
# read image
image = read_image(dataset_dict["file_name"], format=self.data_format)
check_image_size(dataset_dict, image)
if "annotations" in dataset_dict:
annotations = dataset_dict.pop("annotations")
annotations = [
ann for ann in annotations if ann.get("iscrowd", 0) == 0
]
else:
annotations = None
if "sem_seg_file_name" in dataset_dict:
assert annotations is None
annotations = []
with megfile.smart_open(dataset_dict.get("sem_seg_file_name"),
"rb") as f:
sem_seg_gt = Image.open(f)
sem_seg_gt = np.asarray(sem_seg_gt, dtype="uint8")
annotations.append({"sem_seg": sem_seg_gt})
# apply transfrom
image, annotations = self._apply_transforms(image, annotations)
if "sem_seg_file_name" in dataset_dict:
dataset_dict.pop("sem_seg_file_name")
sem_seg_gt = annotations[0].pop("sem_seg")
sem_seg_gt = torch.as_tensor(sem_seg_gt.astype("long"))
dataset_dict["sem_seg"] = sem_seg_gt
annotations = None
if annotations is not None:
image_shape = image.shape[:2] # h, w
instances = annotations_to_instances(annotations,
image_shape,
mask_format=self.mask_format)
# # Create a tight bounding box from masks, useful when image is cropped
# if self.crop_gen and instances.has("gt_masks"):
# instances.gt_boxes = instances.gt_masks.get_bounding_boxes()
dataset_dict["instances"] = filter_empty_instances(instances)
# convert to Instance type
# Pytorch's dataloader is efficient on torch.Tensor due to shared-memory,
# but not efficient on large generic data structures due to the use of pickle & mp.Queue.
# Therefore it's important to use torch.Tensor.
# h, w, c -> c, h, w
dataset_dict["image"] = torch.as_tensor(
np.ascontiguousarray(image.transpose(2, 0, 1)))
return dataset_dict
def evaluate(self):
comm.synchronize()
self._predictions = comm.gather(self._predictions)
self._predictions = list(itertools.chain(*self._predictions))
if not comm.is_main_process():
return
# gt_json = PathManager.get_local_path(self._metadata.panoptic_json)
gt_json = self._metadata.panoptic_json
gt_folder = self._metadata.panoptic_root
with tempfile.TemporaryDirectory(prefix="panoptic_eval") as pred_dir:
logger.info(
"Writing all panoptic predictions to {} ...".format(pred_dir))
for p in self._predictions:
with open(os.path.join(pred_dir, p["file_name"]), "wb") as f:
f.write(p.pop("png_string"))
with open(gt_json, "r") as f:
json_data = json.load(f)
json_data["annotations"] = self._predictions
with megfile.smart_open(self._predictions_json, "w") as f:
f.write(json.dumps(json_data))
from panopticapi.evaluation import pq_compute
with contextlib.redirect_stdout(io.StringIO()):
pq_res = pq_compute(
gt_json,
self._predictions_json,
gt_folder=gt_folder,
pred_folder=pred_dir,
)
res = {}
res["PQ"] = 100 * pq_res["All"]["pq"]
res["SQ"] = 100 * pq_res["All"]["sq"]
res["RQ"] = 100 * pq_res["All"]["rq"]
res["PQ_th"] = 100 * pq_res["Things"]["pq"]
res["SQ_th"] = 100 * pq_res["Things"]["sq"]
res["RQ_th"] = 100 * pq_res["Things"]["rq"]
res["PQ_st"] = 100 * pq_res["Stuff"]["pq"]
res["SQ_st"] = 100 * pq_res["Stuff"]["sq"]
res["RQ_st"] = 100 * pq_res["Stuff"]["rq"]
results = OrderedDict({"panoptic_seg": res})
table = _print_panoptic_results(pq_res)
if self._dump:
dump_info_one_task = {
"task": "panoptic_seg",
"tables": [table],
}
_dump_to_markdown([dump_info_one_task])
return results
def tag_last_checkpoint(self, last_filename_basename: str):
"""
Tag the last checkpoint.
Args:
last_filename_basename (str): the basename of the last filename.
"""
save_file = os.path.join(self.save_dir, "last_checkpoint")
with megfile.smart_open(save_file, "w") as f:
f.write(last_filename_basename)
def __init__(self, json_file, window_size=20):
"""
Args:
json_file (str): path to the json file. New data will be appended if the file exists.
window_size (int): the window size of median smoothing for the scalars whose
`smoothing_hint` are True.
"""
self._file_handle = megfile.smart_open(json_file, "a")
self._window_size = window_size
self._last_write = -1
def load_proposals_into_dataset(dataset_dicts, proposal_file):
r"""
Load precomputed object proposals into the dataset.
The proposal file should be a pickled dict with the following keys:
- "ids": list[int] or list[str], the image ids
- "boxes": list[np.ndarray], each is an Nx4 array of boxes corresponding to the image id
- "objectness_logits": list[np.ndarray], each is an N sized array of objectness scores
corresponding to the boxes.
- "bbox_mode": the BoxMode of the boxes array. Defaults to ``BoxMode.XYXY_ABS``.
Args:
dataset_dicts (list[dict]): annotations in cvpods Dataset format.
proposal_file (str): file path of pre-computed proposals, in pkl format.
Returns:
list[dict]: the same format as dataset_dicts, but added proposal field.
"""
logger.info("Loading proposals from: {}".format(proposal_file))
with megfile.smart_open(proposal_file, "rb") as f:
proposals = pickle.load(f, encoding="latin1")
# Rename the key names in D1 proposal files
rename_keys = {"indexes": "ids", "scores": "objectness_logits"}
for key in rename_keys:
if key in proposals:
proposals[rename_keys[key]] = proposals.pop(key)
# Fetch the indexes of all proposals that are in the dataset
# Convert image_id to str since they could be int.
img_ids = set({str(record["image_id"]) for record in dataset_dicts})
id_to_index = {
str(id): i
for i, id in enumerate(proposals["ids"]) if str(id) in img_ids
}
# Assuming default bbox_mode of precomputed proposals are 'XYXY_ABS'
bbox_mode = BoxMode(proposals["bbox_mode"]) if "bbox_mode" in proposals else BoxMode.XYXY_ABS
for record in dataset_dicts:
# Get the index of the proposal
i = id_to_index[str(record["image_id"])]
boxes = proposals["boxes"][i]
objectness_logits = proposals["objectness_logits"][i]
# Sort the proposals in descending order of the scores
inds = objectness_logits.argsort()[::-1]
record["proposal_boxes"] = boxes[inds]
record["proposal_objectness_logits"] = objectness_logits[inds]
record["proposal_bbox_mode"] = bbox_mode
return dataset_dicts
def _eval_predictions(self, tasks):
"""
Evaluate self._predictions on the given tasks.
Fill self._results with the metrics of the tasks.
"""
logger.info("Preparing results for COCO format ...")
self._coco_results = list(
itertools.chain(*[x["instances"] for x in self._predictions]))
# unmap the category ids for COCO
if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"):
reverse_id_mapping = {
v: k
for k, v in
self._metadata.thing_dataset_id_to_contiguous_id.items()
}
for result in self._coco_results:
category_id = result["category_id"]
assert (
category_id in reverse_id_mapping
), "A prediction has category_id={}, which is not available in the dataset.".format(
category_id)
result["category_id"] = reverse_id_mapping[category_id]
if self._output_dir:
file_path = os.path.join(self._output_dir,
"coco_instances_results.json")
logger.info("Saving results to {}".format(file_path))
with megfile.smart_open(file_path, "w") as f:
f.write(json.dumps(self._coco_results))
f.flush()
if not self._do_evaluation:
logger.info("Annotations are not available for evaluation.")
return
logger.info("Evaluating predictions ...")
for task in sorted(tasks):
coco_eval, summary = (
_evaluate_predictions_on_coco(
self._coco_api,
self._coco_results,
task,
kpt_oks_sigmas=self._kpt_oks_sigmas)
if len(self._coco_results) > 0 else
None # cocoapi does not handle empty results very well
)
logger.info("\n" + summary.getvalue())
res = self._derive_coco_results(
coco_eval,
task,
summary,
class_names=self._metadata.thing_classes)
self._results[task] = res
def open(path: str, mode: str = "r"):
"""
Open a stream to a URI, similar to the built-in `open`.
Args:
path (str): A URI supported by this PathHandler
Returns:
file: a file-like object.
"""
return megfile.smart_open(path, mode)
def inner_func(path, *args, **kwargs):
cache_dir = get_cache_dir()
protocol, path_without_protocol = megfile.SmartPath._extract_protocol(
path)
local_path = os.path.join(cache_dir, path_without_protocol)
if megfile.smart_exists(local_path): # already cached
return megfile.smart_open(local_path, *args, **kwargs)
# caching logic
if protocol == "s3":
with file_lock(local_path):
megfile.s3_download(path, local_path)
elif protocol == "http" or protocol == "https":
with file_lock(local_path):
if not isinstance(path, str):
path = path.abspath()
download(path,
os.path.dirname(local_path),
filename=os.path.basename(local_path))
return megfile.smart_open(local_path, *args, **kwargs)
def _load_file(self, filename):
"""
Args:
filename (str): load checkpoint file from local or oss. checkpoint can be of type
pkl, pth
"""
if filename.endswith(".pkl"):
with megfile.smart_open(filename, "rb") as f:
data = pickle.load(f, encoding="latin1")
if "model" in data and "__author__" in data:
# file is in cvpods model zoo format
logger.info("Reading a file from '{}'".format(
data["__author__"]))
return data
else:
# assume file is from Caffe2 / Detectron1 model zoo
if "blobs" in data:
# Detection models have "blobs", but ImageNet models don't
data = data["blobs"]
data = {
k: v
for k, v in data.items() if not k.endswith("_momentum")
}
return {
"model": data,
"__author__": "Caffe2",
"matching_heuristics": True
}
elif filename.endswith(".pth"):
if filename.startswith("s3://"):
with megfile.smart_open(filename, "rb") as f:
loaded = torch.load(f, map_location=torch.device("cpu"))
else:
loaded = super()._load_file(
filename) # load native pth checkpoint
if "model" not in loaded:
loaded = {"model": loaded}
return loaded
def get_checkpoint_file(self):
"""
Returns:
str: The latest checkpoint file in target directory.
"""
save_file = os.path.join(self.save_dir, "last_checkpoint")
try:
with megfile.smart_open(save_file, "r") as f:
last_saved = f.read().strip()
except IOError:
# if file doesn't exist, maybe because it has just been
# deleted by a separate process
return ""
return os.path.join(self.save_dir, last_saved)
def build_darknet_backbone(cfg, input_shape):
depth = cfg.MODEL.DARKNET.DEPTH
stem_channels = cfg.MODEL.DARKNET.STEM_OUT_CHANNELS
output_features = cfg.MODEL.DARKNET.OUT_FEATURES
model = Darknet(depth, input_shape.channels, stem_channels,
output_features)
filename = cfg.MODEL.DARKNET.WEIGHTS
if filename.startswith("s3://"):
with megfile.smart_open(filename, "rb") as f:
state_dict = torch.load(f, map_location='cpu')
model.load_state_dict(state_dict)
return model
def _eval_predictions(self, tasks):
"""
Evaluate self._predictions on the given tasks.
Fill self._results with the metrics of the tasks.
"""
logger.info("Preparing results for COCO format ...")
self._coco_results = list(
itertools.chain(*[x["instances"] for x in self._predictions]))
# unmap the category ids for COCO
if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"):
reverse_id_mapping = {
v: k
for k, v in
self._metadata.thing_dataset_id_to_contiguous_id.items()
}
for result in self._coco_results:
result["category_id"] = reverse_id_mapping[
result["category_id"]]
if self._output_dir:
file_path = os.path.join(self._output_dir,
"coco_instances_results.json")
logger.info("Saving results to {}".format(file_path))
with megfile.smart_open(file_path, "w") as f:
f.write(json.dumps(self._coco_results))
f.flush()
if not self._do_evaluation:
logger.info("Annotations are not available for evaluation.")
return
logger.info("Evaluating predictions ...")
for task in sorted(tasks):
assert task == "bbox", "Task {} is not supported".format(task)
coco_eval = (
self._evaluate_predictions_on_coco(self._coco_api,
self._coco_results)
if len(self._coco_results) > 0 else
None # cocoapi does not handle empty results very well
)
res = self._derive_coco_results(
coco_eval,
task,
class_names=self._metadata.get("thing_classes"))
self._results[task] = res
def _eval_box_proposals(self):
"""
Evaluate the box proposals in self._predictions.
Fill self._results with the metrics for "box_proposals" task.
"""
if self._output_dir:
# Saving generated box proposals to file.
# Predicted box_proposals are in XYXY_ABS mode.
bbox_mode = BoxMode.XYXY_ABS.value
ids, boxes, objectness_logits = [], [], []
for prediction in self._predictions:
ids.append(prediction["image_id"])
boxes.append(
prediction["proposals"].proposal_boxes.tensor.numpy())
objectness_logits.append(
prediction["proposals"].objectness_logits.numpy())
proposal_data = {
"boxes": boxes,
"objectness_logits": objectness_logits,
"ids": ids,
"bbox_mode": bbox_mode,
}
with megfile.smart_open(
os.path.join(self._output_dir, "box_proposals.pkl"),
"wb") as f:
pickle.dump(proposal_data, f)
if not self._do_evaluation:
logger.info("Annotations are not available for evaluation.")
return
logger.info("Evaluating bbox proposals ...")
res = {}
areas = {"all": "", "small": "s", "medium": "m", "large": "l"}
for limit in [100, 1000]:
for area, suffix in areas.items():
stats = _evaluate_box_proposals(self._predictions,
self._coco_api,
area=area,
limit=limit)
key = "AR{}@{:d}".format(suffix, limit)
res[key] = float(stats["ar"].item() * 100)
logger.info("Proposal metrics: \n" + create_small_table(res))
self._results["box_proposals"] = res
def _eval_predictions(self, tasks):
"""
Evaluate self._predictions on the given tasks.
Fill self._results with the metrics of the tasks.
"""
logger.info("Preparing results in the LVIS format ...")
self._lvis_results = list(itertools.chain(*[x["instances"] for x in self._predictions]))
# LVIS evaluator can be used to evaluate results for COCO dataset categories.
# In this case `_metadata` variable will have a field with COCO-specific category mapping.
if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"):
reverse_id_mapping = {
v: k for k, v in self._metadata.thing_dataset_id_to_contiguous_id.items()
}
for result in self._lvis_results:
result["category_id"] = reverse_id_mapping[result["category_id"]]
else:
# unmap the category ids for LVIS (from 0-indexed to 1-indexed)
for result in self._lvis_results:
result["category_id"] += 1
if self._output_dir:
file_path = os.path.join(self._output_dir, "lvis_instances_results.json")
logger.info("Saving results to {}".format(file_path))
with megfile.smart_open(file_path, "w") as f:
f.write(json.dumps(self._lvis_results))
f.flush()
if not self._do_evaluation:
logger.info("Annotations are not available for evaluation.")
return
logger.info("Evaluating predictions ...")
for task in sorted(tasks):
lvis_eval, summary = (
_evaluate_predictions_on_lvis(
self._lvis_api, self._lvis_results, task
)
if len(self._lvis_results) > 0
else None
)
logger.info("\n" + summary.getvalue())
res = self._derive_lvis_results(lvis_eval, task, summary)
self._results[task] = res
def default_setup(cfg, args):
"""
Perform some basic common setups at the beginning of a job, including:
1. Set up the cvpods logger
2. Log basic information about environment, cmdline arguments, and config
3. Backup the config to the output directory
Args:
cfg (BaseConfig): the full config to be used
args (argparse.NameSpace): the command line arguments to be logged
"""
output_dir = cfg.OUTPUT_DIR
if comm.is_main_process() and output_dir:
ensure_dir(output_dir)
rank = comm.get_rank()
# setup_logger(output_dir, distributed_rank=rank, name="cvpods")
setup_logger(output_dir, distributed_rank=rank)
logger.info("Rank of current process: {}. World size: {}".format(
rank, comm.get_world_size()))
logger.info("Environment info:\n" + collect_env_info())
logger.info("Command line arguments: " + str(args))
if hasattr(args, "config_file") and args.config_file != "":
logger.info("Contents of args.config_file={}:\n{}".format(
args.config_file,
megfile.smart_open(args.config_file, "r").read()))
adjust_config(cfg)
# make sure each worker has a different, yet deterministic seed if specified
seed = seed_all_rng(None if cfg.SEED < 0 else cfg.SEED + rank)
# save seed to config for dump
cfg.SEED = seed
# cudnn benchmark has large overhead. It shouldn't be used considering the small size of
# typical validation set.
if not (hasattr(args, "eval_only") and args.eval_only):
torch.backends.cudnn.benchmark = cfg.CUDNN_BENCHMARK
return cfg
def read_image(file_name, format=None):
"""
Read an image into the given format.
Will apply rotation and flipping if the image has such exif information.
Args:
file_name (str): image file path
format (str): one of the supported image modes in PIL, or "BGR" or "YUV-BT.601".
Returns:
image (np.ndarray): an HWC image in the given format, which is 0-255, uint8 for
supported image modes in PIL or "BGR"; float (0-1 for Y) for YUV-BT.601.
"""
with megfile.smart_open(file_name, "rb") as f:
image = Image.open(f)
if format == "RGB":
image = image.convert(
format) # avoid warnings caused by _apply_exif_orientation
return np.array(image)
else:
# work around this bug: https://github.com/python-pillow/Pillow/issues/3973
image = _apply_exif_orientation(image)
return convert_PIL_to_numpy(image, format)
def evaluate(self):
"""
Evaluates standard semantic segmentation metrics (http://cocodataset.org/#stuff-eval):
* Mean intersection-over-union averaged across classes (mIoU)
* Frequency Weighted IoU (fwIoU)
* Mean pixel accuracy averaged across classes (mACC)
* Pixel Accuracy (pACC)
"""
if self._distributed:
comm.synchronize()
conf_matrix_list = comm.all_gather(self._conf_matrix)
self._predictions = comm.all_gather(self._predictions)
self._predictions = list(itertools.chain(*self._predictions))
if not comm.is_main_process():
return
self._conf_matrix = np.zeros_like(self._conf_matrix)
for conf_matrix in conf_matrix_list:
self._conf_matrix += conf_matrix
if self._output_dir:
ensure_dir(self._output_dir)
file_path = os.path.join(self._output_dir,
"sem_seg_predictions.json")
with megfile.smart_open(file_path, "w") as f:
f.write(json.dumps(self._predictions))
acc = np.zeros(self._num_classes, dtype=np.float)
iou = np.zeros(self._num_classes, dtype=np.float)
tp = self._conf_matrix.diagonal()[:-1].astype(np.float)
pos_gt = np.sum(self._conf_matrix[:-1, :-1], axis=0).astype(np.float)
class_weights = pos_gt / np.sum(pos_gt)
pos_pred = np.sum(self._conf_matrix[:-1, :-1], axis=1).astype(np.float)
acc_valid = pos_gt > 0
acc[acc_valid] = tp[acc_valid] / pos_gt[acc_valid]
iou_valid = (pos_gt + pos_pred) > 0
union = pos_gt + pos_pred - tp
iou[acc_valid] = tp[acc_valid] / union[acc_valid]
macc = np.sum(acc) / np.sum(acc_valid)
miou = np.sum(iou) / np.sum(iou_valid)
fiou = np.sum(iou * class_weights)
pacc = np.sum(tp) / np.sum(pos_gt)
res = {}
res["mIoU"] = 100 * miou
res["fwIoU"] = 100 * fiou
res["mACC"] = 100 * macc
res["pACC"] = 100 * pacc
if self._output_dir:
file_path = os.path.join(self._output_dir,
"sem_seg_evaluation.pth")
with megfile.smart_open(file_path, "wb") as f:
torch.save(res, f)
results = OrderedDict({"sem_seg": res})
small_table = create_small_table(res)
logger.info("Evaluation results for sem_seg: \n" + small_table)
if self._dump:
dump_info_one_task = {
"task": "sem_seg",
"tables": [small_table],
}
_dump_to_markdown([dump_info_one_task])
return results
def is_file(self):
d2_path = self.DETECTRON2_PREFIX + self.path_without_protocol
return megfile.smart_open(d2_path)
def open(self, mode: str = 'r', **kwargs) -> IO[AnyStr]:
# TODO add cache logic
d2_path = self.DETECTRON2_PREFIX + self.path_without_protocol
return megfile.smart_open(d2_path, mode=mode, **kwargs)
def open(self, mode="r", **kwargs):
catalog_path = ModelCatalog.get(self.path_without_protocol)
logger.info("Catalog entry {} points to {}".format(
self.path, catalog_path))
return megfile.smart_open(catalog_path, mode, **kwargs)
"--config",
required=True,
help="path to a python file with a definition of `config`")
parser.add_argument(
"--dataset",
help="name of the dataset. Use DATASETS.TEST[0] if not specified.",
default="")
parser.add_argument("--conf-threshold",
default=0.5,
type=float,
help="confidence threshold")
args = parser.parse_args()
logger = setup_logger()
cfg = setup_cfg(args.config, logger)
with megfile.smart_open(args.input, "r") as f:
predictions = json.load(f)
pred_by_image = defaultdict(list)
for p in predictions:
pred_by_image[p["image_id"]].append(p)
# TODO: add DatasetCatalog, MetadataCatalog
dataset = build_dataset(
cfg, [args.dataset] if args.dataset else [cfg.DATASETS.TEST[0]],
transforms=[],
is_train=False)
dicts = dataset.datasets[0].dataset_dicts
metadata = dataset.meta
if hasattr(metadata, "thing_dataset_id_to_contiguous_id"):
def cityscapes_files_to_dict(files, from_json, to_polygons):
"""
Parse cityscapes annotation files to a dict.
Args:
files (tuple): consists of (image_file, instance_id_file, label_id_file, json_file)
from_json (bool): whether to read annotations from the raw json file or the png files.
to_polygons (bool): whether to represent the segmentation as polygons
(COCO's format) instead of masks (cityscapes's format).
Returns:
A dict in cvpods Dataset format.
"""
from cityscapesscripts.helpers.labels import id2label, name2label
image_file, instance_id_file, _, json_file = files
annos = []
if from_json:
from shapely.geometry import MultiPolygon, Polygon
with megfile.smart_open(json_file, "r") as f:
jsonobj = json.load(f)
ret = {
"file_name": image_file,
"image_id": os.path.basename(image_file),
"height": jsonobj["imgHeight"],
"width": jsonobj["imgWidth"],
}
# `polygons_union` contains the union of all valid polygons.
polygons_union = Polygon()
# CityscapesScripts draw the polygons in sequential order
# and each polygon *overwrites* existing ones. See
# (https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/preparation/json2instanceImg.py) # noqa
# We use reverse order, and each polygon *avoids* early ones.
# This will resolve the ploygon overlaps in the same way as CityscapesScripts.
for obj in jsonobj["objects"][::-1]:
if "deleted" in obj: # cityscapes data format specific
continue
label_name = obj["label"]
try:
label = name2label[label_name]
except KeyError:
if label_name.endswith("group"): # crowd area
label = name2label[label_name[:-len("group")]]
else:
raise
if label.id < 0: # cityscapes data format
continue
# Cityscapes's raw annotations uses integer coordinates
# Therefore +0.5 here
poly_coord = np.asarray(obj["polygon"], dtype="f4") + 0.5
# CityscapesScript uses PIL.ImageDraw.polygon to rasterize
# polygons for evaluation. This function operates in integer space
# and draws each pixel whose center falls into the polygon.
# Therefore it draws a polygon which is 0.5 "fatter" in expectation.
# We therefore dilate the input polygon by 0.5 as our input.
poly = Polygon(poly_coord).buffer(0.5, resolution=4)
if not label.hasInstances or label.ignoreInEval:
# even if we won't store the polygon it still contributes to overlaps resolution
polygons_union = polygons_union.union(poly)
continue
# Take non-overlapping part of the polygon
poly_wo_overlaps = poly.difference(polygons_union)
if poly_wo_overlaps.is_empty:
continue
polygons_union = polygons_union.union(poly)
anno = {}
anno["iscrowd"] = label_name.endswith("group")
anno["category_id"] = label.id
if isinstance(poly_wo_overlaps, Polygon):
poly_list = [poly_wo_overlaps]
elif isinstance(poly_wo_overlaps, MultiPolygon):
poly_list = poly_wo_overlaps.geoms
else:
raise NotImplementedError(
"Unknown geometric structure {}".format(poly_wo_overlaps))
poly_coord = []
for poly_el in poly_list:
# COCO API can work only with exterior boundaries now, hence we store only them.
# TODO: store both exterior and interior boundaries once other parts of the
# codebase support holes in polygons.
poly_coord.append(list(chain(*poly_el.exterior.coords)))
anno["segmentation"] = poly_coord
(xmin, ymin, xmax, ymax) = poly_wo_overlaps.bounds
anno["bbox"] = (xmin, ymin, xmax, ymax)
anno["bbox_mode"] = BoxMode.XYXY_ABS
annos.append(anno)
else:
# See also the official annotation parsing scripts at
# https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/instances2dict.py # noqa
with megfile.smart_open(instance_id_file, "rb") as f:
inst_image = np.asarray(Image.open(f), order="F")
# ids < 24 are stuff labels (filtering them first is about 5% faster)
flattened_ids = np.unique(inst_image[inst_image >= 24])
ret = {
"file_name": image_file,
"image_id": os.path.basename(image_file),
"height": inst_image.shape[0],
"width": inst_image.shape[1],
}
for instance_id in flattened_ids:
# For non-crowd annotations, instance_id // 1000 is the label_id
# Crowd annotations have <1000 instance ids
label_id = instance_id // 1000 if instance_id >= 1000 else instance_id
label = id2label[label_id]
if not label.hasInstances or label.ignoreInEval:
continue
anno = {}
anno["iscrowd"] = instance_id < 1000
anno["category_id"] = label.id
mask = np.asarray(inst_image == instance_id,
dtype=np.uint8,
order="F")
inds = np.nonzero(mask)
ymin, ymax = inds[0].min(), inds[0].max()
xmin, xmax = inds[1].min(), inds[1].max()
anno["bbox"] = (xmin, ymin, xmax, ymax)
if xmax <= xmin or ymax <= ymin:
continue
anno["bbox_mode"] = BoxMode.XYXY_ABS
if to_polygons:
# This conversion comes from D4809743 and D5171122,
# when Mask-RCNN was first developed.
contours = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_NONE)[-2]
polygons = [
c.reshape(-1).tolist() for c in contours if len(c) >= 3
]
# opencv's can produce invalid polygons
if len(polygons) == 0:
continue
anno["segmentation"] = polygons
else:
anno["segmentation"] = mask_util.encode(mask[:, :, None])[0]
annos.append(anno)
ret["annotations"] = annos
return ret
