def load_mask(path, inverse_colormap=None):
mask = load_image(path)
mask = mask.astype(np.uint8)
if inverse_colormap is not None:
if len(mask.shape) == 3 and mask.shape[2] != 1:
mask = unpaint_mask(mask, inverse_colormap)
return mask
def _load_items(self):
items = {}
image_path_by_id = {}
if self._images_dir:
image_path_by_id = {
self._get_id_from_image_path(p): p
for p in find_images(self._images_dir, recursive=True)
}
masks = glob.glob(osp.join(
self._gt_anns_dir, '**',
f'*{CityscapesPath.LABEL_TRAIN_IDS_SUFFIX}'),
recursive=True)
mask_suffix = CityscapesPath.LABEL_TRAIN_IDS_SUFFIX
if not masks:
masks = glob.glob(osp.join(
self._gt_anns_dir, '**',
f'*{CityscapesPath.GT_INSTANCE_MASK_SUFFIX}'),
recursive=True)
mask_suffix = CityscapesPath.GT_INSTANCE_MASK_SUFFIX
for mask_path in masks:
item_id = self._get_id_from_mask_path(mask_path, mask_suffix)
anns = []
instances_mask = load_image(mask_path, dtype=np.int32)
segm_ids = np.unique(instances_mask)
for segm_id in segm_ids:
# either is_crowd or ann_id should be set
if segm_id < 1000:
label_id = segm_id
is_crowd = True
ann_id = None
else:
label_id = segm_id // 1000
is_crowd = False
ann_id = segm_id % 1000
anns.append(
Mask(image=self._lazy_extract_mask(instances_mask,
segm_id),
label=label_id,
id=ann_id,
attributes={'is_crowd': is_crowd}))
items[item_id] = DatasetItem(id=item_id,
subset=self._subset,
image=image_path_by_id.pop(
item_id, None),
annotations=anns)
for item_id, path in image_path_by_id.items():
items[item_id] = DatasetItem(id=item_id,
subset=self._subset,
image=path)
self._categories = self._load_categories(self._path,
use_train_label_map= \
mask_suffix is CityscapesPath.LABEL_TRAIN_IDS_SUFFIX)
return items
def _load_instances_items(self):
items = {}
instances_dir = osp.join(self._annotations_dir, MapillaryVistasPath.INSTANCES_DIR)
for instance_path in find_images(instances_dir, recursive=True):
item_id = osp.splitext(osp.relpath(instance_path, instances_dir))[0]
mask = load_image(instance_path, dtype=np.uint32)
annotations = []
for uval in np.unique(mask):
label_id, instance_id = uval >> 8, uval & 255
annotations.append(
Mask(
image=self._lazy_extract_mask(mask, uval),
label=label_id, id=instance_id
)
)
items[item_id] = DatasetItem(id=item_id, subset=self._subset,
annotations=annotations)
class_dir = osp.join(self._annotations_dir, MapillaryVistasPath.CLASS_DIR)
for class_path in find_images(class_dir, recursive=True):
item_id = osp.splitext(osp.relpath(class_path, class_dir))[0]
if item_id in items:
continue
from PIL import Image as PILImage
class_mask = np.array(PILImage.open(class_path))
classes = np.unique(class_mask)
annotations = []
for label_id in classes:
annotations.append(Mask(label=label_id,
image=self._lazy_extract_mask(class_mask, label_id))
)
items[item_id] = DatasetItem(id=item_id, subset=self._subset,
annotations=annotations)
for image_path in find_images(self._images_dir, recursive=True):
item_id = osp.splitext(osp.relpath(image_path, self._images_dir))[0]
image = Image(path=image_path)
if item_id in items:
items[item_id].image = image
else:
items[item_id] = DatasetItem(id=item_id, subset=self._subset,
image=image)
self._load_polygons(items)
return items.values()
def _test_can_save_and_load(self,
src_image,
path,
save_backend=None,
load_backend=None):
if save_backend:
image_module._IMAGE_BACKEND = save_backend
image_module.save_image(path, src_image)
if load_backend:
image_module._IMAGE_BACKEND = load_backend
dst_image = image_module.load_image(path)
self.assertTrue(np.all(src_image == dst_image), 'save: %s, load: %s' % \
(save_backend, load_backend))
def test_save_and_load_backends(self):
backends = image_module._IMAGE_BACKENDS
for save_backend, load_backend, c in product(backends, backends, [1, 3]):
with TestDir() as test_dir:
if c == 1:
src_image = np.random.randint(0, 255 + 1, (2, 4))
else:
src_image = np.random.randint(0, 255 + 1, (2, 4, c))
path = osp.join(test_dir, 'img.png') # lossless
image_module._IMAGE_BACKEND = save_backend
image_module.save_image(path, src_image, jpeg_quality=100)
image_module._IMAGE_BACKEND = load_backend
dst_image = image_module.load_image(path)
self.assertTrue(np.array_equal(src_image, dst_image),
'save: %s, load: %s' % (save_backend, load_backend))
def _parse_annotations(self, path):
combined_mask = load_image(path, dtype=np.uint16)
masks = []
for obj_id in np.unique(combined_mask):
class_id, instance_id = divmod(obj_id, MotsPath.MAX_INSTANCES)
z_order = 0
if class_id == 0:
continue # background
if class_id == 10 and \
len(self._categories[AnnotationType.label]) < 10:
z_order = 1
class_id = self._categories[AnnotationType.label].find(
MotsLabels.ignored.name)[0]
else:
class_id -= 1
masks.append(
Mask(self._lazy_extract_mask(combined_mask, obj_id),
label=class_id,
z_order=z_order,
attributes={'track_id': instance_id}))
return masks
def _load_pan_mask(path):
mask = load_image(path)
mask = bgr2index(mask)
return mask
def is_image_path(value):
try:
return load_image(value) is not None
except Exception:
return False
def _image_loader(item_id, extractor):
if not extractor._is_image_cached(item_id):
extractor._download_image(item_id)
local_path = extractor._image_local_path(item_id)
return load_image(local_path)
def explain_command(args):
project_path = args.project_dir
if is_project_path(project_path):
project = Project.load(project_path)
else:
project = None
args.target = target_selector(
ProjectTarget(is_default=True, project=project),
SourceTarget(project=project), ImageTarget())(args.target)
if args.target[0] == TargetKinds.project:
if is_project_path(args.target[1]):
args.project_dir = osp.dirname(osp.abspath(args.target[1]))
import cv2
from matplotlib import cm
project = load_project(args.project_dir)
model = project.make_executable_model(args.model)
if str(args.algorithm).lower() != 'rise':
raise NotImplementedError()
from datumaro.components.algorithms.rise import RISE
rise = RISE(model,
max_samples=args.max_samples,
mask_width=args.mask_width,
mask_height=args.mask_height,
prob=args.prob,
iou_thresh=args.iou_thresh,
nms_thresh=args.nms_iou_thresh,
det_conf_thresh=args.det_conf_thresh,
batch_size=args.batch_size)
if args.target[0] == TargetKinds.image:
image_path = args.target[1]
image = load_image(image_path)
log.info("Running inference explanation for '%s'" % image_path)
heatmap_iter = rise.apply(image, progressive=args.display)
image = image / 255.0
file_name = osp.splitext(osp.basename(image_path))[0]
if args.display:
for i, heatmaps in enumerate(heatmap_iter):
for j, heatmap in enumerate(heatmaps):
hm_painted = cm.jet(heatmap)[:, :, 2::-1]
disp = (image + hm_painted) / 2
cv2.imshow('heatmap-%s' % j, hm_painted)
cv2.imshow(file_name + '-heatmap-%s' % j, disp)
cv2.waitKey(10)
print("Iter", i, "of", args.max_samples, end='\r')
else:
heatmaps = next(heatmap_iter)
if args.save_dir is not None:
log.info("Saving inference heatmaps at '%s'" % args.save_dir)
os.makedirs(args.save_dir, exist_ok=True)
for j, heatmap in enumerate(heatmaps):
save_path = osp.join(args.save_dir,
file_name + '-heatmap-%s.png' % j)
save_image(save_path, heatmap * 255.0)
else:
for j, heatmap in enumerate(heatmaps):
disp = (image + cm.jet(heatmap)[:, :, 2::-1]) / 2
cv2.imshow(file_name + '-heatmap-%s' % j, disp)
cv2.waitKey(0)
elif args.target[0] == TargetKinds.source or \
args.target[0] == TargetKinds.project:
if args.target[0] == TargetKinds.source:
source_name = args.target[1]
dataset = project.make_source_project(source_name).make_dataset()
log.info("Running inference explanation for '%s'" % source_name)
else:
project_name = project.config.project_name
dataset = project.make_dataset()
log.info("Running inference explanation for '%s'" % project_name)
for item in dataset:
image = item.image.data
if image is None:
log.warn(
"Dataset item %s does not have image data. Skipping." % \
(item.id))
continue
heatmap_iter = rise.apply(image)
image = image / 255.0
heatmaps = next(heatmap_iter)
if args.save_dir is not None:
log.info("Saving inference heatmaps to '%s'" % args.save_dir)
os.makedirs(args.save_dir, exist_ok=True)
for j, heatmap in enumerate(heatmaps):
save_image(osp.join(args.save_dir,
item.id + '-heatmap-%s.png' % j),
heatmap * 255.0,
create_dir=True)
if not args.save_dir or args.display:
for j, heatmap in enumerate(heatmaps):
disp = (image + cm.jet(heatmap)[:, :, 2::-1]) / 2
cv2.imshow(item.id + '-heatmap-%s' % j, disp)
cv2.waitKey(0)
else:
raise NotImplementedError()
return 0
def _load_items(self, root_dir):
image_dir = osp.join(root_dir, SynthiaPath.IMAGES_DIR)
if osp.isdir(image_dir):
images = {
osp.splitext(osp.relpath(p, image_dir))[0].replace('\\', '/'):
p
for p in find_images(image_dir, recursive=True)
}
else:
images = {}
items = {}
inst_dir = osp.join(root_dir, SynthiaPath.LABELS_SEGM_DIR)
if osp.isdir(inst_dir):
gt_images = find_images(inst_dir, recursive=True)
for gt_img in gt_images:
item_id = osp.splitext(osp.relpath(gt_img,
inst_dir))[0].replace(
'\\', '/')
anno = []
labels_mask = load_image(gt_img, dtype=np.uint16)
dynamic_objects = np.unique(labels_mask[:, :, 1])
labels_mask = labels_mask[:, :, 2]
segm_ids = np.unique(labels_mask)
for segm_id in segm_ids:
attr = {'dynamic_object': False}
if segm_id != 0 and segm_id in dynamic_objects:
attr['dynamic_object'] = True
anno.append(
Mask(image=self._lazy_extract_mask(
labels_mask, segm_id),
label=segm_id,
attributes=attr))
items[item_id] = DatasetItem(id=item_id,
image=images[item_id],
annotations=anno)
elif osp.isdir(osp.join(root_dir, SynthiaPath.SEMANTIC_SEGM_DIR)):
gt_dir = osp.join(root_dir, SynthiaPath.SEMANTIC_SEGM_DIR)
gt_images = find_images(gt_dir, recursive=True)
for gt_img in gt_images:
item_id = osp.splitext(osp.relpath(gt_img, gt_dir))[0].replace(
'\\', '/')
anno = []
inverse_cls_colormap = \
self._categories[AnnotationType.mask].inverse_colormap
color_mask = lazy_mask(gt_img, inverse_cls_colormap)
color_mask = color_mask()
classes = np.unique(color_mask)
for label_id in classes:
anno.append(
Mask(image=self._lazy_extract_mask(
color_mask, label_id),
label=label_id))
items[item_id] = DatasetItem(id=item_id,
image=images[item_id],
annotations=anno)
return items
def _load_and_resize_mask(path, size):
raw = load_image(path, dtype=np.uint8)
resized = cv2.resize(raw, (size[1], size[0]),
interpolation=cv2.INTER_NEAREST)
return resized.astype(bool)
def _load_items(self):
items = {}
image_dir = osp.join(self._path, KittiPath.IMAGES_DIR)
image_path_by_id = {
osp.splitext(osp.relpath(p, image_dir))[0]: p
for p in find_images(image_dir, recursive=True)
}
segm_dir = osp.join(self._path, KittiPath.INSTANCES_DIR)
if self._task == KittiTask.segmentation:
for instances_path in find_images(segm_dir,
exts=KittiPath.MASK_EXT,
recursive=True):
item_id = osp.splitext(osp.relpath(instances_path,
segm_dir))[0]
anns = []
instances_mask = load_image(instances_path, dtype=np.int32)
segm_ids = np.unique(instances_mask)
for segm_id in segm_ids:
semantic_id = segm_id >> 8
ann_id = int(segm_id % 256)
isCrowd = (ann_id == 0)
anns.append(
Mask(image=self._lazy_extract_mask(
instances_mask, segm_id),
label=semantic_id,
id=ann_id,
attributes={'is_crowd': isCrowd}))
items[item_id] = DatasetItem(id=item_id,
annotations=anns,
image=image_path_by_id.pop(
item_id, None),
subset=self._subset)
det_dir = osp.join(self._path, KittiPath.LABELS_DIR)
if self._task == KittiTask.detection:
for labels_path in sorted(
glob.glob(osp.join(det_dir, '**', '*.txt'),
recursive=True)):
item_id = osp.splitext(osp.relpath(labels_path, det_dir))[0]
anns = []
with open(labels_path, 'r', encoding='utf-8') as f:
lines = f.readlines()
for line_idx, line in enumerate(lines):
line = line.split()
assert len(line) == 15 or len(line) == 16
x1, y1 = float(line[4]), float(line[5])
x2, y2 = float(line[6]), float(line[7])
attributes = {}
attributes['truncated'] = float(line[1]) != 0
attributes['occluded'] = int(line[2]) != 0
if len(line) == 16:
attributes['score'] = float(line[15])
label_id = self.categories()[AnnotationType.label].find(
line[0])[0]
if label_id is None:
label_id = self.categories()[AnnotationType.label].add(
line[0])
anns.append(
Bbox(
x=x1,
y=y1,
w=x2 - x1,
h=y2 - y1,
id=line_idx,
attributes=attributes,
label=label_id,
))
items[item_id] = DatasetItem(id=item_id,
annotations=anns,
image=image_path_by_id.pop(
item_id, None),
subset=self._subset)
for item_id, image_path in image_path_by_id.items():
items[item_id] = DatasetItem(id=item_id,
subset=self._subset,
image=image_path)
return items
def _load_class_mask(path):
mask = load_image(path)
mask = ((mask[:, :, 2] / 10).astype(np.int32) << 8) \
+ mask[:, :, 1].astype(np.int32)
return mask
def _load_instance_mask(path):
mask = load_image(path)
_, instance_mask = np.unique(mask, return_inverse=True)
instance_mask = instance_mask.reshape(mask.shape)
return instance_mask
def explain_command(args):
from matplotlib import cm
import cv2
project = scope_add(load_project(args.project_dir))
model = project.working_tree.models.make_executable_model(args.model)
if str(args.algorithm).lower() != 'rise':
raise NotImplementedError()
from datumaro.components.algorithms.rise import RISE
rise = RISE(model,
max_samples=args.max_samples,
mask_width=args.mask_width,
mask_height=args.mask_height,
prob=args.prob,
iou_thresh=args.iou_thresh,
nms_thresh=args.nms_iou_thresh,
det_conf_thresh=args.det_conf_thresh,
batch_size=args.batch_size)
if args.target and is_image(args.target):
image_path = args.target
image = load_image(image_path)
log.info("Running inference explanation for '%s'" % image_path)
heatmap_iter = rise.apply(image, progressive=args.display)
image = image / 255.0
file_name = osp.splitext(osp.basename(image_path))[0]
if args.display:
for i, heatmaps in enumerate(heatmap_iter):
for j, heatmap in enumerate(heatmaps):
hm_painted = cm.jet(heatmap)[:, :, 2::-1]
disp = (image + hm_painted) / 2
cv2.imshow('heatmap-%s' % j, hm_painted)
cv2.imshow(file_name + '-heatmap-%s' % j, disp)
cv2.waitKey(10)
print("Iter", i, "of", args.max_samples, end='\r')
else:
heatmaps = next(heatmap_iter)
if args.save_dir is not None:
log.info("Saving inference heatmaps at '%s'" % args.save_dir)
os.makedirs(args.save_dir, exist_ok=True)
for j, heatmap in enumerate(heatmaps):
save_path = osp.join(args.save_dir,
file_name + '-heatmap-%s.png' % j)
save_image(save_path, heatmap * 255.0)
else:
for j, heatmap in enumerate(heatmaps):
disp = (image + cm.jet(heatmap)[:, :, 2::-1]) / 2
cv2.imshow(file_name + '-heatmap-%s' % j, disp)
cv2.waitKey(0)
else:
dataset, target_project = \
parse_full_revpath(args.target or 'project', project)
if target_project:
scope_add(target_project)
log.info("Running inference explanation for '%s'" % args.target)
for item in dataset:
image = item.image.data
if image is None:
log.warning("Item %s does not have image data. Skipping.",
item.id)
continue
heatmap_iter = rise.apply(image)
image = image / 255.0
heatmaps = next(heatmap_iter)
if args.save_dir is not None:
log.info("Saving inference heatmaps to '%s'" % args.save_dir)
os.makedirs(args.save_dir, exist_ok=True)
for j, heatmap in enumerate(heatmaps):
save_image(osp.join(args.save_dir,
item.id + '-heatmap-%s.png' % j),
heatmap * 255.0,
create_dir=True)
if not args.save_dir or args.display:
for j, heatmap in enumerate(heatmaps):
disp = (image + cm.jet(heatmap)[:, :, 2::-1]) / 2
cv2.imshow(item.id + '-heatmap-%s' % j, disp)
cv2.waitKey(0)
return 0
