def convert_single_core(proc_id,
image_set,
categories,
source_folder,
segmentations_folder,
VOID=0):
annotations = []
for working_idx, image_info in enumerate(image_set):
if working_idx % 100 == 0:
print('Core: {}, {} from {} images converted'.format(
proc_id, working_idx, len(image_set)))
file_name = '{}.png'.format(image_info['file_name'].rsplit('.')[0])
try:
original_format = np.array(Image.open(
os.path.join(source_folder, file_name)),
dtype=np.uint32)
except IOError:
raise KeyError('no prediction png file for id: {}'.format(
image_info['id']))
pan = OFFSET * original_format[:, :, 0] + original_format[:, :, 1]
pan_format = np.zeros(
(original_format.shape[0], original_format.shape[1], 3),
dtype=np.uint8)
id_generator = IdGenerator(categories)
l = np.unique(pan)
segm_info = []
for el in l:
sem = el // OFFSET
if sem == VOID:
continue
if sem not in categories:
raise KeyError('Unknown semantic label {}'.format(sem))
mask = pan == el
segment_id, color = id_generator.get_id_and_color(sem)
pan_format[mask] = color
segm_info.append({"id": segment_id, "category_id": sem})
annotations.append({
'image_id': image_info['id'],
'file_name': file_name,
"segments_info": segm_info
})
Image.fromarray(pan_format).save(
os.path.join(segmentations_folder, file_name))
print('Core: {}, all {} images processed'.format(proc_id, len(image_set)))
return annotations
def convert_detection_to_panoptic_coco_format_single_core(
proc_id, coco_detection, img_ids, categories, segmentations_folder):
id_generator = IdGenerator(categories)
annotations_panoptic = []
for working_idx, img_id in enumerate(img_ids):
if working_idx % 100 == 0:
print('Core: {}, {} from {} images processed'.format(
proc_id, working_idx, len(img_ids)))
img = coco_detection.loadImgs(int(img_id))[0]
pan_format = np.zeros((img['height'], img['width'], 3), dtype=np.uint8)
overlaps_map = np.zeros((img['height'], img['width']), dtype=np.uint32)
anns_ids = coco_detection.getAnnIds(img_id)
anns = coco_detection.loadAnns(anns_ids)
panoptic_record = {}
panoptic_record['image_id'] = img_id
file_name = '{}.png'.format(img['file_name'].rsplit('.')[0])
panoptic_record['file_name'] = file_name
segments_info = []
for ann in anns:
if ann['category_id'] not in categories:
raise Exception(
'Panoptic coco categories file does not contain \
category with id: {}'.format(ann['category_id']))
segment_id, color = id_generator.get_id_and_color(
ann['category_id'])
mask = coco_detection.annToMask(ann)
overlaps_map += mask
pan_format[mask == 1] = color
ann.pop('segmentation')
ann.pop('image_id')
ann['id'] = segment_id
segments_info.append(ann)
if np.sum(overlaps_map > 1) != 0:
raise Exception(
"Segments for image {} overlap each other.".format(img_id))
panoptic_record['segments_info'] = segments_info
annotations_panoptic.append(panoptic_record)
Image.fromarray(pan_format).save(
os.path.join(segmentations_folder, file_name))
print('Core: {}, all {} images processed'.format(proc_id, len(img_ids)))
return annotations_panoptic
def do_panoptic_test(cfg, model):
categoryies=json.load(open("../data/panoptic_coco_categories.json",'r'))
os.makedirs("../data/detectron2_panoptic", exist_ok=True)
categoryies_dict={}
for category in categoryies:
categoryies_dict[category['id']]=category
id_generator=IdGenerator(categoryies_dict)
image_dict = {}
error_list=[]
for dataset_name in ['viroi_test']:#,'viroi_train']:#cfg.DATASETS.TRAIN:#cfg.DATASETS.TEST:
# data_loader = build_detection_test_loader(cfg, dataset_name)
thing_id_map = MetadataCatalog.get(dataset_name).get("thing_contiguous_id_to_class_id")
stuff_id_map = MetadataCatalog.get(dataset_name).get("stuff_contiguous_id_to_class_id")
test_images_dict=json.load(open(MetadataCatalog.get(dataset_name).get("instance_json_file"),'r'))
image_path = MetadataCatalog.get(dataset_name).get("image_path")
predictor=DefaultPredictor(cfg)
total = len(test_images_dict)
count=0
# for idx, inputs in enumerate(data_loader):
for image_id in test_images_dict:
image_info=test_images_dict[image_id]
img=read_image(image_path+"/"+image_info['image_name'],format="BGR")
count+=1
print(str(count)+"/"+str(total))
if True:
# try:
# print(inputs[0])
# predictions = model(inputs, "panoptic")[0] # 'sem_seg', 'instances', 'panoptic_seg'
predictions = predictor(img,0)
panoptic_seg, segments_info = predictions["panoptic_seg"] # seg, info
panoptic_seg=panoptic_seg.data.cpu().numpy()
panoptic_color_seg = np.zeros((panoptic_seg.shape[0], panoptic_seg.shape[1], 3)) #tensor
instance_dict={}
for info in segments_info:
if 'score' in info:
del info['score']
if 'area' in info:
del info['area']
bbox = info['box'] # x1,y1,x2,y2->y1,x1,y2,x2
info['bbox'] = [int(bbox[1]), int(bbox[0]), int(bbox[3]), int(bbox[2])]
del info['box']
class_id=info['class_id']
del info['category_id']
mask = info['mask'].data.cpu().numpy()
mask = np.asfortranarray(mask)
segmentation = maskUtils.encode(mask)
segmentation['counts'] = segmentation['counts'].decode('utf8')
info['segmentation'] = segmentation
instance_id, panoptic_color_seg[mask] = id_generator.get_id_and_color(categoryies[class_id - 1]['id'])
info['instance_id'] = instance_id
del info['mask']
instance_dict[str(instance_id)]=info
image_dict[image_id]={'instances':instance_dict,
"image_id":image_info['image_id'],
"height":image_info['height'],
"width":image_info['width'],
"image_name":image_info['image_name']
}
# print(image_dict)
Image.fromarray(panoptic_color_seg.astype(np.uint8)).save("../data/detectron2_panoptic/"+image_info["image_name"].replace("jpg","png"))
# except:
# print("ERROR - "+image_info['image_name'])
# error_list.append(image_info['image_name'])
json.dump(image_dict,open("../data/viroi_json/detectron2_"+dataset_name+"_images_dict.json",'w'))
json.dump(error_list,open("error_list.json",'w'))
def process_image(working_idx):
global file_list, categories_dic, output_folder
f = file_list[working_idx]
# print(f)
images = []
img = Image.open(f)
img = img.resize((1280, 720))
original_format = np.array(img)
# print("Processing file", f)
file_name = f.split('/')[-1]
image_id = file_name.rsplit('_', 2)[0]
image_filename = '{}_{}_gtFine_panopticlevel3Ids.png'.format(
f.split('/')[-2], image_id)
# pdb.set_trace()
# image entry, id for image is its filename without extension
image = {
"id": image_filename,
"width": original_format.shape[1],
"height": original_format.shape[0],
"file_name": image_filename
}
pan_format = np.zeros(
(original_format.shape[0], original_format.shape[1], 3),
dtype=np.uint8)
id_generator = IdGenerator(categories_dict)
idx = 0
l = np.unique(original_format)
segm_info = []
for el in l:
if el < 1000:
semantic_id = el
is_crowd = 1
else:
semantic_id = el // 1000
is_crowd = 0
if semantic_id not in categories_dict:
continue
if categories_dict[semantic_id]['isthing'] == 0:
is_crowd = 0
mask = original_format == el
segment_id, color = id_generator.get_id_and_color(semantic_id)
pan_format[mask] = color
area = np.sum(mask) # segment area computation
# bbox computation for a segment
hor = np.sum(mask, axis=0)
hor_idx = np.nonzero(hor)[0]
x = hor_idx[0]
width = hor_idx[-1] - x + 1
vert = np.sum(mask, axis=1)
vert_idx = np.nonzero(vert)[0]
y = vert_idx[0]
height = vert_idx[-1] - y + 1
bbox = [x, y, width, height]
segm_info.append({
"id": int(segment_id),
"category_id": int(semantic_id),
"area": int(area),
"bbox": [int(x) for x in bbox],
"iscrowd": is_crowd
})
Image.fromarray(pan_format).save(
os.path.join(output_folder, image_filename))
return image, segm_info
def panoptic_converter(original_format_folder, out_folder, out_file):
if not os.path.isdir(out_folder):
print("Creating folder {} for panoptic segmentation PNGs".format(
out_folder))
os.mkdir(out_folder)
categories = []
for idx, el in enumerate(labels):
if el.ignoreInEval:
continue
categories.append({
'id': el.id,
'name': el.name,
'color': el.color,
'supercategory': el.category,
'isthing': 1 if el.hasInstances else 0
})
categories_dict = {cat['id']: cat for cat in categories}
file_list = sorted(
glob.glob(
os.path.join(original_format_folder,
'*/*_gtFine_instanceIds.png')))
images = []
annotations = []
for working_idx, f in enumerate(file_list):
if working_idx % 10 == 0:
print(working_idx, len(file_list))
original_format = np.array(Image.open(f))
file_name = f.split('/')[-1]
image_id = file_name.rsplit('_', 2)[0]
image_filename = '{}_leftImg8bit.png'.format(image_id)
# image entry, id for image is its filename without extension
images.append({
"id": image_id,
"width": original_format.shape[1],
"height": original_format.shape[0],
"file_name": image_filename
})
pan_format = np.zeros(
(original_format.shape[0], original_format.shape[1], 3),
dtype=np.uint8)
id_generator = IdGenerator(categories_dict)
ll = np.unique(original_format)
segm_info = []
for el in ll:
if el < 1000:
semantic_id = el
is_crowd = 1
else:
semantic_id = el // 1000
is_crowd = 0
if semantic_id not in categories_dict:
continue
if categories_dict[semantic_id]['isthing'] == 0:
is_crowd = 0
mask = original_format == el
segment_id, color = id_generator.get_id_and_color(semantic_id)
pan_format[mask] = color
area = np.sum(mask) # segment area computation
# bbox computation for a segment
hor = np.sum(mask, axis=0)
hor_idx = np.nonzero(hor)[0]
x = hor_idx[0]
width = hor_idx[-1] - x + 1
vert = np.sum(mask, axis=1)
vert_idx = np.nonzero(vert)[0]
y = vert_idx[0]
height = vert_idx[-1] - y + 1
bbox = [x, y, width, height]
segm_info.append({
"id": int(segment_id),
"category_id": int(semantic_id),
"area": area,
"bbox": bbox,
"iscrowd": is_crowd
})
annotations.append({
'image_id': image_id,
'file_name': file_name,
"segments_info": segm_info
})
Image.fromarray(pan_format).save(os.path.join(out_folder, file_name))
d = {
'images': images,
'annotations': annotations,
'categories': categories,
}
save_json(d, out_file)
def generate_occlusion_ground_truth_single_core(proc_id, coco_detection,
annotations_set, categories,
segmentations_folder,
threshold):
id_generator = IdGenerator(categories)
occlusion_gt = []
for working_idx, annotation in enumerate(annotations_set):
if working_idx % 100 == 0:
print('Core: {}, {} from {} images processed'.format(
proc_id, working_idx, len(annotations_set)))
img_id = annotation['image_id']
img = coco_detection.loadImgs(int(img_id))[0]
overlaps_map = np.zeros((img['height'], img['width']), dtype=np.uint32)
anns_ids = coco_detection.getAnnIds(img_id)
anns = coco_detection.loadAnns(anns_ids)
occlusion_record = dict()
occlusion_record['image_id'] = img_id
file_name = '{}.png'.format(annotation['file_name'].rsplit('.')[0])
# Read instance segments from panoptic segmentation gt
try:
pan_format = np.array(Image.open(
os.path.join(segmentations_folder, file_name)),
dtype=np.uint32)
except IOError:
raise KeyError('no prediction png file for id: {}'.format(
annotation['image_id']))
pan = rgb2id(pan_format)
pan_mask = {}
for segm_info in annotation['segments_info']:
if categories[segm_info['category_id']]['isthing'] != 1:
continue
mask = (pan == segm_info['id']).astype(np.uint8)
pan_mask[segm_info['id']] = mask
# Read instance segments from instance segmentation gt
segments_info = []
ins_mask = {}
overlap_pairs = []
for ann in anns:
if ann['category_id'] not in categories:
raise Exception(
'Panoptic coco categories file does not contain \
category with id: {}'.format(ann['category_id']))
_, color = id_generator.get_id_and_color(ann['category_id'])
mask = coco_detection.annToMask(ann)
overlaps_map += mask
pan_format[mask == 1] = color
ann.pop('segmentation')
ann.pop('image_id')
# ann['id'] kept as the same
segments_info.append(ann)
ins_mask[ann['id']] = mask
# match segment ID in instance segmentation and panoptic segmentation by IOU
ins2pan = {}
pan2ins = {}
for pan_id in pan_mask:
iou_max = 0
match = None
for ins_id in ins_mask:
if ins_id in ins2pan:
continue
mask_sum = pan_mask[pan_id] + ins_mask[ins_id]
iou = np.sum(mask_sum > 1) / np.sum(mask_sum > 0)
if iou > iou_max:
iou_max = iou
match = ins_id
if not match:
print(
"Inconsistent panoptic annotation and instance annotation")
else:
ins2pan[match] = pan_id
pan2ins[pan_id] = match
if np.sum(overlaps_map > 1) != 0:
for i, _ in enumerate(segments_info):
for j in range(i + 1, len(segments_info)):
id_i = segments_info[i]['id']
id_j = segments_info[j]['id']
mask_i = ins_mask[id_i]
mask_j = ins_mask[id_j]
mask_merge = mask_i + mask_j
r_i = np.sum(mask_merge > 1) / np.sum(mask_i)
r_j = np.sum(mask_merge > 1) / np.sum(mask_j)
if r_i >= threshold or r_j >= threshold:
if id_i not in ins2pan or id_j not in ins2pan:
continue
pan_id_i = ins2pan[id_i]
pan_id_j = ins2pan[id_j]
pan_id_top = None
max_cnt = 0
pan_intersection = pan[mask_merge > 1]
candidate_ids, candidate_cnts = np.unique(
pan_intersection, return_counts=True
) # count the number of different segments
if candidate_ids.size == 0:
print("candidate_ids: ")
print(candidate_ids)
print("candidate_cnts: ")
print(candidate_cnts)
print("filename: ")
print(file_name)
print("imgid={} ".format(img_id))
raise Exception("Wrong intersection.")
for it in range(candidate_ids.size):
if candidate_ids[it] == 0: # remove background 0
continue
if candidate_cnts[it] > max_cnt:
max_cnt = candidate_cnts[it]
pan_id_top = int(candidate_ids[it])
if pan_id_top and pan_id_top in [pan_id_i, pan_id_j]:
# overlap_pairs.append((pan_id_i, pan_id_j, pan_id_top))
overlap_pairs.append(
(pan2ins[pan_id_i], pan2ins[pan_id_j],
pan2ins[pan_id_top]))
occlusion_record['overlap_pairs'] = overlap_pairs
occlusion_gt.append(occlusion_record)
print('Core: {}, all {} images processed'.format(proc_id,
len(annotations_set)))
return occlusion_gt
def panoptic_video_converter():
original_format_folder = os.path.join(ROOT_DIR, MODE, 'panoptic_inst')
# folder to store panoptic PNGs
out_folder = os.path.join(ROOT_DIR, MODE, 'panoptic_video')
out_file = os.path.join(ROOT_DIR, 'panoptic_gt_%s_city_vps.json' % (MODE))
if not os.path.isdir(out_folder):
os.makedirs(out_folder)
categories = CATEGORIES
categories_dict = {el['id']: el for el in CATEGORIES}
file_list = sorted(glob.glob(os.path.join(original_format_folder,
'*.png')))
images = []
annotations = []
instid2color = {}
videos = []
id_generator = IdGenerator(categories_dict)
print('==> %s/panoptic_video/ ...' % (MODE))
for idx in trange(len(file_list)):
f = file_list[idx]
original_format = np.array(Image.open(f))
file_name = f.split('/')[-1]
image_id = file_name.rsplit('_', 2)[0]
video_id = image_id[:4]
if video_id not in videos:
videos.append(video_id)
instid2color = {}
image_filename = file_name.replace('final_mask', 'newImg8bit').replace(
'gtFine_color', 'leftImg8bit')
# image entry, id for image is its filename without extension
images.append({
"id": image_id,
"width": original_format.shape[1],
"height": original_format.shape[0],
"file_name": image_filename
})
pan_format = np.zeros(
(original_format.shape[0], original_format.shape[1], 3),
dtype=np.uint8)
l = np.unique(original_format)
segm_info = {}
for el in l:
if el < 1000:
semantic_id = el
is_crowd = 1
else:
semantic_id = el // 1000
is_crowd = 0
if semantic_id not in categories_dict:
continue
if categories_dict[semantic_id]['isthing'] == 0:
is_crowd = 0
mask = (original_format == el)
if el not in instid2color:
segment_id, color = id_generator.get_id_and_color(semantic_id)
instid2color[el] = (segment_id, color)
else:
segment_id, color = instid2color[el]
pan_format[mask] = color
# area = np.sum(mask) # segment area computation
# # bbox computation for a segment
# hor = np.sum(mask, axis=0)
# hor_idx = np.nonzero(hor)[0]
# x = hor_idx[0]
# width = hor_idx[-1] - x + 1
# vert = np.sum(mask, axis=1)
# vert_idx = np.nonzero(vert)[0]
# y = vert_idx[0]
# height = vert_idx[-1] - y + 1
# bbox = [int(x), int(y), int(width), int(height)]
segm_info[int(segment_id)] = \
{"id": int(segment_id),
"category_id": int(semantic_id),
# "area": int(area),
"iscrowd": is_crowd}
Image.fromarray(pan_format).save(os.path.join(out_folder, file_name))
# segment sanity check, area recalculation
gt_pan = np.uint32(pan_format)
pan_gt = gt_pan[:, :,
0] + gt_pan[:, :, 1] * 256 + gt_pan[:, :,
2] * 256 * 256
labels, labels_cnt = np.unique(pan_gt, return_counts=True)
gt_labels = [_ for _ in segm_info.keys()]
gt_labels_set = set(gt_labels)
for label, area in zip(labels, labels_cnt):
if label == 0:
continue
if label not in gt_labels and label > 0:
print('png label not in json labels.')
segm_info[label]["area"] = int(area)
gt_labels_set.remove(label)
if len(gt_labels_set) != 0:
raise KeyError('remaining gt_labels json')
segm_info = [v for k, v in segm_info.items()]
annotations.append({
'image_id': image_id,
'file_name': file_name,
"segments_info": segm_info
})
d = {
'images': images,
'annotations': annotations,
'categories': categories,
}
save_json(d, out_file)
print('==> Saved json file at %s' % (out_file))