def convert_single_core(proc_id, image_set, categories, source_folder, segmentations_folder, VOID=0): annotations = [] for working_idx, image_info in enumerate(image_set): if working_idx % 100 == 0: print('Core: {}, {} from {} images converted'.format( proc_id, working_idx, len(image_set))) file_name = '{}.png'.format(image_info['file_name'].rsplit('.')[0]) try: original_format = np.array(Image.open( os.path.join(source_folder, file_name)), dtype=np.uint32) except IOError: raise KeyError('no prediction png file for id: {}'.format( image_info['id'])) pan = OFFSET * original_format[:, :, 0] + original_format[:, :, 1] pan_format = np.zeros( (original_format.shape[0], original_format.shape[1], 3), dtype=np.uint8) id_generator = IdGenerator(categories) l = np.unique(pan) segm_info = [] for el in l: sem = el // OFFSET if sem == VOID: continue if sem not in categories: raise KeyError('Unknown semantic label {}'.format(sem)) mask = pan == el segment_id, color = id_generator.get_id_and_color(sem) pan_format[mask] = color segm_info.append({"id": segment_id, "category_id": sem}) annotations.append({ 'image_id': image_info['id'], 'file_name': file_name, "segments_info": segm_info }) Image.fromarray(pan_format).save( os.path.join(segmentations_folder, file_name)) print('Core: {}, all {} images processed'.format(proc_id, len(image_set))) return annotations
def convert_detection_to_panoptic_coco_format_single_core( proc_id, coco_detection, img_ids, categories, segmentations_folder): id_generator = IdGenerator(categories) annotations_panoptic = [] for working_idx, img_id in enumerate(img_ids): if working_idx % 100 == 0: print('Core: {}, {} from {} images processed'.format( proc_id, working_idx, len(img_ids))) img = coco_detection.loadImgs(int(img_id))[0] pan_format = np.zeros((img['height'], img['width'], 3), dtype=np.uint8) overlaps_map = np.zeros((img['height'], img['width']), dtype=np.uint32) anns_ids = coco_detection.getAnnIds(img_id) anns = coco_detection.loadAnns(anns_ids) panoptic_record = {} panoptic_record['image_id'] = img_id file_name = '{}.png'.format(img['file_name'].rsplit('.')[0]) panoptic_record['file_name'] = file_name segments_info = [] for ann in anns: if ann['category_id'] not in categories: raise Exception( 'Panoptic coco categories file does not contain \ category with id: {}'.format(ann['category_id'])) segment_id, color = id_generator.get_id_and_color( ann['category_id']) mask = coco_detection.annToMask(ann) overlaps_map += mask pan_format[mask == 1] = color ann.pop('segmentation') ann.pop('image_id') ann['id'] = segment_id segments_info.append(ann) if np.sum(overlaps_map > 1) != 0: raise Exception( "Segments for image {} overlap each other.".format(img_id)) panoptic_record['segments_info'] = segments_info annotations_panoptic.append(panoptic_record) Image.fromarray(pan_format).save( os.path.join(segmentations_folder, file_name)) print('Core: {}, all {} images processed'.format(proc_id, len(img_ids))) return annotations_panoptic
def do_panoptic_test(cfg, model): categoryies=json.load(open("../data/panoptic_coco_categories.json",'r')) os.makedirs("../data/detectron2_panoptic", exist_ok=True) categoryies_dict={} for category in categoryies: categoryies_dict[category['id']]=category id_generator=IdGenerator(categoryies_dict) image_dict = {} error_list=[] for dataset_name in ['viroi_test']:#,'viroi_train']:#cfg.DATASETS.TRAIN:#cfg.DATASETS.TEST: # data_loader = build_detection_test_loader(cfg, dataset_name) thing_id_map = MetadataCatalog.get(dataset_name).get("thing_contiguous_id_to_class_id") stuff_id_map = MetadataCatalog.get(dataset_name).get("stuff_contiguous_id_to_class_id") test_images_dict=json.load(open(MetadataCatalog.get(dataset_name).get("instance_json_file"),'r')) image_path = MetadataCatalog.get(dataset_name).get("image_path") predictor=DefaultPredictor(cfg) total = len(test_images_dict) count=0 # for idx, inputs in enumerate(data_loader): for image_id in test_images_dict: image_info=test_images_dict[image_id] img=read_image(image_path+"/"+image_info['image_name'],format="BGR") count+=1 print(str(count)+"/"+str(total)) if True: # try: # print(inputs[0]) # predictions = model(inputs, "panoptic")[0] # 'sem_seg', 'instances', 'panoptic_seg' predictions = predictor(img,0) panoptic_seg, segments_info = predictions["panoptic_seg"] # seg, info panoptic_seg=panoptic_seg.data.cpu().numpy() panoptic_color_seg = np.zeros((panoptic_seg.shape[0], panoptic_seg.shape[1], 3)) #tensor instance_dict={} for info in segments_info: if 'score' in info: del info['score'] if 'area' in info: del info['area'] bbox = info['box'] # x1,y1,x2,y2->y1,x1,y2,x2 info['bbox'] = [int(bbox[1]), int(bbox[0]), int(bbox[3]), int(bbox[2])] del info['box'] class_id=info['class_id'] del info['category_id'] mask = info['mask'].data.cpu().numpy() mask = np.asfortranarray(mask) segmentation = maskUtils.encode(mask) segmentation['counts'] = segmentation['counts'].decode('utf8') info['segmentation'] = segmentation instance_id, panoptic_color_seg[mask] = id_generator.get_id_and_color(categoryies[class_id - 1]['id']) info['instance_id'] = instance_id del info['mask'] instance_dict[str(instance_id)]=info image_dict[image_id]={'instances':instance_dict, "image_id":image_info['image_id'], "height":image_info['height'], "width":image_info['width'], "image_name":image_info['image_name'] } # print(image_dict) Image.fromarray(panoptic_color_seg.astype(np.uint8)).save("../data/detectron2_panoptic/"+image_info["image_name"].replace("jpg","png")) # except: # print("ERROR - "+image_info['image_name']) # error_list.append(image_info['image_name']) json.dump(image_dict,open("../data/viroi_json/detectron2_"+dataset_name+"_images_dict.json",'w')) json.dump(error_list,open("error_list.json",'w'))
def process_image(working_idx): global file_list, categories_dic, output_folder f = file_list[working_idx] # print(f) images = [] img = Image.open(f) img = img.resize((1280, 720)) original_format = np.array(img) # print("Processing file", f) file_name = f.split('/')[-1] image_id = file_name.rsplit('_', 2)[0] image_filename = '{}_{}_gtFine_panopticlevel3Ids.png'.format( f.split('/')[-2], image_id) # pdb.set_trace() # image entry, id for image is its filename without extension image = { "id": image_filename, "width": original_format.shape[1], "height": original_format.shape[0], "file_name": image_filename } pan_format = np.zeros( (original_format.shape[0], original_format.shape[1], 3), dtype=np.uint8) id_generator = IdGenerator(categories_dict) idx = 0 l = np.unique(original_format) segm_info = [] for el in l: if el < 1000: semantic_id = el is_crowd = 1 else: semantic_id = el // 1000 is_crowd = 0 if semantic_id not in categories_dict: continue if categories_dict[semantic_id]['isthing'] == 0: is_crowd = 0 mask = original_format == el segment_id, color = id_generator.get_id_and_color(semantic_id) pan_format[mask] = color area = np.sum(mask) # segment area computation # bbox computation for a segment hor = np.sum(mask, axis=0) hor_idx = np.nonzero(hor)[0] x = hor_idx[0] width = hor_idx[-1] - x + 1 vert = np.sum(mask, axis=1) vert_idx = np.nonzero(vert)[0] y = vert_idx[0] height = vert_idx[-1] - y + 1 bbox = [x, y, width, height] segm_info.append({ "id": int(segment_id), "category_id": int(semantic_id), "area": int(area), "bbox": [int(x) for x in bbox], "iscrowd": is_crowd }) Image.fromarray(pan_format).save( os.path.join(output_folder, image_filename)) return image, segm_info
def panoptic_converter(original_format_folder, out_folder, out_file): if not os.path.isdir(out_folder): print("Creating folder {} for panoptic segmentation PNGs".format( out_folder)) os.mkdir(out_folder) categories = [] for idx, el in enumerate(labels): if el.ignoreInEval: continue categories.append({ 'id': el.id, 'name': el.name, 'color': el.color, 'supercategory': el.category, 'isthing': 1 if el.hasInstances else 0 }) categories_dict = {cat['id']: cat for cat in categories} file_list = sorted( glob.glob( os.path.join(original_format_folder, '*/*_gtFine_instanceIds.png'))) images = [] annotations = [] for working_idx, f in enumerate(file_list): if working_idx % 10 == 0: print(working_idx, len(file_list)) original_format = np.array(Image.open(f)) file_name = f.split('/')[-1] image_id = file_name.rsplit('_', 2)[0] image_filename = '{}_leftImg8bit.png'.format(image_id) # image entry, id for image is its filename without extension images.append({ "id": image_id, "width": original_format.shape[1], "height": original_format.shape[0], "file_name": image_filename }) pan_format = np.zeros( (original_format.shape[0], original_format.shape[1], 3), dtype=np.uint8) id_generator = IdGenerator(categories_dict) ll = np.unique(original_format) segm_info = [] for el in ll: if el < 1000: semantic_id = el is_crowd = 1 else: semantic_id = el // 1000 is_crowd = 0 if semantic_id not in categories_dict: continue if categories_dict[semantic_id]['isthing'] == 0: is_crowd = 0 mask = original_format == el segment_id, color = id_generator.get_id_and_color(semantic_id) pan_format[mask] = color area = np.sum(mask) # segment area computation # bbox computation for a segment hor = np.sum(mask, axis=0) hor_idx = np.nonzero(hor)[0] x = hor_idx[0] width = hor_idx[-1] - x + 1 vert = np.sum(mask, axis=1) vert_idx = np.nonzero(vert)[0] y = vert_idx[0] height = vert_idx[-1] - y + 1 bbox = [x, y, width, height] segm_info.append({ "id": int(segment_id), "category_id": int(semantic_id), "area": area, "bbox": bbox, "iscrowd": is_crowd }) annotations.append({ 'image_id': image_id, 'file_name': file_name, "segments_info": segm_info }) Image.fromarray(pan_format).save(os.path.join(out_folder, file_name)) d = { 'images': images, 'annotations': annotations, 'categories': categories, } save_json(d, out_file)
def generate_occlusion_ground_truth_single_core(proc_id, coco_detection, annotations_set, categories, segmentations_folder, threshold): id_generator = IdGenerator(categories) occlusion_gt = [] for working_idx, annotation in enumerate(annotations_set): if working_idx % 100 == 0: print('Core: {}, {} from {} images processed'.format( proc_id, working_idx, len(annotations_set))) img_id = annotation['image_id'] img = coco_detection.loadImgs(int(img_id))[0] overlaps_map = np.zeros((img['height'], img['width']), dtype=np.uint32) anns_ids = coco_detection.getAnnIds(img_id) anns = coco_detection.loadAnns(anns_ids) occlusion_record = dict() occlusion_record['image_id'] = img_id file_name = '{}.png'.format(annotation['file_name'].rsplit('.')[0]) # Read instance segments from panoptic segmentation gt try: pan_format = np.array(Image.open( os.path.join(segmentations_folder, file_name)), dtype=np.uint32) except IOError: raise KeyError('no prediction png file for id: {}'.format( annotation['image_id'])) pan = rgb2id(pan_format) pan_mask = {} for segm_info in annotation['segments_info']: if categories[segm_info['category_id']]['isthing'] != 1: continue mask = (pan == segm_info['id']).astype(np.uint8) pan_mask[segm_info['id']] = mask # Read instance segments from instance segmentation gt segments_info = [] ins_mask = {} overlap_pairs = [] for ann in anns: if ann['category_id'] not in categories: raise Exception( 'Panoptic coco categories file does not contain \ category with id: {}'.format(ann['category_id'])) _, color = id_generator.get_id_and_color(ann['category_id']) mask = coco_detection.annToMask(ann) overlaps_map += mask pan_format[mask == 1] = color ann.pop('segmentation') ann.pop('image_id') # ann['id'] kept as the same segments_info.append(ann) ins_mask[ann['id']] = mask # match segment ID in instance segmentation and panoptic segmentation by IOU ins2pan = {} pan2ins = {} for pan_id in pan_mask: iou_max = 0 match = None for ins_id in ins_mask: if ins_id in ins2pan: continue mask_sum = pan_mask[pan_id] + ins_mask[ins_id] iou = np.sum(mask_sum > 1) / np.sum(mask_sum > 0) if iou > iou_max: iou_max = iou match = ins_id if not match: print( "Inconsistent panoptic annotation and instance annotation") else: ins2pan[match] = pan_id pan2ins[pan_id] = match if np.sum(overlaps_map > 1) != 0: for i, _ in enumerate(segments_info): for j in range(i + 1, len(segments_info)): id_i = segments_info[i]['id'] id_j = segments_info[j]['id'] mask_i = ins_mask[id_i] mask_j = ins_mask[id_j] mask_merge = mask_i + mask_j r_i = np.sum(mask_merge > 1) / np.sum(mask_i) r_j = np.sum(mask_merge > 1) / np.sum(mask_j) if r_i >= threshold or r_j >= threshold: if id_i not in ins2pan or id_j not in ins2pan: continue pan_id_i = ins2pan[id_i] pan_id_j = ins2pan[id_j] pan_id_top = None max_cnt = 0 pan_intersection = pan[mask_merge > 1] candidate_ids, candidate_cnts = np.unique( pan_intersection, return_counts=True ) # count the number of different segments if candidate_ids.size == 0: print("candidate_ids: ") print(candidate_ids) print("candidate_cnts: ") print(candidate_cnts) print("filename: ") print(file_name) print("imgid={} ".format(img_id)) raise Exception("Wrong intersection.") for it in range(candidate_ids.size): if candidate_ids[it] == 0: # remove background 0 continue if candidate_cnts[it] > max_cnt: max_cnt = candidate_cnts[it] pan_id_top = int(candidate_ids[it]) if pan_id_top and pan_id_top in [pan_id_i, pan_id_j]: # overlap_pairs.append((pan_id_i, pan_id_j, pan_id_top)) overlap_pairs.append( (pan2ins[pan_id_i], pan2ins[pan_id_j], pan2ins[pan_id_top])) occlusion_record['overlap_pairs'] = overlap_pairs occlusion_gt.append(occlusion_record) print('Core: {}, all {} images processed'.format(proc_id, len(annotations_set))) return occlusion_gt
def panoptic_video_converter(): original_format_folder = os.path.join(ROOT_DIR, MODE, 'panoptic_inst') # folder to store panoptic PNGs out_folder = os.path.join(ROOT_DIR, MODE, 'panoptic_video') out_file = os.path.join(ROOT_DIR, 'panoptic_gt_%s_city_vps.json' % (MODE)) if not os.path.isdir(out_folder): os.makedirs(out_folder) categories = CATEGORIES categories_dict = {el['id']: el for el in CATEGORIES} file_list = sorted(glob.glob(os.path.join(original_format_folder, '*.png'))) images = [] annotations = [] instid2color = {} videos = [] id_generator = IdGenerator(categories_dict) print('==> %s/panoptic_video/ ...' % (MODE)) for idx in trange(len(file_list)): f = file_list[idx] original_format = np.array(Image.open(f)) file_name = f.split('/')[-1] image_id = file_name.rsplit('_', 2)[0] video_id = image_id[:4] if video_id not in videos: videos.append(video_id) instid2color = {} image_filename = file_name.replace('final_mask', 'newImg8bit').replace( 'gtFine_color', 'leftImg8bit') # image entry, id for image is its filename without extension images.append({ "id": image_id, "width": original_format.shape[1], "height": original_format.shape[0], "file_name": image_filename }) pan_format = np.zeros( (original_format.shape[0], original_format.shape[1], 3), dtype=np.uint8) l = np.unique(original_format) segm_info = {} for el in l: if el < 1000: semantic_id = el is_crowd = 1 else: semantic_id = el // 1000 is_crowd = 0 if semantic_id not in categories_dict: continue if categories_dict[semantic_id]['isthing'] == 0: is_crowd = 0 mask = (original_format == el) if el not in instid2color: segment_id, color = id_generator.get_id_and_color(semantic_id) instid2color[el] = (segment_id, color) else: segment_id, color = instid2color[el] pan_format[mask] = color # area = np.sum(mask) # segment area computation # # bbox computation for a segment # hor = np.sum(mask, axis=0) # hor_idx = np.nonzero(hor)[0] # x = hor_idx[0] # width = hor_idx[-1] - x + 1 # vert = np.sum(mask, axis=1) # vert_idx = np.nonzero(vert)[0] # y = vert_idx[0] # height = vert_idx[-1] - y + 1 # bbox = [int(x), int(y), int(width), int(height)] segm_info[int(segment_id)] = \ {"id": int(segment_id), "category_id": int(semantic_id), # "area": int(area), "iscrowd": is_crowd} Image.fromarray(pan_format).save(os.path.join(out_folder, file_name)) # segment sanity check, area recalculation gt_pan = np.uint32(pan_format) pan_gt = gt_pan[:, :, 0] + gt_pan[:, :, 1] * 256 + gt_pan[:, :, 2] * 256 * 256 labels, labels_cnt = np.unique(pan_gt, return_counts=True) gt_labels = [_ for _ in segm_info.keys()] gt_labels_set = set(gt_labels) for label, area in zip(labels, labels_cnt): if label == 0: continue if label not in gt_labels and label > 0: print('png label not in json labels.') segm_info[label]["area"] = int(area) gt_labels_set.remove(label) if len(gt_labels_set) != 0: raise KeyError('remaining gt_labels json') segm_info = [v for k, v in segm_info.items()] annotations.append({ 'image_id': image_id, 'file_name': file_name, "segments_info": segm_info }) d = { 'images': images, 'annotations': annotations, 'categories': categories, } save_json(d, out_file) print('==> Saved json file at %s' % (out_file))