def convert_single_core(proc_id, image_set, categories, source_folder, segmentations_folder, VOID=0): annotations = [] for working_idx, image_info in enumerate(image_set): if working_idx % 100 == 0: print('Core: {}, {} from {} images converted'.format( proc_id, working_idx, len(image_set))) file_name = '{}.png'.format(image_info['file_name'].rsplit('.')[0]) try: original_format = np.array(Image.open( os.path.join(source_folder, file_name)), dtype=np.uint32) except IOError: raise KeyError('no prediction png file for id: {}'.format( image_info['id'])) pan = OFFSET * original_format[:, :, 0] + original_format[:, :, 1] pan_format = np.zeros( (original_format.shape[0], original_format.shape[1], 3), dtype=np.uint8) id_generator = IdGenerator(categories) l = np.unique(pan) segm_info = [] for el in l: sem = el // OFFSET if sem == VOID: continue if sem not in categories: raise KeyError('Unknown semantic label {}'.format(sem)) mask = pan == el segment_id, color = id_generator.get_id_and_color(sem) pan_format[mask] = color segm_info.append({"id": segment_id, "category_id": sem}) annotations.append({ 'image_id': image_info['id'], 'file_name': file_name, "segments_info": segm_info }) Image.fromarray(pan_format).save( os.path.join(segmentations_folder, file_name)) print('Core: {}, all {} images processed'.format(proc_id, len(image_set))) return annotations
def infer_panoptic_mask(self, instance_mask_only=False): segments = [] meta = { 'image_id': None, # not needed for now 'file_name': None, 'segments_info': segments } id_gen = IdGenerator(self.categories) instance_tsr, _, sem_cats = self.get_instance_tsr(resolve_overlap=True) assert instance_tsr.sum( 0).max() <= 1, 'contested pixs should not exist' sem_cats = sem_cats.tolist() sem_cats = [self.trainId_2_catId[el] for el in sem_cats] iids = [] for _cat, _ins_mask in zip(sem_cats, instance_tsr): _id = id_gen.get_id(_cat) iids.append(_id) segments.append({ 'id': _id, 'category_id': _cat, 'isthing': 1, 'bbox': [ int(elem) for elem in get_xywh_bbox_from_binary_mask(_ins_mask) ], 'area': int(_ins_mask.sum()) }) # note numpy.int64 is not json serializable mask = (instance_tsr * np.array(iids).reshape(-1, 1, 1)).sum(axis=0) mask = mask.astype(np.uint32) if instance_mask_only: return mask, meta sem_remain = self.sem_decision.copy() sem_remain[mask > 0] = -1 for trainId in np.unique(sem_remain): if trainId < 0: continue _cat = self.trainId_2_catId[trainId] if self.categories[_cat]['isthing']: continue # abstain on ungrouped instance pixels _id = id_gen.get_id(_cat) segments.append({'id': _id, 'category_id': _cat, 'isthing': 0}) mask[sem_remain == trainId] = _id for seg in meta['segments_info']: assert seg['id'] in mask return mask, meta
def show_semseg_result(semseg_json_file, categories_json_file, out_image_file): sem_by_image = defaultdict(list) with open(semseg_json_file, 'r') as f: sem_results = json.load(f) print("Semantic segmentation:") print("\tJSON file: {}".format(semseg_json_file)) with open(categories_json_file, 'r') as f: categories_list = json.load(f) categories = {el['id']: el for el in categories_list} for sem in sem_results: img_id = sem['image_id'] sem_by_image[sem['image_id']].append(sem) id_generator = IdGenerator(categories) #pan_segm_id = np.zeros((mask.shape[0], mask.shape[1]), dtype=np.uint32) pan_segm_id = np.zeros((480, 640), dtype=np.uint32) # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # To Do: change logic of multiple annotations case, for now, it is override # but we can easily learn it from panoptic combine script # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! for ann in sem_by_image[img_id]: mask = COCOmask.decode(ann['segmentation']) #print(mask.shape()) plt.imshow(mask) plt.show() segment_id = id_generator.get_id(ann['category_id']) print("id: ") print(ann['category_id']) pan_segm_id[mask==1] = segment_id print("segment_id: ") print(segment_id) print(id2rgb(pan_segm_id).shape) #print(sem_by_image) Image.fromarray(id2rgb(pan_segm_id)).save( os.path.join("/home/zhiliu/Documents/Panoptic_Segement/Cocopanopticapi/VanillaPanopticSeg/data/predictions/test_result_for_vox/", out_image_file) )
def get_gt(pan_gt_json_file=None, pan_gt_folder=None): if pan_gt_json_file is None: pan_gt_json_file = self.panoptic_json_file if pan_gt_folder is None: pan_gt_folder = self.panoptic_gt_folder with open(pan_gt_json_file, 'r') as f: pan_gt_json = json.load(f) files = [item['file_name'] for item in pan_gt_json['images']] if 'viper' in pan_gt_folder: files = [ _.split('/')[-1].replace('.jpg', '.png') for _ in files ] cpu_num = multiprocessing.cpu_count() files_split = np.array_split(files, cpu_num) workers = multiprocessing.Pool(processes=cpu_num) processes = [] for proc_id, files_set in enumerate(files_split): p = workers.apply_async(BaseDataset._load_image_single_core, (proc_id, files_set, pan_gt_folder)) processes.append(p) workers.close() workers.join() pan_gt_all = [] for p in processes: pan_gt_all.extend(p.get()) categories = pan_gt_json['categories'] categories = {el['id']: el for el in categories} color_gererator = IdGenerator(categories) return pan_gt_all, pan_gt_json, categories, color_gererator
def convert_detection_to_panoptic_coco_format_single_core( proc_id, coco_detection, img_ids, categories, segmentations_folder): id_generator = IdGenerator(categories) annotations_panoptic = [] for working_idx, img_id in enumerate(img_ids): if working_idx % 100 == 0: print('Core: {}, {} from {} images processed'.format( proc_id, working_idx, len(img_ids))) img = coco_detection.loadImgs(int(img_id))[0] pan_format = np.zeros((img['height'], img['width'], 3), dtype=np.uint8) overlaps_map = np.zeros((img['height'], img['width']), dtype=np.uint32) anns_ids = coco_detection.getAnnIds(img_id) anns = coco_detection.loadAnns(anns_ids) panoptic_record = {} panoptic_record['image_id'] = img_id file_name = '{}.png'.format(img['file_name'].rsplit('.')[0]) panoptic_record['file_name'] = file_name segments_info = [] for ann in anns: if ann['category_id'] not in categories: raise Exception( 'Panoptic coco categories file does not contain \ category with id: {}'.format(ann['category_id'])) segment_id, color = id_generator.get_id_and_color( ann['category_id']) mask = coco_detection.annToMask(ann) overlaps_map += mask pan_format[mask == 1] = color ann.pop('segmentation') ann.pop('image_id') ann['id'] = segment_id segments_info.append(ann) if np.sum(overlaps_map > 1) != 0: raise Exception( "Segments for image {} overlap each other.".format(img_id)) panoptic_record['segments_info'] = segments_info annotations_panoptic.append(panoptic_record) Image.fromarray(pan_format).save( os.path.join(segmentations_folder, file_name)) print('Core: {}, all {} images processed'.format(proc_id, len(img_ids))) return annotations_panoptic
def get_gt(pan_gt_json_file=None): if pan_gt_json_file is None: pan_gt_json_file = self.panoptic_json_file with open(pan_gt_json_file, 'r') as f: pan_gt_json = json.load(f) files = [item['file_name'] for item in pan_gt_json['images']] categories = pan_gt_json['categories'] categories = {el['id']: el for el in categories} #breakpoint() color_gererator = IdGenerator(categories) return pan_gt_json, categories, color_gererator
def do_panoptic_test(cfg, model): categoryies=json.load(open("../data/panoptic_coco_categories.json",'r')) os.makedirs("../data/detectron2_panoptic", exist_ok=True) categoryies_dict={} for category in categoryies: categoryies_dict[category['id']]=category id_generator=IdGenerator(categoryies_dict) image_dict = {} error_list=[] for dataset_name in ['viroi_test']:#,'viroi_train']:#cfg.DATASETS.TRAIN:#cfg.DATASETS.TEST: # data_loader = build_detection_test_loader(cfg, dataset_name) thing_id_map = MetadataCatalog.get(dataset_name).get("thing_contiguous_id_to_class_id") stuff_id_map = MetadataCatalog.get(dataset_name).get("stuff_contiguous_id_to_class_id") test_images_dict=json.load(open(MetadataCatalog.get(dataset_name).get("instance_json_file"),'r')) image_path = MetadataCatalog.get(dataset_name).get("image_path") predictor=DefaultPredictor(cfg) total = len(test_images_dict) count=0 # for idx, inputs in enumerate(data_loader): for image_id in test_images_dict: image_info=test_images_dict[image_id] img=read_image(image_path+"/"+image_info['image_name'],format="BGR") count+=1 print(str(count)+"/"+str(total)) if True: # try: # print(inputs[0]) # predictions = model(inputs, "panoptic")[0] # 'sem_seg', 'instances', 'panoptic_seg' predictions = predictor(img,0) panoptic_seg, segments_info = predictions["panoptic_seg"] # seg, info panoptic_seg=panoptic_seg.data.cpu().numpy() panoptic_color_seg = np.zeros((panoptic_seg.shape[0], panoptic_seg.shape[1], 3)) #tensor instance_dict={} for info in segments_info: if 'score' in info: del info['score'] if 'area' in info: del info['area'] bbox = info['box'] # x1,y1,x2,y2->y1,x1,y2,x2 info['bbox'] = [int(bbox[1]), int(bbox[0]), int(bbox[3]), int(bbox[2])] del info['box'] class_id=info['class_id'] del info['category_id'] mask = info['mask'].data.cpu().numpy() mask = np.asfortranarray(mask) segmentation = maskUtils.encode(mask) segmentation['counts'] = segmentation['counts'].decode('utf8') info['segmentation'] = segmentation instance_id, panoptic_color_seg[mask] = id_generator.get_id_and_color(categoryies[class_id - 1]['id']) info['instance_id'] = instance_id del info['mask'] instance_dict[str(instance_id)]=info image_dict[image_id]={'instances':instance_dict, "image_id":image_info['image_id'], "height":image_info['height'], "width":image_info['width'], "image_name":image_info['image_name'] } # print(image_dict) Image.fromarray(panoptic_color_seg.astype(np.uint8)).save("../data/detectron2_panoptic/"+image_info["image_name"].replace("jpg","png")) # except: # print("ERROR - "+image_info['image_name']) # error_list.append(image_info['image_name']) json.dump(image_dict,open("../data/viroi_json/detectron2_"+dataset_name+"_images_dict.json",'w')) json.dump(error_list,open("error_list.json",'w'))
def inference_panoptic_video( self, pred_pans_2ch, output_dir, # pan_im_json_file, categories, names, n_video=0): from panopticapi.utils import IdGenerator # Sample only frames with GT annotations. pred_pans_2ch = pred_pans_2ch[(self.labeled_fid // self.lambda_)::self.lambda_] categories = {el['id']: el for el in categories} color_generator = IdGenerator(categories) def get_pred_large(pan_2ch_all, vid_num, nframes_per_video=6): vid_num = len(pan_2ch_all) // nframes_per_video # 10 cpu_num = multiprocessing.cpu_count() // 2 # 32 --> 16 nprocs = min(vid_num, cpu_num) # 10 max_nframes = cpu_num * nframes_per_video nsplits = (len(pan_2ch_all) - 1) // max_nframes + 1 annotations, pan_all = [], [] for i in range(0, len(pan_2ch_all), max_nframes): print('==> Read and convert VPS output - split %d/%d' % ((i // max_nframes) + 1, nsplits)) pan_2ch_part = pan_2ch_all[i:min(i + max_nframes, len(pan_2ch_all ))] pan_2ch_split = np.array_split(pan_2ch_part, nprocs) workers = multiprocessing.Pool(processes=nprocs) processes = [] for proc_id, pan_2ch_set in enumerate(pan_2ch_split): p = workers.apply_async( self.converter_2ch_track_core, (proc_id, pan_2ch_set, color_generator)) processes.append(p) workers.close() workers.join() for p in processes: p = p.get() annotations.extend(p[0]) pan_all.extend(p[1]) pan_json = {'annotations': annotations} return pan_all, pan_json def save_image(images, save_folder, names, colors=None): os.makedirs(save_folder, exist_ok=True) names = [ osp.join( save_folder, name.replace('_leftImg8bit', '').replace('_newImg8bit', '').replace( 'jpg', 'png').replace('jpeg', 'png')) for name in names ] cpu_num = multiprocessing.cpu_count() // 2 images_split = np.array_split(images, cpu_num) names_split = np.array_split(names, cpu_num) workers = multiprocessing.Pool(processes=cpu_num) for proc_id, (images_set, names_set) in enumerate( zip(images_split, names_split)): workers.apply_async(BaseDataset._save_image_single_core, (proc_id, images_set, names_set, colors)) workers.close() workers.join() # inference_panoptic_video pred_pans, pred_json = get_pred_large(pred_pans_2ch, vid_num=n_video) print('--------------------------------------') print('==> Saving VPS output png files') os.makedirs(output_dir, exist_ok=True) save_image(pred_pans_2ch, osp.join(output_dir, 'pan_2ch'), names) save_image(pred_pans, osp.join(output_dir, 'pan_pred'), names) print('==> Saving pred.jsons file') json.dump(pred_json, open(osp.join(output_dir, 'pred.json'), 'w')) print('--------------------------------------') return pred_pans, pred_json
def combine_to_panoptic_single_core(proc_id, img_ids, img_id2img, inst_by_image, sem_by_image, segmentations_folder, overlap_thr, stuff_area_limit, categories): panoptic_json = [] id_generator = IdGenerator(categories) for idx, img_id in enumerate(img_ids): img = img_id2img[img_id] if idx % 100 == 0: print('Core: {}, {} from {} images processed.'.format( proc_id, idx, len(img_ids))) pan_segm_id = np.zeros((img['height'], img['width']), dtype=np.uint32) used = None annotation = {} try: annotation['image_id'] = int(img_id) except Exception: annotation['image_id'] = img_id annotation['file_name'] = img['file_name'].replace('.jpg', '.png') segments_info = [] for ann in inst_by_image[img_id]: area = COCOmask.area(ann['segmentation']) if area == 0: continue if used is None: intersect = 0 used = copy.deepcopy(ann['segmentation']) else: intersect = COCOmask.area( COCOmask.merge([used, ann['segmentation']], intersect=True)) if intersect / area > overlap_thr: continue used = COCOmask.merge([used, ann['segmentation']], intersect=False) mask = COCOmask.decode(ann['segmentation']) == 1 if intersect != 0: mask = np.logical_and(pan_segm_id == 0, mask) segment_id = id_generator.get_id(ann['category_id']) panoptic_ann = {} panoptic_ann['id'] = segment_id panoptic_ann['category_id'] = ann['category_id'] pan_segm_id[mask] = segment_id segments_info.append(panoptic_ann) for ann in sem_by_image[img_id]: mask = COCOmask.decode(ann['segmentation']) == 1 mask_left = np.logical_and(pan_segm_id == 0, mask) if mask_left.sum() < stuff_area_limit: continue segment_id = id_generator.get_id(ann['category_id']) panoptic_ann = {} panoptic_ann['id'] = segment_id panoptic_ann['category_id'] = ann['category_id'] pan_segm_id[mask_left] = segment_id segments_info.append(panoptic_ann) annotation['segments_info'] = segments_info panoptic_json.append(annotation) Image.fromarray(id2rgb(pan_segm_id)).save( os.path.join(segmentations_folder, annotation['file_name'])) return panoptic_json
def process_image(working_idx): global file_list, categories_dic, output_folder f = file_list[working_idx] # print(f) images = [] img = Image.open(f) img = img.resize((1280, 720)) original_format = np.array(img) # print("Processing file", f) file_name = f.split('/')[-1] image_id = file_name.rsplit('_', 2)[0] image_filename = '{}_{}_gtFine_panopticlevel3Ids.png'.format( f.split('/')[-2], image_id) # pdb.set_trace() # image entry, id for image is its filename without extension image = { "id": image_filename, "width": original_format.shape[1], "height": original_format.shape[0], "file_name": image_filename } pan_format = np.zeros( (original_format.shape[0], original_format.shape[1], 3), dtype=np.uint8) id_generator = IdGenerator(categories_dict) idx = 0 l = np.unique(original_format) segm_info = [] for el in l: if el < 1000: semantic_id = el is_crowd = 1 else: semantic_id = el // 1000 is_crowd = 0 if semantic_id not in categories_dict: continue if categories_dict[semantic_id]['isthing'] == 0: is_crowd = 0 mask = original_format == el segment_id, color = id_generator.get_id_and_color(semantic_id) pan_format[mask] = color area = np.sum(mask) # segment area computation # bbox computation for a segment hor = np.sum(mask, axis=0) hor_idx = np.nonzero(hor)[0] x = hor_idx[0] width = hor_idx[-1] - x + 1 vert = np.sum(mask, axis=1) vert_idx = np.nonzero(vert)[0] y = vert_idx[0] height = vert_idx[-1] - y + 1 bbox = [x, y, width, height] segm_info.append({ "id": int(segment_id), "category_id": int(semantic_id), "area": int(area), "bbox": [int(x) for x in bbox], "iscrowd": is_crowd }) Image.fromarray(pan_format).save( os.path.join(output_folder, image_filename)) return image, segm_info
def visualize(image_path, label_path): """ Visualizes in a pyplot window an image and a label pair from provided paths. For reading Pillow is used so all paths and formats must be Pillow-compatible. Args: image_path: an image path provided to Pillow.Image.open label_path: a label path provided to Pillow.Image.open """ assert op.exists(image_path) assert op.exists(label_path) # Prepare canvases and decode the labels. image = np.array(Image.open(image_path), dtype=np.uint8) label = np.array(Image.open(label_path), dtype=np.int32) uids_unique_org = np.unique(label) semantic_segmentation = np.zeros((image.shape[0], image.shape[1], 3), dtype=np.uint8) instance_segmentation = np.zeros((image.shape[0], image.shape[1], 3), dtype=np.uint8) parts_segmentation = np.zeros((image.shape[0], image.shape[1], 3), dtype=np.uint8) sids, iids, _ = decode_uids(label) # Color at the semantic level. color_generator = IdGenerator(CATEGORIES) for sid in np.unique(sids): mask = np.equal(sids, sid) color = CATEGORIES[sid]['color'] semantic_segmentation[mask] = color # Color at the semantic and instance level and find the instance-level boundaries. sids_only = np.where(iids < 0, sids, np.zeros_like(iids)) for sid in np.unique(sids_only): mask = np.equal(sids_only, sid) color = color_generator.get_color(sid) instance_segmentation[mask] = color sid_iids = np.where(iids >= 0, sids * 10**3 + iids, np.zeros_like(iids)) boundaries = np.full(sid_iids.shape, False) for sid_iid in np.unique(sid_iids): if sid_iid != 0: mask = np.equal(sid_iids, sid_iid) color = color_generator.get_color(sid_iid // 1000) instance_segmentation[mask] = color boundary_horizon = ndimage.sobel(mask, 0) boundary_vertical = ndimage.sobel(mask, 1) boundaries = np.logical_or( np.hypot(boundary_horizon, boundary_vertical), boundaries) # Color at the part level. # Conver the original labels into the form for visualization with IdGenerator. for uid in uids_unique_org: # If uid is sid or sid_iid, encode them as they are. if uid <= 99_999: sid_iid = uid # If uid is sid_iid_pid, map sid_pid to its corresponding sid and create new label as sid_iid. else: sid, iid, pid = decode_uids(uid) sid_pid = sid * 10**2 + pid if sid_pid in SID_PID2PARTS_CID: sid_iid = SID_PID2PARTS_CID[sid_pid] * 10**3 + iid else: sid_iid = sid * 10**3 + iid label[label == uid] = sid_iid color_generator = IdGenerator(CATEGORIES_PARTS) for sid_iid in np.unique(label): # If sid_iid is in the format of sid , use sid for color generation (things and stuff classes differentiated by IdGenerator inherently). if sid_iid <= 99: id_ = sid_iid # If sid_iid is in the format of sid_iid, get sid. else: id_ = sid_iid // 1000 mask = label == sid_iid color = color_generator.get_color(id_) parts_segmentation[mask] = color # Depict boundaries. instance_segmentation[boundaries] = [255, 255, 255] parts_segmentation[boundaries] = [255, 255, 255] # plot # initialize figure for plotting _, ((ax1, ax2), (ax3, ax4)) = plt.subplots(nrows=2, ncols=2) # for ax in axes: # ax.set_axis_off() ax1.imshow(image) ax1.set_title('image') ax2.imshow(semantic_segmentation) ax2.set_title('labels colored on semantic level') ax3.imshow(instance_segmentation) ax3.set_title('labels colored on semantic and instance levels') ax4.imshow(parts_segmentation) ax4.set_title('labels colored on semantic, instance, and parts levels') plt.show()
def generate_occlusion_ground_truth_single_core(proc_id, coco_detection, annotations_set, categories, segmentations_folder, threshold): id_generator = IdGenerator(categories) occlusion_gt = [] for working_idx, annotation in enumerate(annotations_set): if working_idx % 100 == 0: print('Core: {}, {} from {} images processed'.format( proc_id, working_idx, len(annotations_set))) img_id = annotation['image_id'] img = coco_detection.loadImgs(int(img_id))[0] overlaps_map = np.zeros((img['height'], img['width']), dtype=np.uint32) anns_ids = coco_detection.getAnnIds(img_id) anns = coco_detection.loadAnns(anns_ids) occlusion_record = dict() occlusion_record['image_id'] = img_id file_name = '{}.png'.format(annotation['file_name'].rsplit('.')[0]) # Read instance segments from panoptic segmentation gt try: pan_format = np.array(Image.open( os.path.join(segmentations_folder, file_name)), dtype=np.uint32) except IOError: raise KeyError('no prediction png file for id: {}'.format( annotation['image_id'])) pan = rgb2id(pan_format) pan_mask = {} for segm_info in annotation['segments_info']: if categories[segm_info['category_id']]['isthing'] != 1: continue mask = (pan == segm_info['id']).astype(np.uint8) pan_mask[segm_info['id']] = mask # Read instance segments from instance segmentation gt segments_info = [] ins_mask = {} overlap_pairs = [] for ann in anns: if ann['category_id'] not in categories: raise Exception( 'Panoptic coco categories file does not contain \ category with id: {}'.format(ann['category_id'])) _, color = id_generator.get_id_and_color(ann['category_id']) mask = coco_detection.annToMask(ann) overlaps_map += mask pan_format[mask == 1] = color ann.pop('segmentation') ann.pop('image_id') # ann['id'] kept as the same segments_info.append(ann) ins_mask[ann['id']] = mask # match segment ID in instance segmentation and panoptic segmentation by IOU ins2pan = {} pan2ins = {} for pan_id in pan_mask: iou_max = 0 match = None for ins_id in ins_mask: if ins_id in ins2pan: continue mask_sum = pan_mask[pan_id] + ins_mask[ins_id] iou = np.sum(mask_sum > 1) / np.sum(mask_sum > 0) if iou > iou_max: iou_max = iou match = ins_id if not match: print( "Inconsistent panoptic annotation and instance annotation") else: ins2pan[match] = pan_id pan2ins[pan_id] = match if np.sum(overlaps_map > 1) != 0: for i, _ in enumerate(segments_info): for j in range(i + 1, len(segments_info)): id_i = segments_info[i]['id'] id_j = segments_info[j]['id'] mask_i = ins_mask[id_i] mask_j = ins_mask[id_j] mask_merge = mask_i + mask_j r_i = np.sum(mask_merge > 1) / np.sum(mask_i) r_j = np.sum(mask_merge > 1) / np.sum(mask_j) if r_i >= threshold or r_j >= threshold: if id_i not in ins2pan or id_j not in ins2pan: continue pan_id_i = ins2pan[id_i] pan_id_j = ins2pan[id_j] pan_id_top = None max_cnt = 0 pan_intersection = pan[mask_merge > 1] candidate_ids, candidate_cnts = np.unique( pan_intersection, return_counts=True ) # count the number of different segments if candidate_ids.size == 0: print("candidate_ids: ") print(candidate_ids) print("candidate_cnts: ") print(candidate_cnts) print("filename: ") print(file_name) print("imgid={} ".format(img_id)) raise Exception("Wrong intersection.") for it in range(candidate_ids.size): if candidate_ids[it] == 0: # remove background 0 continue if candidate_cnts[it] > max_cnt: max_cnt = candidate_cnts[it] pan_id_top = int(candidate_ids[it]) if pan_id_top and pan_id_top in [pan_id_i, pan_id_j]: # overlap_pairs.append((pan_id_i, pan_id_j, pan_id_top)) overlap_pairs.append( (pan2ins[pan_id_i], pan2ins[pan_id_j], pan2ins[pan_id_top])) occlusion_record['overlap_pairs'] = overlap_pairs occlusion_gt.append(occlusion_record) print('Core: {}, all {} images processed'.format(proc_id, len(annotations_set))) return occlusion_gt
def sa_pixel_to_coco_panoptic(dataset_name, export_root, thing_ids): os.makedirs(os.path.join(dataset_name, "annotations"), exist_ok=True) info = { 'description': 'This is stable 1.0 version of the ' + dataset_name + ' dataset.', 'url': 'https://superannotate.ai', 'version': '1.0', 'year': 2019, 'contributor': 'Annotator LLC', 'date_created': '2019-11-15 11:47:32.67823' } licences = [ { 'url': 'https://superannotate.ai', 'id': 1, 'name': 'Superannotate License' } ] categories = [] dbid_to_catid = {} classes = json.load( open(os.path.join(export_root, "classes", "classes.json")) ) for idx, dbclass in enumerate(classes, 1): category = { "id": idx, "name": dbclass["name"], "supercategory": dbclass["name"], "isthing": dbclass["id"] in thing_ids, "color": id2rgb(int(dbclass["color"][1:], 16)) } dbid_to_catid[dbclass["id"]] = category["id"] categories.append(category) print("Converting annotations for {} dataset ...".format(dataset_name)) id_generator = IdGenerator({cat['id']: cat for cat in categories}) panoptic_root = os.path.join( dataset_name, "panoptic_{}".format(dataset_name) ) os.makedirs(panoptic_root, exist_ok=True) jsons = glob.glob(os.path.join(export_root, "*.json")) images = [] annotations = [] for idx, filepath in tqdm(enumerate(jsons, 1)): filename = os.path.basename(filepath) imagename = filename[:-len('___pixel.json')] + '___lores.jpg' width, height = Image.open(os.path.join(export_root, imagename)).size image_info = { "id": idx, "file_name": imagename, "height": height, "width": width, "license": 1 } images.append(image_info) segments_info = [] sa_ann_json = json.load(open(os.path.join(export_root, filename))) sa_bluemask_path = os.path.join( export_root, filename[:-len('___pixel.json')] + '___save.png' ) sa_bluemask_rgb = np.asarray( Image.open(sa_bluemask_path).convert('RGB'), dtype=np.uint32 ) ann_mask = np.zeros((height, width), dtype=np.uint32) flat_mask = (sa_bluemask_rgb[:, :, 0] << 16) | (sa_bluemask_rgb[:, :, 1] << 8) | (sa_bluemask_rgb[:, :, 2]) for instance in sa_ann_json: parts = [int(part["color"][1:], 16) for part in instance["parts"]] category_id = dbid_to_catid[instance["classId"]] instance_bitmask = np.isin(flat_mask, parts) segment_id = id_generator.get_id(category_id) ann_mask[instance_bitmask] = segment_id coco_instance_mask = cocomask.encode( np.asfortranarray(instance_bitmask) ) bbox = cocomask.toBbox(coco_instance_mask).tolist() area = int(cocomask.area(coco_instance_mask)) segment_info = { "id": segment_id, "category_id": category_id, "area": area, "bbox": bbox, "iscrowd": 0 } segments_info.append(segment_info) panopticmask = imagename[:-len("jpg")] + "png" Image.fromarray(id2rgb(ann_mask)).save( os.path.join(panoptic_root, panopticmask) ) annotation = { "image_id": idx, "file_name": panopticmask, "segments_info": segments_info } annotations.append(annotation) panoptic_data = { "info": info, "licences": licences, "images": images, "annotations": annotations, "categories": categories } json_data = json.dumps(panoptic_data, indent=4) with open( os.path.join( dataset_name, "annotations", "panoptic_{}.json".format(dataset_name) ), "w+" ) as coco_json: coco_json.write(json_data)
def build_panoptic_area(img_id, output_path, detection_path, segmentation_path): """ Build panoptic segmentation of the specified image. Sort segments by area: write first smaller objects to avoid overlapping. :param img_id: image identifier (for retrieving file name) :param output_path: path to store panoptic segmentation results (png) :param detection_path: input path for detection :param segmentation_path: input path for semantic segmentation :return: the json annotation with class information for each panoptic segment. """ if not os.path.exists(output_path): os.makedirs(output_path) #Read categories and create IdGenerator (official panopticapi repository) categories = load_panoptic_category_info() id_generator = IdGenerator(categories) #Parameters: overlap_thr = 0.5 stuff_area_limit = 64 * 64 #read segmentation data segm_probs = json.load( open(segmentation_path + '/' + img_id + '_prob.json', 'r')) segm_labelmap = np.array( Image.open(segmentation_path + '/' + img_id + '_0.png'), np.uint8) #.labelmap.astype(np.uint8)).save() #read detection data detection = json.load( open(detection_path + '/' + img_id + '_prob.json', 'r')) pan_segm_id = np.zeros(segm_labelmap.shape, dtype=np.uint32) used = np.full(segm_labelmap.shape, False) annotation = {} try: annotation['image_id'] = int(img_id) except Exception: annotation['image_id'] = img_id annotation['file_name'] = img_id + '.png' segments_info = [] for obj in detection: #for ann in ... obj_mask = extract_mask_bool(obj['mask']) obj_area = np.count_nonzero(obj_mask) obj['area'] = obj_area obj['mask'] = obj_mask detection.sort(key=lambda x: x['area'], reverse=False) ##First smaller, than bigger for obj in detection: #for ann in ... obj_mask = obj['mask'] #extract_mask_bool(obj['mask']) obj_area = obj['area'] #np.count_nonzero(obj_mask) if obj_area == 0: continue #Filter out objects with intersection > 50% with used area intersection_mask = used & obj_mask intersect_area = np.count_nonzero(intersection_mask) if 1.0 * intersect_area / obj_area > overlap_thr: continue used = used | obj_mask segment_id = id_generator.get_id(obj['class']) panoptic_ann = {} panoptic_ann['id'] = segment_id panoptic_ann['category_id'] = obj['class'] if intersect_area > 0: pan_segm_id[obj_mask & (~intersection_mask)] = segment_id else: pan_segm_id[obj_mask] = segment_id segments_info.append(panoptic_ann) # # for segm_class in np.unique(segm_labelmap): segm_class = int(segm_class) if segm_class == 183: #void class continue #Check class: exclude non-stuff objects category = categories[segm_class] if category['isthing'] == 1: continue segm_mask = (segm_labelmap == segm_class) mask_left = segm_mask & (~used) # Filter out segments with small area if np.count_nonzero(mask_left) < stuff_area_limit: continue segment_id = id_generator.get_id(segm_class) panoptic_ann = {} panoptic_ann['id'] = segment_id panoptic_ann['category_id'] = segm_class used = used | mask_left pan_segm_id[mask_left] = segment_id segments_info.append(panoptic_ann) annotation['segments_info'] = segments_info # Save annotated image Image.fromarray(id2rgb(pan_segm_id)).save( os.path.join(output_path, annotation['file_name'])) ############## ##remove segments with zero area ids = set(np.unique(pan_segm_id)) segments_info_cleaned = [] for seg in segments_info: if seg['id'] in ids: segments_info_cleaned.append(seg) annotation['segments_info'] = segments_info_cleaned ################## return annotation
def panoptic_video_converter(): original_format_folder = os.path.join(ROOT_DIR, MODE, 'panoptic_inst') # folder to store panoptic PNGs out_folder = os.path.join(ROOT_DIR, MODE, 'panoptic_video') out_file = os.path.join(ROOT_DIR, 'panoptic_gt_%s_city_vps.json' % (MODE)) if not os.path.isdir(out_folder): os.makedirs(out_folder) categories = CATEGORIES categories_dict = {el['id']: el for el in CATEGORIES} file_list = sorted(glob.glob(os.path.join(original_format_folder, '*.png'))) images = [] annotations = [] instid2color = {} videos = [] id_generator = IdGenerator(categories_dict) print('==> %s/panoptic_video/ ...' % (MODE)) for idx in trange(len(file_list)): f = file_list[idx] original_format = np.array(Image.open(f)) file_name = f.split('/')[-1] image_id = file_name.rsplit('_', 2)[0] video_id = image_id[:4] if video_id not in videos: videos.append(video_id) instid2color = {} image_filename = file_name.replace('final_mask', 'newImg8bit').replace( 'gtFine_color', 'leftImg8bit') # image entry, id for image is its filename without extension images.append({ "id": image_id, "width": original_format.shape[1], "height": original_format.shape[0], "file_name": image_filename }) pan_format = np.zeros( (original_format.shape[0], original_format.shape[1], 3), dtype=np.uint8) l = np.unique(original_format) segm_info = {} for el in l: if el < 1000: semantic_id = el is_crowd = 1 else: semantic_id = el // 1000 is_crowd = 0 if semantic_id not in categories_dict: continue if categories_dict[semantic_id]['isthing'] == 0: is_crowd = 0 mask = (original_format == el) if el not in instid2color: segment_id, color = id_generator.get_id_and_color(semantic_id) instid2color[el] = (segment_id, color) else: segment_id, color = instid2color[el] pan_format[mask] = color # area = np.sum(mask) # segment area computation # # bbox computation for a segment # hor = np.sum(mask, axis=0) # hor_idx = np.nonzero(hor)[0] # x = hor_idx[0] # width = hor_idx[-1] - x + 1 # vert = np.sum(mask, axis=1) # vert_idx = np.nonzero(vert)[0] # y = vert_idx[0] # height = vert_idx[-1] - y + 1 # bbox = [int(x), int(y), int(width), int(height)] segm_info[int(segment_id)] = \ {"id": int(segment_id), "category_id": int(semantic_id), # "area": int(area), "iscrowd": is_crowd} Image.fromarray(pan_format).save(os.path.join(out_folder, file_name)) # segment sanity check, area recalculation gt_pan = np.uint32(pan_format) pan_gt = gt_pan[:, :, 0] + gt_pan[:, :, 1] * 256 + gt_pan[:, :, 2] * 256 * 256 labels, labels_cnt = np.unique(pan_gt, return_counts=True) gt_labels = [_ for _ in segm_info.keys()] gt_labels_set = set(gt_labels) for label, area in zip(labels, labels_cnt): if label == 0: continue if label not in gt_labels and label > 0: print('png label not in json labels.') segm_info[label]["area"] = int(area) gt_labels_set.remove(label) if len(gt_labels_set) != 0: raise KeyError('remaining gt_labels json') segm_info = [v for k, v in segm_info.items()] annotations.append({ 'image_id': image_id, 'file_name': file_name, "segments_info": segm_info }) d = { 'images': images, 'annotations': annotations, 'categories': categories, } save_json(d, out_file) print('==> Saved json file at %s' % (out_file))
img = np.array( Image.open(os.path.join(img_folder, image_info['file_name']))) except BaseException: print("Undable to find correspoding input image.") break segmentation = np.array(Image.open( os.path.join(segmentations_folder, ann['file_name'])), dtype=np.uint8) segmentation_id = rgb2id(segmentation) # find segments boundaries boundaries = find_boundaries(segmentation_id, mode='thick') if generate_new_colors: segmentation[:, :, :] = 0 color_generator = IdGenerator(categegories) for segment_info in ann['segments_info']: color = color_generator.get_color(segment_info['category_id']) mask = segmentation_id == segment_info['id'] segmentation[mask] = color # depict boundaries segmentation[boundaries] = [0, 0, 0] if img is None: plt.figure() plt.imshow(segmentation) plt.axis('off') else: plt.figure(figsize=(9, 5)) plt.subplot(121)
def panoptic_converter(original_format_folder, out_folder, out_file): if not os.path.isdir(out_folder): print("Creating folder {} for panoptic segmentation PNGs".format( out_folder)) os.mkdir(out_folder) categories = [] for idx, el in enumerate(labels): if el.ignoreInEval: continue categories.append({ 'id': el.id, 'name': el.name, 'color': el.color, 'supercategory': el.category, 'isthing': 1 if el.hasInstances else 0 }) categories_dict = {cat['id']: cat for cat in categories} file_list = sorted( glob.glob( os.path.join(original_format_folder, '*/*_gtFine_instanceIds.png'))) images = [] annotations = [] for working_idx, f in enumerate(file_list): if working_idx % 10 == 0: print(working_idx, len(file_list)) original_format = np.array(Image.open(f)) file_name = f.split('/')[-1] image_id = file_name.rsplit('_', 2)[0] image_filename = '{}_leftImg8bit.png'.format(image_id) # image entry, id for image is its filename without extension images.append({ "id": image_id, "width": original_format.shape[1], "height": original_format.shape[0], "file_name": image_filename }) pan_format = np.zeros( (original_format.shape[0], original_format.shape[1], 3), dtype=np.uint8) id_generator = IdGenerator(categories_dict) ll = np.unique(original_format) segm_info = [] for el in ll: if el < 1000: semantic_id = el is_crowd = 1 else: semantic_id = el // 1000 is_crowd = 0 if semantic_id not in categories_dict: continue if categories_dict[semantic_id]['isthing'] == 0: is_crowd = 0 mask = original_format == el segment_id, color = id_generator.get_id_and_color(semantic_id) pan_format[mask] = color area = np.sum(mask) # segment area computation # bbox computation for a segment hor = np.sum(mask, axis=0) hor_idx = np.nonzero(hor)[0] x = hor_idx[0] width = hor_idx[-1] - x + 1 vert = np.sum(mask, axis=1) vert_idx = np.nonzero(vert)[0] y = vert_idx[0] height = vert_idx[-1] - y + 1 bbox = [x, y, width, height] segm_info.append({ "id": int(segment_id), "category_id": int(semantic_id), "area": area, "bbox": bbox, "iscrowd": is_crowd }) annotations.append({ 'image_id': image_id, 'file_name': file_name, "segments_info": segm_info }) Image.fromarray(pan_format).save(os.path.join(out_folder, file_name)) d = { 'images': images, 'annotations': annotations, 'categories': categories, } save_json(d, out_file)