def infer_panoptic_mask(self, instance_mask_only=False): segments = [] meta = { 'image_id': None, # not needed for now 'file_name': None, 'segments_info': segments } id_gen = IdGenerator(self.categories) instance_tsr, _, sem_cats = self.get_instance_tsr(resolve_overlap=True) assert instance_tsr.sum( 0).max() <= 1, 'contested pixs should not exist' sem_cats = sem_cats.tolist() sem_cats = [self.trainId_2_catId[el] for el in sem_cats] iids = [] for _cat, _ins_mask in zip(sem_cats, instance_tsr): _id = id_gen.get_id(_cat) iids.append(_id) segments.append({ 'id': _id, 'category_id': _cat, 'isthing': 1, 'bbox': [ int(elem) for elem in get_xywh_bbox_from_binary_mask(_ins_mask) ], 'area': int(_ins_mask.sum()) }) # note numpy.int64 is not json serializable mask = (instance_tsr * np.array(iids).reshape(-1, 1, 1)).sum(axis=0) mask = mask.astype(np.uint32) if instance_mask_only: return mask, meta sem_remain = self.sem_decision.copy() sem_remain[mask > 0] = -1 for trainId in np.unique(sem_remain): if trainId < 0: continue _cat = self.trainId_2_catId[trainId] if self.categories[_cat]['isthing']: continue # abstain on ungrouped instance pixels _id = id_gen.get_id(_cat) segments.append({'id': _id, 'category_id': _cat, 'isthing': 0}) mask[sem_remain == trainId] = _id for seg in meta['segments_info']: assert seg['id'] in mask return mask, meta
def show_semseg_result(semseg_json_file, categories_json_file, out_image_file): sem_by_image = defaultdict(list) with open(semseg_json_file, 'r') as f: sem_results = json.load(f) print("Semantic segmentation:") print("\tJSON file: {}".format(semseg_json_file)) with open(categories_json_file, 'r') as f: categories_list = json.load(f) categories = {el['id']: el for el in categories_list} for sem in sem_results: img_id = sem['image_id'] sem_by_image[sem['image_id']].append(sem) id_generator = IdGenerator(categories) #pan_segm_id = np.zeros((mask.shape[0], mask.shape[1]), dtype=np.uint32) pan_segm_id = np.zeros((480, 640), dtype=np.uint32) # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # To Do: change logic of multiple annotations case, for now, it is override # but we can easily learn it from panoptic combine script # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! for ann in sem_by_image[img_id]: mask = COCOmask.decode(ann['segmentation']) #print(mask.shape()) plt.imshow(mask) plt.show() segment_id = id_generator.get_id(ann['category_id']) print("id: ") print(ann['category_id']) pan_segm_id[mask==1] = segment_id print("segment_id: ") print(segment_id) print(id2rgb(pan_segm_id).shape) #print(sem_by_image) Image.fromarray(id2rgb(pan_segm_id)).save( os.path.join("/home/zhiliu/Documents/Panoptic_Segement/Cocopanopticapi/VanillaPanopticSeg/data/predictions/test_result_for_vox/", out_image_file) )
def combine_to_panoptic_single_core(proc_id, img_ids, img_id2img, inst_by_image, sem_by_image, segmentations_folder, overlap_thr, stuff_area_limit, categories): panoptic_json = [] id_generator = IdGenerator(categories) for idx, img_id in enumerate(img_ids): img = img_id2img[img_id] if idx % 100 == 0: print('Core: {}, {} from {} images processed.'.format( proc_id, idx, len(img_ids))) pan_segm_id = np.zeros((img['height'], img['width']), dtype=np.uint32) used = None annotation = {} try: annotation['image_id'] = int(img_id) except Exception: annotation['image_id'] = img_id annotation['file_name'] = img['file_name'].replace('.jpg', '.png') segments_info = [] for ann in inst_by_image[img_id]: area = COCOmask.area(ann['segmentation']) if area == 0: continue if used is None: intersect = 0 used = copy.deepcopy(ann['segmentation']) else: intersect = COCOmask.area( COCOmask.merge([used, ann['segmentation']], intersect=True)) if intersect / area > overlap_thr: continue used = COCOmask.merge([used, ann['segmentation']], intersect=False) mask = COCOmask.decode(ann['segmentation']) == 1 if intersect != 0: mask = np.logical_and(pan_segm_id == 0, mask) segment_id = id_generator.get_id(ann['category_id']) panoptic_ann = {} panoptic_ann['id'] = segment_id panoptic_ann['category_id'] = ann['category_id'] pan_segm_id[mask] = segment_id segments_info.append(panoptic_ann) for ann in sem_by_image[img_id]: mask = COCOmask.decode(ann['segmentation']) == 1 mask_left = np.logical_and(pan_segm_id == 0, mask) if mask_left.sum() < stuff_area_limit: continue segment_id = id_generator.get_id(ann['category_id']) panoptic_ann = {} panoptic_ann['id'] = segment_id panoptic_ann['category_id'] = ann['category_id'] pan_segm_id[mask_left] = segment_id segments_info.append(panoptic_ann) annotation['segments_info'] = segments_info panoptic_json.append(annotation) Image.fromarray(id2rgb(pan_segm_id)).save( os.path.join(segmentations_folder, annotation['file_name'])) return panoptic_json
def sa_pixel_to_coco_panoptic(dataset_name, export_root, thing_ids): os.makedirs(os.path.join(dataset_name, "annotations"), exist_ok=True) info = { 'description': 'This is stable 1.0 version of the ' + dataset_name + ' dataset.', 'url': 'https://superannotate.ai', 'version': '1.0', 'year': 2019, 'contributor': 'Annotator LLC', 'date_created': '2019-11-15 11:47:32.67823' } licences = [ { 'url': 'https://superannotate.ai', 'id': 1, 'name': 'Superannotate License' } ] categories = [] dbid_to_catid = {} classes = json.load( open(os.path.join(export_root, "classes", "classes.json")) ) for idx, dbclass in enumerate(classes, 1): category = { "id": idx, "name": dbclass["name"], "supercategory": dbclass["name"], "isthing": dbclass["id"] in thing_ids, "color": id2rgb(int(dbclass["color"][1:], 16)) } dbid_to_catid[dbclass["id"]] = category["id"] categories.append(category) print("Converting annotations for {} dataset ...".format(dataset_name)) id_generator = IdGenerator({cat['id']: cat for cat in categories}) panoptic_root = os.path.join( dataset_name, "panoptic_{}".format(dataset_name) ) os.makedirs(panoptic_root, exist_ok=True) jsons = glob.glob(os.path.join(export_root, "*.json")) images = [] annotations = [] for idx, filepath in tqdm(enumerate(jsons, 1)): filename = os.path.basename(filepath) imagename = filename[:-len('___pixel.json')] + '___lores.jpg' width, height = Image.open(os.path.join(export_root, imagename)).size image_info = { "id": idx, "file_name": imagename, "height": height, "width": width, "license": 1 } images.append(image_info) segments_info = [] sa_ann_json = json.load(open(os.path.join(export_root, filename))) sa_bluemask_path = os.path.join( export_root, filename[:-len('___pixel.json')] + '___save.png' ) sa_bluemask_rgb = np.asarray( Image.open(sa_bluemask_path).convert('RGB'), dtype=np.uint32 ) ann_mask = np.zeros((height, width), dtype=np.uint32) flat_mask = (sa_bluemask_rgb[:, :, 0] << 16) | (sa_bluemask_rgb[:, :, 1] << 8) | (sa_bluemask_rgb[:, :, 2]) for instance in sa_ann_json: parts = [int(part["color"][1:], 16) for part in instance["parts"]] category_id = dbid_to_catid[instance["classId"]] instance_bitmask = np.isin(flat_mask, parts) segment_id = id_generator.get_id(category_id) ann_mask[instance_bitmask] = segment_id coco_instance_mask = cocomask.encode( np.asfortranarray(instance_bitmask) ) bbox = cocomask.toBbox(coco_instance_mask).tolist() area = int(cocomask.area(coco_instance_mask)) segment_info = { "id": segment_id, "category_id": category_id, "area": area, "bbox": bbox, "iscrowd": 0 } segments_info.append(segment_info) panopticmask = imagename[:-len("jpg")] + "png" Image.fromarray(id2rgb(ann_mask)).save( os.path.join(panoptic_root, panopticmask) ) annotation = { "image_id": idx, "file_name": panopticmask, "segments_info": segments_info } annotations.append(annotation) panoptic_data = { "info": info, "licences": licences, "images": images, "annotations": annotations, "categories": categories } json_data = json.dumps(panoptic_data, indent=4) with open( os.path.join( dataset_name, "annotations", "panoptic_{}.json".format(dataset_name) ), "w+" ) as coco_json: coco_json.write(json_data)
def build_panoptic_area(img_id, output_path, detection_path, segmentation_path): """ Build panoptic segmentation of the specified image. Sort segments by area: write first smaller objects to avoid overlapping. :param img_id: image identifier (for retrieving file name) :param output_path: path to store panoptic segmentation results (png) :param detection_path: input path for detection :param segmentation_path: input path for semantic segmentation :return: the json annotation with class information for each panoptic segment. """ if not os.path.exists(output_path): os.makedirs(output_path) #Read categories and create IdGenerator (official panopticapi repository) categories = load_panoptic_category_info() id_generator = IdGenerator(categories) #Parameters: overlap_thr = 0.5 stuff_area_limit = 64 * 64 #read segmentation data segm_probs = json.load( open(segmentation_path + '/' + img_id + '_prob.json', 'r')) segm_labelmap = np.array( Image.open(segmentation_path + '/' + img_id + '_0.png'), np.uint8) #.labelmap.astype(np.uint8)).save() #read detection data detection = json.load( open(detection_path + '/' + img_id + '_prob.json', 'r')) pan_segm_id = np.zeros(segm_labelmap.shape, dtype=np.uint32) used = np.full(segm_labelmap.shape, False) annotation = {} try: annotation['image_id'] = int(img_id) except Exception: annotation['image_id'] = img_id annotation['file_name'] = img_id + '.png' segments_info = [] for obj in detection: #for ann in ... obj_mask = extract_mask_bool(obj['mask']) obj_area = np.count_nonzero(obj_mask) obj['area'] = obj_area obj['mask'] = obj_mask detection.sort(key=lambda x: x['area'], reverse=False) ##First smaller, than bigger for obj in detection: #for ann in ... obj_mask = obj['mask'] #extract_mask_bool(obj['mask']) obj_area = obj['area'] #np.count_nonzero(obj_mask) if obj_area == 0: continue #Filter out objects with intersection > 50% with used area intersection_mask = used & obj_mask intersect_area = np.count_nonzero(intersection_mask) if 1.0 * intersect_area / obj_area > overlap_thr: continue used = used | obj_mask segment_id = id_generator.get_id(obj['class']) panoptic_ann = {} panoptic_ann['id'] = segment_id panoptic_ann['category_id'] = obj['class'] if intersect_area > 0: pan_segm_id[obj_mask & (~intersection_mask)] = segment_id else: pan_segm_id[obj_mask] = segment_id segments_info.append(panoptic_ann) # # for segm_class in np.unique(segm_labelmap): segm_class = int(segm_class) if segm_class == 183: #void class continue #Check class: exclude non-stuff objects category = categories[segm_class] if category['isthing'] == 1: continue segm_mask = (segm_labelmap == segm_class) mask_left = segm_mask & (~used) # Filter out segments with small area if np.count_nonzero(mask_left) < stuff_area_limit: continue segment_id = id_generator.get_id(segm_class) panoptic_ann = {} panoptic_ann['id'] = segment_id panoptic_ann['category_id'] = segm_class used = used | mask_left pan_segm_id[mask_left] = segment_id segments_info.append(panoptic_ann) annotation['segments_info'] = segments_info # Save annotated image Image.fromarray(id2rgb(pan_segm_id)).save( os.path.join(output_path, annotation['file_name'])) ############## ##remove segments with zero area ids = set(np.unique(pan_segm_id)) segments_info_cleaned = [] for seg in segments_info: if seg['id'] in ids: segments_info_cleaned.append(seg) annotation['segments_info'] = segments_info_cleaned ################## return annotation