def show_semseg_result(semseg_json_file, categories_json_file, out_image_file):
sem_by_image = defaultdict(list)
with open(semseg_json_file, 'r') as f:
sem_results = json.load(f)
print("Semantic segmentation:")
print("\tJSON file: {}".format(semseg_json_file))
with open(categories_json_file, 'r') as f:
categories_list = json.load(f)
categories = {el['id']: el for el in categories_list}
for sem in sem_results:
img_id = sem['image_id']
sem_by_image[sem['image_id']].append(sem)
id_generator = IdGenerator(categories)
#pan_segm_id = np.zeros((mask.shape[0], mask.shape[1]), dtype=np.uint32)
pan_segm_id = np.zeros((480, 640), dtype=np.uint32)
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# To Do: change logic of multiple annotations case, for now, it is override
# but we can easily learn it from panoptic combine script
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
for ann in sem_by_image[img_id]:
mask = COCOmask.decode(ann['segmentation'])
#print(mask.shape())
plt.imshow(mask)
plt.show()
segment_id = id_generator.get_id(ann['category_id'])
print("id: ")
print(ann['category_id'])
pan_segm_id[mask==1] = segment_id
print("segment_id: ")
print(segment_id)
print(id2rgb(pan_segm_id).shape)
#print(sem_by_image)
Image.fromarray(id2rgb(pan_segm_id)).save(
os.path.join("/home/zhiliu/Documents/Panoptic_Segement/Cocopanopticapi/VanillaPanopticSeg/data/predictions/test_result_for_vox/", out_image_file)
)
def get_ids_area(masks, scores, dedup=False):
# This helper function creates the final panoptic segmentation image
# It also returns the area of the masks that appears on the image
m_id = masks.transpose(0, 1).softmax(-1)
if m_id.shape[-1] == 0:
# We didn't detect any mask :(
m_id = torch.zeros((h, w), dtype=torch.long, device=m_id.device)
else:
m_id = m_id.argmax(-1).view(h, w)
if dedup:
# Merge the masks corresponding to the same stuff class
for equiv in stuff_equiv_classes.values():
if len(equiv) > 1:
for eq_id in equiv:
m_id.masked_fill_(m_id.eq(eq_id), equiv[0])
final_h, final_w = to_tuple(target_size)
seg_img = Image.fromarray(id2rgb(m_id.view(h, w).cpu().numpy()))
seg_img = seg_img.resize(size=(final_w, final_h), resample=Image.NEAREST)
np_seg_img = (
torch.ByteTensor(torch.ByteStorage.from_buffer(seg_img.tobytes())).view(final_h, final_w, 3).numpy()
)
m_id = torch.from_numpy(rgb2id(np_seg_img))
area = []
for i in range(len(scores)):
area.append(m_id.eq(i).sum().item())
return area, seg_img
def process(self, inputs, outputs):
from panopticapi.utils import id2rgb
for input, output in zip(inputs, outputs):
panoptic_img, segments_info = output["panoptic_seg"]
panoptic_img = panoptic_img.cpu().numpy()
file_name = os.path.basename(input["file_name"])
file_name_png = os.path.splitext(file_name)[0] + ".png"
with io.BytesIO() as out:
Image.fromarray(id2rgb(panoptic_img)).save(out, format="PNG")
segments_info = [
self._convert_category_id(x) for x in segments_info
]
if self.gen_png:
png = self.gen_2ch_pngs(panoptic_img, segments_info)
png_dir = os.path.join(self.output_dir, '2ch_png')
os.makedirs(png_dir, exist_ok=True)
Image.fromarray(png).save(
os.path.join(png_dir, file_name_png))
self._predictions.append({
"image_id": input["image_id"],
"file_name": file_name_png,
"png_string": out.getvalue(),
"segments_info": segments_info,
})
def update(self, panoptic, image_filename=None, image_id=None):
from panopticapi.utils import id2rgb
if image_filename is None:
raise ValueError('Need to provide image_filename.')
if image_id is None:
raise ValueError('Need to provide image_id.')
# Change void region.
panoptic[panoptic == self._void_label] = 0
segments_info = []
for pan_lab in np.unique(panoptic):
pred_class = pan_lab // self._label_divisor
if self._train_id_to_eval_id is not None:
pred_class = self._train_id_to_eval_id[pred_class]
segments_info.append(
{
'id': int(pan_lab),
'category_id': int(pred_class),
}
)
save_annotation(id2rgb(panoptic), self._panoptic_dir, image_filename, add_colormap=False)
self._predictions.append(
{
'image_id': int(image_id),
'file_name': image_filename + '.png',
'segments_info': segments_info,
}
)
def _create_single_category(self, item):
category = {
'id': item.id,
'name': item.class_name,
'supercategory': item.class_name,
'isthing': 1,
'color': id2rgb(item.id)
}
return category
def process(self, inx, model_outputs, inputs, dset):
from panoptic.entry import gt_tsr_res_reduction # HELPPPPPPPPPPPPPPPPPPPPPPPPPPPPP
sem_pd, vote_pd = model_outputs
sem_pd, vote_pd = F.softmax(sem_pd, dim=1), F.softmax(vote_pd, dim=1)
model = self.model
oracle_mode = self.oracle_mode
oracle_res = 4
imgMeta, segments_info, _, pan_gt_mask = dset.pan_getitem(
inx, apply_trans=False)
if dset.transforms is not None:
_, trans_pan_gt = dset.transforms(_, pan_gt_mask)
else:
trans_pan_gt = pan_gt_mask.copy()
pan_gt_ann = {
'image_id': imgMeta['id'],
# shameful mogai; can only access image f_name here. alas...
'file_name': imgMeta['file_name'].split('.')[0] + '.png',
'segments_info': list(segments_info.values())
}
if oracle_mode is not None:
sem_ora, vote_ora = gt_tsr_res_reduction(oracle_res,
dset.gt_prod_handle,
dset.meta, model.pcv,
trans_pan_gt,
segments_info)
if oracle_mode == 'vote':
pass # if using model sem pd, maintain stuff pred thresh
else: # sem or full oracle, using gt sem pd, do not filter
self.infer_cfg['stuff_pred_thresh'] = -1
if oracle_mode == 'sem':
sem_pd = sem_ora
elif oracle_mode == 'vote':
vote_pd = vote_ora
else:
sem_pd, vote_pd = sem_ora, vote_ora
pan_pd_mask, pan_pd_ann = model.stitch_pan_mask(
self.infer_cfg, sem_pd, vote_pd, pan_gt_mask.size)
pan_pd_ann['image_id'] = pan_gt_ann['image_id']
pan_pd_ann['file_name'] = pan_gt_ann['file_name']
self.overall_pred_meta['annotations'].append(pan_pd_ann)
self.metric.update(pan_gt_ann, rgb2id(np.array(pan_gt_mask)),
pan_pd_ann, pan_pd_mask)
pan_pd_mask = Image.fromarray(id2rgb(pan_pd_mask))
fname = osp.join(self.pan_mask_dir, pan_pd_ann['file_name'])
os.makedirs(osp.dirname(fname), exist_ok=True) # make region subdir
pan_pd_mask.save(fname)
def coco_panoptic_segmentation_to_sa_pixel(coco_path, images_path):
coco_json = json.load(open(coco_path))
hex_colors = blue_color_generator(len(coco_json["categories"]))
annotate_list = coco_json["annotations"]
cat_id_to_cat = {}
for cat in coco_json['categories']:
cat_id_to_cat[cat['id']] = cat['name']
sa_jsons = {}
for annotate in tqdm(annotate_list, "Converting"):
annot_name = Path(annotate["file_name"]).stem
img_cv = cv2.imread(str(images_path / (annot_name + ".png")))
if img_cv is None:
print(
"'{}' file dosen't exist!".format(
images_path / (annot_name + ".png")
)
)
continue
img = cv2.cvtColor(img_cv, cv2.COLOR_BGR2RGB)
H, W, C = img.shape
img = img.reshape((H * W, C))
segments = annotate["segments_info"]
hex_colors = blue_color_generator(len(segments))
out_json = []
for i, seg in enumerate(segments):
img[np.all(img == id2rgb(seg["id"]),
axis=1)] = hex_to_rgb(hex_colors[i])
dd = {
"classId": seg["category_id"],
'className': cat_id_to_cat[seg["category_id"]],
"probability": 100,
"visible": True,
"parts": [{
"color": hex_colors[i]
}],
"attributes": [],
"attributeNames": [],
"imageId": annotate["image_id"]
}
out_json.append(dd)
img = cv2.cvtColor(img.reshape((H, W, C)), cv2.COLOR_RGB2BGR)
cv2.imwrite(str(images_path / (annot_name + ".jpg___save.png")), img)
file_name = annot_name + ".jpg___pixel.json"
sa_jsons[file_name] = out_json
(images_path / (annot_name + ".png")).unlink()
return sa_jsons
def generate_pred_panoptic(cfg, outputs):
"""
Take all output of a model and save a json file with al predictions as
well as all panoptic image prediction.
This is done in order to use `pq_compute` function from panoptic api
Args:
- cfg (Config) : config object
- outputs (list[dict]) : List of a full epoch of outputs
"""
# Create prediction dir if needed
pred_dir = os.path.join(cfg.DATASET_PATH, cfg.PRED_DIR)
if not os.path.exists(pred_dir): os.makedirs(pred_dir)
annotations = []
print("Saving panoptic prediction to compute validation metrics")
# Loop on each validation output
for output in tqdm(outputs):
# Loop on each image of the batch
for img_panoptic, image_id in zip(output['panoptic'],
output['image_id']):
img_data = dict()
img_data['image_id'] = image_id
# Resize the image to original size
img_panoptic = F.interpolate(
img_panoptic.unsqueeze(0).unsqueeze(0).float(),
size=(1024, 2048),
mode='nearest')[0, 0, ...]
# Create segment_info data
img_data['segments_info'] = []
img_panoptic = img_panoptic.cpu().numpy()
for instance in np.unique(img_panoptic):
if instance == 0:
continue
img_data['segments_info'].append({
'id':
int(instance),
'category_id':
int(instance) if instance < 1000 else int(instance / 1000)
})
# Save panotic_pred
img_data['file_name'] = "{}_preds_panoptic.png".format(image_id)
img = id2rgb(img_panoptic)
img_to_save = Image.fromarray(img)
img_to_save.save(os.path.join(pred_dir, img_data['file_name']))
# Add annotation of a one image
annotations.append(img_data)
save_json_file(cfg, annotations)
def pq_compute_single_core(proc_id, annotation_set, gt_folder, pred_folder,
categories):
pq_stat = PQStat()
idx = 0
for gt_ann, pred_ann in annotation_set:
if idx % 100 == 0:
print('Core: {}, {} from {} images processed'.format(
proc_id, idx, len(annotation_set)))
idx += 1
pan_gt = np.array(Image.open(
os.path.join(gt_folder, gt_ann['file_name'])),
dtype=np.uint32)
pan_gt = rgb2id(pan_gt)
pan_pred = np.array(Image.open(
os.path.join(pred_folder, pred_ann['file_name'])),
dtype=np.uint32)
pan_pred = rgb2id(pan_pred)
# downsize pred and upsize back; resolve the lost segments
ratio = 8
if ratio > 1:
pan_pred = id2rgb(pan_pred)
pan_pred = Image.fromarray(pan_pred)
h, w = pan_pred.size
pan_pred = pan_pred\
.resize((h // ratio, w // ratio), Image.NEAREST)\
.resize((h, w), Image.NEAREST)
pan_pred = np.array(pan_pred, dtype=np.uint32)
pan_pred = rgb2id(pan_pred)
acc = []
_p_segs = pred_ann['segments_info']
for el in _p_segs:
iid = el['id']
if iid not in pan_pred:
continue
acc.append(el)
pred_ann['segments_info'] = acc
_single_img_stat = pq_compute_single_img(categories, gt_ann, pan_gt,
pred_ann, pan_pred)
pq_stat += _single_img_stat
print('Core: {}, all {} images processed'.format(proc_id,
len(annotation_set)))
return pq_stat
def coco_panoptic_segmentation_to_sa_pixel(coco_path, images_path):
coco_json = json.load(open(coco_path))
hex_colors = _blue_color_generator(len(coco_json["categories"]))
annotate_list = coco_json["annotations"]
for annotate in tqdm(annotate_list, "Converting"):
annot_name = os.path.splitext(annotate["file_name"])[0]
img_cv = cv2.imread(os.path.join(images_path, annot_name + ".png"))
if img_cv is None:
print("'{}' file dosen't exist!".format(
os.path.join(images_path, annot_name + ".png")))
continue
img = cv2.cvtColor(img_cv, cv2.COLOR_BGR2RGB)
H, W, C = img.shape
img = img.reshape((H * W, C))
segments = annotate["segments_info"]
hex_colors = _blue_color_generator(len(segments) + 1)
out_json = []
for i, seg in enumerate(segments):
img[np.all(img == id2rgb(seg["id"]),
axis=1)] = _hex_to_rgb(hex_colors[i + 1])
dd = {
"classId": seg["category_id"],
"probability": 100,
"visible": True,
"parts": [{
"color": hex_colors[i + 1]
}],
"attributes": [],
"attributeNames": [],
"imageId": annotate["image_id"]
}
out_json.append(dd)
with open(os.path.join(images_path, annot_name + ".jpg___pixel.json"),
"w") as writer:
json.dump(out_json, writer, indent=2)
img = cv2.cvtColor(img.reshape((H, W, C)), cv2.COLOR_RGB2BGR)
cv2.imwrite(os.path.join(images_path, annot_name + ".jpg___save.png"),
img)
os.remove(os.path.join(images_path, annot_name + ".png"))
def process(self, inputs, outputs):
from panopticapi.utils import id2rgb
for input, output in zip(inputs, outputs):
panoptic_img, segments_info = output["panoptic_seg"]
panoptic_img = panoptic_img.cpu().numpy()
if segments_info is None:
# If "segments_info" is None, we assume "panoptic_img" is a
# H*W int32 image storing the panoptic_id in the format of
# category_id * label_divisor + instance_id. We reserve -1 for
# VOID label, and add 1 to panoptic_img since the official
# evaluation script uses 0 for VOID label.
label_divisor = self._metadata.label_divisor
segments_info = []
for panoptic_label in np.unique(panoptic_img):
if panoptic_label == -1:
# VOID region.
continue
pred_class = panoptic_label // label_divisor
isthing = (
pred_class in self._metadata.thing_dataset_id_to_contiguous_id.values()
)
segments_info.append(
{
"id": int(panoptic_label) + 1,
"category_id": int(pred_class),
"isthing": bool(isthing),
}
)
# Official evaluation script uses 0 for VOID label.
panoptic_img += 1
file_name = os.path.basename(input["file_name"])
file_name_png = os.path.splitext(file_name)[0] + ".png"
with io.BytesIO() as out:
Image.fromarray(id2rgb(panoptic_img)).save(out, format="PNG")
segments_info = [self._convert_category_id(x) for x in segments_info]
self._predictions.append(
{
"image_id": input["image_id"],
"file_name": file_name_png,
"png_string": out.getvalue(),
"segments_info": segments_info,
}
)
def sa_to_output_format(self):
out_json = self._create_skeleton()
out_json['categories'] = self._create_categories(
os.path.join(self.export_root, 'classes_mapper.json')
)
panoptic_root = os.path.join(
self.dataset_name, "panoptic_{}".format(self.dataset_name)
)
images = []
annotations = []
id_generator = self._make_id_generator()
jsons = glob.glob(
os.path.join(self.export_root, '*pixel.json'), recursive=True
)
for id_, json_ in tqdm(enumerate(jsons, 1)):
res = self._sa_to_coco_single(id_, json_, id_generator)
panoptic_mask = json_[:-len('___pixel.json')] + '.png'
Image.fromarray(id2rgb(res[2])).save(panoptic_mask)
annotation = {
'image_id': res[0]['id'],
'file_name': panoptic_mask,
'segments_info': res[1]
}
annotations.append(annotation)
images.append(res[0])
out_json['annotations'] = annotations
out_json['images'] = images
json_data = json.dumps(out_json, indent=4)
with open(
os.path.join(self.output_dir, '{}.json'.format(self.dataset_name)),
'w+'
) as coco_json:
coco_json.write(json_data)
self.set_num_converted(len(jsons))
def _pan2json(self, results, outfile_prefix):
"""Convert panoptic results to COCO panoptic json style."""
label2cat = dict((v, k) for (k, v) in self.cat2label.items())
pred_annotations = []
outdir = os.path.join(os.path.dirname(outfile_prefix), 'panoptic')
for idx in range(len(self)):
img_id = self.img_ids[idx]
segm_file = self.data_infos[idx]['segm_file']
pan = results[idx]
pan_labels = np.unique(pan)
segm_info = []
for pan_label in pan_labels:
sem_label = pan_label % INSTANCE_OFFSET
# We reserve the length of self.CLASSES for VOID label
if sem_label == len(self.CLASSES):
continue
# convert sem_label to json label
cat_id = label2cat[sem_label]
is_thing = self.categories[cat_id]['isthing']
mask = pan == pan_label
area = mask.sum()
segm_info.append({
'id': int(pan_label),
'category_id': cat_id,
'isthing': is_thing,
'area': int(area)
})
# evaluation script uses 0 for VOID label.
pan[pan % INSTANCE_OFFSET == len(self.CLASSES)] = VOID
pan = id2rgb(pan).astype(np.uint8)
mmcv.imwrite(pan[:, :, ::-1], os.path.join(outdir, segm_file))
record = {
'image_id': img_id,
'segments_info': segm_info,
'file_name': segm_file
}
pred_annotations.append(record)
pan_json_results = dict(annotations=pred_annotations)
return pan_json_results
def sa_to_output_format(self):
out_json = self._create_skeleton()
out_json['categories'] = self._create_categories(
self.export_root / 'classes_mapper.json'
)
cat_id_map = json.load(open(self.export_root / 'classes_mapper.json'))
images = []
annotations = []
id_generator = self._make_id_generator()
jsons_gen = self.export_root.glob('*pixel.json')
jsons = [path for path in jsons_gen]
for id_, json_ in tqdm(enumerate(jsons, 1)):
res = self._sa_to_coco_single(id_, json_, id_generator, cat_id_map)
panoptic_mask = str(json_)[:-len('___pixel.json')] + '.png'
Image.fromarray(id2rgb(res[2])).save(panoptic_mask)
annotation = {
'image_id': res[0]['id'],
'file_name': panoptic_mask,
'segments_info': res[1]
}
annotations.append(annotation)
images.append(res[0])
out_json['annotations'] = annotations
out_json['images'] = images
json_data = json.dumps(out_json, indent=4)
with open(
self.output_dir / '{}.json'.format(self.dataset_name), 'w+'
) as coco_json:
coco_json.write(json_data)
self.set_num_converted(len(jsons))
def process(self, inputs, outputs):
from panopticapi.utils import id2rgb
preds = []
for input, output in zip(inputs, outputs):
panoptic_img, segments_info = output["panoptic_seg"]
panoptic_img = panoptic_img.cpu().numpy()
file_name = os.path.basename(input["file_name"])
file_name_png = os.path.splitext(file_name)[0] + ".png"
with io.BytesIO() as out:
Image.fromarray(id2rgb(panoptic_img)).save(out, format="PNG")
segments_info = [
self._convert_category_id(x) for x in segments_info
]
preds.append({
"image_id": input["image_id"],
"file_name": file_name_png,
"png_string": out.getvalue(),
"segments_info": segments_info,
})
return {self.evaluator_name: preds}
def _create_panoptic_gt_annotations(ann_file):
categories = [{
'id': 0,
'name': 'person',
'supercategory': 'person',
'isthing': 1
}, {
'id': 1,
'name': 'dog',
'supercategory': 'dog',
'isthing': 1
}, {
'id': 2,
'name': 'wall',
'supercategory': 'wall',
'isthing': 0
}]
images = [{
'id': 0,
'width': 80,
'height': 60,
'file_name': 'fake_name1.jpg',
}]
annotations = [{
'segments_info': [{
'id': 1,
'category_id': 0,
'area': 400,
'bbox': [10, 10, 10, 40],
'iscrowd': 0
}, {
'id': 2,
'category_id': 0,
'area': 400,
'bbox': [30, 10, 10, 40],
'iscrowd': 0
}, {
'id': 3,
'category_id': 1,
'iscrowd': 0,
'bbox': [50, 10, 10, 5],
'area': 50
}, {
'id': 4,
'category_id': 2,
'iscrowd': 0,
'bbox': [0, 0, 80, 60],
'area': 3950
}],
'file_name':
'fake_name1.png',
'image_id':
0
}]
gt_json = {
'images': images,
'annotations': annotations,
'categories': categories
}
# 4 is the id of the background class annotation.
gt = np.zeros((60, 80), dtype=np.int64) + 4
gt_bboxes = np.array([[10, 10, 10, 40], [30, 10, 10, 40], [50, 10, 10, 5]],
dtype=np.int64)
for i in range(3):
x, y, w, h = gt_bboxes[i]
gt[y:y + h, x:x + w] = i + 1 # id starts from 1
gt = id2rgb(gt).astype(np.uint8)
img_path = osp.join(osp.dirname(ann_file), 'fake_name1.png')
mmcv.imwrite(gt[:, :, ::-1], img_path)
mmcv.dump(gt_json, ann_file)
return gt_json
# panoptic
elif 'panoptic' in str(coco_json_file):
copy_png()
rename_png()
pan_loader = []
for annot in json_data['annotations']:
for cat in json_data['categories']:
for si in annot['segments_info']:
if cat['id'] == si['category_id']:
sa_dict = {
'classId': cat['id'],
'probability': 100,
'parts': [{
'color': rgb_to_hex(tuple(id2rgb(si['id'])))
}],
'attributes': [],
'attributeNames': [],
'imageId': annot['image_id']
}
pan_loader.append((sa_dict['imageId'], sa_dict))
for img in json_data['images']:
f_loader = []
for img_id, img_data in pan_loader:
if img['id'] == img_id:
f_loader.append(img_data)
with open(
os.path.join(main_dir,
os.path.join(coco_path_folder,
os.path.join("panoptic_masks",
img_name + ".png")))
if img_cv is None:
print("Error: '{}' file dosen't exist!".format(
os.path.join("panoptic_masks", img_name + ".png")))
break
img = cv2.cvtColor(img_cv, cv2.COLOR_BGR2RGB)
H, W, C = img.shape
img = img.reshape((H * W, C))
segments = annotate["segments_info"]
hex_colors = blue_color_generator(len(segments))
out_json = []
for i, seg in enumerate(segments):
img[np.all(img == id2rgb(seg["id"]),
axis=1)] = hex_to_rgb(hex_colors[i])
dd = {
"classId": seg["category_id"],
"probability": 100,
"visible": True,
"parts": [{
"color": hex_colors[i]
}],
"attributes": [],
"attributeNames": [],
"imageId": annotate["image_id"]
}
out_json.append(dd)
with open(os.path.join(sa_dir, img_name + ".jpg___pixel.json"),
"w") as writer:
def PQ_eval_dirty(
self, oracle_res=4, save_output=False,
oracle_mode=None
# oracle=False, semantic_oracle=False, vote_oracle=False
):
"""
optionally save all predicted files through output writer
"""
_VALID_ORACLE_MODES = ('full', 'sem', 'vote')
assert not self.mp_distributed
model = self.model
loader = self.val_loader
dset = loader.dataset
eval_at = model.curr_epoch - 1
logger.info("Panoptic Quality eval at epoch {}".format(eval_at))
# setup space to save pan predictions
PRED_OUT_NAME = 'pred'
dump_root = osp.join(self.output_mngr.root, 'pd_dump')
pred_meta_fname = osp.join(dump_root, '{}.json'.format(PRED_OUT_NAME))
pred_mask_dir = osp.join(dump_root, PRED_OUT_NAME)
del dump_root
overall_pred_meta = {
'images': list(dset.imgs.values()),
'categories': list(dset.meta['cats'].values()),
'annotations': []
}
metric = PQMetric(dset.meta)
timer = CompoundTimer()
timer.data.tic()
# sota_sseg = SOTASSeg(dset.name)
# do nasty
upsnet_pred_json = json.load(open('/home-nfs/whc/panout/upsnet/coco/val/pred.json', 'r'))
upsnet_pred_ann = {_['image_id']: _ for _ in upsnet_pred_json['annotations']}
ups_thing_cat_ids = [_['id'] for _ in upsnet_pred_json['categories'] if _['isthing']]
ups_stuff_cat_ids = [_['id'] for _ in upsnet_pred_json['categories'] if _['isthing'] == 0]
def load_pred_mask(filename):
return rgb2id(np.array(Image.open(f'/home-nfs/whc/panout/upsnet/coco/val/pred/{filename}').convert('RGB')))
for i, inputs in enumerate(loader):
x = inputs[0].cuda()
del inputs
if self.debug and (i > 5):
break
imgMeta, segments_info, _, pan_gt_mask = dset.pan_getitem(
i, apply_trans=False)
# rt change
# to get ground truth instance centroid
# lo_pan_mask = pan_gt_mask.resize(
# np.array(_.size, dtype=np.int) // 4, resample=Image.NEAREST
# )
# tmp_handle = dset.gt_prod_handle(dset.meta, dset.pcv,
# lo_pan_mask, segments_info)
# tmp_handle.generate_gt()
# gt_ins_centroids = tmp_handle.ins_centroids
if dset.transforms is not None:
_, trans_pan_gt = dset.transforms(_, pan_gt_mask)
else:
trans_pan_gt = pan_gt_mask.copy()
pan_gt_mask = pan_gt_mask
pan_gt_ann = {
'image_id': imgMeta['id'],
# shameful mogai; can only access image f_name here. alas...
'file_name': imgMeta['file_name'].split('.')[0] + '.png',
'segments_info': list(segments_info.values())
}
# torch.cuda.synchronize()
timer.data.toc()
timer.compute.tic()
if oracle_mode != 'full': # way too ugly
from fabric.utils.timer import global_timer
global_timer.network.tic()
sem_pd, vote_pd = model.infer(x, softmax_normalize=True)
global_timer.network.toc()
if oracle_mode is not None:
assert oracle_mode in _VALID_ORACLE_MODES
sem_ora, vote_ora = gt_tsr_res_reduction(
oracle_res, self.gt_prod_handle,
dset.meta, model.pcv, trans_pan_gt, segments_info
)
if oracle_mode == 'vote':
pass # if using model sem pd, maintain stuff pred thresh
else: # sem or full oracle, using gt sem pd, do not filter
model.dset_meta['stuff_pred_thresh'] = -1
if oracle_mode == 'sem':
sem_pd = sem_ora
elif oracle_mode == 'vote':
vote_pd = vote_ora
else:
sem_pd, vote_pd = sem_ora, vote_ora
pan_pd_mask, pan_pd_ann = model.stitch_pan_mask(
cfg.pcv, sem_pd, vote_pd, pan_gt_mask.size
)
# if oracle:
# sem_pd, vote_pd = gt_tsr_res_reduction(
# oracle_res, self.gt_prod_handle,
# dset.meta, model.pcv, trans_pan_gt, segments_info
# )
# model.dset_meta['stuff_pred_thresh'] = -1
# pan_pd_mask, pan_pd_ann = model.stitch_pan_mask(
# sem_pd, vote_pd, pan_gt_mask.size)
# elif semantic_oracle:
# sem_pd, vote_pd = model.infer(x, softmax_normalize=True)
# sem_pd, _ = gt_tsr_res_reduction(
# oracle_res, self.gt_prod_handle,
# dset.meta, model.pcv, trans_pan_gt, segments_info
# )
# model.dset_meta['stuff_pred_thresh'] = -1
# pan_pd_mask, pan_pd_ann = model.stitch_pan_mask(
# sem_pd, vote_pd, pan_gt_mask.size)
# elif vote_oracle:
# sem_pd, vote_pd = model.infer(x, softmax_normalize=True)
# _, vote_pd = gt_tsr_res_reduction(
# oracle_res, self.gt_prod_handle,
# dset.meta, model.pcv, trans_pan_gt, segments_info
# )
# pan_pd_mask, pan_pd_ann = model.stitch_pan_mask(
# sem_pd, vote_pd, pan_gt_mask.size)
# else:
# # _sem_pd, _vote_pd = gt_tsr_res_reduction(
# # oracle_res, self.gt_prod_handle,
# # dset.meta, model.pcv, trans_pan_gt, segments_info
# # )
# sem_pd, vote_pd = model.infer(x, softmax_normalize=True)
# # if hasattr(self, 'model2'):
# # _sem_pd, _vote_pd = self.model2.infer(x, softmax_normalize=True)
# # sem_pd = (sem_pd + _sem_pd) / 2
# # vote_pd = (vote_pd + _vote_pd) / 2
# # sem_pd, vote_pd = model.flip_infer(x, softmax_normalize=True)
# # sem_pd = sota_sseg.get(i, sem_pd.shape[-2:])
# # model.dset_meta['tmp_gt_ins_centroids'] = gt_ins_centroids # rt change
# pan_pd_mask, pan_pd_ann = model.stitch_pan_mask(
# sem_pd, vote_pd, pan_gt_mask.size)
pan_pd_ann['image_id'] = pan_gt_ann['image_id']
pan_pd_ann['file_name'] = pan_gt_ann['file_name']
overall_pred_meta['annotations'].append(pan_pd_ann)
metric.update(
pan_gt_ann, rgb2id(np.array(pan_gt_mask)), pan_pd_ann, pan_pd_mask
)
if save_output:
# pan_pd_mask = Image.fromarray(id2rgb(pan_pd_mask))
# fname = osp.join(pred_mask_dir, pan_pd_ann['file_name'])
# os.makedirs(osp.dirname(fname), exist_ok=True) # make region subdir
# pan_pd_mask.save(fname)
fname = osp.join(pred_mask_dir, pan_pd_ann['file_name'])
os.makedirs(osp.dirname(fname), exist_ok=True) # make region subdir
os.makedirs(osp.dirname(fname.replace(pred_mask_dir, pred_mask_dir+'_d2')), exist_ok=True) # make region subdir
# os.makedirs(osp.dirname(fname.replace(pred_mask_dir, pred_mask_dir+'_d2_ups')), exist_ok=True) # make region subdir
from .vis import d2_vis
im = dset.pan_getitem(i, apply_trans=False)[2]
if cfg.data.dataset.params['caffe_mode']:
im = np.array(im)[:, :, ::-1]
im = np.array(im)
Image.fromarray(
d2_vis(
dset.meta,
pan_pd_mask,
pan_pd_ann,
im
)
).save(fname.replace(pred_mask_dir, pred_mask_dir+'_d2'))
from .vis import d2_vis
im = dset.pan_getitem(i, apply_trans=False)[2]
if cfg.data.dataset.params['caffe_mode']:
im = np.array(im)[:, :, ::-1]
im = np.array(im)
tmp_ann = upsnet_pred_ann[pan_gt_ann['image_id']]
for seg in tmp_ann['segments_info']:
if seg['category_id'] in ups_thing_cat_ids:
seg['isthing'] = 1
else:
seg['isthing'] = 0
Image.fromarray(
d2_vis(
dset.meta,
load_pred_mask(pan_gt_ann['file_name']),
tmp_ann,
im
)
).save(fname.replace(pred_mask_dir, pred_mask_dir+'_d2').replace('.png', '_ups.png'))
pan_pd_mask = Image.fromarray(id2rgb(pan_pd_mask))
pan_pd_mask.save(fname)
# torch.cuda.synchronize()
timer.compute.toc()
if (i % 50) == 0:
logger.info(timer)
logger.info(
"iter {}, eta to val end {}".format(i, timer.eta(i, len(loader)))
)
print(metric)
timer.data.tic()
if save_output:
with open(pred_meta_fname, 'w') as f:
json.dump(overall_pred_meta, f)
self.output_mngr.save_f(
metric.results, osp.join(self.output_mngr.root, 'score.pkl')
)
logger.info("eval complete \n{}".format(metric))
return metric
def PQ_test(self, save_output=False):
assert not self.mp_distributed
model = self.model
loader = self.test_loader
dset = loader.dataset
eval_at = model.curr_epoch - 1
logger.info("Panoptic Quality eval at epoch {}".format(eval_at))
# PRED_OUT_NAME = 'panoptic_{}2017_pcv_results'.format(dset.split)
PRED_OUT_NAME = 'pred'
dump_root = osp.join(self.output_mngr.root, 'pd_dump')
pred_meta_fname = osp.join(dump_root, '{}.json'.format(PRED_OUT_NAME))
pred_mask_dir = osp.join(dump_root, PRED_OUT_NAME)
overall_pred_meta = {
'images': list(dset.imgs.values()),
'categories': list(dset.meta['cats'].values()),
'annotations': []
}
timer = CompoundTimer()
timer.data.tic()
for i, inputs in enumerate(loader):
if self.debug and (i > 5):
break
x = inputs[0].cuda()
del inputs
imgMeta, _ = dset.get_meta(i)
timer.data.toc()
timer.compute.tic()
sem_pd, vote_pd = model.infer(x, softmax_normalize=True)
# if hasattr(self, 'model2'):
# _sem_pd, _vote_pd = self.model2.infer(x, softmax_normalize=True)
# sem_pd = (sem_pd + _sem_pd) / 2
# vote_pd = (vote_pd + _vote_pd) / 2
img_original_size = (imgMeta['width'], imgMeta['height'])
pan_pd_mask, pan_pd_ann = model.stitch_pan_mask(
cfg.pcv, sem_pd, vote_pd, img_original_size
)
pan_pd_ann['image_id'] = imgMeta['id']
pan_pd_ann['file_name'] = imgMeta['file_name'].split('.')[0] + '.png'
overall_pred_meta['annotations'].append(pan_pd_ann)
if save_output:
fname = osp.join(pred_mask_dir, pan_pd_ann['file_name'])
os.makedirs(osp.dirname(fname), exist_ok=True) # make region subdir
os.makedirs(osp.dirname(fname.replace(pred_mask_dir, pred_mask_dir+'_d2')), exist_ok=True) # make region subdir
from .vis import d2_vis
im = dset.pan_getitem(i, apply_trans=False)[2]
if cfg.data.dataset.params['caffe_mode']:
im = np.array(im)[:, :, ::-1]
im = np.array(im)
Image.fromarray(
d2_vis(
dset.meta,
pan_pd_mask,
pan_pd_ann,
im
)
).save(fname.replace(pred_mask_dir, pred_mask_dir+'_d2'))
pan_pd_mask = Image.fromarray(id2rgb(pan_pd_mask))
pan_pd_mask.save(fname)
# torch.cuda.synchronize()
timer.compute.toc()
if (i % 50) == 0:
logger.info(timer)
logger.info(
"iter {}/{}, eta to val end {}".format(
i, len(loader), timer.eta(i, len(loader))
)
)
timer.data.tic()
if save_output:
with open(pred_meta_fname, 'w') as f:
json.dump(overall_pred_meta, f)
logger.info("test eval complete")
def combine_to_panoptic_single_core(proc_id, img_ids, img_id2img,
inst_by_image, sem_by_image,
segmentations_folder, overlap_thr,
stuff_area_limit, categories):
panoptic_json = []
id_generator = IdGenerator(categories)
for idx, img_id in enumerate(img_ids):
img = img_id2img[img_id]
if idx % 100 == 0:
print('Core: {}, {} from {} images processed.'.format(
proc_id, idx, len(img_ids)))
pan_segm_id = np.zeros((img['height'], img['width']), dtype=np.uint32)
used = None
annotation = {}
try:
annotation['image_id'] = int(img_id)
except Exception:
annotation['image_id'] = img_id
annotation['file_name'] = img['file_name'].replace('.jpg', '.png')
segments_info = []
for ann in inst_by_image[img_id]:
area = COCOmask.area(ann['segmentation'])
if area == 0:
continue
if used is None:
intersect = 0
used = copy.deepcopy(ann['segmentation'])
else:
intersect = COCOmask.area(
COCOmask.merge([used, ann['segmentation']],
intersect=True))
if intersect / area > overlap_thr:
continue
used = COCOmask.merge([used, ann['segmentation']], intersect=False)
mask = COCOmask.decode(ann['segmentation']) == 1
if intersect != 0:
mask = np.logical_and(pan_segm_id == 0, mask)
segment_id = id_generator.get_id(ann['category_id'])
panoptic_ann = {}
panoptic_ann['id'] = segment_id
panoptic_ann['category_id'] = ann['category_id']
pan_segm_id[mask] = segment_id
segments_info.append(panoptic_ann)
for ann in sem_by_image[img_id]:
mask = COCOmask.decode(ann['segmentation']) == 1
mask_left = np.logical_and(pan_segm_id == 0, mask)
if mask_left.sum() < stuff_area_limit:
continue
segment_id = id_generator.get_id(ann['category_id'])
panoptic_ann = {}
panoptic_ann['id'] = segment_id
panoptic_ann['category_id'] = ann['category_id']
pan_segm_id[mask_left] = segment_id
segments_info.append(panoptic_ann)
annotation['segments_info'] = segments_info
panoptic_json.append(annotation)
Image.fromarray(id2rgb(pan_segm_id)).save(
os.path.join(segmentations_folder, annotation['file_name']))
return panoptic_json
def build_panoptic_area(img_id, output_path, detection_path,
segmentation_path):
"""
Build panoptic segmentation of the specified image.
Sort segments by area: write first smaller objects to avoid overlapping.
:param img_id: image identifier (for retrieving file name)
:param output_path: path to store panoptic segmentation results (png)
:param detection_path: input path for detection
:param segmentation_path: input path for semantic segmentation
:return: the json annotation with class information for each panoptic segment.
"""
if not os.path.exists(output_path):
os.makedirs(output_path)
#Read categories and create IdGenerator (official panopticapi repository)
categories = load_panoptic_category_info()
id_generator = IdGenerator(categories)
#Parameters:
overlap_thr = 0.5
stuff_area_limit = 64 * 64
#read segmentation data
segm_probs = json.load(
open(segmentation_path + '/' + img_id + '_prob.json', 'r'))
segm_labelmap = np.array(
Image.open(segmentation_path + '/' + img_id + '_0.png'),
np.uint8) #.labelmap.astype(np.uint8)).save()
#read detection data
detection = json.load(
open(detection_path + '/' + img_id + '_prob.json', 'r'))
pan_segm_id = np.zeros(segm_labelmap.shape, dtype=np.uint32)
used = np.full(segm_labelmap.shape, False)
annotation = {}
try:
annotation['image_id'] = int(img_id)
except Exception:
annotation['image_id'] = img_id
annotation['file_name'] = img_id + '.png'
segments_info = []
for obj in detection: #for ann in ...
obj_mask = extract_mask_bool(obj['mask'])
obj_area = np.count_nonzero(obj_mask)
obj['area'] = obj_area
obj['mask'] = obj_mask
detection.sort(key=lambda x: x['area'],
reverse=False) ##First smaller, than bigger
for obj in detection: #for ann in ...
obj_mask = obj['mask'] #extract_mask_bool(obj['mask'])
obj_area = obj['area'] #np.count_nonzero(obj_mask)
if obj_area == 0:
continue
#Filter out objects with intersection > 50% with used area
intersection_mask = used & obj_mask
intersect_area = np.count_nonzero(intersection_mask)
if 1.0 * intersect_area / obj_area > overlap_thr:
continue
used = used | obj_mask
segment_id = id_generator.get_id(obj['class'])
panoptic_ann = {}
panoptic_ann['id'] = segment_id
panoptic_ann['category_id'] = obj['class']
if intersect_area > 0:
pan_segm_id[obj_mask & (~intersection_mask)] = segment_id
else:
pan_segm_id[obj_mask] = segment_id
segments_info.append(panoptic_ann)
#
#
for segm_class in np.unique(segm_labelmap):
segm_class = int(segm_class)
if segm_class == 183: #void class
continue
#Check class: exclude non-stuff objects
category = categories[segm_class]
if category['isthing'] == 1:
continue
segm_mask = (segm_labelmap == segm_class)
mask_left = segm_mask & (~used)
# Filter out segments with small area
if np.count_nonzero(mask_left) < stuff_area_limit:
continue
segment_id = id_generator.get_id(segm_class)
panoptic_ann = {}
panoptic_ann['id'] = segment_id
panoptic_ann['category_id'] = segm_class
used = used | mask_left
pan_segm_id[mask_left] = segment_id
segments_info.append(panoptic_ann)
annotation['segments_info'] = segments_info
# Save annotated image
Image.fromarray(id2rgb(pan_segm_id)).save(
os.path.join(output_path, annotation['file_name']))
##############
##remove segments with zero area
ids = set(np.unique(pan_segm_id))
segments_info_cleaned = []
for seg in segments_info:
if seg['id'] in ids:
segments_info_cleaned.append(seg)
annotation['segments_info'] = segments_info_cleaned
##################
return annotation
def sa_pixel_to_coco_panoptic(dataset_name, export_root, thing_ids):
os.makedirs(os.path.join(dataset_name, "annotations"), exist_ok=True)
info = {
'description':
'This is stable 1.0 version of the ' + dataset_name + ' dataset.',
'url':
'https://superannotate.ai',
'version':
'1.0',
'year':
2019,
'contributor':
'Annotator LLC',
'date_created':
'2019-11-15 11:47:32.67823'
}
licences = [
{
'url': 'https://superannotate.ai',
'id': 1,
'name': 'Superannotate License'
}
]
categories = []
dbid_to_catid = {}
classes = json.load(
open(os.path.join(export_root, "classes", "classes.json"))
)
for idx, dbclass in enumerate(classes, 1):
category = {
"id": idx,
"name": dbclass["name"],
"supercategory": dbclass["name"],
"isthing": dbclass["id"] in thing_ids,
"color": id2rgb(int(dbclass["color"][1:], 16))
}
dbid_to_catid[dbclass["id"]] = category["id"]
categories.append(category)
print("Converting annotations for {} dataset ...".format(dataset_name))
id_generator = IdGenerator({cat['id']: cat for cat in categories})
panoptic_root = os.path.join(
dataset_name, "panoptic_{}".format(dataset_name)
)
os.makedirs(panoptic_root, exist_ok=True)
jsons = glob.glob(os.path.join(export_root, "*.json"))
images = []
annotations = []
for idx, filepath in tqdm(enumerate(jsons, 1)):
filename = os.path.basename(filepath)
imagename = filename[:-len('___pixel.json')] + '___lores.jpg'
width, height = Image.open(os.path.join(export_root, imagename)).size
image_info = {
"id": idx,
"file_name": imagename,
"height": height,
"width": width,
"license": 1
}
images.append(image_info)
segments_info = []
sa_ann_json = json.load(open(os.path.join(export_root, filename)))
sa_bluemask_path = os.path.join(
export_root, filename[:-len('___pixel.json')] + '___save.png'
)
sa_bluemask_rgb = np.asarray(
Image.open(sa_bluemask_path).convert('RGB'), dtype=np.uint32
)
ann_mask = np.zeros((height, width), dtype=np.uint32)
flat_mask = (sa_bluemask_rgb[:, :, 0] <<
16) | (sa_bluemask_rgb[:, :, 1] <<
8) | (sa_bluemask_rgb[:, :, 2])
for instance in sa_ann_json:
parts = [int(part["color"][1:], 16) for part in instance["parts"]]
category_id = dbid_to_catid[instance["classId"]]
instance_bitmask = np.isin(flat_mask, parts)
segment_id = id_generator.get_id(category_id)
ann_mask[instance_bitmask] = segment_id
coco_instance_mask = cocomask.encode(
np.asfortranarray(instance_bitmask)
)
bbox = cocomask.toBbox(coco_instance_mask).tolist()
area = int(cocomask.area(coco_instance_mask))
segment_info = {
"id": segment_id,
"category_id": category_id,
"area": area,
"bbox": bbox,
"iscrowd": 0
}
segments_info.append(segment_info)
panopticmask = imagename[:-len("jpg")] + "png"
Image.fromarray(id2rgb(ann_mask)).save(
os.path.join(panoptic_root, panopticmask)
)
annotation = {
"image_id": idx,
"file_name": panopticmask,
"segments_info": segments_info
}
annotations.append(annotation)
panoptic_data = {
"info": info,
"licences": licences,
"images": images,
"annotations": annotations,
"categories": categories
}
json_data = json.dumps(panoptic_data, indent=4)
with open(
os.path.join(
dataset_name, "annotations",
"panoptic_{}.json".format(dataset_name)
), "w+"
) as coco_json:
coco_json.write(json_data)
def PQ_eval(
self, oracle_res=4, save_output=False,
oracle_mode=None
# oracle=False, semantic_oracle=False, vote_oracle=False
):
"""
optionally save all predicted files through output writer
"""
_VALID_ORACLE_MODES = ('full', 'sem', 'vote')
assert not self.mp_distributed
model = self.model
loader = self.val_loader
dset = loader.dataset
eval_at = model.curr_epoch - 1
logger.info("Panoptic Quality eval at epoch {}".format(eval_at))
# setup space to save pan predictions
PRED_OUT_NAME = 'pred'
dump_root = osp.join(self.output_mngr.root, 'pd_dump')
pred_meta_fname = osp.join(dump_root, '{}.json'.format(PRED_OUT_NAME))
pred_mask_dir = osp.join(dump_root, PRED_OUT_NAME)
del dump_root
overall_pred_meta = {
'images': list(dset.imgs.values()),
'categories': list(dset.meta['cats'].values()),
'annotations': []
}
metric = PQMetric(dset.meta)
timer = CompoundTimer()
timer.data.tic()
for i, inputs in enumerate(loader):
x = inputs[0].cuda()
del inputs
if self.debug and (i > 5):
break
imgMeta, segments_info, _, pan_gt_mask = dset.pan_getitem(
i, apply_trans=False)
if dset.transforms is not None:
_, trans_pan_gt = dset.transforms(_, pan_gt_mask)
else:
trans_pan_gt = pan_gt_mask.copy()
pan_gt_mask = pan_gt_mask
pan_gt_ann = {
'image_id': imgMeta['id'],
# shameful mogai; can only access image f_name here. alas...
'file_name': imgMeta['file_name'].split('.')[0] + '.png',
'segments_info': list(segments_info.values())
}
# torch.cuda.synchronize()
timer.data.toc()
timer.compute.tic()
if oracle_mode != 'full': # way too ugly
# from fabric.utils.timer import global_timer
# global_timer.network.tic()
sem_pd, vote_pd = model.infer(x, softmax_normalize=True)
# global_timer.network.toc()
if oracle_mode is not None:
assert oracle_mode in _VALID_ORACLE_MODES
sem_ora, vote_ora = gt_tsr_res_reduction(
oracle_res, self.gt_prod_handle,
dset.meta, model.pcv, trans_pan_gt, segments_info
)
if oracle_mode == 'vote':
pass # if using model sem pd, maintain stuff pred thresh
else: # sem or full oracle, using gt sem pd, do not filter
model.dset_meta['stuff_pred_thresh'] = -1
if oracle_mode == 'sem':
sem_pd = sem_ora
elif oracle_mode == 'vote':
vote_pd = vote_ora
else:
sem_pd, vote_pd = sem_ora, vote_ora
pan_pd_mask, pan_pd_ann = model.stitch_pan_mask(
cfg.pcv, sem_pd, vote_pd, pan_gt_mask.size
)
pan_pd_ann['image_id'] = pan_gt_ann['image_id']
pan_pd_ann['file_name'] = pan_gt_ann['file_name']
overall_pred_meta['annotations'].append(pan_pd_ann)
metric.update(
pan_gt_ann, rgb2id(np.array(pan_gt_mask)), pan_pd_ann, pan_pd_mask
)
if save_output:
# pan_pd_mask = Image.fromarray(id2rgb(pan_pd_mask))
# fname = osp.join(pred_mask_dir, pan_pd_ann['file_name'])
# os.makedirs(osp.dirname(fname), exist_ok=True) # make region subdir
# pan_pd_mask.save(fname)
fname = osp.join(pred_mask_dir, pan_pd_ann['file_name'])
os.makedirs(osp.dirname(fname), exist_ok=True) # make region subdir
pan_pd_mask = Image.fromarray(id2rgb(pan_pd_mask))
pan_pd_mask.save(fname)
# torch.cuda.synchronize()
timer.compute.toc()
if (i % 50) == 0:
logger.info(timer)
logger.info(
"iter {}, eta to val end {}".format(i, timer.eta(i, len(loader)))
)
print(metric)
timer.data.tic()
if save_output:
with open(pred_meta_fname, 'w') as f:
json.dump(overall_pred_meta, f)
self.output_mngr.save_f(
metric.results, osp.join(self.output_mngr.root, 'score.pkl')
)
logger.info("eval complete \n{}".format(metric))
return metric
