def convert_cityscapes_instance_only(data_dir, out_dir):
"""Convert from cityscapes format to COCO instance seg format - polygons"""
sets = [
#'gtFine_val'
'gtFine_train',
# 'gtFine_test',
# 'gtCoarse_train',
# 'gtCoarse_val',
# 'gtCoarse_train_extra'
]
ann_dirs = [
#'gtFine_trainvaltest/gtFine/val'
'gtFine_trainvaltest/gtFine/train',
# 'gtFine_trainvaltest/gtFine/test',
# 'gtCoarse/train',
# 'gtCoarse/train_extra',
# 'gtCoarse/val'
]
json_name = 'instancesonly_filtered_%s.json'
ends_in = '%s_polygons.json'
img_id = 0
ann_id = 0
cat_id = 1
category_dict = {}
category_instancesonly = [
'person',
'rider',
'car',
'truck',
'bus',
'caravan',
'trailer',
'train',
'motorcycle',
'bicycle',
]
for data_set, ann_dir in zip(sets, ann_dirs):
print('Starting %s' % data_set)
ann_dict = {}
images = []
annotations = []
ann_dir = os.path.join(data_dir, ann_dir)
for root, _, files in os.walk(ann_dir):
for filename in files:
#print("*******", filename)
if filename.endswith(ends_in % data_set.split('_')[0]):
if len(images) % 50 == 0:
print("Processed %s images, %s annotations" %
(len(images), len(annotations)))
json_ann = json.load(open(os.path.join(root, filename)))
image = {}
image['id'] = img_id
img_id += 1
image['width'] = json_ann['imgWidth']
image['height'] = json_ann['imgHeight']
image['file_name'] = filename[:-len(
ends_in % data_set.split('_')[0])] + 'leftImg8bit.png'
image['seg_file_name'] = filename[:-len(
ends_in % data_set.split('_')[0]
)] + '%s_instanceIds.png' % data_set.split('_')[0]
images.append(image)
fullname = os.path.join(
root, image['seg_file_name']) # instance name
objects = cs.instances2dict_with_polygons(
[fullname], False)[os.path.abspath(fullname)]
#print(fullname)
#print(objects)
for object_cls in objects:
if object_cls not in category_instancesonly:
continue # skip non-instance categories
for obj in objects[object_cls]:
if obj['contours'] == []:
print('Warning: empty contours.')
continue # skip non-instance categories
len_p = [len(p) for p in obj['contours']]
if min(len_p) <= 4:
print('Warning: invalid contours.')
continue # skip non-instance categories
ann = {}
ann['id'] = ann_id
ann_id += 1
ann['image_id'] = image['id']
ann['segmentation'] = obj['contours']
if object_cls not in category_dict:
category_dict[object_cls] = cat_id
cat_id += 1
ann['category_id'] = category_dict[object_cls]
ann['iscrowd'] = 0
ann['area'] = obj['pixelCount']
ann['bbox'] = bboxs_util.xyxy_to_xywh(
segms_util.polys_to_boxes(
[ann['segmentation']])).tolist()[0]
annotations.append(ann)
ann_dict['images'] = images
categories = [{
"id": category_dict[name],
"name": name
} for name in category_dict]
ann_dict['categories'] = categories
ann_dict['annotations'] = annotations
print("Num categories: %s" % len(categories))
print("Num images: %s" % len(images))
print("Num annotations: %s" % len(annotations))
with open(os.path.join(out_dir, json_name % data_set), 'w') as outfile:
outfile.write(json.dumps(ann_dict))
def convert_cityscapes_instance_only(data_dir, out_dir):
"""Convert from cityscapes format to COCO instance seg format - polygons"""
sets = [
"gtFine_val",
"gtFine_train",
"gtFine_test",
# 'gtCoarse_train',
# 'gtCoarse_val',
# 'gtCoarse_train_extra'
]
ann_dirs = [
"gtFine_trainvaltest/gtFine/val",
"gtFine_trainvaltest/gtFine/train",
"gtFine_trainvaltest/gtFine/test",
# 'gtCoarse/train',
# 'gtCoarse/train_extra',
# 'gtCoarse/val'
]
json_name = "instancesonly_filtered_%s.json"
ends_in = "%s_polygons.json"
img_id = 0
ann_id = 0
cat_id = 1
category_dict = {}
category_instancesonly = [
"person",
"rider",
"car",
"truck",
"bus",
"train",
"motorcycle",
"bicycle",
]
for data_set, ann_dir in zip(sets, ann_dirs):
print("Starting %s" % data_set)
ann_dict = {}
images = []
annotations = []
ann_dir = os.path.join(data_dir, ann_dir)
for root, _, files in os.walk(ann_dir):
for filename in files:
if filename.endswith(ends_in % data_set.split("_")[0]):
if len(images) % 50 == 0:
print("Processed %s images, %s annotations" %
(len(images), len(annotations)))
json_ann = json.load(open(os.path.join(root, filename)))
image = {}
image["id"] = img_id
img_id += 1
image["width"] = json_ann["imgWidth"]
image["height"] = json_ann["imgHeight"]
image["file_name"] = (
filename[:-len(ends_in % data_set.split("_")[0])] +
"leftImg8bit.png")
image["seg_file_name"] = (
filename[:-len(ends_in % data_set.split("_")[0])] +
"%s_instanceIds.png" % data_set.split("_")[0])
images.append(image)
fullname = os.path.join(root, image["seg_file_name"])
objects = cs.instances2dict_with_polygons(
[fullname], verbose=False)[fullname]
for object_cls in objects:
if object_cls not in category_instancesonly:
continue # skip non-instance categories
for obj in objects[object_cls]:
if obj["contours"] == []:
print("Warning: empty contours.")
continue # skip non-instance categories
len_p = [len(p) for p in obj["contours"]]
if min(len_p) <= 4:
print("Warning: invalid contours.")
continue # skip non-instance categories
ann = {}
ann["id"] = ann_id
ann_id += 1
ann["image_id"] = image["id"]
ann["segmentation"] = obj["contours"]
if object_cls not in category_dict:
category_dict[object_cls] = cat_id
cat_id += 1
ann["category_id"] = category_dict[object_cls]
ann["iscrowd"] = 0
ann["area"] = obj["pixelCount"]
ann["bbox"] = bboxs_util.xyxy_to_xywh(
segms_util.polys_to_boxes(
[ann["segmentation"]])).tolist()[0]
annotations.append(ann)
ann_dict["images"] = images
categories = [{
"id": category_dict[name],
"name": name
} for name in category_dict]
ann_dict["categories"] = categories
ann_dict["annotations"] = annotations
print("Num categories: %s" % len(categories))
print("Num images: %s" % len(images))
print("Num annotations: %s" % len(annotations))
with open(os.path.join(out_dir, json_name % data_set), "w") as outfile:
outfile.write(json.dumps(ann_dict))
def convert_cityscapes_instance_only(
data_dir, out_dir):
"""Convert from cityscapes format to COCO instance seg format - polygons"""
sets = [
'train',
'val'
# 'images/100k/train',
# 'images/100k/val'
# 'gtFine_train',
# 'gtCoarse_train',
# 'gtCoarse_val',
# 'gtCoarse_train_extra'
]
ann_dirs = [
'annotation_train',
'annotation_val'
# 'labels/100k/train',
# 'labels/100k/val'
# 'gtFine_trainvaltest/gtFine/train',
# 'gtFine_trainvaltest/gtFine/test',
# 'gtCoarse/train',
# 'gtCoarse/train_extra',
# 'gtCoarse/val'
]
json_name = 'instancesonly_filtered_%s.json'
ends_in = '.json'
img_id = 0
ann_id = 0
cat_id = 1
category_dict = {}
category_instancesonly = ['__background__',
"bike",
"bus",
"car",
# "motor",
"person",
"rider",
"traffic light",
"traffic sign",
# "train",
"truck",
"area/alternative",
"area/drivable",
# "lane/crosswalk",
# "lane/double other",
# "lane/double white",
# "lane/double yellow",
# "lane/road curb",
# "lane/single other",
# "lane/single white",
# "lane/single yellow"
]#--------------------------------------------------------------------------------------
# Write "info"
infodata = {'info': {'description': 'This is stable 1.0 version of the 2014 MS COCO dataset.', 'url': u'http://mscoco.org', 'version': u'1.0', 'year': 2014, 'contributor': 'Microsoft COCO group', 'date_created': '2015-01-27 09:11:52.357475'}}
for data_set, ann_dir in zip(sets, ann_dirs):
print('Starting %s' % data_set)
ann_dict = {}
ann_dict["info"] = infodata["info"]
ann_dict["type"] = 'instances'
annPath = os.path.join(data_dir, 'coco_ref',
'instances_' + data_set + '2014.json')
with open(annPath) as annFile:
print ("open " + str(annFile))
cocodata = json.load(annFile)
licdata = [i for i in cocodata['licenses']]
ann_dict["licenses"] = licdata
images = []
annotations = []
ann_dir = os.path.join(data_dir, ann_dir)
for root, _, files in os.walk(ann_dir):
for filename in files:
if filename.endswith(ends_in):
if len(images) % 50 == 0:
print("Processed %s images, %s annotations" % (
len(images), len(annotations)))
json_ann = json.load(open(os.path.join(root, filename)))
image = {}
image['id'] = img_id
img_id += 1
# im = Image.open(filename)
# (width, height) = im.size
image['width'] = 1280
image['height'] = 720
outmask = np.zeros((image['height'], image['width']), np.uint8)
img_dir = os.path.join(data_dir, data_set)
# image['file_name'] = img_dir + "/" + filename.split('.')[0] + ".jpg"
image['file_name'] = filename.split('.')[0] + ".jpg"
images.append(image)
# fullname = os.path.join(root, image['seg_file_name'])
# objects = cs.instances2dict_with_polygons(
# [fullname], verbose=False)[fullname]
objects = json_ann["frames"][0]["objects"]
for obj in objects:
if obj["category"] not in category_instancesonly:
continue # skip non-instance categories
index = category_instancesonly.index(obj["category"])# + 184
seg_points = getPointByObj(obj)#[[[point1],[point2]]]
seg = []
for seg_poit in seg_points:
seg.extend(sum(seg_poit, []))
if len(seg) == 0:
print('Warning: invalid segmentation.')
continue
ann = {}
ann['id'] = ann_id
ann_id += 1
ann['image_id'] = image['id']
category_dict[obj["category"]] = index
ann['category_id'] = index
ann['iscrowd'] = 0
if obj.has_key("box2d"):
ann['bbox'] = getBoxByObj(obj)
else:
ann['area'], ann['segmentation'] = getAreaByObj(seg_points, image['height'], image['width'], ann['category_id'])
ann['bbox'] = bboxs_util.xyxy_to_xywh(segms_util.polys_to_boxes(
[ann['segmentation']])).tolist()[0]
annotations.append(ann)
# break
ann_dict['images'] = images
# category_dict.values()
# categories = [{"id": category_dict[name], "name": name} for name in category_dict]
categories = []
for index, value in enumerate(category_instancesonly):
categories.append({"id": index, "name": value})
categories = categories[1:]
ann_dict['categories'] = categories
ann_dict['annotations'] = annotations
print("Num categories: %s" % len(categories))
print("Num images: %s" % len(images))
print("Num annotations: %s" % len(annotations))
with open(os.path.join(out_dir, json_name % data_set), 'wb') as outfile:
outfile.write(json.dumps(ann_dict))
def add_mask_rcnn_blobs(blobs, sampled_boxes, roidb, im_scale, batch_idx):
"""Add Mask R-CNN specific blobs to the input blob dictionary."""
# Prepare the mask targets by associating one gt mask to each training roi
# that has a fg (non-bg) class label.
M = cfg.MRCNN.RESOLUTION
input_w = roidb['input_width']
input_h = roidb['input_height']
polys_gt_inds = np.where((roidb['gt_classes'] > 0)
& (roidb['is_crowd'] == 0))[0]
polys_gt = [roidb['segms'][i] for i in polys_gt_inds]
boxes_from_polys = segm_utils.polys_to_boxes(polys_gt)
fg_inds = np.where(blobs['labels_int32'] > 0)[0]
roi_has_mask = blobs['labels_int32'].copy()
roi_has_mask[roi_has_mask > 0] = 1
mask_fg_rois_per_this_image = cfg.MRCNN.MAX_ROIS_PER_IM
if fg_inds.shape[0] > 0:
if fg_inds.size > mask_fg_rois_per_this_image:
fg_inds = np.random.choice(fg_inds,
size=mask_fg_rois_per_this_image,
replace=False)
# Class labels for the foreground rois
mask_class_labels = blobs['labels_int32'][fg_inds]
masks = blob_utils.zeros((fg_inds.shape[0], M**2), int32=True)
all_person_masks = np.zeros(
(int(input_h / im_scale), int(input_w / im_scale)),
dtype=np.float32)
# Find overlap between all foreground rois and the bounding boxes
# enclosing each segmentation
rois_fg = sampled_boxes[fg_inds]
overlaps_bbfg_bbpolys = box_utils.bbox_overlaps(
rois_fg.astype(np.float32, copy=False),
boxes_from_polys.astype(np.float32, copy=False))
# Map from each fg rois to the index of the mask with highest overlap
# (measured by bbox overlap)
fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1)
# add fg targets
for i in range(rois_fg.shape[0]):
fg_polys_ind = fg_polys_inds[i]
poly_gt = polys_gt[fg_polys_ind]
roi_fg = rois_fg[i]
# Rasterize the portion of the polygon mask within the given fg roi
# to an M x M binary image
mask = segm_utils.polys_to_mask_wrt_box(poly_gt, roi_fg, M)
mask = np.array(mask > 0, dtype=np.int32) # Ensure it's binary
masks[i, :] = np.reshape(mask, M**2)
# to an box_h x box_w binary image
mask_wrt_bbox = segm_utils.convert_polys_to_mask_wrt_box(
poly_gt, roi_fg)
start_y, start_x = int(roi_fg[1]), int(roi_fg[0])
end_y, end_x = start_y + mask_wrt_bbox.shape[
0], start_x + mask_wrt_bbox.shape[1]
all_person_masks[start_y:end_y, start_x:end_x] = mask_wrt_bbox
proposal_all_mask = blob_utils.zeros((fg_inds.shape[0], M, M),
int32=True)
for i in range(rois_fg.shape[0]):
roi_fg = rois_fg[i]
w = roi_fg[2] - roi_fg[0]
h = roi_fg[3] - roi_fg[1]
w = int(np.maximum(w, 1))
h = int(np.maximum(h, 1))
proposal_mask = all_person_masks[int(roi_fg[1]):int(roi_fg[1]) + h,
int(roi_fg[0]):int(roi_fg[0]) + w]
# proposal_mask = proposal_mask.astype(np.float32)
proposal_mask = cv2.resize(proposal_mask, (M, M))
proposal_mask = (proposal_mask > 0.5).astype(np.int32)
proposal_all_mask[i] = proposal_mask
else: # If there are no fg masks (it does happen)
# The network cannot handle empty blobs, so we must provide a mask
# We simply take the first bg roi, given it an all -1's mask (ignore
# label), and label it with class zero (bg).
bg_inds = np.where(blobs['labels_int32'] == 0)[0]
# rois_fg is actually one background roi, but that's ok because ...
rois_fg = sampled_boxes[bg_inds[0]].reshape((1, -1))
# We give it an -1's blob (ignore label)
masks = -blob_utils.ones((1, M**2), int32=True)
# We label it with class = 0 (background)
mask_class_labels = blob_utils.zeros((1, ))
# Mark that the first roi has a mask
roi_has_mask[0] = 1
proposal_all_mask = -blob_utils.ones((1, M, M), int32=True)
if cfg.MRCNN.CLS_SPECIFIC_MASK:
masks = _expand_to_class_specific_mask_targets(masks,
mask_class_labels)
# Scale rois_fg and format as (batch_idx, x1, y1, x2, y2)
rois_fg *= im_scale
repeated_batch_idx = batch_idx * blob_utils.ones((rois_fg.shape[0], 1))
rois_fg = np.hstack((repeated_batch_idx, rois_fg))
# Update blobs dict with Mask R-CNN blobs
blobs['mask_rois'] = rois_fg
blobs['roi_has_mask_int32'] = roi_has_mask
blobs['masks_int32'] = masks
# blobs['mask_labels'] = np.argmax(masks.reshape((-1,cfg.MODEL.NUM_CLASSES,M,M)),axis=1).reshape((-1,M,M)).astype(np.int32)
# blobs['mask_weights'] = np.ones(blobs['mask_labels'].shape, dtype=np.float32)
# add by wxh
if cfg.MRCNN.USE_CLS_EMBS:
fg_embs, bg_embs, fg_weights, bg_weights = masks_to_embs(
masks.reshape((-1, cfg.MODEL.NUM_CLASSES, M, M)))
# print('fg',fg_embs.max(), fg_embs.min())
# print('bg',bg_embs.max(), bg_embs.min())
fg_norms = np.sum(fg_embs, axis=(1, 2))
fg_norms[fg_norms != 0] = 28. * 28. / (fg_norms[fg_norms != 0] + 1e-6)
bg_norms = np.sum(bg_embs, axis=(1, 2))
bg_norms[bg_norms != 0] = 28. * 28. / (bg_norms[bg_norms != 0] + 1e-6)
blobs['fg_mask'] = np.repeat(np.reshape(fg_embs, (-1, 1, M, M)),
2,
axis=1)
blobs['bg_mask'] = np.repeat(np.reshape(bg_embs, (-1, 1, M, M)),
2,
axis=1)
blobs['fg_norm'] = np.repeat(np.reshape(fg_norms, (-1, 1)), 2, axis=1)
blobs['bg_norm'] = np.repeat(np.reshape(bg_norms, (-1, 1)), 2, axis=1)
blobs['mask_emb_fg_labels'] = np.ones((fg_embs.shape[0], 1),
dtype=np.int32)
blobs['mask_emb_bg_labels'] = np.zeros((bg_embs.shape[0], 1),
dtype=np.int32)
# blobs['mask_emb_weights'] = np.vstack([fg_weights, bg_weights]).reshape((-1,1)).astype(np.float32)
if cfg.MRCNN.BBOX_CASCADE_MASK_ON:
blobs['inter_masks_int32'] = proposal_all_mask
def convert_cityscapes_instance_only(
data_dir, out_dir):
"""Convert from cityscapes format to COCO instance seg format - polygons"""
sets = [
'gtFine_val',
# 'gtFine_train',
# 'gtFine_test',
# 'gtCoarse_train',
# 'gtCoarse_val',
# 'gtCoarse_train_extra'
]
ann_dirs = [
'gtFine_trainvaltest/gtFine/val',
# 'gtFine_trainvaltest/gtFine/train',
# 'gtFine_trainvaltest/gtFine/test',
# 'gtCoarse/train',
# 'gtCoarse/train_extra',
# 'gtCoarse/val'
]
json_name = 'instancesonly_filtered_%s.json'
ends_in = '%s_polygons.json'
img_id = 0
ann_id = 0
cat_id = 1
category_dict = {}
category_instancesonly = [
'person',
'rider',
'car',
'truck',
'bus',
'train',
'motorcycle',
'bicycle',
]
for data_set, ann_dir in zip(sets, ann_dirs):
print('Starting %s' % data_set)
ann_dict = {}
images = []
annotations = []
ann_dir = os.path.join(data_dir, ann_dir)
for root, _, files in os.walk(ann_dir):
for filename in files:
if filename.endswith(ends_in % data_set.split('_')[0]):
if len(images) % 50 == 0:
print("Processed %s images, %s annotations" % (
len(images), len(annotations)))
json_ann = json.load(open(os.path.join(root, filename)))
image = {}
image['id'] = img_id
img_id += 1
image['width'] = json_ann['imgWidth']
image['height'] = json_ann['imgHeight']
image['file_name'] = filename[:-len(
ends_in % data_set.split('_')[0])] + 'leftImg8bit.png'
image['seg_file_name'] = filename[:-len(
ends_in % data_set.split('_')[0])] + \
'%s_instanceIds.png' % data_set.split('_')[0]
images.append(image)
fullname = os.path.join(root, image['seg_file_name'])
objects = cs.instances2dict_with_polygons(
[fullname], verbose=False)[fullname]
for object_cls in objects:
if object_cls not in category_instancesonly:
continue # skip non-instance categories
for obj in objects[object_cls]:
if obj['contours'] == []:
print('Warning: empty contours.')
continue # skip non-instance categories
len_p = [len(p) for p in obj['contours']]
if min(len_p) <= 4:
print('Warning: invalid contours.')
continue # skip non-instance categories
ann = {}
ann['id'] = ann_id
ann_id += 1
ann['image_id'] = image['id']
ann['segmentation'] = obj['contours']
if object_cls not in category_dict:
category_dict[object_cls] = cat_id
cat_id += 1
ann['category_id'] = category_dict[object_cls]
ann['iscrowd'] = 0
ann['area'] = obj['pixelCount']
ann['bbox'] = bboxs_util.xyxy_to_xywh(
segms_util.polys_to_boxes(
[ann['segmentation']])).tolist()[0]
annotations.append(ann)
ann_dict['images'] = images
categories = [{"id": category_dict[name], "name": name} for name in
category_dict]
ann_dict['categories'] = categories
ann_dict['annotations'] = annotations
print("Num categories: %s" % len(categories))
print("Num images: %s" % len(images))
print("Num annotations: %s" % len(annotations))
with open(os.path.join(out_dir, json_name % data_set), 'wb') as outfile:
outfile.write(json.dumps(ann_dict))
def convert_cityscapes_instance_only(data_dir, out_dir):
"""Convert from cityscapes format to COCO instance seg format - polygons"""
sets = args.set.split(',')
for i in sets:
if i == 'train' or i == 'val':
ann_dirs = ['train/images', 'val/images']
elif i == 'test': # NEED TEST MASK ANNOTATIONS
ann_dirs = ['test/images']
else:
print('Invalid input')
json_name = 'instancesonly_filtered_%s.json'
img_id = 0
ann_id = 0
cat_id = 1
category_dict = {}
category_instancesonly = [
'unlabeled', 'ship', 'storage_tank', 'baseball_diamond',
'tennis_court', 'basketball_court', 'Ground_Track_Field', 'Bridge',
'Large_Vehicle', 'Small_Vehicle', 'Helicopter', 'Swimming_pool',
'Roundabout', 'Soccer_ball_field', 'plane', 'Harbor'
]
for data_set, ann_dir in zip(sets, ann_dirs):
print('Starting %s' % data_set)
ann_dict = {}
images = []
annotations = []
ann_dir = os.path.join(data_dir, ann_dir)
print(ann_dir)
img_id = 0 # for every image_id with different indexing
c_images = 0
for root, _, files in os.walk(ann_dir):
for filename in natsorted(files):
if filename.endswith(
'_color_RGB.png'
): #if re.match(r'\w*\d+.png', filename) or filename.split('.')[0].count('_')==4:
#import pdb;pdb.set_trace()
c_images += 1
filename = ''.join(filename)
filename = filename.split('_')[:-3]
if len(filename) > 1:
filename = '_'.join(filename)
else:
filename = ''.join(filename)
filename = filename + '.png'
print("Processed %s images" % (c_images))
image_dim = cv2.imread(os.path.join(root, filename))
imgHeight, imgWidth, _ = image_dim.shape
image = {}
image['id'] = img_id
img_id += 1
image['width'] = imgWidth
image['height'] = imgHeight
print("Processing Image", filename)
image['file_name'] = filename.split('.')[0] + '.png'
print("Processing Image", image['file_name'])
image['ins_file_name'] = filename.split(
'.')[0] + '_instance_id_RGB.png'
image['seg_file_name'] = filename.split(
'.')[0] + '_instance_color_RGB.png'
images.append(image)
#import pdb;pdb.set_trace()
seg_fullname = os.path.join(root, image['seg_file_name'])
inst_fullname = os.path.join(root, image['ins_file_name'])
if not os.path.exists(seg_fullname):
print("YOU DONT HAVE TEST MASKS")
sys.exit(0)
objects = cs.instances2dict_with_polygons([seg_fullname],
[inst_fullname],
verbose=True)
for k, v in objects.items():
for object_cls in list(v.keys()):
if object_cls not in category_instancesonly: #to get the labels only mentioned in category_instancesonly
continue
for obj in v[object_cls]:
if obj['contours'] == []:
print('Warning: empty contours.')
continue
len_p = [len(p) for p in obj['contours']]
if min(len_p) <= 4:
print('Warning: invalid contours.')
continue
ann = {}
ann['id'] = ann_id
ann_id += 1
ann['image_id'] = image['id']
ann['segmentation'] = obj['contours']
if object_cls not in category_dict:
category_dict[object_cls] = label2id[
object_cls]
ann['category_id'] = category_dict[object_cls]
ann['category_name'] = object_cls
ann['iscrowd'] = 0
ann['area'] = obj['pixelCount']
ann['bbox'] = bboxs_util.xyxy_to_xywh(
segms_util.polys_to_boxes(
[ann['segmentation']])).tolist()[0]
#annotations.append(ann)
if ann['area'] > 10:
annotations.append(ann)
ann_dict['images'] = images
categories = [{
"id": category_dict[name],
"name": name
} for name in category_dict]
ann_dict['categories'] = categories
ann_dict['annotations'] = annotations
print("Num categories: %s" % len(categories))
print("Num images: %s" % len(images))
print("Num annotations: %s" % len(annotations))
with open(
os.path.join(os.path.join(out_dir, data_set),
json_name % data_set), "w") as outfile:
outfile.write(json.dumps(ann_dict))
def convert_cityscapes_instance_only(data_dir, out_dir):
"""Convert from cityscapes format to COCO instance seg format - polygons"""
sets = [
'gtFine_train',
'gtFine_val',
# 'gtFine_test',
# 'gtCoarse_train',
# 'gtCoarse_val',
# 'gtCoarse_train_extra'
]
ann_dirs = [
'labels_train',
'labels_val',
# 'gtFine_trainvaltest/gtFine/test',
# 'gtCoarse/train',
# 'gtCoarse/train_extra',
# 'gtCoarse/val'
]
json_name = 'instancesonly_filtered_%s.json'
ends_in = '.json'
img_id = 0
ann_id = 0
cat_id = 1
category_dict = {}
category_instancesonly = [
'__background__', 'person', 'rider', 'car', 'truck', 'bus',
'motorcycle', 'bicycle', 'ground', 'road', 'sky'
]
for data_set, ann_dir in zip(sets, ann_dirs):
print('Starting %s' % data_set)
ann_dict = {}
images = []
annotations = []
ann_dir = os.path.join(data_dir, ann_dir)
for root, _, files in os.walk(ann_dir):
for filename in files:
if filename.endswith(ends_in):
if len(images) % 50 == 0:
print("Processed %s images, %s annotations" %
(len(images), len(annotations)))
json_ann = json.load(open(os.path.join(root, filename)))
image = {}
image['id'] = img_id
img_id += 1
image['width'] = json_ann['imgWidth']
image['height'] = json_ann['imgHeight']
sub_name = filename.split('_')[0:3]
image['file_name'] = sub_name[0] + "_" + sub_name[
1] + "_" + sub_name[2] + "_" + 'leftImg8bit.png'
images.append(image)
objects = json_ann["objects"]
for obj in objects:
if obj["label"] not in category_instancesonly:
continue # skip non-instance categories
index = category_instancesonly.index(
obj["label"]) # + 184
ann = {}
ann['id'] = ann_id
ann_id += 1
ann['image_id'] = image['id']
ann['segmentation'] = [sum(obj['polygon'], [])]
ann['category_id'] = index
ann['iscrowd'] = 0
seg_points = obj["polygon"]
ann['area'] = getAreaByPoint(seg_points,
image['height'],
image['width'],
ann['category_id'])
ann['bbox'] = bboxs_util.xyxy_to_xywh(
segms_util.polys_to_boxes([ann['segmentation']
])).tolist()[0]
annotations.append(ann)
# break
ann_dict['images'] = images
categories = []
for index, value in enumerate(category_instancesonly):
categories.append({"id": index, "name": value})
categories = categories[1:]
ann_dict['categories'] = categories
ann_dict['annotations'] = annotations
print("Num categories: %s" % len(categories))
print("Num images: %s" % len(images))
print("Num annotations: %s" % len(annotations))
with open(os.path.join(out_dir, json_name % data_set),
'wb') as outfile:
outfile.write(json.dumps(ann_dict))
def add_mask_rcnn_blobs(blobs, sampled_boxes, roidb, im_scale, batch_idx):
"""Add Mask R-CNN specific blobs to the input blob dictionary."""
# Prepare the mask targets by associating one gt mask to each training roi
# that has a fg (non-bg) class label.
M = cfg.MRCNN.RESOLUTION
polys_gt_inds = np.where(
(roidb['gt_classes'] > 0) & (roidb['is_crowd'] == 0)
)[0]
polys_gt = [roidb['segms'][i] for i in polys_gt_inds]
boxes_from_polys = segm_utils.polys_to_boxes(polys_gt)
fg_inds = np.where(blobs['labels_int32'] > 0)[0]
roi_has_mask = blobs['labels_int32'].copy()
roi_has_mask[roi_has_mask > 0] = 1
if fg_inds.shape[0] > 0:
# Class labels for the foreground rois
mask_class_labels = blobs['labels_int32'][fg_inds]
masks = blob_utils.zeros((fg_inds.shape[0], M**2), int32=True)
# Find overlap between all foreground rois and the bounding boxes
# enclosing each segmentation
rois_fg = sampled_boxes[fg_inds]
overlaps_bbfg_bbpolys = box_utils.bbox_overlaps(
rois_fg.astype(np.float32, copy=False),
boxes_from_polys.astype(np.float32, copy=False)
)
# Map from each fg rois to the index of the mask with highest overlap
# (measured by bbox overlap)
fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1)
# add fg targets
for i in range(rois_fg.shape[0]):
fg_polys_ind = fg_polys_inds[i]
poly_gt = polys_gt[fg_polys_ind]
roi_fg = rois_fg[i]
# Rasterize the portion of the polygon mask within the given fg roi
# to an M x M binary image
mask = segm_utils.polys_to_mask_wrt_box(poly_gt, roi_fg, M)
mask = np.array(mask > 0, dtype=np.int32) # Ensure it's binary
masks[i, :] = np.reshape(mask, M**2)
else: # If there are no fg masks (it does happen)
# The network cannot handle empty blobs, so we must provide a mask
# We simply take the first bg roi, given it an all -1's mask (ignore
# label), and label it with class zero (bg).
bg_inds = np.where(blobs['labels_int32'] == 0)[0]
# rois_fg is actually one background roi, but that's ok because ...
rois_fg = sampled_boxes[bg_inds[0]].reshape((1, -1))
# We give it an -1's blob (ignore label)
masks = -blob_utils.ones((1, M**2), int32=True)
# We label it with class = 0 (background)
mask_class_labels = blob_utils.zeros((1, ))
# Mark that the first roi has a mask
roi_has_mask[0] = 1
if cfg.MRCNN.CLS_SPECIFIC_MASK:
masks = _expand_to_class_specific_mask_targets(masks, mask_class_labels)
# Scale rois_fg and format as (batch_idx, x1, y1, x2, y2)
rois_fg *= im_scale
repeated_batch_idx = batch_idx * blob_utils.ones((rois_fg.shape[0], 1))
rois_fg = np.hstack((repeated_batch_idx, rois_fg))
# Update blobs dict with Mask R-CNN blobs
blobs['mask_rois'] = rois_fg
blobs['roi_has_mask_int32'] = roi_has_mask
blobs['masks_int32'] = masks
def add_mask_rcnn_blobs(blobs, sampled_boxes, roidb, im_scale, batch_idx):
"""Add Mask R-CNN specific blobs to the input blob dictionary."""
# Prepare the mask targets by associating one gt mask to each training roi
# that has a fg (non-bg) class label.
M = cfg.MRCNN.RESOLUTION
polys_gt_inds = np.where(
(roidb['gt_classes'] > 0) & (roidb['is_crowd'] == 0)
)[0]
polys_gt = [roidb['segms'][i] for i in polys_gt_inds]
boxes_from_polys = segm_utils.polys_to_boxes(polys_gt)
fg_inds = np.where(blobs['labels_int32'] > 0)[0]
roi_has_mask = blobs['labels_int32'].copy()
roi_has_mask[roi_has_mask > 0] = 1
if fg_inds.shape[0] > 0:
# Class labels for the foreground rois
mask_class_labels = blobs['labels_int32'][fg_inds]
masks = blob_utils.zeros((fg_inds.shape[0], M**2), int32=True)
# Find overlap between all foreground rois and the bounding boxes
# enclosing each segmentation
rois_fg = sampled_boxes[fg_inds]
overlaps_bbfg_bbpolys = box_utils.bbox_overlaps(
rois_fg.astype(np.float32, copy=False),
boxes_from_polys.astype(np.float32, copy=False)
)
# Map from each fg rois to the index of the mask with highest overlap
# (measured by bbox overlap)
fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1)
# add fg targets
for i in range(rois_fg.shape[0]):
fg_polys_ind = fg_polys_inds[i]
poly_gt = polys_gt[fg_polys_ind]
roi_fg = rois_fg[i]
# Rasterize the portion of the polygon mask within the given fg roi
# to an M x M binary image
mask = segm_utils.polys_to_mask_wrt_box(poly_gt, roi_fg, M)
mask = np.array(mask > 0, dtype=np.int32) # Ensure it's binary
masks[i, :] = np.reshape(mask, M**2)
else: # If there are no fg masks (it does happen)
# The network cannot handle empty blobs, so we must provide a mask
# We simply take the first bg roi, given it an all -1's mask (ignore
# label), and label it with class zero (bg).
bg_inds = np.where(blobs['labels_int32'] == 0)[0]
# rois_fg is actually one background roi, but that's ok because ...
rois_fg = sampled_boxes[bg_inds[0]].reshape((1, -1))
# We give it an -1's blob (ignore label)
masks = -blob_utils.ones((1, M**2), int32=True)
# We label it with class = 0 (background)
mask_class_labels = blob_utils.zeros((1, ))
# Mark that the first roi has a mask
roi_has_mask[0] = 1
if cfg.MRCNN.CLS_SPECIFIC_MASK:
masks = _expand_to_class_specific_mask_targets(masks, mask_class_labels)
# Scale rois_fg and format as (batch_idx, x1, y1, x2, y2)
rois_fg *= im_scale
repeated_batch_idx = batch_idx * blob_utils.ones((rois_fg.shape[0], 1))
rois_fg = np.hstack((repeated_batch_idx, rois_fg))
# Update blobs dict with Mask R-CNN blobs
blobs['mask_rois'] = rois_fg
blobs['roi_has_mask_int32'] = roi_has_mask
blobs['masks_int32'] = masks
def convert_cityscapes_instance_only(data_dir, out_dir):
"""Convert from cityscapes format to COCO instance seg format - polygons"""
sets = [
'image_train',
'image_val',
]
ann_dirs = [
'gtFine/train',
'gtFine/val',
]
json_name = 'instancesonly_filtered_%s.json'
ends_in = '_polygons.json'
img_id = 0
ann_id = 0
cat_id = 1
category_dict = {}
add_instancesonly = ['__background__', 'person', 'car']
category_instancesonly = [
'__background__', 'guard rail', 'car', 'dashed', 'solid',
'solid solid', 'dashed dashed', 'dashed-solid', 'solid-dashed',
'yellow dashed', 'yellow solid', 'yellow solid solid',
'yellow dashed dashed', 'yellow dashed-solid', 'yellow solid-dashed',
'boundary', 'fork_line', 'fork_edge', 'arrow_s', 'arrow_r', 'arrow_l',
'arrow_lr', 'arrow_inclined_r', 'arrow_r_s', 'arrow_l_s', 'sidewalk',
'handrail'
]
category_instancesonly = [
'__background__',
'person',
'bicycle',
'car',
'motorcycle',
'airplane',
'bus',
'train',
'truck',
'boat',
'traffic light',
'fire hydrant',
'stop sign',
'parking meter',
'bench',
'bird',
'cat',
'dog',
'horse',
'sheep',
'cow',
'elephant',
'bear',
'zebra',
'giraffe',
'backpack',
'umbrella',
'handbag',
'tie',
'suitcase',
'frisbee',
'skis',
'snowboard',
'sports ball',
'kite',
'baseball bat',
'baseball glove',
'skateboard',
'surfboard',
'tennis racket',
'bottle',
'wine glass',
'cup',
'fork',
'knife',
'spoon',
'bowl',
'banana',
'apple',
'sandwich',
'orange',
'broccoli',
'carrot',
'hot dog',
'pizza',
'donut',
'cake',
'chair',
'couch',
'potted plant',
'bed',
'dining table',
'toilet',
'tv',
'laptop',
'mouse',
'remote',
'keyboard',
'cell phone',
'microwave',
'oven',
'toaster',
'sink',
'refrigerator',
'book',
'clock',
'vase',
'scissors',
'teddy bear',
'hair drier',
'toothbrush',
'guard rail',
# 'car',
'dashed',
'solid',
'solid solid',
'dashed dashed',
'dashed-solid',
'solid-dashed',
'yellow dashed',
'yellow solid',
'yellow solid solid',
'yellow dashed dashed',
'yellow dashed-solid',
'yellow solid-dashed',
'boundary',
'fork_line',
'fork_edge',
'arrow_s',
'arrow_r',
'arrow_l',
'arrow_lr',
'arrow_inclined_r',
'arrow_r_s',
'arrow_l_s',
'sidewalk',
'handrail'
]
for data_set, ann_dir in zip(sets, ann_dirs):
print('Starting %s' % data_set)
ann_dict = {}
images = []
annotations = []
ann_dir = os.path.join(data_dir, ann_dir)
file_list = []
for root, sub_dirs, files in os.walk(ann_dir):
for filename in files:
file_list.append([root, filename])
random.shuffle(file_list)
for file_item in file_list:
root = file_item[0]
filename = file_item[1]
if filename.endswith(ends_in):
if filename.startswith("shcity_000875_000002"):
pass
if len(images) % 50 == 0:
print("Processed %s images, %s annotations" %
(len(images), len(annotations)))
json_ann = json.load(open(os.path.join(root, filename)))
image = {}
image['id'] = img_id
img_id += 1
image['width'] = json_ann['imgWidth']
image['height'] = json_ann['imgHeight']
sub_file_name = filename.split('_')
image['file_name'] = os.path.join(
sub_file_name[0],
'_'.join(sub_file_name[:-2]) + '_leftImg8bit.png')
image['seg_file_name'] = '_'.join(
filename.split('_')[:-1]) + '_instanceIds.png'
images.append(image)
fullname = os.path.join(root, image['seg_file_name'])
print("fullname:" + fullname)
objects = cs.instances2dict_with_polygons(
[fullname], verbose=False)[fullname]
for object_cls in objects:
# if object_cls not in add_instancesonly:
# continue
if object_cls not in category_instancesonly:
continue # skip non-instance categories
for obj in objects[object_cls]:
if obj['contours'] == []:
print('Warning: empty contours.')
continue # skip non-instance categories
index = category_instancesonly.index(
object_cls) # + 184
good_area = [p for p in obj['contours'] if len(p) > 4]
if len(good_area) == 0:
print('Warning: invalid contours.')
continue # skip non-instance categories
ann = {}
ann['id'] = ann_id
ann_id += 1
ann['image_id'] = image['id']
ann['segmentation'] = good_area
ann['category_id'] = index
ann['iscrowd'] = 0
ann['area'] = obj['pixelCount']
ann['bbox'] = bboxs_util.xyxy_to_xywh(
segms_util.polys_to_boxes([ann['segmentation']
])).tolist()[0]
annotations.append(ann)
ann_dict['images'] = images
categories = []
for index, value in enumerate(category_instancesonly):
categories.append({"id": index, "name": value})
categories = categories[1:]
ann_dict['categories'] = categories
ann_dict['annotations'] = annotations
print("Num categories: %s" % len(categories))
print("Num images: %s" % len(images))
print("Num annotations: %s" % len(annotations))
with open(os.path.join(out_dir, json_name % data_set),
'wb') as outfile:
outfile.write(json.dumps(ann_dict))