def _toMask(objs, coco):
# modify segmentation by reference
for obj in objs:
t = coco.imgs[obj['image_id']]
for region in obj['regions']:
if type(region['segmentation']) == list:
# format is xyxy, convert to RLE
region['segmentation'] = mask.frPyObjects(
[region['segmentation']], t['height'], t['width'])
if len(region['segmentation']) == 1:
region['segmentation'] = region['segmentation'][0]
else:
# an object can have multiple polygon regions
# merge them into one RLE mask
region['segmentation'] = mask.merge(
obj['segmentation'])
if 'area' not in region:
region['area'] = mask.area(
[region['segmentation']])[0]
elif type(region['segmentation']) == dict and type(
region['segmentation']['counts']) == list:
region['segmentation'] = mask.frPyObjects(
[region['segmentation']], t['height'],
t['width'])[0]
elif type(region['segmentation']) == dict and \
type(region['segmentation']['counts'] == unicode or type(region['segmentation']['counts']) == str):
# format is already RLE, do nothing
if 'area' not in region:
region['area'] = mask.area(
[region['segmentation']])[0]
else:
raise Exception('segmentation format not supported.')
def showAnns(self, anns):
"""
Display the specified annotations.
:param anns (array of object): annotations to display
:return: None
"""
if len(anns) == 0:
return 0
if 'segmentation' in anns[0]:
datasetType = 'instances'
elif 'caption' in anns[0]:
datasetType = 'captions'
if datasetType == 'instances':
ax = plt.gca()
ax.set_autoscale_on(False)
polygons = []
color = []
for ann in anns:
c = (np.random.random((1, 3))*0.6+0.4).tolist()[0]
if type(ann['segmentation']) == list:
# polygon
for seg in ann['segmentation']:
poly = np.array(seg).reshape((len(seg)/2, 2))
polygons.append(Polygon(poly))
color.append(c)
else:
# mask
t = self.imgs[ann['image_id']]
if type(ann['segmentation']['counts']) == list:
rle = mask.frPyObjects([ann['segmentation']], t['height'], t['width'])
else:
rle = [ann['segmentation']]
m = mask.decode(rle)
img = np.ones( (m.shape[0], m.shape[1], 3) )
if ann['iscrowd'] == 1:
color_mask = np.array([2.0,166.0,101.0])/255
if ann['iscrowd'] == 0:
color_mask = np.random.random((1, 3)).tolist()[0]
for i in range(3):
img[:,:,i] = color_mask[i]
ax.imshow(np.dstack( (img, m*0.5) ))
if 'keypoints' in ann and type(ann['keypoints']) == list:
# turn skeleton into zero-based index
sks = np.array(self.loadCats(ann['category_id'])[0]['skeleton'])-1
kp = np.array(ann['keypoints'])
x = kp[0::3]
y = kp[1::3]
v = kp[2::3]
for sk in sks:
if np.all(v[sk]>0):
plt.plot(x[sk],y[sk], linewidth=3, color=c)
plt.plot(x[v==1], y[v==1],'o',markersize=8, markerfacecolor=c, markeredgecolor='k',markeredgewidth=2)
plt.plot(x[v==2], y[v==2],'o',markersize=8, markerfacecolor=c, markeredgecolor=c, markeredgewidth=2)
p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4)
ax.add_collection(p)
p = PatchCollection(polygons, facecolor="none", edgecolors=color, linewidths=2)
ax.add_collection(p)
elif datasetType == 'captions':
for ann in anns:
print ann['caption']
def showAnns(self, anns):
"""
Display the specified annotations.
:param anns (array of object): annotations to display
:return: None
"""
if len(anns) == 0:
return 0
if 'segmentation' in anns[0]:
datasetType = 'instances'
elif 'caption' in anns[0]:
datasetType = 'captions'
if datasetType == 'instances':
ax = plt.gca()
polygons = []
color = []
for ann in anns:
c = np.random.random((1, 3)).tolist()[0]
if type(ann['segmentation']) == list:
# polygon
for seg in ann['segmentation']:
poly = np.array(seg).reshape((len(seg) / 2, 2))
polygons.append(Polygon(poly, True, alpha=0.4))
color.append(c)
else:
# mask
t = self.imgs[ann['image_id']]
if type(ann['segmentation']['counts']) == list:
rle = mask.frPyObjects([ann['segmentation']],
t['height'], t['width'])
else:
rle = [ann['segmentation']]
m = mask.decode(rle)
img = np.ones((m.shape[0], m.shape[1], 3))
if ann['iscrowd'] == 1:
color_mask = np.array([2.0, 166.0, 101.0]) / 255
if ann['iscrowd'] == 0:
color_mask = np.random.random((1, 3)).tolist()[0]
for i in range(3):
img[:, :, i] = color_mask[i]
ax.imshow(np.dstack((img, m * 0.5)))
p = PatchCollection(polygons,
facecolors=color,
edgecolors=(0, 0, 0, 1),
linewidths=3,
alpha=0.4)
ax.add_collection(p)
elif datasetType == 'captions':
n = 0
cap = [None] * 5
for ann in anns:
#print ann['caption']
if n < 5:
cap[n] = ann['caption']
#print cap[n]
n = n + 1
print n
print cap
return cap
def annToRLE(self, ann, height, width):
"""
Convert annotation which can be polygons, uncompressed RLE to RLE.
:return: binary mask (numpy 2D array)
"""
segm = ann['segmentation']
if isinstance(segm, list):
# polygon -- a single object might consist of multiple parts
# we merge all parts into one mask rle code
rles = maskUtils.frPyObjects(segm, height, width)
rle = maskUtils.merge(rles)
elif isinstance(segm['counts'], list):
# uncompressed RLE
rle = maskUtils.frPyObjects(segm, height, width)
else:
# rle
rle = ann['segmentation']
return rle
def annToMask(segm, h, w):
"""
Convert annotation which can be polygons, uncompressed RLE to RLE.
:return: binary mask (numpy 2D array)
"""
if type(segm) == list:
# polygon -- a single object might consist of multiple parts
# we merge all parts into one mask rle code
rles = maskUtils.frPyObjects(segm, h, w)
rle = maskUtils.merge(rles)
elif type(segm['counts']) == list:
# uncompressed RLE
rle = maskUtils.frPyObjects(segm, h, w)
else:
# rle
rle = segm
return maskUtils.decode(rle)
def annToRLE(self, ann):
"""
Convert annotation which can be polygons, uncompressed RLE to RLE.
:return: binary mask (numpy 2D array)
"""
t = self.imgs[ann['image_id']]
h, w = t['height'], t['width']
segm = ann['segmentation']
if type(segm) == list:
# polygon -- a single object might consist of multiple parts
# we merge all parts into one mask rle code
rles = maskUtils.frPyObjects(segm, h, w)
rle = maskUtils.merge(rles)
elif type(segm['counts']) == list:
# uncompressed RLE
rle = maskUtils.frPyObjects(segm, h, w)
else:
# rle
rle = ann['segmentation']
return rle
def showAnns(self, anns):
"""
Display the specified annotations.
:param anns (array of object): annotations to display
:return: None
"""
if len(anns) == 0:
return 0
if 'segmentation' in anns[0]:
datasetType = 'instances'
elif 'caption' in anns[0]:
datasetType = 'captions'
if datasetType == 'instances':
ax = plt.gca()
polygons = []
color = []
for ann in anns:
c = np.random.random((1, 3)).tolist()[0]
if type(ann['segmentation']) == list:
# polygon
for seg in ann['segmentation']:
poly = np.array(seg).reshape((len(seg)/2, 2))
polygons.append(Polygon(poly, True,alpha=0.4))
color.append(c)
else:
# mask
t = self.imgs[ann['image_id']]
if type(ann['segmentation']['counts']) == list:
rle = mask.frPyObjects([ann['segmentation']], t['height'], t['width'])
else:
rle = [ann['segmentation']]
m = mask.decode(rle)
img = np.ones( (m.shape[0], m.shape[1], 3) )
if ann['iscrowd'] == 1:
color_mask = np.array([2.0,166.0,101.0])/255
if ann['iscrowd'] == 0:
color_mask = np.random.random((1, 3)).tolist()[0]
for i in range(3):
img[:,:,i] = color_mask[i]
ax.imshow(np.dstack( (img, m*0.5) ))
p = PatchCollection(polygons, facecolors=color, edgecolors=(0,0,0,1), linewidths=3, alpha=0.4)
ax.add_collection(p)
elif datasetType == 'captions':
n=0
cap= [None] * 5
for ann in anns:
#print ann['caption']
if n<5:
cap[n]=ann['caption']
#print cap[n]
n = n + 1
print n
print cap
return cap
def _toMask(objs, coco):
# modify segmentation by reference
for obj in objs:
t = coco.imgs[obj['image_id']]
if type(obj['segmentation']) == list:
if type(obj['segmentation'][0]) == dict:
print 'debug'
obj['segmentation'] = mask.frPyObjects(obj['segmentation'],t['height'],t['width'])
if len(obj['segmentation']) == 1:
obj['segmentation'] = obj['segmentation'][0]
else:
# an object can have multiple polygon regions
# merge them into one RLE mask
obj['segmentation'] = mask.merge(obj['segmentation'])
elif type(obj['segmentation']) == dict and type(obj['segmentation']['counts']) == list:
obj['segmentation'] = mask.frPyObjects([obj['segmentation']],t['height'],t['width'])[0]
elif type(obj['segmentation']) == dict and \
type(obj['segmentation']['counts'] == unicode or type(obj['segmentation']['counts']) == str):
pass
else:
raise Exception('segmentation format not supported.')
def _toMask(objs, coco):
# modify segmentation by reference
for obj in objs:
t = coco.imgs[obj['image_id']]
if type(obj['segmentation']) == list:
if type(obj['segmentation'][0]) == dict:
print('debug')
obj['segmentation'] = mask.frPyObjects(obj['segmentation'],t['height'],t['width'])
if len(obj['segmentation']) == 1:
obj['segmentation'] = obj['segmentation'][0]
else:
# an object can have multiple polygon regions
# merge them into one RLE mask
obj['segmentation'] = mask.merge(obj['segmentation'])
elif type(obj['segmentation']) == dict and type(obj['segmentation']['counts']) == list:
obj['segmentation'] = mask.frPyObjects([obj['segmentation']],t['height'],t['width'])[0]
elif type(obj['segmentation']) == dict and \
type(obj['segmentation']['counts'] == unicode or type(obj['segmentation']['counts']) == str):
pass
else:
raise Exception('segmentation format not supported.')
def drawSegmentation(image, anns, img):
"""
draws segmentation on input image
:param anns (array of object): annotations to display
:return: None
"""
if len(anns) == 0:
return False
if 'segmentation' in anns[0]:
# sort annotations from biggest to smallest to avoid occlusions
anns.sort(key=lambda x: x['area'], reverse=True)
if anns[len(anns)-1]['area'] < 200:
return False
for ann in anns:
# open file making the conversion MSCOCO classes -> VOC classes
f = open('classes.txt', 'r')
for line in f:
splt = line.split('\t')
if ann['category_id'] == int(splt[0]):
pixelvalue = int(splt[1])
break
f.close()
c = [pixelvalue, pixelvalue, pixelvalue]
if type(ann['segmentation']) == list:
# polygon
for seg in ann['segmentation']:
poly = np.array(seg).reshape((len(seg)/2, 2))
pts = np.array(poly, np.int32)
pts.reshape((-1,1,2))
cv2.polylines(image,[pts],True,(255,255,255), 3)
cv2.fillPoly(image, [pts], c)
else:
# mask
t = coco.imgs[ann['image_id']]
if type(ann['segmentation']['counts']) == list:
rle = mask.frPyObjects([ann['segmentation']], t['height'], t['width'])
else:
rle = [ann['segmentation']]
m = mask.decode(rle)
img = np.ones( (m.shape[0], m.shape[1], 3) )
for i in range(3):
img[:,:,i] = pixelvalue
mask2 = np.dstack( (img, m) )
for x in range(img.shape[0]):
for y in range(img.shape[1]):
if not mask2[x][y][3] == 0:
image[x][y] = c
return True
def _toMask(objs, coco):
# modify segmentation by reference
for obj in objs:
t = coco.imgs[obj['image_id']]
for region in obj['regions']:
if type(region['segmentation']) == list:
# format is xyxy, convert to RLE
region['segmentation'] = mask.frPyObjects(
[region['segmentation']], t['height'], t['width'])
region['segmentation'][0]['counts'] = region[
'segmentation'][0]['counts'].decode('utf-8')
if len(region['segmentation']) == 1:
region['segmentation'] = region['segmentation'][0]
else:
# an object can have multiple polygon regions
# merge them into one RLE mask
region['segmentation'] = mask.merge(
obj['segmentation'])
if 'area' not in region:
region['area'] = mask.area(
[region['segmentation']])[0]
elif type(region['segmentation']) == dict and type(
region['segmentation']['counts']) == list:
region['segmentation'] = mask.frPyObjects(
[region['segmentation']], t['height'],
t['width'])[0]
region['segmentation']['counts'] = region[
'segmentation']['counts'].decode('utf-8')
elif type(region['segmentation']) == dict and \
type(region['segmentation']['counts']) == str or type(region['segmentation']['counts']) == bytes:
# Python 3 renamed the unicode type to str, the old str type has been replaced by bytes.
# format is already RLE, do nothing
if 'area' not in region:
region['area'] = mask.area(
[region['segmentation']])[0]
else:
print('???')
raise Exception('segmentation format not supported.')
def showAnns(self, anns):
"""
Display the specified annotations.
:param anns (array of object): annotations to display
:return: None
"""
if len(anns) == 0:
return 0
if 'segmentation' in anns[0] or 'keypoints' in anns[0]:
if 'bbox' in anns[0]:
datasetType = 'detections'
else:
datasetType = 'instances'
elif 'caption' in anns[0]:
datasetType = 'captions'
else:
raise Exception("datasetType not supported")
if datasetType == 'instances':
ax = plt.gca()
ax.set_autoscale_on(False)
polygons = []
color = []
for ann in anns:
c = (np.random.random((1, 3))*0.6+0.4).tolist()[0]
if 'segmentation' in ann:
if type(ann['segmentation']) == list:
# polygon
for seg in ann['segmentation']:
poly = np.array(seg).reshape((len(seg)/2, 2))
polygons.append(Polygon(poly))
color.append(c)
else:
# mask
t = self.imgs[ann['image_id']]
if type(ann['segmentation']['counts']) == list:
rle = mask.frPyObjects([ann['segmentation']], t['height'], t['width'])
else:
rle = [ann['segmentation']]
m = mask.decode(rle)
img = np.ones( (m.shape[0], m.shape[1], 3) )
if ann['iscrowd'] == 1:
color_mask = np.array([2.0,166.0,101.0])/255
if ann['iscrowd'] == 0:
color_mask = np.random.random((1, 3)).tolist()[0]
for i in range(3):
img[:,:,i] = color_mask[i]
ax.imshow(np.dstack( (img, m*0.5) ))
if 'keypoints' in ann and type(ann['keypoints']) == list:
# turn skeleton into zero-based index
sks = np.array(self.loadCats(ann['category_id'])[0]['skeleton'])-1
kp = np.array(ann['keypoints'])
x = kp[0::3]
y = kp[1::3]
v = kp[2::3]
for sk in sks:
if np.all(v[sk]>0):
plt.plot(x[sk],y[sk], linewidth=3, color=c)
plt.plot(x[v>0], y[v>0],'o',markersize=8, markerfacecolor=c, markeredgecolor='k',markeredgewidth=2)
plt.plot(x[v>1], y[v>1],'o',markersize=8, markerfacecolor=c, markeredgecolor=c, markeredgewidth=2)
p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4)
ax.add_collection(p)
p = PatchCollection(polygons, facecolor="none", edgecolors=color, linewidths=2)
ax.add_collection(p)
elif datasetType == 'detections':
ax = plt.gca()
colors = plt.cm.hsv(np.linspace(0, 1, 91)).tolist()
for ann in anns:
cat_id = ann['category_id']
color = colors[cat_id]
bbox = ann['bbox']
coords = (bbox[0], bbox[1]), bbox[2], bbox[3]
ax.add_patch(plt.Rectangle(*coords, fill=False, edgecolor=color, linewidth=3))
name = 'Unknown'
for cat in self.dataset['categories']:
if ann['category_id'] == cat['id']:
name = cat['name']
if 'score' in ann:
score = ann['score']
display_text = '%s: %.2f' % (name, score)
else:
display_text = name
ax.text(bbox[0], bbox[1], display_text, bbox={'facecolor':color, 'alpha':0.5})
elif datasetType == 'captions':
for ann in anns:
print ann['caption']
im = misc.imread(im_name)
im_category_id = annotations[annotation_number]['category_id']
annotation_ii_to_category_ii = category_ids.index(im_category_id)
im_categories = categories[annotation_ii_to_category_ii]
misc.imsave(output_images + '/image_' + str(i) + '_999.jpg', im)
training_image_category_id.append(im_category_id)
training_image_category_name.append(im_categories['supercategory'])
temp_content_ids = []
temp_content_names = []
num_annotations = len(annotation_seg)
if num_annotations > 0:
for s in range(0, num_annotations):
# poly = np.array(s).reshape((len(s)/2, 2))
# polygons.append(Polygon(poly))
# if type(s['counts']) == list:
rle = mask.frPyObjects([annotation_seg[s]], im_h, im_w)
# else:
# rle = [s]
m = mask.decode(rle)
if len(im.shape) == 2:
im = im[:, :, None]
im = np.tile(im, [1, 1, 3])
m_im = np.tile(m, [1, 1, 3])
masked_im = np.multiply(im, m_im)
misc.imsave(
output_images + '/image_' +
str(training_image_category_id[0]) + '_' + str(i) + '_' +
str(s) + '.jpg', masked_im)
temp_content_ids.append(im_categories['id'])
temp_content_names.append(im_categories['name'])
training_content_category_id.append(temp_content_ids)
