def test_objectFields(self):
cursor = self.conn.cursor()
cursor.execute('SELECT * FROM objects WHERE objectid=1')
entry = cursor.fetchone()
self.assertEqual(backendDb.objectField(entry, 'objectid'), 1,
str(entry))
self.assertEqual(backendDb.objectField(entry, 'imagefile'),
'images/000000.jpg', str(entry))
self.assertEqual(backendDb.objectField(entry, 'x1'), 225, str(entry))
self.assertEqual(backendDb.objectField(entry, 'y1'), 134, str(entry))
self.assertEqual(backendDb.objectField(entry, 'width'), 356,
str(entry))
self.assertEqual(backendDb.objectField(entry, 'height'), 377,
str(entry))
self.assertEqual(backendDb.objectField(entry, 'name'), 'car',
str(entry))
self.assertAlmostEqual(backendDb.objectField(entry, 'score'),
0.606193,
msg=str(entry))
with self.assertRaises(KeyError):
backendDb.objectField(entry, 'dummy')
# Multiple fields.
self.assertEqual(backendDb.objectFields(entry, ['objectid', 'x1']),
[1, 225], str(entry))
with self.assertRaises(KeyError):
self.assertEqual(backendDb.objectFields(entry, ['x1', 'dummy']))
def _exportLabel(c, out_path_noext, imagefile):
'''
Writes objects to json file.
Please use https://github.com/mcordts/cityscapesScripts to generate masks.
'''
c.execute('SELECT width,height FROM images WHERE imagefile=?',
(imagefile, ))
imgWidth, imgHeight = c.fetchone()
data = {'imgWidth': imgWidth, 'imgHeight': imgHeight, 'objects': []}
c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, ))
for object_ in c.fetchall():
objectid = backendDb.objectField(object_, 'objectid')
name = backendDb.objectField(object_, 'name')
# Polygon = [[x1, y1], [x2, y2], ...].
# Check if the object has a polygon.
c.execute('SELECT x,y FROM polygons WHERE objectid=?', (objectid, ))
polygon = c.fetchall()
if len(polygon) == 0:
# If there is no polygon, make one from bounding box.
[y1, x1, y2, x2] = backendDb.objectField(object_, 'roi')
polygon = [(x1, y1), (x2, y1), (x2, y2), (x1, y2)]
color = (255, )
cv2.rectangle(mask, (x1, y1), (x2, y2), color, -1)
data['objects'].append({'label': name, 'polygon': polygon})
json_path = '%s_polygons.json' % out_path_noext
with open(json_path, 'w') as f:
json.dump(data, f, indent=4)
def buildObjectSample(object_entry, c, imreader):
'''
Load images and get necessary information from object_entry to make a frame.
Args:
object_entry: (tuple) tuple with all fields from "objects" table.
c: (cursor) sqlite3 cursor.
imreader: (lib.backend.backendMedia.MediaReader)
Returns a dict with keys:
image: (np.uint8) The array corresponding to an image.
mask: (np.uint8) The array corresponding to a mask if exists, or None.
objectid: (int) The primary key in the "objects" table.
name: (string) The "name" field in the "objects" table.
score: (float) The "score" field in the "objects" table.
imagefile: (string) The image id.
All key-value pairs from the "properties" table for this objectid.
'''
objectid = backendDb.objectField(object_entry, 'objectid')
imagefile = backendDb.objectField(object_entry, 'imagefile')
name = backendDb.objectField(object_entry, 'name')
score = backendDb.objectField(object_entry, 'score')
c.execute('SELECT maskfile FROM images WHERE imagefile=?', (imagefile, ))
maskfile = c.fetchone()[0]
logging.debug('Reading object %d from %s imagefile' %
(objectid, imagefile))
try:
image = imreader.imread(imagefile)
mask = imreader.maskread(maskfile) if maskfile is not None else None
except ValueError:
traceback.print_exc()
logging.error('Reading image or mask failed. Returning None.')
return None
roi = backendDb.objectField(object_entry, 'roi')
logging.debug('Roi: %s' % roi)
roi = [int(x) for x in roi]
image = image[roi[0]:roi[2], roi[1]:roi[3]]
mask = mask[roi[0]:roi[2], roi[1]:roi[3]] if mask is not None else None
sample = {
'image': image,
'mask': mask,
'name': name,
'score': score,
'imagefile': imagefile,
'objectid': objectid
}
# Add properties.
c.execute('SELECT key,value FROM properties WHERE objectid=?',
(objectid, ))
property_entries = c.fetchall()
sample.update(dict(property_entries))
return sample
def test_record_objects(self):
out_db_file = op.join(self.work_dir, 'out.db')
self.writer = DatasetWriter(out_db_file)
# Require imagefile.
with self.assertRaises(KeyError):
objectid = self.writer.addObject({})
objectid1 = self.writer.addObject({
'imagefile': 'myimagefile1',
'x1': 10,
'y1': 20,
'width': 30,
'height': 40,
'name': 'car',
'score': 0.5
})
objectid2 = self.writer.addObject({
'imagefile': 'myimagefile2',
})
self.writer.close()
self.assertTrue(op.exists(out_db_file))
self.conn = sqlite3.connect('file:%s?mode=ro' % out_db_file, uri=True)
c = self.conn.cursor()
c.execute('SELECT * FROM objects')
object_entries = c.fetchall()
self.assertEqual(len(object_entries), 2)
self.assertEqual(objectid1, 1)
self.assertEqual(objectField(object_entries[0], 'objectid'), 1)
self.assertEqual(objectField(object_entries[0], 'imagefile'),
'myimagefile1')
self.assertEqual(objectField(object_entries[0], 'x1'), 10)
self.assertEqual(objectField(object_entries[0], 'y1'), 20)
self.assertEqual(objectField(object_entries[0], 'width'), 30)
self.assertEqual(objectField(object_entries[0], 'height'), 40)
self.assertEqual(objectField(object_entries[0], 'name'), 'car')
self.assertEqual(objectField(object_entries[0], 'score'), 0.5)
self.assertEqual(objectid2, 2)
self.assertEqual(objectField(object_entries[1], 'objectid'), 2)
self.assertEqual(objectField(object_entries[1], 'imagefile'),
'myimagefile2')
def getIntersectingObjects(objects1, objects2, IoU_threshold, same_id_ok=True):
'''
Given two lists of objects find pairs that intersect by IoU_threshold.
Objects are assumed to be in the same image.
Args:
objects1, objects2: A list of object entries.
Each entry is the whole row in the 'objects' table.
IoU_threshold: A float in range [0, 1].
Returns:
A list of tuples. Each tuple has objectid of an entry in 'objects1' and
objectid of an entry in 'objects2'.
'''
# Compute pairwise distances between rectangles.
# TODO: possibly can optimize in future to avoid O(N^2) complexity.
pairwise_IoU = np.zeros(shape=(len(objects1), len(objects2)), dtype=float)
for i1, object1 in enumerate(objects1):
for i2, object2 in enumerate(objects2):
# Do not merge an object with itself.
objectid1 = backendDb.objectField(object1, 'objectid')
objectid2 = backendDb.objectField(object2, 'objectid')
if objectid1 == objectid2 and not same_id_ok:
pairwise_IoU[i1, i2] = np.nan
else:
roi1 = backendDb.objectField(object1, 'roi')
roi2 = backendDb.objectField(object2, 'roi')
pairwise_IoU[i1, i2] = utilBoxes.getIoU(roi1, roi2)
logging.debug('Pairwise_IoU is:\n%s', pprint.pformat(pairwise_IoU))
# Greedy search for pairs.
pairs_to_merge = []
for _ in range(min(len(objects1), len(objects2))):
i1, i2 = np.unravel_index(np.argmax(pairwise_IoU), pairwise_IoU.shape)
IoU = pairwise_IoU[i1, i2]
logging.debug('Next object at indices [%d, %d]. IoU: %s', i1, i2,
str(IoU))
# Stop if no more good pairs.
if np.isnan(IoU) or IoU < IoU_threshold:
break
# Disable these objects for the next step.
pairwise_IoU[i1, :] = 0.
pairwise_IoU[:, i2] = 0.
# Add a pair to the list.
objectid1 = backendDb.objectField(objects1[i1], 'objectid')
objectid2 = backendDb.objectField(objects2[i2], 'objectid')
pairs_to_merge.append((objectid1, objectid2))
name1 = backendDb.objectField(objects1[i1], 'name')
name2 = backendDb.objectField(objects2[i2], 'name')
logging.debug('Will merge objects %d (%s) and %d (%s) with IoU %f.',
objectid1, name1, objectid2, name2, IoU)
return pairs_to_merge
def bboxes2polygons(cursor, objectid):
''' A rectangular polygon is added to objects that are missing polygons. '''
# If there are already polygon entries, do nothing.
cursor.execute('SELECT COUNT(1) FROM polygons WHERE objectid=?',
(objectid, ))
if cursor.fetchone()[0] > 0:
return
cursor.execute('SELECT * FROM objects WHERE objectid=?', (objectid, ))
object_entry = cursor.fetchone()
if object_entry is None:
raise ValueError('Objectid %d does not exist.' % objectid)
y1, x1, y2, x2 = backendDb.objectField(object_entry, 'roi')
for x, y in [(x1, y1), (x2, y1), (x2, y2), (x1, y2)]:
cursor.execute(
'INSERT INTO polygons(objectid,x,y,name) VALUES (?,?,?,"bounding box")',
(objectid, x, y))
logging.debug(
'Added polygon from bbox y1=%d,x1=%d,y2=%d,x2=%d to objectid %d', y1,
x1, y2, x2, objectid)
def displayImagesPlt(c, args):
c.execute('SELECT * FROM images WHERE (%s)' % args.where_image)
image_entries = c.fetchall()
logging.info('%d images found.', len(image_entries))
if len(image_entries) == 0:
logging.error('There are no images. Exiting.')
return
if args.shuffle:
np.random.shuffle(image_entries)
if len(image_entries) < args.limit:
image_entries = image_entries[:args.limit]
def _getNumRows(total, ncols):
nrows = int((total - 1) / ncols) + 1
logging.info('Grid: %dx%d from the total of %d', nrows, ncols, total)
return nrows
if args.limit < len(image_entries):
image_entries = image_entries[:args.limit]
nrows = _getNumRows(len(image_entries), args.ncols)
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For overlaying masks.
labelmap = literal_eval(
args.mask_mapping_dict) if args.mask_mapping_dict else None
logging.info('Parsed mask_mapping_dict to %s', pformat(labelmap))
for i_image, image_entry in enumerate(image_entries):
ax = plt.subplot(nrows, args.ncols, i_image + 1)
imagefile = backendDb.imageField(image_entry, 'imagefile')
maskfile = backendDb.imageField(image_entry, 'maskfile')
imname = backendDb.imageField(image_entry, 'name')
imscore = backendDb.imageField(image_entry, 'score')
logging.info('Imagefile "%s"', imagefile)
logging.debug('Image name="%s", score=%s', imname, imscore)
image = imreader.imread(imagefile)
# Overlay the mask.
if maskfile is not None:
mask = imreader.maskread(maskfile)
if args.mask_aside:
image = util.drawMaskAside(image, mask, labelmap=labelmap)
elif args.mask_alpha is not None:
image = util.drawMaskOnImage(image,
mask,
alpha=args.mask_alpha,
labelmap=labelmap)
else:
logging.info('No mask for this image.')
# Put the objects on top of the image.
if args.with_objects:
c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, ))
object_entries = c.fetchall()
logging.info('Found %d objects for image %s', len(object_entries),
imagefile)
for object_entry in object_entries:
objectid = backendDb.objectField(object_entry, 'objectid')
roi = backendDb.objectField(object_entry, 'roi')
score = backendDb.objectField(object_entry, 'score')
name = backendDb.objectField(object_entry, 'name')
logging.info('objectid: %d, roi: %s, score: %s, name: %s',
objectid, roi, score, name)
c.execute('SELECT * FROM polygons WHERE objectid=?',
(objectid, ))
polygon_entries = c.fetchall()
if len(polygon_entries) > 0:
logging.info('showing object with a polygon.')
polygon = [(int(backendDb.polygonField(p, 'x')),
int(backendDb.polygonField(p, 'y')))
for p in polygon_entries]
drawScoredPolygon(ax, polygon, label=name, score=score)
elif roi is not None:
logging.info('showing object with a bounding box.')
drawScoredRoi(ax, roi, label=name, score=score)
else:
logging.warning(
'Neither polygon, nor bbox is available for objectid %d',
objectid)
# Overlay imagefile.
title = ""
if args.with_imagefile:
title += '%s ' % op.basename(
backendMedia.normalizeSeparators(imagefile))
# Overlay score.
if args.with_score:
title += '%.3f ' % imscore if imscore is not None else ''
ax.set_title(title)
# Display
ax.imshow(image)
ax.axis('off')
plt.tight_layout()
plt.show()
def _findPressedObject(x, y, cars):
for i in range(len(cars)):
roi = backendDb.objectField(cars[i], 'roi')
if x >= roi[1] and x < roi[3] and y >= roi[0] and y < roi[2]:
return i
return None
def labelObjects(c, args):
cv2.namedWindow("labelObjects")
c.execute('SELECT COUNT(*) FROM objects WHERE (%s) ' % args.where_object)
logging.info('Found %d objects in db.' % c.fetchone()[0])
c.execute('SELECT * FROM objects WHERE (%s)' % args.where_object)
object_entries = c.fetchall()
logging.info('Found %d objects in db.' % len(object_entries))
if len(object_entries) == 0:
return
if args.shuffle:
np.random.shuffle(object_entries)
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For parsing keys.
key_reader = KeyReader(args.key_dict)
button = 0
index_object = 0
another_object = True
while button != 27:
go_next_object = False
if another_object:
another_object = False
logging.info(' ')
logging.info('Object %d out of %d' %
(index_object, len(object_entries)))
object_entry = object_entries[index_object]
objectid = backendDb.objectField(object_entry, 'objectid')
bbox = backendDb.objectField(object_entry, 'bbox')
roi = backendDb.objectField(object_entry, 'roi')
imagefile = backendDb.objectField(object_entry, 'imagefile')
logging.info('imagefile: %s' % imagefile)
image = imreader.imread(imagefile)
# Display an image, wait for the key from user, and parse that key.
scale = float(args.winsize) / max(image.shape[0:2])
logging.debug('Will resize image and annotations with scale: %f' %
scale)
image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale)
logging.info('objectid: %d, roi: %s' % (objectid, roi))
c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, ))
polygon_entries = c.fetchall()
if len(polygon_entries) > 0:
logging.info('showing object with a polygon.')
polygon = [(backendDb.polygonField(p, 'x'),
backendDb.polygonField(p, 'y'))
for p in polygon_entries]
logging.debug('nonscaled polygon: %s' % pformat(polygon))
polygon = [(int(scale * p[0]), int(scale * p[1]))
for p in polygon]
logging.debug('scaled polygon: %s' % pformat(polygon))
util.drawScoredPolygon(image, polygon, label=None, score=None)
elif roi is not None:
logging.info('showing object with a bounding box.')
logging.debug('nonscaled roi: %s' % pformat(roi))
roi = [int(scale * r)
for r in roi] # For displaying the scaled image.
logging.debug('scaled roi: %s' % pformat(roi))
util.drawScoredRoi(image, roi, label=None, score=None)
else:
raise Exception(
'Neither polygon, nor bbox is available for objectid %d' %
objectid)
c.execute(
'SELECT key,value FROM properties WHERE objectid=? AND key=?',
(objectid, args.property))
# TODO: Multiple properties are possible because there is no
# contraint on uniqueness on table properties(objectid,key).
# Change when the uniqueness constraint is added to the
# database schema. On the other hand, it's a feature.
properties = c.fetchall()
if len(properties) > 1:
logging.warning(
'Multiple values for object %s and property %s. '
'If reassigned, both will be changed' %
(objectid, args.property))
for iproperty, (key, value) in enumerate(properties):
cv2.putText(image, '%s: %s' % (key, value),
(10, util.SCALE * (iproperty + 1)), util.FONT,
util.FONT_SIZE, (0, 0, 0), util.THICKNESS)
cv2.putText(image, '%s: %s' % (key, value),
(10, util.SCALE * (iproperty + 1)), util.FONT,
util.FONT_SIZE, (255, 255, 255),
util.THICKNESS - 1)
logging.info('objectid: %d. %s = %s.' % (objectid, key, value))
cv2.imshow('labelObjects', image[:, :, ::-1])
action = key_reader.parse(cv2.waitKey(-1))
if action == 'exit':
break
elif action == 'delete_label' and any_object_in_focus:
logging.info('Remove label from objectid "%s"' % objectid)
c.execute('DELETE FROM properties WHERE objectid=? AND key=?',
(objectid, args.property))
go_next_object = True
elif action is not None and action not in ['previous', 'next']:
# User pressed something else which has an assigned action,
# assume it is a new value.
logging.info('Setting label "%s" to objectid "%s"' %
(action, objectid))
if len(properties) > 0:
c.execute('DELETE FROM properties WHERE objectid=? AND key=?',
(objectid, args.property))
c.execute(
'INSERT INTO properties(objectid,key,value) VALUES (?,?,?)',
(objectid, args.property, str(action)))
go_next_object = True
# Navigation.
if action == 'previous':
logging.debug('previous object')
another_object = True
if index_object > 0:
index_object -= 1
else:
logging.warning('Already at the first object.')
elif action == 'next' or go_next_object == True:
logging.debug('next object')
another_object = True
if index_object < len(object_entries) - 1:
index_object += 1
else:
logging.warning(
'Already at the last object. Press Esc to save and exit.')
cv2.destroyWindow("labelObjects")
def importKitti(c, args):
if args.with_display:
imreader = backendMedia.MediaReader(args.rootdir)
image_paths = sorted(glob(op.join(args.images_dir, '*.png')))
logging.info('Found %d PNG images in %s' %
(len(image_paths), args.images_dir))
for image_path in progressbar(image_paths):
filename = op.splitext(op.basename(image_path))[0]
logging.debug('Processing image: "%s"' % filename)
# Add image to the database.
imheight, imwidth = backendMedia.getPictureSize(image_path)
imagefile = op.relpath(image_path, args.rootdir)
c.execute('INSERT INTO images(imagefile,width,height) VALUES (?,?,?)',
(imagefile, imwidth, imheight))
if args.with_display:
img = imreader.imread(imagefile)
# Detection annotations.
if args.detection_dir:
detection_path = op.join(args.detection_dir, '%s.txt' % filename)
if not op.exists(detection_path):
raise FileNotFoundError('Annotation file not found at "%s".' %
detection_path)
# Read annotation file.
with open(detection_path) as f:
lines = f.read().splitlines()
logging.debug('Read %d lines from detection file "%s"' %
(len(lines), detection_path))
for line in lines:
objectid = _parseObject(c, line, imagefile)
if args.with_display:
c.execute('SELECT * FROM objects WHERE objectid=?',
(objectid, ))
object_entry = c.fetchone()
name = backendDb.objectField(object_entry, 'name')
roi = backendDb.objectField(object_entry, 'roi')
score = backendDb.objectField(object_entry, 'score')
util.drawScoredRoi(img, roi, name, score=score)
# Segmentation annotations.
if args.segmentation_dir:
segmentation_path = op.join(args.segmentation_dir,
'%s.png' % filename)
if not op.exists(segmentation_path):
raise FileNotFoundError('Annotation file not found at "%s".' %
segmentation_path)
# Add image to the database.
maskfile = op.relpath(segmentation_path, args.rootdir)
c.execute('UPDATE images SET maskfile=? WHERE imagefile=?',
(maskfile, imagefile))
if args.with_display:
mask = imreader.maskread(maskfile)
img = util.drawMaskAside(img, mask, labelmap=None)
# Maybe display.
if args.with_display:
cv2.imshow('importKitti', img[:, :, ::-1])
if cv2.waitKey(-1) == 27:
args.with_display = False
cv2.destroyWindow('importKitti')
def filterObjectsInsideCertainObjects(c, args):
c.execute('SELECT imagefile FROM images')
for imagefile, in progressbar(c.fetchall()):
# Shadow objects.
c.execute(
'SELECT * FROM objects WHERE imagefile=? AND (%s)' %
args.where_shadowing_objects, (imagefile, ))
shadow_object_entries = c.fetchall()
logging.info('Found %d shadowing objects.', len(shadow_object_entries))
# Populate polygons of the shadow objects.
shadow_object_polygons = []
for shadow_object_entry in shadow_object_entries:
shadow_objectid = backendDb.objectField(shadow_object_entry,
'objectid')
c.execute('SELECT y,x FROM polygons WHERE objectid=?',
(shadow_objectid, ))
shadow_polygon = c.fetchall()
shadow_object_polygons.append(shadow_polygon)
shadow_object_ids_set = set([
backendDb.objectField(entry, 'objectid')
for entry in shadow_object_entries
])
# Get all the objects that can be considered.
c.execute(
'SELECT * FROM objects WHERE imagefile=? AND (%s)' %
args.where_objects, (imagefile, ))
object_entries = c.fetchall()
logging.info('Total %d objects satisfying the condition.',
len(object_entries))
for object_entry in object_entries:
objectid = backendDb.objectField(object_entry, 'objectid')
if objectid in shadow_object_ids_set:
logging.debug('Object %d is in the shadow set', objectid)
continue
c.execute('SELECT AVG(y),AVG(x) FROM polygons WHERE objectid=?',
(objectid, ))
center_yx = c.fetchone()
# If polygon does not exist, use bbox.
if center_yx[0] is None:
roi = utilBoxes.bbox2roi(
backendDb.objectField(object_entry, 'bbox'))
center_yx = (roi[0] + roi[2]) / 2, (roi[1] + roi[3]) / 2
logging.debug('center_yx: %s', str(center_yx))
for shadow_object_entry, shadow_polygon in zip(
shadow_object_entries, shadow_object_polygons):
# Get the shadow roi, or polygon if it exists.
shadow_objectid = backendDb.objectField(
object_entry, 'objectid')
# Check that the center is within the shadow polygon or bbox.
if len(shadow_polygon) > 0:
is_inside = cv2.pointPolygonTest(np.array(shadow_polygon),
center_yx, False) >= 0
else:
shadow_roi = utilBoxes.bbox2roi(
backendDb.objectField(shadow_object_entry, 'bbox'))
is_inside = (center_yx[0] > shadow_roi[0]
and center_yx[0] < shadow_roi[2]
and center_yx[1] > shadow_roi[1]
and center_yx[1] < shadow_roi[3])
if is_inside:
backendDb.deleteObject(c, objectid)
# We do not need to check other shadow_object_entries.
continue
def filterObjectsByIntersection(c, args):
def getRoiIntesection(rioi1, roi2):
dy = min(roi1[2], roi2[2]) - max(roi1[0], roi2[0])
dx = min(roi1[3], roi2[3]) - max(roi1[1], roi2[1])
if dy <= 0 or dx <= 0: return 0
return dy * dx
if args.with_display:
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
c.execute('SELECT imagefile FROM images')
for imagefile, in progressbar(c.fetchall()):
if args.with_display:
image = imreader.imread(imagefile)
c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, ))
object_entries = c.fetchall()
logging.debug('%d objects found for %s' %
(len(object_entries), imagefile))
good_objects = np.ones(shape=len(object_entries), dtype=bool)
for iobject1, object_entry1 in enumerate(object_entries):
#roi1 = _expandCarBbox_(object_entry1, args)
roi1 = backendDb.objectField(object_entry1, 'roi')
if roi1 is None:
logging.error(
'No roi for objectid %d, intersection on polygons '
'not implemented.', iobject1)
continue
area1 = (roi1[2] - roi1[0]) * (roi1[3] - roi1[1])
if area1 == 0:
logging.warning('An object in %s has area 0. Will delete.' %
imagefile)
good_objects[iobject1] = False
break
for iobject2, object_entry2 in enumerate(object_entries):
if iobject2 == iobject1:
continue
roi2 = backendDb.objectField(object_entry2, 'roi')
if roi2 is None:
continue
intersection = getRoiIntesection(roi1, roi2) / float(area1)
if intersection > args.intersection_thresh:
good_objects[iobject1] = False
break
if args.with_display:
image = imreader.imread(imagefile)
util.drawScoredRoi(image,
roi1,
score=(1 if good_objects[iobject1] else 0))
for iobject2, object_entry2 in enumerate(object_entries):
if iobject1 == iobject2:
continue
roi2 = backendDb.objectField(object_entry2, 'roi')
if roi2 is None:
continue
util.drawScoredRoi(image, roi2, score=0.5)
cv2.imshow('filterObjectsByIntersection', image[:, :, ::-1])
key = cv2.waitKey(-1)
if key == 27:
cv2.destroyWindow('filterObjectsByIntersection')
args.with_display = 0
for object_entry, is_object_good in zip(object_entries, good_objects):
if not is_object_good:
backendDb.deleteObject(
c, backendDb.objectField(object_entry, 'objectid'))
def filterObjectsAtBorder(c, args):
def isPolygonAtBorder(polygon_entries, width, height, border_thresh_perc):
xs = [backendDb.polygonField(p, 'x') for p in polygon_entries]
ys = [backendDb.polygonField(p, 'y') for p in polygon_entries]
border_thresh = (height + width) / 2 * border_thresh_perc
dist_to_border = min(xs, [width - x for x in xs], ys,
[height - y for y in ys])
num_too_close = sum([x < border_thresh for x in dist_to_border])
return num_too_close >= 2
def isRoiAtBorder(roi, width, height, border_thresh_perc):
border_thresh = (height + width) / 2 * border_thresh_perc
logging.debug('border_thresh: %f' % border_thresh)
return min(roi[0], roi[1], height + 1 - roi[2],
width + 1 - roi[3]) < border_thresh
if args.with_display:
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For the reference.
c.execute('SELECT COUNT(1) FROM objects')
num_before = c.fetchone()[0]
c.execute('SELECT imagefile FROM images')
for imagefile, in progressbar(c.fetchall()):
if args.with_display:
image = imreader.imread(imagefile)
c.execute('SELECT width,height FROM images WHERE imagefile=?',
(imagefile, ))
(imwidth, imheight) = c.fetchone()
c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, ))
object_entries = c.fetchall()
logging.debug('%d objects found for %s' %
(len(object_entries), imagefile))
for object_entry in object_entries:
for_deletion = False
# Get all necessary entries.
objectid = backendDb.objectField(object_entry, 'objectid')
roi = backendDb.objectField(object_entry, 'roi')
c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, ))
polygon_entries = c.fetchall()
# Find if the polygon or the roi is at the border,
# polygon has preference over roi.
if len(polygon_entries) > 0:
if isPolygonAtBorder(polygon_entries, imwidth, imheight,
args.border_thresh):
logging.debug('border polygon %s' % str(polygon))
for_deletion = True
elif roi is not None:
if isRoiAtBorder(roi, imwidth, imheight, args.border_thresh):
logging.debug('border roi %s' % str(roi))
for_deletion = True
else:
logging.error(
'Neither polygon, nor bbox is available for objectid %d' %
objectid)
# Draw polygon or roi.
if args.with_display:
if len(polygon_entries) > 0:
polygon = [(backendDb.polygonField(p, 'x'),
backendDb.polygonField(p, 'y'))
for p in polygon_entries]
util.drawScoredPolygon(image,
polygon,
score=(0 if for_deletion else 1))
elif roi is not None:
util.drawScoredRoi(image,
roi,
score=(0 if for_deletion else 1))
# Delete if necessary
if for_deletion:
backendDb.deleteObject(c, objectid)
if args.with_display:
cv2.imshow('filterObjectsAtBorder', image[:, :, ::-1])
key = cv2.waitKey(-1)
if key == 27:
args.with_display = False
cv2.destroyWindow('filterObjectsAtBorder')
# For the reference.
c.execute('SELECT COUNT(1) FROM objects')
num_after = c.fetchone()[0]
logging.info('Deleted %d out of %d objects.' %
(num_before - num_after, num_before))
def examineImages(c, args):
cv2.namedWindow("examineImages")
c.execute('SELECT * FROM images WHERE (%s)' % args.where_image)
image_entries = c.fetchall()
logging.info('%d images found.' % len(image_entries))
if len(image_entries) == 0:
logging.error('There are no images. Exiting.')
return
if args.shuffle:
np.random.shuffle(image_entries)
if args.snapshot_dir and not op.exists(args.snapshot_dir):
os.makedirs(args.snapshot_dir)
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For parsing keys.
key_reader = KeyReader(args.key_dict)
# For overlaying masks.
labelmap = literal_eval(
args.mask_mapping_dict) if args.mask_mapping_dict else None
logging.info('Parsed mask_mapping_dict to %s' % pformat(labelmap))
index_image = 0
while True: # Until a user hits the key for the "exit" action.
image_entry = image_entries[index_image]
imagefile = backendDb.imageField(image_entry, 'imagefile')
maskfile = backendDb.imageField(image_entry, 'maskfile')
imname = backendDb.imageField(image_entry, 'name')
imscore = backendDb.imageField(image_entry, 'score')
logging.info('Imagefile "%s"' % imagefile)
logging.debug('Image name="%s", score=%s' % (imname, imscore))
image = imreader.imread(imagefile)
# Overlay the mask.
if maskfile is not None:
mask = imreader.maskread(maskfile)
if args.mask_aside:
image = util.drawMaskAside(image, mask, labelmap=labelmap)
elif args.mask_alpha is not None:
image = util.drawMaskOnImage(image,
mask,
alpha=args.mask_alpha,
labelmap=labelmap)
else:
logging.info('No mask for this image.')
# Put the objects on top of the image.
if args.with_objects:
c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, ))
object_entries = c.fetchall()
logging.info('Found %d objects for image %s' %
(len(object_entries), imagefile))
for object_entry in object_entries:
objectid = backendDb.objectField(object_entry, 'objectid')
roi = backendDb.objectField(object_entry, 'roi')
score = backendDb.objectField(object_entry, 'score')
name = backendDb.objectField(object_entry, 'name')
logging.info('objectid: %d, roi: %s, score: %s, name: %s' %
(objectid, roi, score, name))
c.execute('SELECT * FROM polygons WHERE objectid=?',
(objectid, ))
polygon_entries = c.fetchall()
if len(polygon_entries) > 0:
logging.info('showing object with a polygon.')
polygon = [(int(backendDb.polygonField(p, 'x')),
int(backendDb.polygonField(p, 'y')))
for p in polygon_entries]
util.drawScoredPolygon(image,
polygon,
label=name,
score=score)
elif roi is not None:
logging.info('showing object with a bounding box.')
util.drawScoredRoi(image, roi, label=name, score=score)
else:
logging.warning(
'Neither polygon, nor bbox is available for objectid %d'
% objectid)
# Display an image, wait for the key from user, and parse that key.
scale = float(args.winsize) / max(list(image.shape[0:2]))
scaled_image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale)
# Overlay imagefile.
if args.with_imagefile:
util.drawTextOnImage(
scaled_image,
op.basename(backendMedia.normalizeSeparators(imagefile)))
# Overlay score.
# TODO: add y offset, if necessary
if args.with_score:
util.drawTextOnImage(scaled_image, '%.3f' % imscore)
# Display
cv2.imshow('examineImages', scaled_image[:, :, ::-1])
action = key_reader.parse(cv2.waitKey(-1))
if action is None:
# User pressed something which does not have an action.
continue
elif action == 'snapshot':
if args.snapshot_dir:
snaphot_path = op.join(args.snapshot_dir,
'%08d.png' % index_image)
logging.info('Making a snapshot at path: %s' % snaphot_path)
imageio.imwrite(snaphot_path, image)
else:
logging.warning(
'The user pressed a snapshot key, but snapshot_dir is not '
'specified. Will not write a snapshot.')
elif action == 'delete':
backendDb.deleteImage(c, imagefile)
del image_entries[index_image]
if len(image_entries) == 0:
logging.warning('Deleted the last image.')
break
index_image += 1
elif action == 'exit':
break
elif action == 'previous':
index_image -= 1
elif action == 'next':
index_image += 1
else:
# User pressed something else which has an assigned action, assume it is a new name.
logging.info('Setting name "%s" to imagefile "%s"' %
(action, imagefile))
c.execute('UPDATE images SET name=? WHERE imagefile=?' (action,
imagefile))
index_image = index_image % len(image_entries)
cv2.destroyWindow("examineImages")
def _evaluateDetectionForClassSklearn(c, c_gt, class_name, args, sklearn):
''' Helper function for evaluateDetection. '''
# Detected objects sorted by descending score (confidence).
if class_name in ['any', 'ignore']:
c.execute('SELECT * FROM objects ORDER BY score DESC')
else:
c.execute('SELECT * FROM objects WHERE name=? ORDER BY score DESC',
(class_name, ))
entries_det = c.fetchall()
logging.info('Num of positive "%s": %d', class_name, len(entries_det))
# Create arrays 'y_score' with predicted scores, binary 'y_true' for GT,
# and a binary 'y_ignored' for detected objects that are neither TP nor FP.
y_score = np.zeros(len(entries_det), dtype=float)
y_true = np.zeros(len(entries_det), dtype=bool)
y_ignored = np.zeros(len(entries_det), dtype=bool)
# 'already_detected' used to penalize multiple detections of same GT box
already_detected = set()
# Go through each detection.
for idet, entry_det in enumerate(entries_det):
bbox_det = np.array(backendDb.objectField(entry_det, 'bbox'),
dtype=float)
imagefile = backendDb.objectField(entry_det, 'imagefile')
name = backendDb.objectField(entry_det, 'name')
score = backendDb.objectField(entry_det, 'score')
logging.debug('Detected object: %s', entry_det)
y_score[idet] = score
# Get all GT boxes from the same imagefile and of the same class.
if class_name == 'ignore':
c_gt.execute('SELECT * FROM objects WHERE imagefile=?',
(imagefile, ))
else:
c_gt.execute('SELECT * FROM objects WHERE imagefile=? AND name=?',
(imagefile, name))
entries_gt = c_gt.fetchall()
objectids_gt = [
backendDb.objectField(entry, 'objectid') for entry in entries_gt
]
bboxes_gt = np.array(
[backendDb.objectField(entry, 'bbox') for entry in entries_gt],
dtype=float)
logging.debug('- GT objects with the same imagefile and name: %s',
entries_gt)
# Separately manage the case of no GT boxes in this image.
if bboxes_gt.size == 0:
y_score[idet] = False
continue
# Intersection between bbox_det and all bboxes_gt.
ixmin = np.maximum(bboxes_gt[:, 0], bbox_det[0])
iymin = np.maximum(bboxes_gt[:, 1], bbox_det[1])
ixmax = np.minimum(bboxes_gt[:, 0] + bboxes_gt[:, 2],
bbox_det[0] + bbox_det[2])
iymax = np.minimum(bboxes_gt[:, 1] + bboxes_gt[:, 3],
bbox_det[1] + bbox_det[3])
iw = np.maximum(ixmax - ixmin, 0.)
ih = np.maximum(iymax - iymin, 0.)
intersection = iw * ih
logging.debug('- Intersections with GT objects: %s' % intersection)
# Union between bbox_det and all bboxes_gt.
union = (bbox_det[2] * bbox_det[3] +
bboxes_gt[:, 2] * bboxes_gt[:, 3] - intersection)
logging.debug('- Unions with GT objects: %s' % union)
# Compute the best IoU between the bbox_det and all bboxes_gt.
IoUs = intersection / union
max_IoU = np.max(IoUs)
objectid_gt = objectids_gt[np.argmax(IoUs)]
logging.debug('- IoUs with GT objects: %s' % IoUs)
logging.debug('max_IoU=%.3f for idet %d with objectid_gt %d.', max_IoU,
idet, objectid_gt)
# Get all GT objects that are of interest.
if class_name == 'ignore':
c_gt.execute(
'SELECT * FROM objects WHERE imagefile=? AND (%s)' %
args.where_object_gt, (imagefile, ))
else:
c_gt.execute(
'SELECT * FROM objects WHERE imagefile=? AND name=? AND (%s)' %
args.where_object_gt, (imagefile, name))
entries_gt = c_gt.fetchall()
objectids_gt_of_interest = [
backendDb.objectField(entry, 'objectid') for entry in entries_gt
]
# Compute TP and FP. An object is a TP if:
# 1) it has a large enough IoU with a GT object and
# 2) this GT object was not detected before.
if max_IoU > args.IoU_thresh and not objectid_gt in already_detected:
if objectid_gt not in objectids_gt_of_interest:
y_ignored[idet] = True
already_detected.add(objectid_gt)
y_true[idet] = True
else:
y_true[idet] = False
logging.debug('already_detected %d objects.', len(already_detected))
# It doesn't matter if y_ignore'd GT fall into TP or FP. Kick them out.
y_score = y_score[np.bitwise_not(y_ignored)]
y_true = y_true[np.bitwise_not(y_ignored)]
# Find the number of GT of interest.
if class_name in ['any', 'ignore']:
c_gt.execute('SELECT COUNT(1) FROM objects WHERE (%s)' %
args.where_object_gt)
else:
c_gt.execute(
'SELECT COUNT(1) FROM objects WHERE (%s) AND name=?' %
args.where_object_gt, (class_name, ))
num_gt = c_gt.fetchone()[0]
logging.info('Number of ground truth "%s": %d', class_name, num_gt)
# Add FN to y_score and y_true.
num_fn = num_gt - np.count_nonzero(y_true)
logging.info('Number of false negative "%s": %d', class_name, num_fn)
y_score = np.pad(y_score, [0, num_fn], constant_values=0.)
y_true = np.pad(y_true, [0, num_fn], constant_values=True)
# We need the point for threshold=0 to have y=0. Not sure why it's not yet.
# TODO: figure out how to do it properly.
y_score = np.pad(y_score, [0, 1000000], constant_values=0.0001)
y_true = np.pad(y_true, [0, 1000000], constant_values=False)
if 'precision_recall_curve' in args.extra_metrics:
precision, recall, _ = sklearn.metrics.precision_recall_curve(
y_true=y_true, probas_pred=y_score)
if args.out_dir:
plt.clf()
plt.plot(recall, precision)
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.xlabel('Recall')
plt.ylabel('Precision')
_beautifyPlot(plt.gca())
_writeCurveValues(args.out_dir, recall, precision,
'precision-recall', class_name,
'recall precision')
if 'roc_curve' in args.extra_metrics:
fpr, tpr, _ = sklearn.metrics.roc_curve(y_true=y_true,
probas_pred=y_score)
sklearn.metrics.auc(x=fpr, y=tpr)
if args.out_dir:
plt.clf()
plt.plot(fpr, tpr)
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.xlabel('FPR')
plt.ylabel('TPR')
_beautifyPlot(plt.gca())
_writeCurveValues(args.out_dir, fpr, tpr, 'roc', class_name,
'fpr tpr')
# Compute all metrics for this class.
aps = sklearn.metrics.average_precision_score(y_true=y_true,
y_score=y_score)
if class_name in ['any', 'ignore']:
print('Average precision: %.4f' % aps)
else:
print('Average precision for class "%s": %.4f' % (class_name, aps))
return aps
def examineObjects(c, args):
cv2.namedWindow("examineObjects")
c.execute('SELECT COUNT(*) FROM objects WHERE (%s) ' % args.where_object)
logging.info('Found %d objects in db.' % c.fetchone()[0])
c.execute('SELECT DISTINCT imagefile FROM objects WHERE (%s) ' %
args.where_object)
image_entries = c.fetchall()
logging.info('%d images found.' % len(image_entries))
if len(image_entries) == 0:
logging.error('There are no images. Exiting.')
return
if args.shuffle:
np.random.shuffle(image_entries)
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For parsing keys.
key_reader = KeyReader(args.key_dict)
index_image = 0
index_object = 0
# Iterating over images, because for each image we want to show all objects.
while True: # Until a user hits the key for the "exit" action.
(imagefile, ) = image_entries[index_image]
logging.info('Imagefile "%s"' % imagefile)
image = imreader.imread(imagefile)
scale = float(args.winsize) / max(image.shape[0:2])
image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale)
c.execute(
'SELECT * FROM objects WHERE imagefile=? AND (%s)' %
args.where_object, (imagefile, ))
object_entries = c.fetchall()
logging.info('Found %d objects for image %s' %
(len(object_entries), imagefile))
# Put the objects on top of the image.
if len(object_entries) > 0:
assert index_object < len(object_entries)
object_entry = object_entries[index_object]
objectid = backendDb.objectField(object_entry, 'objectid')
roi = backendDb.objectField(object_entry, 'roi')
score = backendDb.objectField(object_entry, 'score')
name = backendDb.objectField(object_entry, 'name')
scaledroi = [int(scale * r)
for r in roi] # For displaying the scaled image.
logging.info('objectid: %d, roi: %s, score: %s, name: %s' %
(objectid, roi, score, name))
c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, ))
polygon_entries = c.fetchall()
if len(polygon_entries) > 0:
logging.info('showing object with a polygon.')
polygon = [(backendDb.polygonField(p, 'x'),
backendDb.polygonField(p, 'y'))
for p in polygon_entries]
logging.debug('nonscaled polygon: %s' % pformat(polygon))
polygon = [(int(scale * x), int(scale * y))
for x, y in polygon]
logging.debug('scaled polygon: %s' % pformat(polygon))
util.drawScoredPolygon(image, polygon, label=None, score=score)
elif roi is not None:
logging.info('showing object with a bounding box.')
util.drawScoredRoi(image, scaledroi, label=None, score=score)
else:
raise Exception(
'Neither polygon, nor bbox is available for objectid %d' %
objectid)
c.execute('SELECT key,value FROM properties WHERE objectid=?',
(objectid, ))
properties = c.fetchall()
if name is not None:
properties.append(('name', name))
if score is not None:
properties.append(('score', score))
for iproperty, (key, value) in enumerate(properties):
cv2.putText(image, '%s: %s' % (key, value),
(scaledroi[3] + 10,
scaledroi[0] - 10 + util.SCALE * (iproperty + 1)),
util.FONT, util.FONT_SIZE, (0, 0, 0),
util.THICKNESS)
cv2.putText(image, '%s: %s' % (key, value),
(scaledroi[3] + 10,
scaledroi[0] - 10 + util.SCALE * (iproperty + 1)),
util.FONT, util.FONT_SIZE, (255, 255, 255),
util.THICKNESS - 1)
logging.info('objectid: %d. %s = %s.' % (objectid, key, value))
# Display an image, wait for the key from user, and parse that key.
cv2.imshow('examineObjects', image[:, :, ::-1])
action = key_reader.parse(cv2.waitKey(-1))
if action is None:
# User pressed something which does not have an action.
continue
elif action == 'exit':
break
elif action == 'previous':
index_object -= 1
if index_object < 0:
index_image -= 1
index_object = 0
elif action == 'next':
index_object += 1
if index_object >= len(object_entries):
index_image += 1
index_object = 0
elif action == 'delete' and len(object_entries) > 0:
backendDb.deleteObject(c, objectid)
del object_entries[index_object]
if index_object >= len(object_entries):
index_image += 1
index_object = 0
elif action == 'unname' and len(object_entries) > 0:
logging.info('Remove the name from objectid "%s"' % objectid)
c.execute('UPDATE objects SET name=NULL WHERE objectid=?',
(objectid, ))
index_image = index_image % len(image_entries)
cv2.destroyWindow("examineObjects")
def _evaluateDetectionForClassPascal(c, c_gt, name, args):
def _voc_ap(rec, prec):
""" Compute VOC AP given precision and recall. """
# First append sentinel values at the end.
mrec = np.concatenate(([0.], rec, [1.]))
mpre = np.concatenate(([0.], prec, [0.]))
# Compute the precision envelope.
for i in range(mpre.size - 1, 0, -1):
mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
# To calculate area under PR curve, look for points
# where X axis (recall) changes value.
i = np.where(mrec[1:] != mrec[:-1])[0]
# Sum (\Delta recall) * prec.
ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
return ap
c.execute('SELECT * FROM objects WHERE name=? ORDER BY score DESC',
(name, ))
entries_det = c.fetchall()
logging.info('Total %d detected objects for class "%s"', len(entries_det),
name)
# Go down dets and mark TPs and FPs.
tp = np.zeros(len(entries_det), dtype=float)
fp = np.zeros(len(entries_det), dtype=float)
# Detected of no interest.
ignored = np.zeros(len(entries_det), dtype=bool)
# 'already_detected' used to penalize multiple detections of same GT box.
already_detected = set()
# Go through each detection.
for idet, entry_det in enumerate(entries_det):
bbox_det = np.array(backendDb.objectField(entry_det, 'bbox'),
dtype=float)
imagefile = backendDb.objectField(entry_det, 'imagefile')
name = backendDb.objectField(entry_det, 'name')
# Get all GT boxes from the same imagefile [of the same class].
c_gt.execute('SELECT * FROM objects WHERE imagefile=? AND name=?',
(imagefile, name))
entries_gt = c_gt.fetchall()
objectids_gt = [
backendDb.objectField(entry, 'objectid') for entry in entries_gt
]
bboxes_gt = np.array(
[backendDb.objectField(entry, 'bbox') for entry in entries_gt],
dtype=float)
# Separately manage no GT boxes.
if bboxes_gt.size == 0:
fp[idet] = 1.
continue
# Intersection between bbox_det and all bboxes_gt.
ixmin = np.maximum(bboxes_gt[:, 0], bbox_det[0])
iymin = np.maximum(bboxes_gt[:, 1], bbox_det[1])
ixmax = np.minimum(bboxes_gt[:, 0] + bboxes_gt[:, 2],
bbox_det[0] + bbox_det[2])
iymax = np.minimum(bboxes_gt[:, 1] + bboxes_gt[:, 3],
bbox_det[1] + bbox_det[3])
iw = np.maximum(ixmax - ixmin, 0.)
ih = np.maximum(iymax - iymin, 0.)
intersection = iw * ih
# Union between bbox_det and all bboxes_gt.
union = (bbox_det[2] * bbox_det[3] +
bboxes_gt[:, 2] * bboxes_gt[:, 3] - intersection)
# IoU and get the best IoU.
IoUs = intersection / union
max_IoU = np.max(IoUs)
objectid_gt = objectids_gt[np.argmax(IoUs)]
logging.debug('max_IoU=%.3f for idet %d with objectid_gt %d.', max_IoU,
idet, objectid_gt)
# Find which objects count towards TP and FN (should be detected).
c_gt.execute(
'SELECT * FROM objects WHERE imagefile=? AND name=? AND %s' %
args.where_object_gt, (imagefile, name))
entries_gt = c_gt.fetchall()
objectids_gt_of_interest = [
backendDb.objectField(entry, 'objectid') for entry in entries_gt
]
# If 1) large enough IoU and
# 2) this GT box was not detected before.
if max_IoU > args.IoU_thresh and not objectid_gt in already_detected:
if objectid_gt in objectids_gt_of_interest:
tp[idet] = 1.
else:
ignored[idet] = True
already_detected.add(objectid_gt)
else:
fp[idet] = 1.
# Find the number of GT of interest.
c_gt.execute(
'SELECT COUNT(1) FROM objects WHERE %s AND name=?' %
args.where_object_gt, (name, ))
n_gt = c_gt.fetchone()[0]
logging.info('Total objects of interest: %d', n_gt)
# Remove dets, neither TP or FP.
tp = tp[np.bitwise_not(ignored)]
fp = fp[np.bitwise_not(ignored)]
logging.info('ignored: %d, tp: %d, fp: %d, gt: %d',
np.count_nonzero(ignored), np.count_nonzero(tp),
np.count_nonzero(fp), n_gt)
assert np.count_nonzero(tp) + np.count_nonzero(fp) + np.count_nonzero(
ignored) == len(entries_det)
fp = np.cumsum(fp)
tp = np.cumsum(tp)
rec = tp / float(n_gt)
# Avoid divide by zero in case the first detection matches a difficult
# ground truth.
prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
aps = _voc_ap(rec, prec)
print('Average precision for class "%s": %.4f' % (name, aps))
return aps
def examineMatches(c, args):
cv2.namedWindow("examineMatches")
c.execute('SELECT DISTINCT(match) FROM matches')
match_entries = c.fetchall()
if args.shuffle:
np.random.shuffle(match_entries)
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For parsing keys.
key_reader = KeyReader(args.key_dict)
index_match = 0
# Iterating over images, because for each image we want to show all objects.
while True: # Until a user hits the key for the "exit" action.
(match, ) = match_entries[index_match]
c.execute(
'SELECT * FROM objects WHERE objectid IN '
'(SELECT objectid FROM matches WHERE match=?)', (match, ))
object_entries = c.fetchall()
logging.info('Found %d objects for match %d' %
(len(object_entries), match))
images = []
for object_entry in object_entries:
imagefile = backendDb.objectField(object_entry, 'imagefile')
objectid = backendDb.objectField(object_entry, 'objectid')
roi = backendDb.objectField(object_entry, 'roi')
score = backendDb.objectField(object_entry, 'score')
image = imreader.imread(imagefile)
util.drawScoredRoi(image, roi, score=score)
scale = float(args.winsize) / max(image.shape[0:2])
image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale)
images.append(image)
# Assume all images have the same size for now.
# Assume there are not so many matches.
image = np.hstack(images)
# Display an image, wait for the key from user, and parse that key.
cv2.imshow('examineMatches', image[:, :, ::-1])
action = key_reader.parse(cv2.waitKey(-1))
if action is None:
# User pressed something which does not have an action.
continue
elif action == 'exit':
break
elif action == 'previous':
index_match -= 1
elif action == 'next':
index_match += 1
else:
# User pressed something else which has an assigned action,
# assume it is a new name.
logging.info('Setting name "%s" to imagefile "%s"' %
(action, imagefile))
c.execute('UPDATE images SET name=? WHERE imagefile=?' (action,
imagefile))
index_match = index_match % len(match_entries)
cv2.destroyWindow("examineMatches")
def exportCoco(c, args):
# Date created.
if args.date_created is None:
date_created = None
else:
try:
date_created = datetime.strptime(args.date_created, '%Y/%m/%d')
except ValueError as e:
raise ValueError('date_created is expected in the form YYYY/MM/DD')
date_created_str = (datetime.strftime(date_created, '%Y/%m/%d')
if date_created is not None else None)
# Info.
info = {
"year": args.year,
"version": args.version,
"description": args.description,
"contributor": args.contributor,
"url": args.url,
"date_created": date_created_str
}
# Licenses.
if args.license_url is None and args.license_name is None:
licenses = []
else:
license_url = args.license_url if args.license_url is not None else ""
license_name = args.license_name if args.license_name is not None else ""
licenses = [{"url": license_url, "id": 1, "name": license_name}]
# Images.
new_image_dir = op.join(args.coco_dir, 'images', args.subset)
if not op.exists(op.dirname(new_image_dir)):
os.makedirs(op.dirname(new_image_dir))
if not args.symlink_image_folder and not op.exists(new_image_dir):
os.makedirs(new_image_dir)
logging.info('Writing images.')
images = [] # big list of images.
imageids = {} # necessary to retrieve imageid by imagefile.
c.execute('SELECT * FROM images')
for imageid, image_entry in progressbar(enumerate(c.fetchall())):
imagefile = backendDb.imageField(image_entry, 'imagefile')
width = backendDb.imageField(image_entry, 'width')
height = backendDb.imageField(image_entry, 'height')
timestamp = backendDb.imageField(image_entry, 'timestamp')
file_name = op.basename(imagefile)
license = 1 if len(
licenses) == 1 else None # Only one license is supported.
date_captured = datetime.strftime(
backendDb.parseTimeString(timestamp),
'%Y-%m-%d %H:%M:%S') if timestamp is not None else None
# For reference in objects.
imageids[imagefile] = imageid
# Add to the big list of images.
images.append({
"id": imageid,
"width": width,
"height": height,
"file_name": file_name,
"license": license,
"date_captured": date_captured,
})
# Maybe copy or make a symlink for images.
old_image_path = op.join(args.rootdir, imagefile)
if not op.exists(old_image_path):
raise FileNotFoundError(
"Image not found at %s. (Using rootdir %s.)" %
(old_image_path, args.rootdir))
new_image_path = op.join(new_image_dir, file_name)
if args.copy_images:
shutil.copyfile(old_image_path, new_image_path)
elif args.symlink_images:
os.symlink(op.abspath(old_image_path),
new_image_path,
target_is_directory=False)
elif args.symlink_image_folder and not op.exists(new_image_dir):
old_image_dir = op.dirname(old_image_path)
os.symlink(op.abspath(old_image_dir),
new_image_dir,
target_is_directory=True)
# Categories.
categories_coco = [] # big list of categories.
categoryids = {} # necessary to retrieve categoryid by category name.
if args.categories is not None:
categories = args.categories
else:
c.execute('SELECT DISTINCT(name) FROM objects WHERE name IS NOT NULL')
categories = [c for c, in c.fetchall()]
for icategory, category in enumerate(categories):
categoryids[category] = icategory
categories_coco.append({
"supercategory": None,
"id": icategory,
"name": category
})
logging.info('Found %d categories: %s' % (len(categoryids), categories))
# Objects.
logging.info('Writing objects.')
annotations = [] # big list of images.
categories_str = ', '.join(['"%s"' % c for c in categories])
logging.debug('Will execute: "SELECT * FROM objects WHERE name IN (%s)"',
categories_str)
c.execute('SELECT * FROM objects WHERE name IN (%s)' % categories_str)
for object_entry in progressbar(c.fetchall()):
objectid = backendDb.objectField(object_entry, 'objectid')
name = backendDb.objectField(object_entry, 'name')
imagefile = backendDb.objectField(object_entry, 'imagefile')
util.polygons2bboxes(c, objectid) # Get a box if there wasn't one.
bbox = backendDb.objectField(object_entry, 'bbox')
imageid = imageids[imagefile]
# Get polygons.
c.execute('SELECT DISTINCT(name) FROM polygons WHERE objectid=?',
(objectid, ))
polygon_names = c.fetchall()
polygons_coco = []
for polygon_name, in polygon_names:
if polygon_name is None:
c.execute(
'SELECT x,y FROM polygons WHERE objectid=? AND name IS NULL',
(objectid, ))
else:
c.execute(
'SELECT x,y FROM polygons WHERE objectid=? AND name=?',
(objectid, polygon_name))
polygon = c.fetchall()
polygon_coco = np.array(polygon).flatten().astype(
int).tolist() # to [x1, y1, x2, y2, ... xn, yn].
polygons_coco.append(polygon_coco)
# Get area.
mask = util.polygons2mask(c, objectid)
area = np.count_nonzero(mask)
annotations.append({
"id": objectid,
"image_id": imageid,
"category_id": categoryids[name],
"segmentation": polygons_coco,
"area": area,
"bbox": [int(round(x)) for x in bbox],
"iscrowd": 0,
})
data = {
"info": info,
"images": images,
"annotations": annotations,
"categories": categories_coco,
"licenses": licenses,
}
json_path = op.join(args.coco_dir, 'annotations',
'instances_%s.json' % args.subset)
if not op.exists(op.dirname(json_path)):
os.makedirs(op.dirname(json_path))
with open(json_path, 'w') as f:
json.dump(data, f, indent=4)
def labelMatches(c, args):
cv2.namedWindow("labelMatches")
cv2.setMouseCallback('labelMatches', _monitorPressRelease)
global mousePressed, mouseReleased, xpress, ypress
c.execute('SELECT imagefile FROM images WHERE %s' % args.where_image)
image_entries = c.fetchall()
logging.debug('Found %d images' % len(image_entries))
if len(image_entries) < 2:
logging.error('Found only %d images. Quit.' % len(image_entries))
return
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For parsing keys.
key_reader = KeyReader(args.key_dict)
index_image = 1
action = None
while action != 'exit' and index_image < len(image_entries):
(imagefile1, ) = image_entries[index_image - 1]
(imagefile2, ) = image_entries[index_image]
img1 = imreader.imread(imagefile1)
img2 = imreader.imread(imagefile2)
# offset of the 2nd image, when they are stacked
yoffset = img1.shape[0]
# Make sure images have the same width, so that we can stack them.
# That will happen when the video changes, so we'll skip that pair.
if img1.shape[1] != img2.shape[1]:
logging.warning('Skipping image pair "%s" and "%s" '
'since they are of different width.' %
(imagefile1, imagefile2))
index_image += 1
# get objects from both images
c.execute('SELECT * FROM objects WHERE imagefile=? ', (imagefile1, ))
objects1 = c.fetchall()
logging.info('%d objects found for %s' % (len(objects1), imagefile1))
c.execute('SELECT * FROM objects WHERE imagefile=? ', (imagefile2, ))
objects2 = c.fetchall()
logging.info('%d objects found for %s' % (len(objects2), imagefile2))
# draw cars in both images
for object_ in objects1:
util.drawScoredRoi(img1, backendDb.objectField(object_, 'roi'))
for object_ in objects2:
util.drawScoredRoi(img2, backendDb.objectField(object_, 'roi'))
i1 = i2 = None # Matches selected with a mouse.
selectedMatch = None
needRedraw = True
action = None
# Stay in side the loop until a key is pressed.
while action is None:
img_stack = np.vstack((img1, img2))
if needRedraw:
# find existing matches, and make a map
matchesOf1 = {}
matchesOf2 = {}
for j1 in range(len(objects1)):
object1 = objects1[j1]
for j2 in range(len(objects2)):
object2 = objects2[j2]
c.execute(
'SELECT match FROM matches WHERE objectid = ? INTERSECT '
'SELECT match FROM matches WHERE objectid = ?',
(backendDb.objectField(object1, 'objectid'),
backendDb.objectField(object2, 'objectid')))
matches = c.fetchall()
if len(matches) > 0:
assert len(matches) == 1 # No duplicate matches.
roi1 = backendDb.objectField(object1, 'roi')
roi2 = backendDb.objectField(object2, 'roi')
_drawMatch(img_stack, roi1, roi2, yoffset)
matchesOf1[j1] = matches[0][0]
matchesOf2[j2] = matches[0][0]
# draw image
scale = float(args.winsize) / max(img_stack.shape[0:2])
img_show = cv2.resize(img_stack,
dsize=(0, 0),
fx=scale,
fy=scale)
cv2.imshow('labelMatches', img_show[:, :, ::-1])
logging.info('Will draw %d matches found between the pair' %
len(matchesOf1))
needRedraw = False
# process mouse callback effect (button has been pressed)
if mousePressed:
i2 = None # reset after the last unsuccessful match
logging.debug('Pressed x=%d, y=%d' % (xpress, ypress))
xpress /= scale
ypress /= scale
i1 = _findPressedObject(xpress, ypress, objects1)
if i1 is not None:
logging.debug('Found pressed object: %d' % i1)
mousePressed = False
# process mouse callback effect (button has been released)
if mouseReleased:
logging.debug('released x=%d, y=%d' % (xpress, ypress))
xpress /= scale
ypress /= scale
i2 = _findPressedObject(xpress, ypress - yoffset, objects2)
if i2 is not None:
logging.debug('Found released object: %d' % i2)
mouseReleased = False
# If we could find pressed and released objects, add match
if i1 is not None and i2 is not None:
# If one of the objects is already matched in this image pair, discard.
if i1 in matchesOf1 or i2 in matchesOf2:
logging.warning(
'One or two connected objects is already matched')
i1 = i2 = None
else:
# Add the match to the list.
objectid1 = backendDb.objectField(objects1[i1], 'objectid')
objectid2 = backendDb.objectField(objects2[i2], 'objectid')
logging.debug('i1 = %d, i2 = %d' % (i1, i2))
# Check if this object already in the matches.
c.execute('SELECT match FROM matches WHERE objectid=?',
(objectid1, ))
matches1 = c.fetchall()
c.execute('SELECT match FROM matches WHERE objectid=?',
(objectid2, ))
matches2 = c.fetchall()
if len(matches1) > 1 or len(matches2) > 1:
logging.error(
'One of the objectids %d, %d is in several matches.'
% (objectid1, objectid2))
continue
elif len(matches1) == 1 and len(matches2) == 1:
logging.info(
'Will merge matches of objectids %d and %d' %
(objectid1, objectid2))
c.execute('UPDATE matches SET match=? WHERE match=?',
(matches1[0][0], matches2[0][0]))
elif len(matches1) == 1 and len(matches2) == 0:
logging.info('Add objectid %d to match %d' %
(objectid2, matches1[0][0]))
c.execute(
'INSERT INTO matches(match, objectid) VALUES (?,?)',
(matches1[0][0], objectid2))
elif len(matches1) == 0 and len(matches2) == 1:
logging.info('Add objectid %d to match %d' %
(objectid1, matches2[0][0]))
c.execute(
'INSERT INTO matches(match, objectid) VALUES (?,?)',
(matches2[0][0], objectid1))
elif len(matches1) == 0 and len(matches2) == 0:
logging.info(
'Add a new match between objectids %d and %d.' %
(objectid1, objectid2))
# Find a free match index
c.execute('SELECT MAX(match) FROM matches')
maxmatch = c.fetchone()[0]
match = int(
maxmatch) + 1 if maxmatch is not None else 1
c.execute(
'INSERT INTO matches(match, objectid) VALUES (?,?)',
(match, objectid1))
c.execute(
'INSERT INTO matches(match, objectid) VALUES (?,?)',
(match, objectid2))
else:
assert False
# Reset when a new match is made.
needRedraw = True
i1 = i2 = None
# Stay inside the loop inside one image pair until some button is pressed
action = key_reader.parse(cv2.waitKey(50))
# Process pressed key (all except exit)
if action == 'previous':
logging.info('Previous image pair')
if index_image == 1:
logging.warning('already the first image pair')
else:
index_image -= 1
elif action == 'next':
logging.info('Next image pair')
index_image += 1 # exit at last image pair from outer loop
elif action == 'delete_match':
# if any car was selected, and it is matched
if i1 is not None and i1 in matchesOf1:
match = matchesOf1[i1]
objectid1 = backendDb.objectField(objects1[i1], 'objectid')
logging.info('deleting match %d' % match)
c.execute(
'DELETE FROM matches WHERE match = ? AND objectid = ?',
(match, objectid1))
else:
logging.debug('delete is pressed, but no match is selected')
cv2.destroyWindow("labelMatches")
def labelAzimuth(c, args):
from lib.backendScenes import Window
from scenes.lib.cvScrollZoomWindow import Window
from scenes.lib.homography import getFrameFlattening, getHfromPose
from scenes.lib.cache import PoseCache, MapsCache
from scenes.lib.warp import transformPoint
os.environ['SCENES_PATH'] = args.scenes_dir
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
key_reader = dbGui.KeyReader(args.key_dict)
c.execute('SELECT * FROM objects WHERE (%s)' % args.where_object)
object_entries = c.fetchall()
logging.info('Found %d objects in db.' % len(object_entries))
if len(object_entries) == 0:
return
if args.shuffle:
np.random.shuffle(object_entries)
# Cached poses and azimuth maps.
topdown_azimuths = MapsCache('topdown_azimuth')
poses = PoseCache()
button = 0
index_object = 0
another_object = True
char_list = []
while button != 27:
go_next_object = False
update_yaw_in_db = False
if another_object:
another_object = False
logging.info(' ')
logging.info('Object %d out of %d' %
(index_object, len(object_entries)))
object_entry = object_entries[index_object]
objectid = backendDb.objectField(object_entry, 'objectid')
bbox = backendDb.objectField(object_entry, 'bbox')
roi = backendDb.objectField(object_entry, 'roi')
imagefile = backendDb.objectField(object_entry, 'imagefile')
# Update yaw inside the loop in case it was just assigned.
c.execute(
'SELECT value FROM properties WHERE objectid=? AND key="yaw"',
(objectid, ))
yaw = c.fetchone()
yaw = float(yaw[0]) if yaw is not None else None
y, x = roi[0] * 0.3 + roi[2] * 0.7, roi[1] * 0.5 + roi[3] * 0.5
try:
flattening = _getFlatteningFromImagefile(
poses, imagefile, y, x)
except:
# A hack that allowed me write all the images into one dir but
# keep info about imagefile.
c.execute('SELECT name FROM images WHERE imagefile=?',
(imagefile, ))
image_name = c.fetchone()[0]
flattening = _getFlatteningFromImagefile(
poses, image_name, y, x)
axis_x = np.linalg.norm(np.asarray(bbox[2:4]), ord=2)
axis_y = axis_x * flattening
display = imreader.imread(imagefile)
window = AzimuthWindow(display,
x,
y,
axis_x,
axis_y,
winsize=args.winsize)
if yaw is not None:
logging.info('Yaw is: %.0f' % yaw)
window.yaw = yaw
window.update_cached_zoomed_img()
window.redraw()
button = cv2.waitKey(50)
action = key_reader.parse(button) if button is not -1 else None
if action == 'delete':
c.execute('DELETE FROM properties WHERE objectid=? AND key="yaw"',
(objectid, ))
logging.info('Yaw is deleted.')
go_next_object = True
char_list = []
# Entry in keyboard.
if button >= ord('0') and button <= ord('9') or button == ord('.'):
char_list += chr(button)
logging.debug('Added %s character to number and got %s' %
(chr(button), ''.join(char_list)))
# Enter accepts GUI or keyboard entry.
elif button == 13:
if char_list: # After keyboard entry.
number_str = ''.join(char_list)
char_list = []
try:
logging.info('Accepting entry from the keyboard.')
yaw = float(number_str)
update_yaw_in_db = True
go_next_object = True
except ValueError:
logging.warning('Could not convert entered %s to number.' %
number_str)
continue
else: # Just navigation.
logging.info('A navigation Enter.')
go_next_object = True
# Entry in GUI.
elif window.selected == True:
logging.info('Accepting entry from GUI.')
yaw = window.yaw
update_yaw_in_db = True
go_next_object = True
# No entry:
else:
yaw = None
# Entry happened one way or the other.
# Update the yaw and go to the next object.
if update_yaw_in_db:
c.execute(
'SELECT COUNT(1) FROM properties WHERE objectid=? AND key="yaw"',
(objectid, ))
if c.fetchone()[0] == 0:
c.execute(
'INSERT INTO properties(value,key,objectid) VALUES (?,"yaw",?)',
(str(yaw), objectid))
else:
c.execute(
'UPDATE properties SET value=? WHERE objectid=? AND key="yaw"',
(str(yaw), objectid))
logging.info('Yaw is assigned to %.f' % yaw)
# Navigation.
if action == 'previous':
logging.debug('previous object')
if index_object > 0:
index_object -= 1
another_object = True
else:
logging.warning('Already at the first object.')
elif action == 'next' or go_next_object == True:
logging.debug('next object')
if index_object < len(object_entries) - 1:
index_object += 1
another_object = True
else:
logging.warning(
'Already at the last object. Press Esc to save and exit.')
