def test_objectFields(self): cursor = self.conn.cursor() cursor.execute('SELECT * FROM objects WHERE objectid=1') entry = cursor.fetchone() self.assertEqual(backendDb.objectField(entry, 'objectid'), 1, str(entry)) self.assertEqual(backendDb.objectField(entry, 'imagefile'), 'images/000000.jpg', str(entry)) self.assertEqual(backendDb.objectField(entry, 'x1'), 225, str(entry)) self.assertEqual(backendDb.objectField(entry, 'y1'), 134, str(entry)) self.assertEqual(backendDb.objectField(entry, 'width'), 356, str(entry)) self.assertEqual(backendDb.objectField(entry, 'height'), 377, str(entry)) self.assertEqual(backendDb.objectField(entry, 'name'), 'car', str(entry)) self.assertAlmostEqual(backendDb.objectField(entry, 'score'), 0.606193, msg=str(entry)) with self.assertRaises(KeyError): backendDb.objectField(entry, 'dummy') # Multiple fields. self.assertEqual(backendDb.objectFields(entry, ['objectid', 'x1']), [1, 225], str(entry)) with self.assertRaises(KeyError): self.assertEqual(backendDb.objectFields(entry, ['x1', 'dummy']))
def _exportLabel(c, out_path_noext, imagefile): ''' Writes objects to json file. Please use https://github.com/mcordts/cityscapesScripts to generate masks. ''' c.execute('SELECT width,height FROM images WHERE imagefile=?', (imagefile, )) imgWidth, imgHeight = c.fetchone() data = {'imgWidth': imgWidth, 'imgHeight': imgHeight, 'objects': []} c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, )) for object_ in c.fetchall(): objectid = backendDb.objectField(object_, 'objectid') name = backendDb.objectField(object_, 'name') # Polygon = [[x1, y1], [x2, y2], ...]. # Check if the object has a polygon. c.execute('SELECT x,y FROM polygons WHERE objectid=?', (objectid, )) polygon = c.fetchall() if len(polygon) == 0: # If there is no polygon, make one from bounding box. [y1, x1, y2, x2] = backendDb.objectField(object_, 'roi') polygon = [(x1, y1), (x2, y1), (x2, y2), (x1, y2)] color = (255, ) cv2.rectangle(mask, (x1, y1), (x2, y2), color, -1) data['objects'].append({'label': name, 'polygon': polygon}) json_path = '%s_polygons.json' % out_path_noext with open(json_path, 'w') as f: json.dump(data, f, indent=4)
def buildObjectSample(object_entry, c, imreader): ''' Load images and get necessary information from object_entry to make a frame. Args: object_entry: (tuple) tuple with all fields from "objects" table. c: (cursor) sqlite3 cursor. imreader: (lib.backend.backendMedia.MediaReader) Returns a dict with keys: image: (np.uint8) The array corresponding to an image. mask: (np.uint8) The array corresponding to a mask if exists, or None. objectid: (int) The primary key in the "objects" table. name: (string) The "name" field in the "objects" table. score: (float) The "score" field in the "objects" table. imagefile: (string) The image id. All key-value pairs from the "properties" table for this objectid. ''' objectid = backendDb.objectField(object_entry, 'objectid') imagefile = backendDb.objectField(object_entry, 'imagefile') name = backendDb.objectField(object_entry, 'name') score = backendDb.objectField(object_entry, 'score') c.execute('SELECT maskfile FROM images WHERE imagefile=?', (imagefile, )) maskfile = c.fetchone()[0] logging.debug('Reading object %d from %s imagefile' % (objectid, imagefile)) try: image = imreader.imread(imagefile) mask = imreader.maskread(maskfile) if maskfile is not None else None except ValueError: traceback.print_exc() logging.error('Reading image or mask failed. Returning None.') return None roi = backendDb.objectField(object_entry, 'roi') logging.debug('Roi: %s' % roi) roi = [int(x) for x in roi] image = image[roi[0]:roi[2], roi[1]:roi[3]] mask = mask[roi[0]:roi[2], roi[1]:roi[3]] if mask is not None else None sample = { 'image': image, 'mask': mask, 'name': name, 'score': score, 'imagefile': imagefile, 'objectid': objectid } # Add properties. c.execute('SELECT key,value FROM properties WHERE objectid=?', (objectid, )) property_entries = c.fetchall() sample.update(dict(property_entries)) return sample
def test_record_objects(self): out_db_file = op.join(self.work_dir, 'out.db') self.writer = DatasetWriter(out_db_file) # Require imagefile. with self.assertRaises(KeyError): objectid = self.writer.addObject({}) objectid1 = self.writer.addObject({ 'imagefile': 'myimagefile1', 'x1': 10, 'y1': 20, 'width': 30, 'height': 40, 'name': 'car', 'score': 0.5 }) objectid2 = self.writer.addObject({ 'imagefile': 'myimagefile2', }) self.writer.close() self.assertTrue(op.exists(out_db_file)) self.conn = sqlite3.connect('file:%s?mode=ro' % out_db_file, uri=True) c = self.conn.cursor() c.execute('SELECT * FROM objects') object_entries = c.fetchall() self.assertEqual(len(object_entries), 2) self.assertEqual(objectid1, 1) self.assertEqual(objectField(object_entries[0], 'objectid'), 1) self.assertEqual(objectField(object_entries[0], 'imagefile'), 'myimagefile1') self.assertEqual(objectField(object_entries[0], 'x1'), 10) self.assertEqual(objectField(object_entries[0], 'y1'), 20) self.assertEqual(objectField(object_entries[0], 'width'), 30) self.assertEqual(objectField(object_entries[0], 'height'), 40) self.assertEqual(objectField(object_entries[0], 'name'), 'car') self.assertEqual(objectField(object_entries[0], 'score'), 0.5) self.assertEqual(objectid2, 2) self.assertEqual(objectField(object_entries[1], 'objectid'), 2) self.assertEqual(objectField(object_entries[1], 'imagefile'), 'myimagefile2')
def getIntersectingObjects(objects1, objects2, IoU_threshold, same_id_ok=True): ''' Given two lists of objects find pairs that intersect by IoU_threshold. Objects are assumed to be in the same image. Args: objects1, objects2: A list of object entries. Each entry is the whole row in the 'objects' table. IoU_threshold: A float in range [0, 1]. Returns: A list of tuples. Each tuple has objectid of an entry in 'objects1' and objectid of an entry in 'objects2'. ''' # Compute pairwise distances between rectangles. # TODO: possibly can optimize in future to avoid O(N^2) complexity. pairwise_IoU = np.zeros(shape=(len(objects1), len(objects2)), dtype=float) for i1, object1 in enumerate(objects1): for i2, object2 in enumerate(objects2): # Do not merge an object with itself. objectid1 = backendDb.objectField(object1, 'objectid') objectid2 = backendDb.objectField(object2, 'objectid') if objectid1 == objectid2 and not same_id_ok: pairwise_IoU[i1, i2] = np.nan else: roi1 = backendDb.objectField(object1, 'roi') roi2 = backendDb.objectField(object2, 'roi') pairwise_IoU[i1, i2] = utilBoxes.getIoU(roi1, roi2) logging.debug('Pairwise_IoU is:\n%s', pprint.pformat(pairwise_IoU)) # Greedy search for pairs. pairs_to_merge = [] for _ in range(min(len(objects1), len(objects2))): i1, i2 = np.unravel_index(np.argmax(pairwise_IoU), pairwise_IoU.shape) IoU = pairwise_IoU[i1, i2] logging.debug('Next object at indices [%d, %d]. IoU: %s', i1, i2, str(IoU)) # Stop if no more good pairs. if np.isnan(IoU) or IoU < IoU_threshold: break # Disable these objects for the next step. pairwise_IoU[i1, :] = 0. pairwise_IoU[:, i2] = 0. # Add a pair to the list. objectid1 = backendDb.objectField(objects1[i1], 'objectid') objectid2 = backendDb.objectField(objects2[i2], 'objectid') pairs_to_merge.append((objectid1, objectid2)) name1 = backendDb.objectField(objects1[i1], 'name') name2 = backendDb.objectField(objects2[i2], 'name') logging.debug('Will merge objects %d (%s) and %d (%s) with IoU %f.', objectid1, name1, objectid2, name2, IoU) return pairs_to_merge
def bboxes2polygons(cursor, objectid): ''' A rectangular polygon is added to objects that are missing polygons. ''' # If there are already polygon entries, do nothing. cursor.execute('SELECT COUNT(1) FROM polygons WHERE objectid=?', (objectid, )) if cursor.fetchone()[0] > 0: return cursor.execute('SELECT * FROM objects WHERE objectid=?', (objectid, )) object_entry = cursor.fetchone() if object_entry is None: raise ValueError('Objectid %d does not exist.' % objectid) y1, x1, y2, x2 = backendDb.objectField(object_entry, 'roi') for x, y in [(x1, y1), (x2, y1), (x2, y2), (x1, y2)]: cursor.execute( 'INSERT INTO polygons(objectid,x,y,name) VALUES (?,?,?,"bounding box")', (objectid, x, y)) logging.debug( 'Added polygon from bbox y1=%d,x1=%d,y2=%d,x2=%d to objectid %d', y1, x1, y2, x2, objectid)
def displayImagesPlt(c, args): c.execute('SELECT * FROM images WHERE (%s)' % args.where_image) image_entries = c.fetchall() logging.info('%d images found.', len(image_entries)) if len(image_entries) == 0: logging.error('There are no images. Exiting.') return if args.shuffle: np.random.shuffle(image_entries) if len(image_entries) < args.limit: image_entries = image_entries[:args.limit] def _getNumRows(total, ncols): nrows = int((total - 1) / ncols) + 1 logging.info('Grid: %dx%d from the total of %d', nrows, ncols, total) return nrows if args.limit < len(image_entries): image_entries = image_entries[:args.limit] nrows = _getNumRows(len(image_entries), args.ncols) imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For overlaying masks. labelmap = literal_eval( args.mask_mapping_dict) if args.mask_mapping_dict else None logging.info('Parsed mask_mapping_dict to %s', pformat(labelmap)) for i_image, image_entry in enumerate(image_entries): ax = plt.subplot(nrows, args.ncols, i_image + 1) imagefile = backendDb.imageField(image_entry, 'imagefile') maskfile = backendDb.imageField(image_entry, 'maskfile') imname = backendDb.imageField(image_entry, 'name') imscore = backendDb.imageField(image_entry, 'score') logging.info('Imagefile "%s"', imagefile) logging.debug('Image name="%s", score=%s', imname, imscore) image = imreader.imread(imagefile) # Overlay the mask. if maskfile is not None: mask = imreader.maskread(maskfile) if args.mask_aside: image = util.drawMaskAside(image, mask, labelmap=labelmap) elif args.mask_alpha is not None: image = util.drawMaskOnImage(image, mask, alpha=args.mask_alpha, labelmap=labelmap) else: logging.info('No mask for this image.') # Put the objects on top of the image. if args.with_objects: c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, )) object_entries = c.fetchall() logging.info('Found %d objects for image %s', len(object_entries), imagefile) for object_entry in object_entries: objectid = backendDb.objectField(object_entry, 'objectid') roi = backendDb.objectField(object_entry, 'roi') score = backendDb.objectField(object_entry, 'score') name = backendDb.objectField(object_entry, 'name') logging.info('objectid: %d, roi: %s, score: %s, name: %s', objectid, roi, score, name) c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, )) polygon_entries = c.fetchall() if len(polygon_entries) > 0: logging.info('showing object with a polygon.') polygon = [(int(backendDb.polygonField(p, 'x')), int(backendDb.polygonField(p, 'y'))) for p in polygon_entries] drawScoredPolygon(ax, polygon, label=name, score=score) elif roi is not None: logging.info('showing object with a bounding box.') drawScoredRoi(ax, roi, label=name, score=score) else: logging.warning( 'Neither polygon, nor bbox is available for objectid %d', objectid) # Overlay imagefile. title = "" if args.with_imagefile: title += '%s ' % op.basename( backendMedia.normalizeSeparators(imagefile)) # Overlay score. if args.with_score: title += '%.3f ' % imscore if imscore is not None else '' ax.set_title(title) # Display ax.imshow(image) ax.axis('off') plt.tight_layout() plt.show()
def _findPressedObject(x, y, cars): for i in range(len(cars)): roi = backendDb.objectField(cars[i], 'roi') if x >= roi[1] and x < roi[3] and y >= roi[0] and y < roi[2]: return i return None
def labelObjects(c, args): cv2.namedWindow("labelObjects") c.execute('SELECT COUNT(*) FROM objects WHERE (%s) ' % args.where_object) logging.info('Found %d objects in db.' % c.fetchone()[0]) c.execute('SELECT * FROM objects WHERE (%s)' % args.where_object) object_entries = c.fetchall() logging.info('Found %d objects in db.' % len(object_entries)) if len(object_entries) == 0: return if args.shuffle: np.random.shuffle(object_entries) imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For parsing keys. key_reader = KeyReader(args.key_dict) button = 0 index_object = 0 another_object = True while button != 27: go_next_object = False if another_object: another_object = False logging.info(' ') logging.info('Object %d out of %d' % (index_object, len(object_entries))) object_entry = object_entries[index_object] objectid = backendDb.objectField(object_entry, 'objectid') bbox = backendDb.objectField(object_entry, 'bbox') roi = backendDb.objectField(object_entry, 'roi') imagefile = backendDb.objectField(object_entry, 'imagefile') logging.info('imagefile: %s' % imagefile) image = imreader.imread(imagefile) # Display an image, wait for the key from user, and parse that key. scale = float(args.winsize) / max(image.shape[0:2]) logging.debug('Will resize image and annotations with scale: %f' % scale) image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale) logging.info('objectid: %d, roi: %s' % (objectid, roi)) c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, )) polygon_entries = c.fetchall() if len(polygon_entries) > 0: logging.info('showing object with a polygon.') polygon = [(backendDb.polygonField(p, 'x'), backendDb.polygonField(p, 'y')) for p in polygon_entries] logging.debug('nonscaled polygon: %s' % pformat(polygon)) polygon = [(int(scale * p[0]), int(scale * p[1])) for p in polygon] logging.debug('scaled polygon: %s' % pformat(polygon)) util.drawScoredPolygon(image, polygon, label=None, score=None) elif roi is not None: logging.info('showing object with a bounding box.') logging.debug('nonscaled roi: %s' % pformat(roi)) roi = [int(scale * r) for r in roi] # For displaying the scaled image. logging.debug('scaled roi: %s' % pformat(roi)) util.drawScoredRoi(image, roi, label=None, score=None) else: raise Exception( 'Neither polygon, nor bbox is available for objectid %d' % objectid) c.execute( 'SELECT key,value FROM properties WHERE objectid=? AND key=?', (objectid, args.property)) # TODO: Multiple properties are possible because there is no # contraint on uniqueness on table properties(objectid,key). # Change when the uniqueness constraint is added to the # database schema. On the other hand, it's a feature. properties = c.fetchall() if len(properties) > 1: logging.warning( 'Multiple values for object %s and property %s. ' 'If reassigned, both will be changed' % (objectid, args.property)) for iproperty, (key, value) in enumerate(properties): cv2.putText(image, '%s: %s' % (key, value), (10, util.SCALE * (iproperty + 1)), util.FONT, util.FONT_SIZE, (0, 0, 0), util.THICKNESS) cv2.putText(image, '%s: %s' % (key, value), (10, util.SCALE * (iproperty + 1)), util.FONT, util.FONT_SIZE, (255, 255, 255), util.THICKNESS - 1) logging.info('objectid: %d. %s = %s.' % (objectid, key, value)) cv2.imshow('labelObjects', image[:, :, ::-1]) action = key_reader.parse(cv2.waitKey(-1)) if action == 'exit': break elif action == 'delete_label' and any_object_in_focus: logging.info('Remove label from objectid "%s"' % objectid) c.execute('DELETE FROM properties WHERE objectid=? AND key=?', (objectid, args.property)) go_next_object = True elif action is not None and action not in ['previous', 'next']: # User pressed something else which has an assigned action, # assume it is a new value. logging.info('Setting label "%s" to objectid "%s"' % (action, objectid)) if len(properties) > 0: c.execute('DELETE FROM properties WHERE objectid=? AND key=?', (objectid, args.property)) c.execute( 'INSERT INTO properties(objectid,key,value) VALUES (?,?,?)', (objectid, args.property, str(action))) go_next_object = True # Navigation. if action == 'previous': logging.debug('previous object') another_object = True if index_object > 0: index_object -= 1 else: logging.warning('Already at the first object.') elif action == 'next' or go_next_object == True: logging.debug('next object') another_object = True if index_object < len(object_entries) - 1: index_object += 1 else: logging.warning( 'Already at the last object. Press Esc to save and exit.') cv2.destroyWindow("labelObjects")
def importKitti(c, args): if args.with_display: imreader = backendMedia.MediaReader(args.rootdir) image_paths = sorted(glob(op.join(args.images_dir, '*.png'))) logging.info('Found %d PNG images in %s' % (len(image_paths), args.images_dir)) for image_path in progressbar(image_paths): filename = op.splitext(op.basename(image_path))[0] logging.debug('Processing image: "%s"' % filename) # Add image to the database. imheight, imwidth = backendMedia.getPictureSize(image_path) imagefile = op.relpath(image_path, args.rootdir) c.execute('INSERT INTO images(imagefile,width,height) VALUES (?,?,?)', (imagefile, imwidth, imheight)) if args.with_display: img = imreader.imread(imagefile) # Detection annotations. if args.detection_dir: detection_path = op.join(args.detection_dir, '%s.txt' % filename) if not op.exists(detection_path): raise FileNotFoundError('Annotation file not found at "%s".' % detection_path) # Read annotation file. with open(detection_path) as f: lines = f.read().splitlines() logging.debug('Read %d lines from detection file "%s"' % (len(lines), detection_path)) for line in lines: objectid = _parseObject(c, line, imagefile) if args.with_display: c.execute('SELECT * FROM objects WHERE objectid=?', (objectid, )) object_entry = c.fetchone() name = backendDb.objectField(object_entry, 'name') roi = backendDb.objectField(object_entry, 'roi') score = backendDb.objectField(object_entry, 'score') util.drawScoredRoi(img, roi, name, score=score) # Segmentation annotations. if args.segmentation_dir: segmentation_path = op.join(args.segmentation_dir, '%s.png' % filename) if not op.exists(segmentation_path): raise FileNotFoundError('Annotation file not found at "%s".' % segmentation_path) # Add image to the database. maskfile = op.relpath(segmentation_path, args.rootdir) c.execute('UPDATE images SET maskfile=? WHERE imagefile=?', (maskfile, imagefile)) if args.with_display: mask = imreader.maskread(maskfile) img = util.drawMaskAside(img, mask, labelmap=None) # Maybe display. if args.with_display: cv2.imshow('importKitti', img[:, :, ::-1]) if cv2.waitKey(-1) == 27: args.with_display = False cv2.destroyWindow('importKitti')
def filterObjectsInsideCertainObjects(c, args): c.execute('SELECT imagefile FROM images') for imagefile, in progressbar(c.fetchall()): # Shadow objects. c.execute( 'SELECT * FROM objects WHERE imagefile=? AND (%s)' % args.where_shadowing_objects, (imagefile, )) shadow_object_entries = c.fetchall() logging.info('Found %d shadowing objects.', len(shadow_object_entries)) # Populate polygons of the shadow objects. shadow_object_polygons = [] for shadow_object_entry in shadow_object_entries: shadow_objectid = backendDb.objectField(shadow_object_entry, 'objectid') c.execute('SELECT y,x FROM polygons WHERE objectid=?', (shadow_objectid, )) shadow_polygon = c.fetchall() shadow_object_polygons.append(shadow_polygon) shadow_object_ids_set = set([ backendDb.objectField(entry, 'objectid') for entry in shadow_object_entries ]) # Get all the objects that can be considered. c.execute( 'SELECT * FROM objects WHERE imagefile=? AND (%s)' % args.where_objects, (imagefile, )) object_entries = c.fetchall() logging.info('Total %d objects satisfying the condition.', len(object_entries)) for object_entry in object_entries: objectid = backendDb.objectField(object_entry, 'objectid') if objectid in shadow_object_ids_set: logging.debug('Object %d is in the shadow set', objectid) continue c.execute('SELECT AVG(y),AVG(x) FROM polygons WHERE objectid=?', (objectid, )) center_yx = c.fetchone() # If polygon does not exist, use bbox. if center_yx[0] is None: roi = utilBoxes.bbox2roi( backendDb.objectField(object_entry, 'bbox')) center_yx = (roi[0] + roi[2]) / 2, (roi[1] + roi[3]) / 2 logging.debug('center_yx: %s', str(center_yx)) for shadow_object_entry, shadow_polygon in zip( shadow_object_entries, shadow_object_polygons): # Get the shadow roi, or polygon if it exists. shadow_objectid = backendDb.objectField( object_entry, 'objectid') # Check that the center is within the shadow polygon or bbox. if len(shadow_polygon) > 0: is_inside = cv2.pointPolygonTest(np.array(shadow_polygon), center_yx, False) >= 0 else: shadow_roi = utilBoxes.bbox2roi( backendDb.objectField(shadow_object_entry, 'bbox')) is_inside = (center_yx[0] > shadow_roi[0] and center_yx[0] < shadow_roi[2] and center_yx[1] > shadow_roi[1] and center_yx[1] < shadow_roi[3]) if is_inside: backendDb.deleteObject(c, objectid) # We do not need to check other shadow_object_entries. continue
def filterObjectsByIntersection(c, args): def getRoiIntesection(rioi1, roi2): dy = min(roi1[2], roi2[2]) - max(roi1[0], roi2[0]) dx = min(roi1[3], roi2[3]) - max(roi1[1], roi2[1]) if dy <= 0 or dx <= 0: return 0 return dy * dx if args.with_display: imreader = backendMedia.MediaReader(rootdir=args.rootdir) c.execute('SELECT imagefile FROM images') for imagefile, in progressbar(c.fetchall()): if args.with_display: image = imreader.imread(imagefile) c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, )) object_entries = c.fetchall() logging.debug('%d objects found for %s' % (len(object_entries), imagefile)) good_objects = np.ones(shape=len(object_entries), dtype=bool) for iobject1, object_entry1 in enumerate(object_entries): #roi1 = _expandCarBbox_(object_entry1, args) roi1 = backendDb.objectField(object_entry1, 'roi') if roi1 is None: logging.error( 'No roi for objectid %d, intersection on polygons ' 'not implemented.', iobject1) continue area1 = (roi1[2] - roi1[0]) * (roi1[3] - roi1[1]) if area1 == 0: logging.warning('An object in %s has area 0. Will delete.' % imagefile) good_objects[iobject1] = False break for iobject2, object_entry2 in enumerate(object_entries): if iobject2 == iobject1: continue roi2 = backendDb.objectField(object_entry2, 'roi') if roi2 is None: continue intersection = getRoiIntesection(roi1, roi2) / float(area1) if intersection > args.intersection_thresh: good_objects[iobject1] = False break if args.with_display: image = imreader.imread(imagefile) util.drawScoredRoi(image, roi1, score=(1 if good_objects[iobject1] else 0)) for iobject2, object_entry2 in enumerate(object_entries): if iobject1 == iobject2: continue roi2 = backendDb.objectField(object_entry2, 'roi') if roi2 is None: continue util.drawScoredRoi(image, roi2, score=0.5) cv2.imshow('filterObjectsByIntersection', image[:, :, ::-1]) key = cv2.waitKey(-1) if key == 27: cv2.destroyWindow('filterObjectsByIntersection') args.with_display = 0 for object_entry, is_object_good in zip(object_entries, good_objects): if not is_object_good: backendDb.deleteObject( c, backendDb.objectField(object_entry, 'objectid'))
def filterObjectsAtBorder(c, args): def isPolygonAtBorder(polygon_entries, width, height, border_thresh_perc): xs = [backendDb.polygonField(p, 'x') for p in polygon_entries] ys = [backendDb.polygonField(p, 'y') for p in polygon_entries] border_thresh = (height + width) / 2 * border_thresh_perc dist_to_border = min(xs, [width - x for x in xs], ys, [height - y for y in ys]) num_too_close = sum([x < border_thresh for x in dist_to_border]) return num_too_close >= 2 def isRoiAtBorder(roi, width, height, border_thresh_perc): border_thresh = (height + width) / 2 * border_thresh_perc logging.debug('border_thresh: %f' % border_thresh) return min(roi[0], roi[1], height + 1 - roi[2], width + 1 - roi[3]) < border_thresh if args.with_display: imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For the reference. c.execute('SELECT COUNT(1) FROM objects') num_before = c.fetchone()[0] c.execute('SELECT imagefile FROM images') for imagefile, in progressbar(c.fetchall()): if args.with_display: image = imreader.imread(imagefile) c.execute('SELECT width,height FROM images WHERE imagefile=?', (imagefile, )) (imwidth, imheight) = c.fetchone() c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, )) object_entries = c.fetchall() logging.debug('%d objects found for %s' % (len(object_entries), imagefile)) for object_entry in object_entries: for_deletion = False # Get all necessary entries. objectid = backendDb.objectField(object_entry, 'objectid') roi = backendDb.objectField(object_entry, 'roi') c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, )) polygon_entries = c.fetchall() # Find if the polygon or the roi is at the border, # polygon has preference over roi. if len(polygon_entries) > 0: if isPolygonAtBorder(polygon_entries, imwidth, imheight, args.border_thresh): logging.debug('border polygon %s' % str(polygon)) for_deletion = True elif roi is not None: if isRoiAtBorder(roi, imwidth, imheight, args.border_thresh): logging.debug('border roi %s' % str(roi)) for_deletion = True else: logging.error( 'Neither polygon, nor bbox is available for objectid %d' % objectid) # Draw polygon or roi. if args.with_display: if len(polygon_entries) > 0: polygon = [(backendDb.polygonField(p, 'x'), backendDb.polygonField(p, 'y')) for p in polygon_entries] util.drawScoredPolygon(image, polygon, score=(0 if for_deletion else 1)) elif roi is not None: util.drawScoredRoi(image, roi, score=(0 if for_deletion else 1)) # Delete if necessary if for_deletion: backendDb.deleteObject(c, objectid) if args.with_display: cv2.imshow('filterObjectsAtBorder', image[:, :, ::-1]) key = cv2.waitKey(-1) if key == 27: args.with_display = False cv2.destroyWindow('filterObjectsAtBorder') # For the reference. c.execute('SELECT COUNT(1) FROM objects') num_after = c.fetchone()[0] logging.info('Deleted %d out of %d objects.' % (num_before - num_after, num_before))
def examineImages(c, args): cv2.namedWindow("examineImages") c.execute('SELECT * FROM images WHERE (%s)' % args.where_image) image_entries = c.fetchall() logging.info('%d images found.' % len(image_entries)) if len(image_entries) == 0: logging.error('There are no images. Exiting.') return if args.shuffle: np.random.shuffle(image_entries) if args.snapshot_dir and not op.exists(args.snapshot_dir): os.makedirs(args.snapshot_dir) imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For parsing keys. key_reader = KeyReader(args.key_dict) # For overlaying masks. labelmap = literal_eval( args.mask_mapping_dict) if args.mask_mapping_dict else None logging.info('Parsed mask_mapping_dict to %s' % pformat(labelmap)) index_image = 0 while True: # Until a user hits the key for the "exit" action. image_entry = image_entries[index_image] imagefile = backendDb.imageField(image_entry, 'imagefile') maskfile = backendDb.imageField(image_entry, 'maskfile') imname = backendDb.imageField(image_entry, 'name') imscore = backendDb.imageField(image_entry, 'score') logging.info('Imagefile "%s"' % imagefile) logging.debug('Image name="%s", score=%s' % (imname, imscore)) image = imreader.imread(imagefile) # Overlay the mask. if maskfile is not None: mask = imreader.maskread(maskfile) if args.mask_aside: image = util.drawMaskAside(image, mask, labelmap=labelmap) elif args.mask_alpha is not None: image = util.drawMaskOnImage(image, mask, alpha=args.mask_alpha, labelmap=labelmap) else: logging.info('No mask for this image.') # Put the objects on top of the image. if args.with_objects: c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, )) object_entries = c.fetchall() logging.info('Found %d objects for image %s' % (len(object_entries), imagefile)) for object_entry in object_entries: objectid = backendDb.objectField(object_entry, 'objectid') roi = backendDb.objectField(object_entry, 'roi') score = backendDb.objectField(object_entry, 'score') name = backendDb.objectField(object_entry, 'name') logging.info('objectid: %d, roi: %s, score: %s, name: %s' % (objectid, roi, score, name)) c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, )) polygon_entries = c.fetchall() if len(polygon_entries) > 0: logging.info('showing object with a polygon.') polygon = [(int(backendDb.polygonField(p, 'x')), int(backendDb.polygonField(p, 'y'))) for p in polygon_entries] util.drawScoredPolygon(image, polygon, label=name, score=score) elif roi is not None: logging.info('showing object with a bounding box.') util.drawScoredRoi(image, roi, label=name, score=score) else: logging.warning( 'Neither polygon, nor bbox is available for objectid %d' % objectid) # Display an image, wait for the key from user, and parse that key. scale = float(args.winsize) / max(list(image.shape[0:2])) scaled_image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale) # Overlay imagefile. if args.with_imagefile: util.drawTextOnImage( scaled_image, op.basename(backendMedia.normalizeSeparators(imagefile))) # Overlay score. # TODO: add y offset, if necessary if args.with_score: util.drawTextOnImage(scaled_image, '%.3f' % imscore) # Display cv2.imshow('examineImages', scaled_image[:, :, ::-1]) action = key_reader.parse(cv2.waitKey(-1)) if action is None: # User pressed something which does not have an action. continue elif action == 'snapshot': if args.snapshot_dir: snaphot_path = op.join(args.snapshot_dir, '%08d.png' % index_image) logging.info('Making a snapshot at path: %s' % snaphot_path) imageio.imwrite(snaphot_path, image) else: logging.warning( 'The user pressed a snapshot key, but snapshot_dir is not ' 'specified. Will not write a snapshot.') elif action == 'delete': backendDb.deleteImage(c, imagefile) del image_entries[index_image] if len(image_entries) == 0: logging.warning('Deleted the last image.') break index_image += 1 elif action == 'exit': break elif action == 'previous': index_image -= 1 elif action == 'next': index_image += 1 else: # User pressed something else which has an assigned action, assume it is a new name. logging.info('Setting name "%s" to imagefile "%s"' % (action, imagefile)) c.execute('UPDATE images SET name=? WHERE imagefile=?' (action, imagefile)) index_image = index_image % len(image_entries) cv2.destroyWindow("examineImages")
def _evaluateDetectionForClassSklearn(c, c_gt, class_name, args, sklearn): ''' Helper function for evaluateDetection. ''' # Detected objects sorted by descending score (confidence). if class_name in ['any', 'ignore']: c.execute('SELECT * FROM objects ORDER BY score DESC') else: c.execute('SELECT * FROM objects WHERE name=? ORDER BY score DESC', (class_name, )) entries_det = c.fetchall() logging.info('Num of positive "%s": %d', class_name, len(entries_det)) # Create arrays 'y_score' with predicted scores, binary 'y_true' for GT, # and a binary 'y_ignored' for detected objects that are neither TP nor FP. y_score = np.zeros(len(entries_det), dtype=float) y_true = np.zeros(len(entries_det), dtype=bool) y_ignored = np.zeros(len(entries_det), dtype=bool) # 'already_detected' used to penalize multiple detections of same GT box already_detected = set() # Go through each detection. for idet, entry_det in enumerate(entries_det): bbox_det = np.array(backendDb.objectField(entry_det, 'bbox'), dtype=float) imagefile = backendDb.objectField(entry_det, 'imagefile') name = backendDb.objectField(entry_det, 'name') score = backendDb.objectField(entry_det, 'score') logging.debug('Detected object: %s', entry_det) y_score[idet] = score # Get all GT boxes from the same imagefile and of the same class. if class_name == 'ignore': c_gt.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, )) else: c_gt.execute('SELECT * FROM objects WHERE imagefile=? AND name=?', (imagefile, name)) entries_gt = c_gt.fetchall() objectids_gt = [ backendDb.objectField(entry, 'objectid') for entry in entries_gt ] bboxes_gt = np.array( [backendDb.objectField(entry, 'bbox') for entry in entries_gt], dtype=float) logging.debug('- GT objects with the same imagefile and name: %s', entries_gt) # Separately manage the case of no GT boxes in this image. if bboxes_gt.size == 0: y_score[idet] = False continue # Intersection between bbox_det and all bboxes_gt. ixmin = np.maximum(bboxes_gt[:, 0], bbox_det[0]) iymin = np.maximum(bboxes_gt[:, 1], bbox_det[1]) ixmax = np.minimum(bboxes_gt[:, 0] + bboxes_gt[:, 2], bbox_det[0] + bbox_det[2]) iymax = np.minimum(bboxes_gt[:, 1] + bboxes_gt[:, 3], bbox_det[1] + bbox_det[3]) iw = np.maximum(ixmax - ixmin, 0.) ih = np.maximum(iymax - iymin, 0.) intersection = iw * ih logging.debug('- Intersections with GT objects: %s' % intersection) # Union between bbox_det and all bboxes_gt. union = (bbox_det[2] * bbox_det[3] + bboxes_gt[:, 2] * bboxes_gt[:, 3] - intersection) logging.debug('- Unions with GT objects: %s' % union) # Compute the best IoU between the bbox_det and all bboxes_gt. IoUs = intersection / union max_IoU = np.max(IoUs) objectid_gt = objectids_gt[np.argmax(IoUs)] logging.debug('- IoUs with GT objects: %s' % IoUs) logging.debug('max_IoU=%.3f for idet %d with objectid_gt %d.', max_IoU, idet, objectid_gt) # Get all GT objects that are of interest. if class_name == 'ignore': c_gt.execute( 'SELECT * FROM objects WHERE imagefile=? AND (%s)' % args.where_object_gt, (imagefile, )) else: c_gt.execute( 'SELECT * FROM objects WHERE imagefile=? AND name=? AND (%s)' % args.where_object_gt, (imagefile, name)) entries_gt = c_gt.fetchall() objectids_gt_of_interest = [ backendDb.objectField(entry, 'objectid') for entry in entries_gt ] # Compute TP and FP. An object is a TP if: # 1) it has a large enough IoU with a GT object and # 2) this GT object was not detected before. if max_IoU > args.IoU_thresh and not objectid_gt in already_detected: if objectid_gt not in objectids_gt_of_interest: y_ignored[idet] = True already_detected.add(objectid_gt) y_true[idet] = True else: y_true[idet] = False logging.debug('already_detected %d objects.', len(already_detected)) # It doesn't matter if y_ignore'd GT fall into TP or FP. Kick them out. y_score = y_score[np.bitwise_not(y_ignored)] y_true = y_true[np.bitwise_not(y_ignored)] # Find the number of GT of interest. if class_name in ['any', 'ignore']: c_gt.execute('SELECT COUNT(1) FROM objects WHERE (%s)' % args.where_object_gt) else: c_gt.execute( 'SELECT COUNT(1) FROM objects WHERE (%s) AND name=?' % args.where_object_gt, (class_name, )) num_gt = c_gt.fetchone()[0] logging.info('Number of ground truth "%s": %d', class_name, num_gt) # Add FN to y_score and y_true. num_fn = num_gt - np.count_nonzero(y_true) logging.info('Number of false negative "%s": %d', class_name, num_fn) y_score = np.pad(y_score, [0, num_fn], constant_values=0.) y_true = np.pad(y_true, [0, num_fn], constant_values=True) # We need the point for threshold=0 to have y=0. Not sure why it's not yet. # TODO: figure out how to do it properly. y_score = np.pad(y_score, [0, 1000000], constant_values=0.0001) y_true = np.pad(y_true, [0, 1000000], constant_values=False) if 'precision_recall_curve' in args.extra_metrics: precision, recall, _ = sklearn.metrics.precision_recall_curve( y_true=y_true, probas_pred=y_score) if args.out_dir: plt.clf() plt.plot(recall, precision) plt.xlim([0, 1]) plt.ylim([0, 1]) plt.xlabel('Recall') plt.ylabel('Precision') _beautifyPlot(plt.gca()) _writeCurveValues(args.out_dir, recall, precision, 'precision-recall', class_name, 'recall precision') if 'roc_curve' in args.extra_metrics: fpr, tpr, _ = sklearn.metrics.roc_curve(y_true=y_true, probas_pred=y_score) sklearn.metrics.auc(x=fpr, y=tpr) if args.out_dir: plt.clf() plt.plot(fpr, tpr) plt.xlim([0, 1]) plt.ylim([0, 1]) plt.xlabel('FPR') plt.ylabel('TPR') _beautifyPlot(plt.gca()) _writeCurveValues(args.out_dir, fpr, tpr, 'roc', class_name, 'fpr tpr') # Compute all metrics for this class. aps = sklearn.metrics.average_precision_score(y_true=y_true, y_score=y_score) if class_name in ['any', 'ignore']: print('Average precision: %.4f' % aps) else: print('Average precision for class "%s": %.4f' % (class_name, aps)) return aps
def examineObjects(c, args): cv2.namedWindow("examineObjects") c.execute('SELECT COUNT(*) FROM objects WHERE (%s) ' % args.where_object) logging.info('Found %d objects in db.' % c.fetchone()[0]) c.execute('SELECT DISTINCT imagefile FROM objects WHERE (%s) ' % args.where_object) image_entries = c.fetchall() logging.info('%d images found.' % len(image_entries)) if len(image_entries) == 0: logging.error('There are no images. Exiting.') return if args.shuffle: np.random.shuffle(image_entries) imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For parsing keys. key_reader = KeyReader(args.key_dict) index_image = 0 index_object = 0 # Iterating over images, because for each image we want to show all objects. while True: # Until a user hits the key for the "exit" action. (imagefile, ) = image_entries[index_image] logging.info('Imagefile "%s"' % imagefile) image = imreader.imread(imagefile) scale = float(args.winsize) / max(image.shape[0:2]) image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale) c.execute( 'SELECT * FROM objects WHERE imagefile=? AND (%s)' % args.where_object, (imagefile, )) object_entries = c.fetchall() logging.info('Found %d objects for image %s' % (len(object_entries), imagefile)) # Put the objects on top of the image. if len(object_entries) > 0: assert index_object < len(object_entries) object_entry = object_entries[index_object] objectid = backendDb.objectField(object_entry, 'objectid') roi = backendDb.objectField(object_entry, 'roi') score = backendDb.objectField(object_entry, 'score') name = backendDb.objectField(object_entry, 'name') scaledroi = [int(scale * r) for r in roi] # For displaying the scaled image. logging.info('objectid: %d, roi: %s, score: %s, name: %s' % (objectid, roi, score, name)) c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, )) polygon_entries = c.fetchall() if len(polygon_entries) > 0: logging.info('showing object with a polygon.') polygon = [(backendDb.polygonField(p, 'x'), backendDb.polygonField(p, 'y')) for p in polygon_entries] logging.debug('nonscaled polygon: %s' % pformat(polygon)) polygon = [(int(scale * x), int(scale * y)) for x, y in polygon] logging.debug('scaled polygon: %s' % pformat(polygon)) util.drawScoredPolygon(image, polygon, label=None, score=score) elif roi is not None: logging.info('showing object with a bounding box.') util.drawScoredRoi(image, scaledroi, label=None, score=score) else: raise Exception( 'Neither polygon, nor bbox is available for objectid %d' % objectid) c.execute('SELECT key,value FROM properties WHERE objectid=?', (objectid, )) properties = c.fetchall() if name is not None: properties.append(('name', name)) if score is not None: properties.append(('score', score)) for iproperty, (key, value) in enumerate(properties): cv2.putText(image, '%s: %s' % (key, value), (scaledroi[3] + 10, scaledroi[0] - 10 + util.SCALE * (iproperty + 1)), util.FONT, util.FONT_SIZE, (0, 0, 0), util.THICKNESS) cv2.putText(image, '%s: %s' % (key, value), (scaledroi[3] + 10, scaledroi[0] - 10 + util.SCALE * (iproperty + 1)), util.FONT, util.FONT_SIZE, (255, 255, 255), util.THICKNESS - 1) logging.info('objectid: %d. %s = %s.' % (objectid, key, value)) # Display an image, wait for the key from user, and parse that key. cv2.imshow('examineObjects', image[:, :, ::-1]) action = key_reader.parse(cv2.waitKey(-1)) if action is None: # User pressed something which does not have an action. continue elif action == 'exit': break elif action == 'previous': index_object -= 1 if index_object < 0: index_image -= 1 index_object = 0 elif action == 'next': index_object += 1 if index_object >= len(object_entries): index_image += 1 index_object = 0 elif action == 'delete' and len(object_entries) > 0: backendDb.deleteObject(c, objectid) del object_entries[index_object] if index_object >= len(object_entries): index_image += 1 index_object = 0 elif action == 'unname' and len(object_entries) > 0: logging.info('Remove the name from objectid "%s"' % objectid) c.execute('UPDATE objects SET name=NULL WHERE objectid=?', (objectid, )) index_image = index_image % len(image_entries) cv2.destroyWindow("examineObjects")
def _evaluateDetectionForClassPascal(c, c_gt, name, args): def _voc_ap(rec, prec): """ Compute VOC AP given precision and recall. """ # First append sentinel values at the end. mrec = np.concatenate(([0.], rec, [1.])) mpre = np.concatenate(([0.], prec, [0.])) # Compute the precision envelope. for i in range(mpre.size - 1, 0, -1): mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i]) # To calculate area under PR curve, look for points # where X axis (recall) changes value. i = np.where(mrec[1:] != mrec[:-1])[0] # Sum (\Delta recall) * prec. ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) return ap c.execute('SELECT * FROM objects WHERE name=? ORDER BY score DESC', (name, )) entries_det = c.fetchall() logging.info('Total %d detected objects for class "%s"', len(entries_det), name) # Go down dets and mark TPs and FPs. tp = np.zeros(len(entries_det), dtype=float) fp = np.zeros(len(entries_det), dtype=float) # Detected of no interest. ignored = np.zeros(len(entries_det), dtype=bool) # 'already_detected' used to penalize multiple detections of same GT box. already_detected = set() # Go through each detection. for idet, entry_det in enumerate(entries_det): bbox_det = np.array(backendDb.objectField(entry_det, 'bbox'), dtype=float) imagefile = backendDb.objectField(entry_det, 'imagefile') name = backendDb.objectField(entry_det, 'name') # Get all GT boxes from the same imagefile [of the same class]. c_gt.execute('SELECT * FROM objects WHERE imagefile=? AND name=?', (imagefile, name)) entries_gt = c_gt.fetchall() objectids_gt = [ backendDb.objectField(entry, 'objectid') for entry in entries_gt ] bboxes_gt = np.array( [backendDb.objectField(entry, 'bbox') for entry in entries_gt], dtype=float) # Separately manage no GT boxes. if bboxes_gt.size == 0: fp[idet] = 1. continue # Intersection between bbox_det and all bboxes_gt. ixmin = np.maximum(bboxes_gt[:, 0], bbox_det[0]) iymin = np.maximum(bboxes_gt[:, 1], bbox_det[1]) ixmax = np.minimum(bboxes_gt[:, 0] + bboxes_gt[:, 2], bbox_det[0] + bbox_det[2]) iymax = np.minimum(bboxes_gt[:, 1] + bboxes_gt[:, 3], bbox_det[1] + bbox_det[3]) iw = np.maximum(ixmax - ixmin, 0.) ih = np.maximum(iymax - iymin, 0.) intersection = iw * ih # Union between bbox_det and all bboxes_gt. union = (bbox_det[2] * bbox_det[3] + bboxes_gt[:, 2] * bboxes_gt[:, 3] - intersection) # IoU and get the best IoU. IoUs = intersection / union max_IoU = np.max(IoUs) objectid_gt = objectids_gt[np.argmax(IoUs)] logging.debug('max_IoU=%.3f for idet %d with objectid_gt %d.', max_IoU, idet, objectid_gt) # Find which objects count towards TP and FN (should be detected). c_gt.execute( 'SELECT * FROM objects WHERE imagefile=? AND name=? AND %s' % args.where_object_gt, (imagefile, name)) entries_gt = c_gt.fetchall() objectids_gt_of_interest = [ backendDb.objectField(entry, 'objectid') for entry in entries_gt ] # If 1) large enough IoU and # 2) this GT box was not detected before. if max_IoU > args.IoU_thresh and not objectid_gt in already_detected: if objectid_gt in objectids_gt_of_interest: tp[idet] = 1. else: ignored[idet] = True already_detected.add(objectid_gt) else: fp[idet] = 1. # Find the number of GT of interest. c_gt.execute( 'SELECT COUNT(1) FROM objects WHERE %s AND name=?' % args.where_object_gt, (name, )) n_gt = c_gt.fetchone()[0] logging.info('Total objects of interest: %d', n_gt) # Remove dets, neither TP or FP. tp = tp[np.bitwise_not(ignored)] fp = fp[np.bitwise_not(ignored)] logging.info('ignored: %d, tp: %d, fp: %d, gt: %d', np.count_nonzero(ignored), np.count_nonzero(tp), np.count_nonzero(fp), n_gt) assert np.count_nonzero(tp) + np.count_nonzero(fp) + np.count_nonzero( ignored) == len(entries_det) fp = np.cumsum(fp) tp = np.cumsum(tp) rec = tp / float(n_gt) # Avoid divide by zero in case the first detection matches a difficult # ground truth. prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps) aps = _voc_ap(rec, prec) print('Average precision for class "%s": %.4f' % (name, aps)) return aps
def examineMatches(c, args): cv2.namedWindow("examineMatches") c.execute('SELECT DISTINCT(match) FROM matches') match_entries = c.fetchall() if args.shuffle: np.random.shuffle(match_entries) imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For parsing keys. key_reader = KeyReader(args.key_dict) index_match = 0 # Iterating over images, because for each image we want to show all objects. while True: # Until a user hits the key for the "exit" action. (match, ) = match_entries[index_match] c.execute( 'SELECT * FROM objects WHERE objectid IN ' '(SELECT objectid FROM matches WHERE match=?)', (match, )) object_entries = c.fetchall() logging.info('Found %d objects for match %d' % (len(object_entries), match)) images = [] for object_entry in object_entries: imagefile = backendDb.objectField(object_entry, 'imagefile') objectid = backendDb.objectField(object_entry, 'objectid') roi = backendDb.objectField(object_entry, 'roi') score = backendDb.objectField(object_entry, 'score') image = imreader.imread(imagefile) util.drawScoredRoi(image, roi, score=score) scale = float(args.winsize) / max(image.shape[0:2]) image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale) images.append(image) # Assume all images have the same size for now. # Assume there are not so many matches. image = np.hstack(images) # Display an image, wait for the key from user, and parse that key. cv2.imshow('examineMatches', image[:, :, ::-1]) action = key_reader.parse(cv2.waitKey(-1)) if action is None: # User pressed something which does not have an action. continue elif action == 'exit': break elif action == 'previous': index_match -= 1 elif action == 'next': index_match += 1 else: # User pressed something else which has an assigned action, # assume it is a new name. logging.info('Setting name "%s" to imagefile "%s"' % (action, imagefile)) c.execute('UPDATE images SET name=? WHERE imagefile=?' (action, imagefile)) index_match = index_match % len(match_entries) cv2.destroyWindow("examineMatches")
def exportCoco(c, args): # Date created. if args.date_created is None: date_created = None else: try: date_created = datetime.strptime(args.date_created, '%Y/%m/%d') except ValueError as e: raise ValueError('date_created is expected in the form YYYY/MM/DD') date_created_str = (datetime.strftime(date_created, '%Y/%m/%d') if date_created is not None else None) # Info. info = { "year": args.year, "version": args.version, "description": args.description, "contributor": args.contributor, "url": args.url, "date_created": date_created_str } # Licenses. if args.license_url is None and args.license_name is None: licenses = [] else: license_url = args.license_url if args.license_url is not None else "" license_name = args.license_name if args.license_name is not None else "" licenses = [{"url": license_url, "id": 1, "name": license_name}] # Images. new_image_dir = op.join(args.coco_dir, 'images', args.subset) if not op.exists(op.dirname(new_image_dir)): os.makedirs(op.dirname(new_image_dir)) if not args.symlink_image_folder and not op.exists(new_image_dir): os.makedirs(new_image_dir) logging.info('Writing images.') images = [] # big list of images. imageids = {} # necessary to retrieve imageid by imagefile. c.execute('SELECT * FROM images') for imageid, image_entry in progressbar(enumerate(c.fetchall())): imagefile = backendDb.imageField(image_entry, 'imagefile') width = backendDb.imageField(image_entry, 'width') height = backendDb.imageField(image_entry, 'height') timestamp = backendDb.imageField(image_entry, 'timestamp') file_name = op.basename(imagefile) license = 1 if len( licenses) == 1 else None # Only one license is supported. date_captured = datetime.strftime( backendDb.parseTimeString(timestamp), '%Y-%m-%d %H:%M:%S') if timestamp is not None else None # For reference in objects. imageids[imagefile] = imageid # Add to the big list of images. images.append({ "id": imageid, "width": width, "height": height, "file_name": file_name, "license": license, "date_captured": date_captured, }) # Maybe copy or make a symlink for images. old_image_path = op.join(args.rootdir, imagefile) if not op.exists(old_image_path): raise FileNotFoundError( "Image not found at %s. (Using rootdir %s.)" % (old_image_path, args.rootdir)) new_image_path = op.join(new_image_dir, file_name) if args.copy_images: shutil.copyfile(old_image_path, new_image_path) elif args.symlink_images: os.symlink(op.abspath(old_image_path), new_image_path, target_is_directory=False) elif args.symlink_image_folder and not op.exists(new_image_dir): old_image_dir = op.dirname(old_image_path) os.symlink(op.abspath(old_image_dir), new_image_dir, target_is_directory=True) # Categories. categories_coco = [] # big list of categories. categoryids = {} # necessary to retrieve categoryid by category name. if args.categories is not None: categories = args.categories else: c.execute('SELECT DISTINCT(name) FROM objects WHERE name IS NOT NULL') categories = [c for c, in c.fetchall()] for icategory, category in enumerate(categories): categoryids[category] = icategory categories_coco.append({ "supercategory": None, "id": icategory, "name": category }) logging.info('Found %d categories: %s' % (len(categoryids), categories)) # Objects. logging.info('Writing objects.') annotations = [] # big list of images. categories_str = ', '.join(['"%s"' % c for c in categories]) logging.debug('Will execute: "SELECT * FROM objects WHERE name IN (%s)"', categories_str) c.execute('SELECT * FROM objects WHERE name IN (%s)' % categories_str) for object_entry in progressbar(c.fetchall()): objectid = backendDb.objectField(object_entry, 'objectid') name = backendDb.objectField(object_entry, 'name') imagefile = backendDb.objectField(object_entry, 'imagefile') util.polygons2bboxes(c, objectid) # Get a box if there wasn't one. bbox = backendDb.objectField(object_entry, 'bbox') imageid = imageids[imagefile] # Get polygons. c.execute('SELECT DISTINCT(name) FROM polygons WHERE objectid=?', (objectid, )) polygon_names = c.fetchall() polygons_coco = [] for polygon_name, in polygon_names: if polygon_name is None: c.execute( 'SELECT x,y FROM polygons WHERE objectid=? AND name IS NULL', (objectid, )) else: c.execute( 'SELECT x,y FROM polygons WHERE objectid=? AND name=?', (objectid, polygon_name)) polygon = c.fetchall() polygon_coco = np.array(polygon).flatten().astype( int).tolist() # to [x1, y1, x2, y2, ... xn, yn]. polygons_coco.append(polygon_coco) # Get area. mask = util.polygons2mask(c, objectid) area = np.count_nonzero(mask) annotations.append({ "id": objectid, "image_id": imageid, "category_id": categoryids[name], "segmentation": polygons_coco, "area": area, "bbox": [int(round(x)) for x in bbox], "iscrowd": 0, }) data = { "info": info, "images": images, "annotations": annotations, "categories": categories_coco, "licenses": licenses, } json_path = op.join(args.coco_dir, 'annotations', 'instances_%s.json' % args.subset) if not op.exists(op.dirname(json_path)): os.makedirs(op.dirname(json_path)) with open(json_path, 'w') as f: json.dump(data, f, indent=4)
def labelMatches(c, args): cv2.namedWindow("labelMatches") cv2.setMouseCallback('labelMatches', _monitorPressRelease) global mousePressed, mouseReleased, xpress, ypress c.execute('SELECT imagefile FROM images WHERE %s' % args.where_image) image_entries = c.fetchall() logging.debug('Found %d images' % len(image_entries)) if len(image_entries) < 2: logging.error('Found only %d images. Quit.' % len(image_entries)) return imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For parsing keys. key_reader = KeyReader(args.key_dict) index_image = 1 action = None while action != 'exit' and index_image < len(image_entries): (imagefile1, ) = image_entries[index_image - 1] (imagefile2, ) = image_entries[index_image] img1 = imreader.imread(imagefile1) img2 = imreader.imread(imagefile2) # offset of the 2nd image, when they are stacked yoffset = img1.shape[0] # Make sure images have the same width, so that we can stack them. # That will happen when the video changes, so we'll skip that pair. if img1.shape[1] != img2.shape[1]: logging.warning('Skipping image pair "%s" and "%s" ' 'since they are of different width.' % (imagefile1, imagefile2)) index_image += 1 # get objects from both images c.execute('SELECT * FROM objects WHERE imagefile=? ', (imagefile1, )) objects1 = c.fetchall() logging.info('%d objects found for %s' % (len(objects1), imagefile1)) c.execute('SELECT * FROM objects WHERE imagefile=? ', (imagefile2, )) objects2 = c.fetchall() logging.info('%d objects found for %s' % (len(objects2), imagefile2)) # draw cars in both images for object_ in objects1: util.drawScoredRoi(img1, backendDb.objectField(object_, 'roi')) for object_ in objects2: util.drawScoredRoi(img2, backendDb.objectField(object_, 'roi')) i1 = i2 = None # Matches selected with a mouse. selectedMatch = None needRedraw = True action = None # Stay in side the loop until a key is pressed. while action is None: img_stack = np.vstack((img1, img2)) if needRedraw: # find existing matches, and make a map matchesOf1 = {} matchesOf2 = {} for j1 in range(len(objects1)): object1 = objects1[j1] for j2 in range(len(objects2)): object2 = objects2[j2] c.execute( 'SELECT match FROM matches WHERE objectid = ? INTERSECT ' 'SELECT match FROM matches WHERE objectid = ?', (backendDb.objectField(object1, 'objectid'), backendDb.objectField(object2, 'objectid'))) matches = c.fetchall() if len(matches) > 0: assert len(matches) == 1 # No duplicate matches. roi1 = backendDb.objectField(object1, 'roi') roi2 = backendDb.objectField(object2, 'roi') _drawMatch(img_stack, roi1, roi2, yoffset) matchesOf1[j1] = matches[0][0] matchesOf2[j2] = matches[0][0] # draw image scale = float(args.winsize) / max(img_stack.shape[0:2]) img_show = cv2.resize(img_stack, dsize=(0, 0), fx=scale, fy=scale) cv2.imshow('labelMatches', img_show[:, :, ::-1]) logging.info('Will draw %d matches found between the pair' % len(matchesOf1)) needRedraw = False # process mouse callback effect (button has been pressed) if mousePressed: i2 = None # reset after the last unsuccessful match logging.debug('Pressed x=%d, y=%d' % (xpress, ypress)) xpress /= scale ypress /= scale i1 = _findPressedObject(xpress, ypress, objects1) if i1 is not None: logging.debug('Found pressed object: %d' % i1) mousePressed = False # process mouse callback effect (button has been released) if mouseReleased: logging.debug('released x=%d, y=%d' % (xpress, ypress)) xpress /= scale ypress /= scale i2 = _findPressedObject(xpress, ypress - yoffset, objects2) if i2 is not None: logging.debug('Found released object: %d' % i2) mouseReleased = False # If we could find pressed and released objects, add match if i1 is not None and i2 is not None: # If one of the objects is already matched in this image pair, discard. if i1 in matchesOf1 or i2 in matchesOf2: logging.warning( 'One or two connected objects is already matched') i1 = i2 = None else: # Add the match to the list. objectid1 = backendDb.objectField(objects1[i1], 'objectid') objectid2 = backendDb.objectField(objects2[i2], 'objectid') logging.debug('i1 = %d, i2 = %d' % (i1, i2)) # Check if this object already in the matches. c.execute('SELECT match FROM matches WHERE objectid=?', (objectid1, )) matches1 = c.fetchall() c.execute('SELECT match FROM matches WHERE objectid=?', (objectid2, )) matches2 = c.fetchall() if len(matches1) > 1 or len(matches2) > 1: logging.error( 'One of the objectids %d, %d is in several matches.' % (objectid1, objectid2)) continue elif len(matches1) == 1 and len(matches2) == 1: logging.info( 'Will merge matches of objectids %d and %d' % (objectid1, objectid2)) c.execute('UPDATE matches SET match=? WHERE match=?', (matches1[0][0], matches2[0][0])) elif len(matches1) == 1 and len(matches2) == 0: logging.info('Add objectid %d to match %d' % (objectid2, matches1[0][0])) c.execute( 'INSERT INTO matches(match, objectid) VALUES (?,?)', (matches1[0][0], objectid2)) elif len(matches1) == 0 and len(matches2) == 1: logging.info('Add objectid %d to match %d' % (objectid1, matches2[0][0])) c.execute( 'INSERT INTO matches(match, objectid) VALUES (?,?)', (matches2[0][0], objectid1)) elif len(matches1) == 0 and len(matches2) == 0: logging.info( 'Add a new match between objectids %d and %d.' % (objectid1, objectid2)) # Find a free match index c.execute('SELECT MAX(match) FROM matches') maxmatch = c.fetchone()[0] match = int( maxmatch) + 1 if maxmatch is not None else 1 c.execute( 'INSERT INTO matches(match, objectid) VALUES (?,?)', (match, objectid1)) c.execute( 'INSERT INTO matches(match, objectid) VALUES (?,?)', (match, objectid2)) else: assert False # Reset when a new match is made. needRedraw = True i1 = i2 = None # Stay inside the loop inside one image pair until some button is pressed action = key_reader.parse(cv2.waitKey(50)) # Process pressed key (all except exit) if action == 'previous': logging.info('Previous image pair') if index_image == 1: logging.warning('already the first image pair') else: index_image -= 1 elif action == 'next': logging.info('Next image pair') index_image += 1 # exit at last image pair from outer loop elif action == 'delete_match': # if any car was selected, and it is matched if i1 is not None and i1 in matchesOf1: match = matchesOf1[i1] objectid1 = backendDb.objectField(objects1[i1], 'objectid') logging.info('deleting match %d' % match) c.execute( 'DELETE FROM matches WHERE match = ? AND objectid = ?', (match, objectid1)) else: logging.debug('delete is pressed, but no match is selected') cv2.destroyWindow("labelMatches")
def labelAzimuth(c, args): from lib.backendScenes import Window from scenes.lib.cvScrollZoomWindow import Window from scenes.lib.homography import getFrameFlattening, getHfromPose from scenes.lib.cache import PoseCache, MapsCache from scenes.lib.warp import transformPoint os.environ['SCENES_PATH'] = args.scenes_dir imreader = backendMedia.MediaReader(rootdir=args.rootdir) key_reader = dbGui.KeyReader(args.key_dict) c.execute('SELECT * FROM objects WHERE (%s)' % args.where_object) object_entries = c.fetchall() logging.info('Found %d objects in db.' % len(object_entries)) if len(object_entries) == 0: return if args.shuffle: np.random.shuffle(object_entries) # Cached poses and azimuth maps. topdown_azimuths = MapsCache('topdown_azimuth') poses = PoseCache() button = 0 index_object = 0 another_object = True char_list = [] while button != 27: go_next_object = False update_yaw_in_db = False if another_object: another_object = False logging.info(' ') logging.info('Object %d out of %d' % (index_object, len(object_entries))) object_entry = object_entries[index_object] objectid = backendDb.objectField(object_entry, 'objectid') bbox = backendDb.objectField(object_entry, 'bbox') roi = backendDb.objectField(object_entry, 'roi') imagefile = backendDb.objectField(object_entry, 'imagefile') # Update yaw inside the loop in case it was just assigned. c.execute( 'SELECT value FROM properties WHERE objectid=? AND key="yaw"', (objectid, )) yaw = c.fetchone() yaw = float(yaw[0]) if yaw is not None else None y, x = roi[0] * 0.3 + roi[2] * 0.7, roi[1] * 0.5 + roi[3] * 0.5 try: flattening = _getFlatteningFromImagefile( poses, imagefile, y, x) except: # A hack that allowed me write all the images into one dir but # keep info about imagefile. c.execute('SELECT name FROM images WHERE imagefile=?', (imagefile, )) image_name = c.fetchone()[0] flattening = _getFlatteningFromImagefile( poses, image_name, y, x) axis_x = np.linalg.norm(np.asarray(bbox[2:4]), ord=2) axis_y = axis_x * flattening display = imreader.imread(imagefile) window = AzimuthWindow(display, x, y, axis_x, axis_y, winsize=args.winsize) if yaw is not None: logging.info('Yaw is: %.0f' % yaw) window.yaw = yaw window.update_cached_zoomed_img() window.redraw() button = cv2.waitKey(50) action = key_reader.parse(button) if button is not -1 else None if action == 'delete': c.execute('DELETE FROM properties WHERE objectid=? AND key="yaw"', (objectid, )) logging.info('Yaw is deleted.') go_next_object = True char_list = [] # Entry in keyboard. if button >= ord('0') and button <= ord('9') or button == ord('.'): char_list += chr(button) logging.debug('Added %s character to number and got %s' % (chr(button), ''.join(char_list))) # Enter accepts GUI or keyboard entry. elif button == 13: if char_list: # After keyboard entry. number_str = ''.join(char_list) char_list = [] try: logging.info('Accepting entry from the keyboard.') yaw = float(number_str) update_yaw_in_db = True go_next_object = True except ValueError: logging.warning('Could not convert entered %s to number.' % number_str) continue else: # Just navigation. logging.info('A navigation Enter.') go_next_object = True # Entry in GUI. elif window.selected == True: logging.info('Accepting entry from GUI.') yaw = window.yaw update_yaw_in_db = True go_next_object = True # No entry: else: yaw = None # Entry happened one way or the other. # Update the yaw and go to the next object. if update_yaw_in_db: c.execute( 'SELECT COUNT(1) FROM properties WHERE objectid=? AND key="yaw"', (objectid, )) if c.fetchone()[0] == 0: c.execute( 'INSERT INTO properties(value,key,objectid) VALUES (?,"yaw",?)', (str(yaw), objectid)) else: c.execute( 'UPDATE properties SET value=? WHERE objectid=? AND key="yaw"', (str(yaw), objectid)) logging.info('Yaw is assigned to %.f' % yaw) # Navigation. if action == 'previous': logging.debug('previous object') if index_object > 0: index_object -= 1 another_object = True else: logging.warning('Already at the first object.') elif action == 'next' or go_next_object == True: logging.debug('next object') if index_object < len(object_entries) - 1: index_object += 1 another_object = True else: logging.warning( 'Already at the last object. Press Esc to save and exit.')