def filterBadImages(c, args): imreader = backendMedia.MediaReader(rootdir=args.rootdir) num_deleted_imagefiles = 0 c.execute('SELECT imagefile,maskfile FROM images') image_entries = c.fetchall() if (isinstance(imreader, backendMedia.PictureReader) and not args.force_single_thread): # Can make it parallel. max_workers = multiprocessing.cpu_count() - 1 logging.info('Running filtering with %d workers.', max_workers) with concurrent.futures.ThreadPoolExecutor(max_workers) as executor: futures = [] for imagefile, maskfile in progressbar(image_entries): logging.debug('Imagefile "%s"', imagefile) futures.append( executor.submit(isImageOk, imreader, imagefile, maskfile)) for future in concurrent.futures.as_completed(futures): if not future.result(): backendDb.deleteImage(c, imagefile) num_deleted_imagefiles += 1 else: logging.info('Running filtering in a single thread.') for imagefile, maskfile in progressbar(image_entries): logging.debug('Imagefile "%s"', imagefile) if not isImageOk(imreader, imagefile, maskfile): backendDb.deleteImage(c, imagefile) num_deleted_imagefiles += 1 logging.info("Deleted %d image(s).", num_deleted_imagefiles)
def __init__(self, db_file, rootdir='.', where_image='TRUE', where_object='TRUE', mode='r', copy_to_memory=True, used_keys=None, transform_group=None): ''' Args: db_file: (string) A path to an sqlite3 database file. rootdir: (string) A root path, is pre-appended to "imagefile" entries of "images" table in the sqlite3 database. where_image: (string) The WHERE part of the SQL query on the "images" table, as in: "SELECT * FROM images WHERE ${where_image};" Allows to query only needed images. where_object: (string) The WHERE part of the SQL query on the "objects" table, as in: "SELECT * FROM objects WHERE ${where_object};" Allows to query only needed objects for each image. mode: ("r" or "w") The readonly or write-read mode to open the database. The default is "r", use "w" only if you plan to call addRecord(). copy_to_memory: (bool) Copies database into memory. Only for mode="r". Should be used for python2. used_keys: (None or list of str) Originally __getitem__ returns a dict with keys: 'image', 'mask', 'objects', 'imagefile', 'name', 'score' (see the comments to this class above). Argument `used_keys` determines which of these keys are needed, and which can be disposed of. transform_group: (a callable or a dict string -> callable) Transform(s) to be applied on a sample. If it is a callable, it is applied to the sample. If it is a dict, each key is matched to a key in the sample, and callables are called on the respective elements. ''' self.mode = mode self.conn = utils.openConnection(db_file, mode, copy_to_memory) self.c = self.conn.cursor() utils.checkWhereImage(where_image) self.c.execute('SELECT * FROM images WHERE %s ORDER BY imagefile' % where_image) self.image_entries = self.c.fetchall() self.imreader = backendMedia.MediaReader(rootdir=rootdir) utils.checkWhereObject(where_object) self.where_object = where_object _checkUsedKeys(used_keys) self.used_keys = used_keys utils.checkTransformGroup(transform_group) self.transform_group = transform_group
def test_imreadPicture(self): reader = backendMedia.MediaReader(rootdir='.') img_gt = skimage.io.imread('testdata/cars/images/000000.jpg') img = reader.imread('testdata/cars/images/000000.jpg') self.assertEqual(img.shape, img_gt.shape) self.assertTrue((img == img_gt).all()) # Second read. img = reader.imread('testdata/cars/images/000000.jpg') self.assertEqual(img.shape, img_gt.shape) self.assertTrue((img == img_gt).all())
def test_maskreadPicture(self): reader = backendMedia.MediaReader(rootdir='.') mask_gt = skimage.io.imread('testdata/cars/masks/000000.png') mask = reader.maskread('testdata/cars/masks/000000.png') self.assertEqual(mask.shape, mask_gt.shape) self.assertLess(_diff(mask, mask_gt), 0.02) #self.assertTrue((mask == mask_gt).all()) # Second read. mask = reader.imread('testdata/cars/masks/000000.png') self.assertEqual(mask.shape, mask_gt.shape) self.assertLess(_diff(mask, mask_gt), 0.02)
def _object_dataset_function(db_file, rootdir='.', where_object='TRUE', used_keys=None, transform_group=None): ''' Args: db_file: (string) A path to an sqlite3 database file. rootdir: (string) A root path, is pre-appended to "imagefile" entries of "images" table in the sqlite3 database. where_object: (string) The WHERE part of the SQL query on the "objects" table, as in: "SELECT * FROM objects WHERE ${where_object};" Allows to query only needed objects. used_keys: (a list of strings or None) If specified, use only these keys for every sample, and discard the rest. transform_group: A dict {string: callable}. Each key of this dict should match a key in each sample, and the callables are applied to the respective sample dict values. ''' conn = utils.openConnection(db_file, 'r', copy_to_memory=False) c = conn.cursor() utils.checkWhereObject(where_object) c.execute('SELECT * FROM objects WHERE %s ORDER BY objectid' % where_object) object_entries = c.fetchall() imreader = backendMedia.MediaReader(rootdir=rootdir) _checkUsedKeys(used_keys) utils.checkTransformGroup(transform_group) samples = [] logging.info('Loading samples...') for index in range(len(object_entries)): object_entry = object_entries[index] sample = utils.buildObjectSample(object_entry, c, imreader) if sample is None: continue sample = _filterKeys(used_keys, sample) sample = utils.applyTransformGroup(transform_group, sample) samples.append(sample) logging.info('Loaded %d samples.', len(object_entries)) return samples
def __getitem__(self, index): ''' Get the i-th image entry with all its objects. Args: index: the index of the image entry in the database. Returns: dictionary with the following keys. { 'image': Numpy array. 'mask': Numpy array. 'imagefile' String. 'maskfile' String or None. 'name' String (e.g. when predicted by a classification model). 'objects' Dictionary with the same keys as field names in table "objects" of the Shuffler schema. } ''' if index >= len(self): raise IndexError("Index %d is greater than the dataframe size %d" % (index, len(self))) self.cursor.execute("SELECT imagefile,maskfile,name FROM images") # TODO: figure out how to not load the whole database. image_entries = self.cursor.fetchall() imagefile, maskfile, name = image_entries[index] cols = backendDb.getColumnsInTable(self.cursor, 'objects') self.cursor.execute( "SELECT %s FROM objects WHERE imagefile=?" % ','.join(cols), (imagefile, )) objects = self.cursor.fetchall() imreader = backendMedia.MediaReader(rootdir=self.rootdir) if imagefile is not None: image = imreader.imread(imagefile) if maskfile is not None: mask = imreader.maskread(imagefile) return { 'image': image, 'mask': mask, 'objects': [zip(cols, o) for o in objects], 'imagefile': imagefile, 'maskfile': maskfile, 'name': name }
def __init__(self, db_file, rootdir='.', where_object='TRUE', mode='r', copy_to_memory=True, used_keys=None, transform_group=None, preload_samples=False): ''' Args: db_file: (string) A path to an sqlite3 database file. rootdir: (string) A root path, is pre-appended to "imagefile" entries of "images" table in the sqlite3 database. where_object: (string) The WHERE part of the SQL query on the "objects" table, as in: "SELECT * FROM objects WHERE ${where_object};" Allows to query only needed objects. mode: ("r" or "w") The readonly or write-read mode to open the database. The default is "r", use "w" only if you plan to call addRecord(). copy_to_memory: (bool) Copies database into memory. Only for mode="r". Should be used for python2. used_keys: (a list of strings or None) If specified, use only these keys for every sample, and discard the rest. transform_group: ((1) a callable, or (2) a list of callables, or (3) a dict {string: callable}) Transform(s) to be applied on a sample. (1) A callable: It is applied to the sample. (2) A list of callables: Each callable is applied to the sample sequentially. (3) A dict {string: callable}: Each key of this dict should match a key in each sample, and the callables are applied to the respective sample dict values. preload_samples: (bool) If true, will try to preload all samples (including images) into memory in __init__. ''' self.mode = mode self.conn = utils.openConnection(db_file, mode, copy_to_memory) self.c = self.conn.cursor() utils.checkWhereObject(where_object) self.c.execute('SELECT * FROM objects WHERE %s ORDER BY objectid' % where_object) self.object_entries = self.c.fetchall() self.imreader = backendMedia.MediaReader(rootdir=rootdir) _checkUsedKeys(used_keys) self.used_keys = used_keys utils.checkTransformGroup(transform_group) self.transform_group = transform_group if not preload_samples: self.preloaded_samples = None else: self.preloaded_samples = [] logging.info('Loading samples...') for index in range(len(self)): object_entry = self.object_entries[index] sample = utils.buildObjectSample(object_entry, self.c, self.imreader) if sample is None: logging.warning('Skip bad sample %d', index) else: self.preloaded_samples.append(sample) logging.info('Loaded %d samples.', len(self))
def displayImagesPlt(c, args): c.execute('SELECT * FROM images WHERE (%s)' % args.where_image) image_entries = c.fetchall() logging.info('%d images found.', len(image_entries)) if len(image_entries) == 0: logging.error('There are no images. Exiting.') return if args.shuffle: np.random.shuffle(image_entries) if len(image_entries) < args.limit: image_entries = image_entries[:args.limit] def _getNumRows(total, ncols): nrows = int((total - 1) / ncols) + 1 logging.info('Grid: %dx%d from the total of %d', nrows, ncols, total) return nrows if args.limit < len(image_entries): image_entries = image_entries[:args.limit] nrows = _getNumRows(len(image_entries), args.ncols) imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For overlaying masks. labelmap = literal_eval( args.mask_mapping_dict) if args.mask_mapping_dict else None logging.info('Parsed mask_mapping_dict to %s', pformat(labelmap)) for i_image, image_entry in enumerate(image_entries): ax = plt.subplot(nrows, args.ncols, i_image + 1) imagefile = backendDb.imageField(image_entry, 'imagefile') maskfile = backendDb.imageField(image_entry, 'maskfile') imname = backendDb.imageField(image_entry, 'name') imscore = backendDb.imageField(image_entry, 'score') logging.info('Imagefile "%s"', imagefile) logging.debug('Image name="%s", score=%s', imname, imscore) image = imreader.imread(imagefile) # Overlay the mask. if maskfile is not None: mask = imreader.maskread(maskfile) if args.mask_aside: image = util.drawMaskAside(image, mask, labelmap=labelmap) elif args.mask_alpha is not None: image = util.drawMaskOnImage(image, mask, alpha=args.mask_alpha, labelmap=labelmap) else: logging.info('No mask for this image.') # Put the objects on top of the image. if args.with_objects: c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, )) object_entries = c.fetchall() logging.info('Found %d objects for image %s', len(object_entries), imagefile) for object_entry in object_entries: objectid = backendDb.objectField(object_entry, 'objectid') roi = backendDb.objectField(object_entry, 'roi') score = backendDb.objectField(object_entry, 'score') name = backendDb.objectField(object_entry, 'name') logging.info('objectid: %d, roi: %s, score: %s, name: %s', objectid, roi, score, name) c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, )) polygon_entries = c.fetchall() if len(polygon_entries) > 0: logging.info('showing object with a polygon.') polygon = [(int(backendDb.polygonField(p, 'x')), int(backendDb.polygonField(p, 'y'))) for p in polygon_entries] drawScoredPolygon(ax, polygon, label=name, score=score) elif roi is not None: logging.info('showing object with a bounding box.') drawScoredRoi(ax, roi, label=name, score=score) else: logging.warning( 'Neither polygon, nor bbox is available for objectid %d', objectid) # Overlay imagefile. title = "" if args.with_imagefile: title += '%s ' % op.basename( backendMedia.normalizeSeparators(imagefile)) # Overlay score. if args.with_score: title += '%.3f ' % imscore if imscore is not None else '' ax.set_title(title) # Display ax.imshow(image) ax.axis('off') plt.tight_layout() plt.show()
def evaluateBinarySegmentation(c, args): import pandas as pd # Get corresponding maskfiles from predictions and ground truth. c.execute('ATTACH ? AS "attached"', (args.gt_db_file, )) c.execute('SELECT pr.imagefile,pr.maskfile,gt.maskfile ' 'FROM images pr INNER JOIN attached.images gt ' 'WHERE pr.imagefile=gt.imagefile ' 'AND pr.maskfile IS NOT NULL ' 'AND gt.maskfile IS NOT NULL ' 'AND %s ' 'ORDER BY pr.imagefile ASC' % args.where_image) entries = c.fetchall() logging.info( 'Total %d images in both the open and the ground truth databases.' % len(entries)) logging.debug(pprint.pformat(entries)) imreader = backendMedia.MediaReader(rootdir=args.rootdir) TPs = np.zeros((256, ), dtype=int) FPs = np.zeros((256, ), dtype=int) FNs = np.zeros((256, ), dtype=int) if args.display_images_roc: fig = plt.figure() plt.xlabel('recall') plt.ylabel('precision') plt.xlim(0, 1) plt.ylim(0, 1) for imagefile, maskfile_pr, maskfile_gt in progressbar.progressbar( entries): # Load masks and bring them to comparable form. mask_gt = imreader.maskread(maskfile_gt) mask_pr = imreader.maskread(maskfile_pr) mask_pr = cv2.resize(mask_pr, (mask_gt.shape[1], mask_gt.shape[0]), cv2.INTER_NEAREST) # Some printputs. gt_pos = np.count_nonzero(mask_gt == 255) gt_neg = np.count_nonzero(mask_gt == 0) gt_other = mask_gt.size - gt_pos - gt_neg logging.debug('GT: positive: %d, negative: %d, others: %d.', gt_pos, gt_neg, gt_other) # If there is torch. try: import torch # Use only relevant pixels (not the 'dontcare' class.) relevant = np.bitwise_or(mask_gt == 0, mask_gt == 255) mask_gt = mask_gt[relevant].flatten() mask_pr = mask_pr[relevant].flatten() mask_gt = torch.Tensor(mask_gt) mask_pr = torch.Tensor(mask_pr) try: mask_gt = mask_gt.cuda() mask_pr = mask_pr.cuda() except RuntimeError: pass TP = np.zeros((256, ), dtype=int) FP = np.zeros((256, ), dtype=int) FN = np.zeros((256, ), dtype=int) for val in range(256): tp = torch.nonzero(torch.mul(mask_pr > val, mask_gt == 255)).size()[0] fp = torch.nonzero(torch.mul(mask_pr > val, mask_gt != 255)).size()[0] fn = torch.nonzero(torch.mul(mask_pr <= val, mask_gt == 255)).size()[0] tn = torch.nonzero(torch.mul(mask_pr <= val, mask_gt != 255)).size()[0] TP[val] = tp FP[val] = fp FN[val] = fn TPs[val] += tp FPs[val] += fp FNs[val] += fn ROC, area = getPrecRecall(TP, FP, FN) logging.info('%s\t%.2f' % (op.basename(imagefile), area * 100.)) except ImportError: # TODO: write the same without torch, on CPU raise NotImplementedError( 'Non-torch implementation is still to be implemented.') if args.display_images_roc: plt.plot(ROC[:, 0], ROC[:, 1], 'go-', linewidth=2, markersize=4) plt.pause(0.05) fig.show() # Accumulate into Precision-Recall curve. ROC, area = getPrecRecall(TPs, FPs, FNs) print( "Average across image area under the Precision-Recall curve, perc: %.2f" % (area * 100.)) if args.out_dir is not None: if not op.exists(args.out_dir): os.makedirs(args.out_dir) fig = plt.figure() plt.xlabel('recall') plt.ylabel('precision') plt.xlim(0, 1) plt.ylim(0, 1) plt.plot(ROC[:, 0], ROC[:, 1], 'bo-', linewidth=2, markersize=6) out_plot_path = op.join(args.out_dir, '%srecall-prec.png' % args.out_prefix) fig.savefig(out_plot_path, transparent=True, bbox_inches='tight', pad_inches=0, dpi=300)
def importCityscapes(c, args): if args.with_display: imreader = backendMedia.MediaReader(args.rootdir) logging.info('Will load splits: %s' % args.splits) logging.info('Will load json type: %s' % args.type) logging.info('Will load mask type: %s' % args.mask_type) if not op.exists(args.cityscapes_dir): raise Exception('Cityscape directory "%s" does not exist' % args.cityscapes_dir) # Directory accessed by label type and by split. dirs_by_typesplit = {} for type_ in [args.type, 'leftImg8bit']: type_dir_template = op.join(args.cityscapes_dir, '%s*' % type_) for type_dir in [x for x in glob(type_dir_template) if op.isdir(x)]: logging.debug('Looking for splits in %s' % type_dir) for split in args.splits: typesplit_dir = op.join(type_dir, split) if op.exists(typesplit_dir): logging.debug('Found split %s in %s' % (split, type_dir)) # Add the info into the main dictionary "dirs_by_typesplit". if split not in dirs_by_typesplit: dirs_by_typesplit[split] = {} dirs_by_typesplit[split][type_] = typesplit_dir logging.info('Found the following directories: \n%s' % pformat(dirs_by_typesplit)) for split in args.splits: # List of cities. assert 'leftImg8bit' in dirs_by_typesplit[split] leftImg8bit_dir = dirs_by_typesplit[split]['leftImg8bit'] cities = os.listdir(leftImg8bit_dir) cities = [x for x in cities if op.isdir(op.join(leftImg8bit_dir, x))] logging.info('Found %d cities in %s' % (len(cities), leftImg8bit_dir)) for city in cities: image_names = os.listdir(op.join(leftImg8bit_dir, city)) logging.info('In split "%s", city "%s" has %d images' % (split, city, len(image_names))) for image_name in image_names: # Get the image path. image_path = op.join(leftImg8bit_dir, city, image_name) name_parts = op.splitext(image_name)[0].split('_') if len(name_parts) != 4: raise Exception( 'Expect to have 4 parts in the name of image "%s"' % image_name) if name_parts[0] != city: raise Exception('The first part of name of image "%s" ' 'is expected to be city "%s".' % (image_name, city)) if name_parts[3] != 'leftImg8bit': raise Exception('The last part of name of image "%s" ' 'is expected to be city "leftImg8bit".' % image_name) imheight, imwidth = backendMedia.getPictureSize(image_path) imagefile = op.relpath(image_path, args.rootdir) c.execute( 'INSERT INTO images(imagefile,width,height) VALUES (?,?,?)', (imagefile, imwidth, imheight)) # Get the json label. if args.type in dirs_by_typesplit[split]: city_dir = op.join(dirs_by_typesplit[split][args.type], city) if op.exists(city_dir): json_name = '_'.join(name_parts[:3] + [args.type, 'polygons']) + '.json' json_path = op.join(city_dir, json_name) if op.exists(json_path): _importJson(c, json_path, imagefile, imheight, imwidth) if args.mask_type is not None: mask_name = '_'.join( name_parts[:3] + [args.type, args.mask_type]) + '.png' mask_path = op.join(city_dir, mask_name) if op.exists(mask_path): maskfile = op.relpath(mask_path, args.rootdir) c.execute( 'UPDATE images SET maskfile=? WHERE imagefile=?', (maskfile, imagefile)) if args.with_display: img = imreader.imread(imagefile) c.execute( 'SELECT objectid,name FROM objects WHERE imagefile=?', (imagefile, )) for objectid, name in c.fetchall(): # Draw polygon. c.execute('SELECT x,y FROM polygons WHERE objectid=?', (objectid, )) polygon = c.fetchall() util.drawScoredPolygon(img, [(int(pt[0]), int(pt[1])) for pt in polygon], name) cv2.imshow('importCityscapes', img[:, :, ::-1]) if cv2.waitKey(-1) == 27: args.with_display = False cv2.destroyWindow('importCityscapes') # Statistics. c.execute('SELECT COUNT(1) FROM images') logging.info('Imported %d images' % c.fetchone()[0]) c.execute('SELECT COUNT(1) FROM images WHERE maskfile IS NOT NULL') logging.info('Imported %d masks' % c.fetchone()[0]) c.execute('SELECT COUNT(DISTINCT(imagefile)) FROM objects') logging.info('Objects are found in %d images' % c.fetchone()[0])
def importKitti(c, args): if args.with_display: imreader = backendMedia.MediaReader(args.rootdir) image_paths = sorted(glob(op.join(args.images_dir, '*.png'))) logging.info('Found %d PNG images in %s' % (len(image_paths), args.images_dir)) for image_path in progressbar(image_paths): filename = op.splitext(op.basename(image_path))[0] logging.debug('Processing image: "%s"' % filename) # Add image to the database. imheight, imwidth = backendMedia.getPictureSize(image_path) imagefile = op.relpath(image_path, args.rootdir) c.execute('INSERT INTO images(imagefile,width,height) VALUES (?,?,?)', (imagefile, imwidth, imheight)) if args.with_display: img = imreader.imread(imagefile) # Detection annotations. if args.detection_dir: detection_path = op.join(args.detection_dir, '%s.txt' % filename) if not op.exists(detection_path): raise FileNotFoundError('Annotation file not found at "%s".' % detection_path) # Read annotation file. with open(detection_path) as f: lines = f.read().splitlines() logging.debug('Read %d lines from detection file "%s"' % (len(lines), detection_path)) for line in lines: objectid = _parseObject(c, line, imagefile) if args.with_display: c.execute('SELECT * FROM objects WHERE objectid=?', (objectid, )) object_entry = c.fetchone() name = backendDb.objectField(object_entry, 'name') roi = backendDb.objectField(object_entry, 'roi') score = backendDb.objectField(object_entry, 'score') util.drawScoredRoi(img, roi, name, score=score) # Segmentation annotations. if args.segmentation_dir: segmentation_path = op.join(args.segmentation_dir, '%s.png' % filename) if not op.exists(segmentation_path): raise FileNotFoundError('Annotation file not found at "%s".' % segmentation_path) # Add image to the database. maskfile = op.relpath(segmentation_path, args.rootdir) c.execute('UPDATE images SET maskfile=? WHERE imagefile=?', (maskfile, imagefile)) if args.with_display: mask = imreader.maskread(maskfile) img = util.drawMaskAside(img, mask, labelmap=None) # Maybe display. if args.with_display: cv2.imshow('importKitti', img[:, :, ::-1]) if cv2.waitKey(-1) == 27: args.with_display = False cv2.destroyWindow('importKitti')
def evaluateSegmentationIoU(c, args): import pandas as pd import matplotlib.pyplot as plt # Get corresponding maskfiles from predictions and ground truth. logging.info('Opening ground truth dataset: %s', args.gt_db_file) c.execute('ATTACH ? AS "attached"', (args.gt_db_file, )) c.execute('SELECT pr.imagefile,pr.maskfile,gt.maskfile ' 'FROM images pr INNER JOIN attached.images gt ' 'WHERE pr.imagefile=gt.imagefile AND pr.maskfile IS NOT NULL ' 'AND gt.maskfile IS NOT NULL ' 'AND %s ' 'ORDER BY pr.imagefile ASC' % args.where_image) entries = c.fetchall() logging.info( 'Total %d images in both the open and the ground truth databases.', len(entries)) logging.debug(pprint.pformat(entries)) imreader = backendMedia.MediaReader(rootdir=args.rootdir) labelmap_gt, labelmap_pr, class_names = _label2classMapping( args.gt_mapping_dict, args.pred_mapping_dict) if args.class_to_record_iou is not None and not args.class_to_record_iou in class_names: raise ValueError( 'class_to_record_iou=%s is not among values of gt_mapping_dict=%s' % (args.class_to_record_iou, args.gt_mapping_dict)) hist_all = np.zeros((len(class_names), len(class_names))) for imagefile, maskfile_pr, maskfile_gt in progressbar.progressbar( entries): # Load masks and bring them to comparable form. mask_gt = util.applyLabelMappingToMask(imreader.maskread(maskfile_gt), labelmap_gt) mask_pr = util.applyLabelMappingToMask(imreader.maskread(maskfile_pr), labelmap_pr) mask_pr = cv2.resize(mask_pr, (mask_gt.shape[1], mask_gt.shape[0]), interpolation=cv2.INTER_NEAREST) # Evaluate one image pair. careabout = ~np.isnan(mask_gt) mask_gt = mask_gt[careabout][:].astype(int) mask_pr = mask_pr[careabout][:].astype(int) hist = fast_hist(mask_gt, mask_pr, len(class_names)) hist_all += hist # Compute and record results by image. iou_list = per_class_iu(hist) if args.class_to_record_iou is None: iou = iou_list.mean() else: iou = iou_list[class_names.index(args.class_to_record_iou)] c.execute('UPDATE images SET score=? WHERE imagefile=?', (iou, imagefile)) # Get label distribution. pr_per_class = hist_all.sum(0) gt_per_class = hist_all.sum(1) iou_list = per_class_iu(hist_all) fwIoU = calc_fw_iu(hist_all) pixAcc = calc_pixel_accuracy(hist_all) mAcc = calc_mean_accuracy(hist_all) result_df = pd.DataFrame({ 'class': class_names, 'IoU': iou_list, "pr_distribution": pr_per_class, "gt_distribution": gt_per_class, }) result_df["IoU"] *= 100 # Changing to percent ratio. result_df.set_index("class", inplace=True) print("---- info per class -----") print(result_df) result_ser = pd.Series({ "pixAcc": pixAcc, "mAcc": mAcc, "fwIoU": fwIoU, "mIoU": iou_list.mean() }) result_ser = result_ser[["pixAcc", "mAcc", "fwIoU", "mIoU"]] result_ser *= 100 # change to percent ratio if args.out_dir is not None: if not op.exists(args.out_dir): os.makedirs(args.out_dir) out_summary_path = op.join(args.out_dir, args.out_summary_file) logging.info('Will add summary to: %s', out_summary_path) with open(out_summary_path, 'a') as f: f.write(args.out_prefix + '\t' + '\t'.join(['%.2f' % x for x in result_df['IoU']]) + '\n') # Save confusion matrix fig = plt.figure() normalized_hist = (hist.astype("float") / hist.sum(axis=1)[:, np.newaxis]) plot_confusion_matrix(normalized_hist, classes=class_names) outfigfn = op.join(args.out_dir, "%sconf_mat.pdf" % args.out_prefix) fig.savefig(outfigfn, transparent=True, bbox_inches='tight', pad_inches=0, dpi=300) print("Confusion matrix was saved to %s" % outfigfn) outdffn = op.join(args.out_dir, "%seval_result_df.csv" % args.out_prefix) result_df.to_csv(outdffn) print('Info per class was saved at %s !' % outdffn) outserfn = op.join(args.out_dir, "%seval_result_ser.csv" % args.out_prefix) result_ser.to_csv(outserfn) print('Total result is saved at %s !' % outserfn)
def test_closeUninitWithoutError(self): reader = backendMedia.MediaReader(rootdir='.') reader.close()
def test_closePictureWithoutError(self): reader = backendMedia.MediaReader(rootdir='.') reader.imread('testdata/cars/images/000000.jpg') reader.close()
def labelObjects(c, args): cv2.namedWindow("labelObjects") c.execute('SELECT COUNT(*) FROM objects WHERE (%s) ' % args.where_object) logging.info('Found %d objects in db.' % c.fetchone()[0]) c.execute('SELECT * FROM objects WHERE (%s)' % args.where_object) object_entries = c.fetchall() logging.info('Found %d objects in db.' % len(object_entries)) if len(object_entries) == 0: return if args.shuffle: np.random.shuffle(object_entries) imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For parsing keys. key_reader = KeyReader(args.key_dict) button = 0 index_object = 0 another_object = True while button != 27: go_next_object = False if another_object: another_object = False logging.info(' ') logging.info('Object %d out of %d' % (index_object, len(object_entries))) object_entry = object_entries[index_object] objectid = backendDb.objectField(object_entry, 'objectid') bbox = backendDb.objectField(object_entry, 'bbox') roi = backendDb.objectField(object_entry, 'roi') imagefile = backendDb.objectField(object_entry, 'imagefile') logging.info('imagefile: %s' % imagefile) image = imreader.imread(imagefile) # Display an image, wait for the key from user, and parse that key. scale = float(args.winsize) / max(image.shape[0:2]) logging.debug('Will resize image and annotations with scale: %f' % scale) image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale) logging.info('objectid: %d, roi: %s' % (objectid, roi)) c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, )) polygon_entries = c.fetchall() if len(polygon_entries) > 0: logging.info('showing object with a polygon.') polygon = [(backendDb.polygonField(p, 'x'), backendDb.polygonField(p, 'y')) for p in polygon_entries] logging.debug('nonscaled polygon: %s' % pformat(polygon)) polygon = [(int(scale * p[0]), int(scale * p[1])) for p in polygon] logging.debug('scaled polygon: %s' % pformat(polygon)) util.drawScoredPolygon(image, polygon, label=None, score=None) elif roi is not None: logging.info('showing object with a bounding box.') logging.debug('nonscaled roi: %s' % pformat(roi)) roi = [int(scale * r) for r in roi] # For displaying the scaled image. logging.debug('scaled roi: %s' % pformat(roi)) util.drawScoredRoi(image, roi, label=None, score=None) else: raise Exception( 'Neither polygon, nor bbox is available for objectid %d' % objectid) c.execute( 'SELECT key,value FROM properties WHERE objectid=? AND key=?', (objectid, args.property)) # TODO: Multiple properties are possible because there is no # contraint on uniqueness on table properties(objectid,key). # Change when the uniqueness constraint is added to the # database schema. On the other hand, it's a feature. properties = c.fetchall() if len(properties) > 1: logging.warning( 'Multiple values for object %s and property %s. ' 'If reassigned, both will be changed' % (objectid, args.property)) for iproperty, (key, value) in enumerate(properties): cv2.putText(image, '%s: %s' % (key, value), (10, util.SCALE * (iproperty + 1)), util.FONT, util.FONT_SIZE, (0, 0, 0), util.THICKNESS) cv2.putText(image, '%s: %s' % (key, value), (10, util.SCALE * (iproperty + 1)), util.FONT, util.FONT_SIZE, (255, 255, 255), util.THICKNESS - 1) logging.info('objectid: %d. %s = %s.' % (objectid, key, value)) cv2.imshow('labelObjects', image[:, :, ::-1]) action = key_reader.parse(cv2.waitKey(-1)) if action == 'exit': break elif action == 'delete_label' and any_object_in_focus: logging.info('Remove label from objectid "%s"' % objectid) c.execute('DELETE FROM properties WHERE objectid=? AND key=?', (objectid, args.property)) go_next_object = True elif action is not None and action not in ['previous', 'next']: # User pressed something else which has an assigned action, # assume it is a new value. logging.info('Setting label "%s" to objectid "%s"' % (action, objectid)) if len(properties) > 0: c.execute('DELETE FROM properties WHERE objectid=? AND key=?', (objectid, args.property)) c.execute( 'INSERT INTO properties(objectid,key,value) VALUES (?,?,?)', (objectid, args.property, str(action))) go_next_object = True # Navigation. if action == 'previous': logging.debug('previous object') another_object = True if index_object > 0: index_object -= 1 else: logging.warning('Already at the first object.') elif action == 'next' or go_next_object == True: logging.debug('next object') another_object = True if index_object < len(object_entries) - 1: index_object += 1 else: logging.warning( 'Already at the last object. Press Esc to save and exit.') cv2.destroyWindow("labelObjects")
def labelMatches(c, args): cv2.namedWindow("labelMatches") cv2.setMouseCallback('labelMatches', _monitorPressRelease) global mousePressed, mouseReleased, xpress, ypress c.execute('SELECT imagefile FROM images WHERE %s' % args.where_image) image_entries = c.fetchall() logging.debug('Found %d images' % len(image_entries)) if len(image_entries) < 2: logging.error('Found only %d images. Quit.' % len(image_entries)) return imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For parsing keys. key_reader = KeyReader(args.key_dict) index_image = 1 action = None while action != 'exit' and index_image < len(image_entries): (imagefile1, ) = image_entries[index_image - 1] (imagefile2, ) = image_entries[index_image] img1 = imreader.imread(imagefile1) img2 = imreader.imread(imagefile2) # offset of the 2nd image, when they are stacked yoffset = img1.shape[0] # Make sure images have the same width, so that we can stack them. # That will happen when the video changes, so we'll skip that pair. if img1.shape[1] != img2.shape[1]: logging.warning('Skipping image pair "%s" and "%s" ' 'since they are of different width.' % (imagefile1, imagefile2)) index_image += 1 # get objects from both images c.execute('SELECT * FROM objects WHERE imagefile=? ', (imagefile1, )) objects1 = c.fetchall() logging.info('%d objects found for %s' % (len(objects1), imagefile1)) c.execute('SELECT * FROM objects WHERE imagefile=? ', (imagefile2, )) objects2 = c.fetchall() logging.info('%d objects found for %s' % (len(objects2), imagefile2)) # draw cars in both images for object_ in objects1: util.drawScoredRoi(img1, backendDb.objectField(object_, 'roi')) for object_ in objects2: util.drawScoredRoi(img2, backendDb.objectField(object_, 'roi')) i1 = i2 = None # Matches selected with a mouse. selectedMatch = None needRedraw = True action = None # Stay in side the loop until a key is pressed. while action is None: img_stack = np.vstack((img1, img2)) if needRedraw: # find existing matches, and make a map matchesOf1 = {} matchesOf2 = {} for j1 in range(len(objects1)): object1 = objects1[j1] for j2 in range(len(objects2)): object2 = objects2[j2] c.execute( 'SELECT match FROM matches WHERE objectid = ? INTERSECT ' 'SELECT match FROM matches WHERE objectid = ?', (backendDb.objectField(object1, 'objectid'), backendDb.objectField(object2, 'objectid'))) matches = c.fetchall() if len(matches) > 0: assert len(matches) == 1 # No duplicate matches. roi1 = backendDb.objectField(object1, 'roi') roi2 = backendDb.objectField(object2, 'roi') _drawMatch(img_stack, roi1, roi2, yoffset) matchesOf1[j1] = matches[0][0] matchesOf2[j2] = matches[0][0] # draw image scale = float(args.winsize) / max(img_stack.shape[0:2]) img_show = cv2.resize(img_stack, dsize=(0, 0), fx=scale, fy=scale) cv2.imshow('labelMatches', img_show[:, :, ::-1]) logging.info('Will draw %d matches found between the pair' % len(matchesOf1)) needRedraw = False # process mouse callback effect (button has been pressed) if mousePressed: i2 = None # reset after the last unsuccessful match logging.debug('Pressed x=%d, y=%d' % (xpress, ypress)) xpress /= scale ypress /= scale i1 = _findPressedObject(xpress, ypress, objects1) if i1 is not None: logging.debug('Found pressed object: %d' % i1) mousePressed = False # process mouse callback effect (button has been released) if mouseReleased: logging.debug('released x=%d, y=%d' % (xpress, ypress)) xpress /= scale ypress /= scale i2 = _findPressedObject(xpress, ypress - yoffset, objects2) if i2 is not None: logging.debug('Found released object: %d' % i2) mouseReleased = False # If we could find pressed and released objects, add match if i1 is not None and i2 is not None: # If one of the objects is already matched in this image pair, discard. if i1 in matchesOf1 or i2 in matchesOf2: logging.warning( 'One or two connected objects is already matched') i1 = i2 = None else: # Add the match to the list. objectid1 = backendDb.objectField(objects1[i1], 'objectid') objectid2 = backendDb.objectField(objects2[i2], 'objectid') logging.debug('i1 = %d, i2 = %d' % (i1, i2)) # Check if this object already in the matches. c.execute('SELECT match FROM matches WHERE objectid=?', (objectid1, )) matches1 = c.fetchall() c.execute('SELECT match FROM matches WHERE objectid=?', (objectid2, )) matches2 = c.fetchall() if len(matches1) > 1 or len(matches2) > 1: logging.error( 'One of the objectids %d, %d is in several matches.' % (objectid1, objectid2)) continue elif len(matches1) == 1 and len(matches2) == 1: logging.info( 'Will merge matches of objectids %d and %d' % (objectid1, objectid2)) c.execute('UPDATE matches SET match=? WHERE match=?', (matches1[0][0], matches2[0][0])) elif len(matches1) == 1 and len(matches2) == 0: logging.info('Add objectid %d to match %d' % (objectid2, matches1[0][0])) c.execute( 'INSERT INTO matches(match, objectid) VALUES (?,?)', (matches1[0][0], objectid2)) elif len(matches1) == 0 and len(matches2) == 1: logging.info('Add objectid %d to match %d' % (objectid1, matches2[0][0])) c.execute( 'INSERT INTO matches(match, objectid) VALUES (?,?)', (matches2[0][0], objectid1)) elif len(matches1) == 0 and len(matches2) == 0: logging.info( 'Add a new match between objectids %d and %d.' % (objectid1, objectid2)) # Find a free match index c.execute('SELECT MAX(match) FROM matches') maxmatch = c.fetchone()[0] match = int( maxmatch) + 1 if maxmatch is not None else 1 c.execute( 'INSERT INTO matches(match, objectid) VALUES (?,?)', (match, objectid1)) c.execute( 'INSERT INTO matches(match, objectid) VALUES (?,?)', (match, objectid2)) else: assert False # Reset when a new match is made. needRedraw = True i1 = i2 = None # Stay inside the loop inside one image pair until some button is pressed action = key_reader.parse(cv2.waitKey(50)) # Process pressed key (all except exit) if action == 'previous': logging.info('Previous image pair') if index_image == 1: logging.warning('already the first image pair') else: index_image -= 1 elif action == 'next': logging.info('Next image pair') index_image += 1 # exit at last image pair from outer loop elif action == 'delete_match': # if any car was selected, and it is matched if i1 is not None and i1 in matchesOf1: match = matchesOf1[i1] objectid1 = backendDb.objectField(objects1[i1], 'objectid') logging.info('deleting match %d' % match) c.execute( 'DELETE FROM matches WHERE match = ? AND objectid = ?', (match, objectid1)) else: logging.debug('delete is pressed, but no match is selected') cv2.destroyWindow("labelMatches")
def examineImages(c, args): cv2.namedWindow("examineImages") c.execute('SELECT * FROM images WHERE (%s)' % args.where_image) image_entries = c.fetchall() logging.info('%d images found.' % len(image_entries)) if len(image_entries) == 0: logging.error('There are no images. Exiting.') return if args.shuffle: np.random.shuffle(image_entries) if args.snapshot_dir and not op.exists(args.snapshot_dir): os.makedirs(args.snapshot_dir) imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For parsing keys. key_reader = KeyReader(args.key_dict) # For overlaying masks. labelmap = literal_eval( args.mask_mapping_dict) if args.mask_mapping_dict else None logging.info('Parsed mask_mapping_dict to %s' % pformat(labelmap)) index_image = 0 while True: # Until a user hits the key for the "exit" action. image_entry = image_entries[index_image] imagefile = backendDb.imageField(image_entry, 'imagefile') maskfile = backendDb.imageField(image_entry, 'maskfile') imname = backendDb.imageField(image_entry, 'name') imscore = backendDb.imageField(image_entry, 'score') logging.info('Imagefile "%s"' % imagefile) logging.debug('Image name="%s", score=%s' % (imname, imscore)) image = imreader.imread(imagefile) # Overlay the mask. if maskfile is not None: mask = imreader.maskread(maskfile) if args.mask_aside: image = util.drawMaskAside(image, mask, labelmap=labelmap) elif args.mask_alpha is not None: image = util.drawMaskOnImage(image, mask, alpha=args.mask_alpha, labelmap=labelmap) else: logging.info('No mask for this image.') # Put the objects on top of the image. if args.with_objects: c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, )) object_entries = c.fetchall() logging.info('Found %d objects for image %s' % (len(object_entries), imagefile)) for object_entry in object_entries: objectid = backendDb.objectField(object_entry, 'objectid') roi = backendDb.objectField(object_entry, 'roi') score = backendDb.objectField(object_entry, 'score') name = backendDb.objectField(object_entry, 'name') logging.info('objectid: %d, roi: %s, score: %s, name: %s' % (objectid, roi, score, name)) c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, )) polygon_entries = c.fetchall() if len(polygon_entries) > 0: logging.info('showing object with a polygon.') polygon = [(int(backendDb.polygonField(p, 'x')), int(backendDb.polygonField(p, 'y'))) for p in polygon_entries] util.drawScoredPolygon(image, polygon, label=name, score=score) elif roi is not None: logging.info('showing object with a bounding box.') util.drawScoredRoi(image, roi, label=name, score=score) else: logging.warning( 'Neither polygon, nor bbox is available for objectid %d' % objectid) # Display an image, wait for the key from user, and parse that key. scale = float(args.winsize) / max(list(image.shape[0:2])) scaled_image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale) # Overlay imagefile. if args.with_imagefile: util.drawTextOnImage( scaled_image, op.basename(backendMedia.normalizeSeparators(imagefile))) # Overlay score. # TODO: add y offset, if necessary if args.with_score: util.drawTextOnImage(scaled_image, '%.3f' % imscore) # Display cv2.imshow('examineImages', scaled_image[:, :, ::-1]) action = key_reader.parse(cv2.waitKey(-1)) if action is None: # User pressed something which does not have an action. continue elif action == 'snapshot': if args.snapshot_dir: snaphot_path = op.join(args.snapshot_dir, '%08d.png' % index_image) logging.info('Making a snapshot at path: %s' % snaphot_path) imageio.imwrite(snaphot_path, image) else: logging.warning( 'The user pressed a snapshot key, but snapshot_dir is not ' 'specified. Will not write a snapshot.') elif action == 'delete': backendDb.deleteImage(c, imagefile) del image_entries[index_image] if len(image_entries) == 0: logging.warning('Deleted the last image.') break index_image += 1 elif action == 'exit': break elif action == 'previous': index_image -= 1 elif action == 'next': index_image += 1 else: # User pressed something else which has an assigned action, assume it is a new name. logging.info('Setting name "%s" to imagefile "%s"' % (action, imagefile)) c.execute('UPDATE images SET name=? WHERE imagefile=?' (action, imagefile)) index_image = index_image % len(image_entries) cv2.destroyWindow("examineImages")
def test_maskreadPictureRootdir(self): reader = backendMedia.MediaReader(rootdir='testdata/cars') mask_gt = skimage.io.imread('testdata/cars/masks/000000.png') mask = reader.maskread('masks/000000.png') self.assertEqual(mask.shape, mask_gt.shape) self.assertLess(_diff(mask, mask_gt), 0.02)
def labelAzimuth(c, args): from lib.backendScenes import Window from scenes.lib.cvScrollZoomWindow import Window from scenes.lib.homography import getFrameFlattening, getHfromPose from scenes.lib.cache import PoseCache, MapsCache from scenes.lib.warp import transformPoint os.environ['SCENES_PATH'] = args.scenes_dir imreader = backendMedia.MediaReader(rootdir=args.rootdir) key_reader = dbGui.KeyReader(args.key_dict) c.execute('SELECT * FROM objects WHERE (%s)' % args.where_object) object_entries = c.fetchall() logging.info('Found %d objects in db.' % len(object_entries)) if len(object_entries) == 0: return if args.shuffle: np.random.shuffle(object_entries) # Cached poses and azimuth maps. topdown_azimuths = MapsCache('topdown_azimuth') poses = PoseCache() button = 0 index_object = 0 another_object = True char_list = [] while button != 27: go_next_object = False update_yaw_in_db = False if another_object: another_object = False logging.info(' ') logging.info('Object %d out of %d' % (index_object, len(object_entries))) object_entry = object_entries[index_object] objectid = backendDb.objectField(object_entry, 'objectid') bbox = backendDb.objectField(object_entry, 'bbox') roi = backendDb.objectField(object_entry, 'roi') imagefile = backendDb.objectField(object_entry, 'imagefile') # Update yaw inside the loop in case it was just assigned. c.execute( 'SELECT value FROM properties WHERE objectid=? AND key="yaw"', (objectid, )) yaw = c.fetchone() yaw = float(yaw[0]) if yaw is not None else None y, x = roi[0] * 0.3 + roi[2] * 0.7, roi[1] * 0.5 + roi[3] * 0.5 try: flattening = _getFlatteningFromImagefile( poses, imagefile, y, x) except: # A hack that allowed me write all the images into one dir but # keep info about imagefile. c.execute('SELECT name FROM images WHERE imagefile=?', (imagefile, )) image_name = c.fetchone()[0] flattening = _getFlatteningFromImagefile( poses, image_name, y, x) axis_x = np.linalg.norm(np.asarray(bbox[2:4]), ord=2) axis_y = axis_x * flattening display = imreader.imread(imagefile) window = AzimuthWindow(display, x, y, axis_x, axis_y, winsize=args.winsize) if yaw is not None: logging.info('Yaw is: %.0f' % yaw) window.yaw = yaw window.update_cached_zoomed_img() window.redraw() button = cv2.waitKey(50) action = key_reader.parse(button) if button is not -1 else None if action == 'delete': c.execute('DELETE FROM properties WHERE objectid=? AND key="yaw"', (objectid, )) logging.info('Yaw is deleted.') go_next_object = True char_list = [] # Entry in keyboard. if button >= ord('0') and button <= ord('9') or button == ord('.'): char_list += chr(button) logging.debug('Added %s character to number and got %s' % (chr(button), ''.join(char_list))) # Enter accepts GUI or keyboard entry. elif button == 13: if char_list: # After keyboard entry. number_str = ''.join(char_list) char_list = [] try: logging.info('Accepting entry from the keyboard.') yaw = float(number_str) update_yaw_in_db = True go_next_object = True except ValueError: logging.warning('Could not convert entered %s to number.' % number_str) continue else: # Just navigation. logging.info('A navigation Enter.') go_next_object = True # Entry in GUI. elif window.selected == True: logging.info('Accepting entry from GUI.') yaw = window.yaw update_yaw_in_db = True go_next_object = True # No entry: else: yaw = None # Entry happened one way or the other. # Update the yaw and go to the next object. if update_yaw_in_db: c.execute( 'SELECT COUNT(1) FROM properties WHERE objectid=? AND key="yaw"', (objectid, )) if c.fetchone()[0] == 0: c.execute( 'INSERT INTO properties(value,key,objectid) VALUES (?,"yaw",?)', (str(yaw), objectid)) else: c.execute( 'UPDATE properties SET value=? WHERE objectid=? AND key="yaw"', (str(yaw), objectid)) logging.info('Yaw is assigned to %.f' % yaw) # Navigation. if action == 'previous': logging.debug('previous object') if index_object > 0: index_object -= 1 another_object = True else: logging.warning('Already at the first object.') elif action == 'next' or go_next_object == True: logging.debug('next object') if index_object < len(object_entries) - 1: index_object += 1 another_object = True else: logging.warning( 'Already at the last object. Press Esc to save and exit.')
def test_imreadNotPictureOrVideoRootdir(self): reader = backendMedia.MediaReader(rootdir='testdata/cars') with self.assertRaises(TypeError): reader.imread('images/000005.txt')
def test_imreadNotExistingRootdir(self): reader = backendMedia.MediaReader(rootdir='testdata/cars') with self.assertRaises(TypeError): reader.imread('noExisting.jpg')
def examineObjects(c, args): cv2.namedWindow("examineObjects") c.execute('SELECT COUNT(*) FROM objects WHERE (%s) ' % args.where_object) logging.info('Found %d objects in db.' % c.fetchone()[0]) c.execute('SELECT DISTINCT imagefile FROM objects WHERE (%s) ' % args.where_object) image_entries = c.fetchall() logging.info('%d images found.' % len(image_entries)) if len(image_entries) == 0: logging.error('There are no images. Exiting.') return if args.shuffle: np.random.shuffle(image_entries) imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For parsing keys. key_reader = KeyReader(args.key_dict) index_image = 0 index_object = 0 # Iterating over images, because for each image we want to show all objects. while True: # Until a user hits the key for the "exit" action. (imagefile, ) = image_entries[index_image] logging.info('Imagefile "%s"' % imagefile) image = imreader.imread(imagefile) scale = float(args.winsize) / max(image.shape[0:2]) image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale) c.execute( 'SELECT * FROM objects WHERE imagefile=? AND (%s)' % args.where_object, (imagefile, )) object_entries = c.fetchall() logging.info('Found %d objects for image %s' % (len(object_entries), imagefile)) # Put the objects on top of the image. if len(object_entries) > 0: assert index_object < len(object_entries) object_entry = object_entries[index_object] objectid = backendDb.objectField(object_entry, 'objectid') roi = backendDb.objectField(object_entry, 'roi') score = backendDb.objectField(object_entry, 'score') name = backendDb.objectField(object_entry, 'name') scaledroi = [int(scale * r) for r in roi] # For displaying the scaled image. logging.info('objectid: %d, roi: %s, score: %s, name: %s' % (objectid, roi, score, name)) c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, )) polygon_entries = c.fetchall() if len(polygon_entries) > 0: logging.info('showing object with a polygon.') polygon = [(backendDb.polygonField(p, 'x'), backendDb.polygonField(p, 'y')) for p in polygon_entries] logging.debug('nonscaled polygon: %s' % pformat(polygon)) polygon = [(int(scale * x), int(scale * y)) for x, y in polygon] logging.debug('scaled polygon: %s' % pformat(polygon)) util.drawScoredPolygon(image, polygon, label=None, score=score) elif roi is not None: logging.info('showing object with a bounding box.') util.drawScoredRoi(image, scaledroi, label=None, score=score) else: raise Exception( 'Neither polygon, nor bbox is available for objectid %d' % objectid) c.execute('SELECT key,value FROM properties WHERE objectid=?', (objectid, )) properties = c.fetchall() if name is not None: properties.append(('name', name)) if score is not None: properties.append(('score', score)) for iproperty, (key, value) in enumerate(properties): cv2.putText(image, '%s: %s' % (key, value), (scaledroi[3] + 10, scaledroi[0] - 10 + util.SCALE * (iproperty + 1)), util.FONT, util.FONT_SIZE, (0, 0, 0), util.THICKNESS) cv2.putText(image, '%s: %s' % (key, value), (scaledroi[3] + 10, scaledroi[0] - 10 + util.SCALE * (iproperty + 1)), util.FONT, util.FONT_SIZE, (255, 255, 255), util.THICKNESS - 1) logging.info('objectid: %d. %s = %s.' % (objectid, key, value)) # Display an image, wait for the key from user, and parse that key. cv2.imshow('examineObjects', image[:, :, ::-1]) action = key_reader.parse(cv2.waitKey(-1)) if action is None: # User pressed something which does not have an action. continue elif action == 'exit': break elif action == 'previous': index_object -= 1 if index_object < 0: index_image -= 1 index_object = 0 elif action == 'next': index_object += 1 if index_object >= len(object_entries): index_image += 1 index_object = 0 elif action == 'delete' and len(object_entries) > 0: backendDb.deleteObject(c, objectid) del object_entries[index_object] if index_object >= len(object_entries): index_image += 1 index_object = 0 elif action == 'unname' and len(object_entries) > 0: logging.info('Remove the name from objectid "%s"' % objectid) c.execute('UPDATE objects SET name=NULL WHERE objectid=?', (objectid, )) index_image = index_image % len(image_entries) cv2.destroyWindow("examineObjects")
def __init__(self, db_file, rootdir='.', where_image='TRUE', where_object='TRUE', mode='r', copy_to_memory=True, used_keys=None, transform_group=None, batch_size=1, shuffle=False): ''' Args: db_file: (string) A path to an sqlite3 database file. rootdir: (string) A root path, is pre-appended to "imagefile" entries of "images" table in the sqlite3 database. where_image: (string) The WHERE part of the SQL query on the "images" table, as in: "SELECT * FROM images WHERE ${where_image};" Allows to query only needed images. where_object: (string) The WHERE part of the SQL query on the "objects" table, as in: "SELECT * FROM objects WHERE ${where_object};" Allows to query only needed objects for each image. mode: ("r" or "w") The readonly or write-read mode to open the database. The default is "r", use "w" only if you plan to call addRecord(). copy_to_memory: (bool) Copies database into memory. Only for mode="r". Should be used for python2. used_keys: (None, list of str, tuple of str, or dict str -> str) Originally __getitem__ returns a dict with keys: 'image', 'mask', 'objects', 'imagefile', 'name', 'score' (see the comments to this class above). Argument `used_keys` determines which of these keys are needed, and which can be disposed of. Options for `used_keys`. 1) None. Each sample is unchanged dict. 2) List of str. Each str is a key. __getitem__ returns a list. 3) Tuple of str. Same as above. __getitem__ returns a tuple. 4) Dict str -> str. The key is the key in the database, the value is the key in the output dict. __getitem__ returns a dict. transform_group: (a callable or a dict string -> callable) Transform(s) to be applied on a sample. 1) If it is a callable, it is applied to the sample. 2) If it is a dict, each key is matched to a key in the sample (after used dict), and callables are called on the respective elements. ''' self.batch_size = batch_size self.shuffle = shuffle if not op.exists(db_file): raise ValueError('db_file does not exist: %s' % db_file) self.mode = mode self.conn = utils.openConnection(db_file, mode, copy_to_memory) self.c = self.conn.cursor() self.c.execute('SELECT * FROM images WHERE %s ORDER BY imagefile' % where_image) self.image_entries = self.c.fetchall() self.imreader = backendMedia.MediaReader(rootdir=rootdir) self.where_object = where_object _checkUsedKeys(used_keys) self.used_keys = used_keys utils.checkTransformGroup(transform_group) self.transform_group = transform_group self.on_epoch_end()
def examineMatches(c, args): cv2.namedWindow("examineMatches") c.execute('SELECT DISTINCT(match) FROM matches') match_entries = c.fetchall() if args.shuffle: np.random.shuffle(match_entries) imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For parsing keys. key_reader = KeyReader(args.key_dict) index_match = 0 # Iterating over images, because for each image we want to show all objects. while True: # Until a user hits the key for the "exit" action. (match, ) = match_entries[index_match] c.execute( 'SELECT * FROM objects WHERE objectid IN ' '(SELECT objectid FROM matches WHERE match=?)', (match, )) object_entries = c.fetchall() logging.info('Found %d objects for match %d' % (len(object_entries), match)) images = [] for object_entry in object_entries: imagefile = backendDb.objectField(object_entry, 'imagefile') objectid = backendDb.objectField(object_entry, 'objectid') roi = backendDb.objectField(object_entry, 'roi') score = backendDb.objectField(object_entry, 'score') image = imreader.imread(imagefile) util.drawScoredRoi(image, roi, score=score) scale = float(args.winsize) / max(image.shape[0:2]) image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale) images.append(image) # Assume all images have the same size for now. # Assume there are not so many matches. image = np.hstack(images) # Display an image, wait for the key from user, and parse that key. cv2.imshow('examineMatches', image[:, :, ::-1]) action = key_reader.parse(cv2.waitKey(-1)) if action is None: # User pressed something which does not have an action. continue elif action == 'exit': break elif action == 'previous': index_match -= 1 elif action == 'next': index_match += 1 else: # User pressed something else which has an assigned action, # assume it is a new name. logging.info('Setting name "%s" to imagefile "%s"' % (action, imagefile)) c.execute('UPDATE images SET name=? WHERE imagefile=?' (action, imagefile)) index_match = index_match % len(match_entries) cv2.destroyWindow("examineMatches")
def filterObjectsAtBorder(c, args): def isPolygonAtBorder(polygon_entries, width, height, border_thresh_perc): xs = [backendDb.polygonField(p, 'x') for p in polygon_entries] ys = [backendDb.polygonField(p, 'y') for p in polygon_entries] border_thresh = (height + width) / 2 * border_thresh_perc dist_to_border = min(xs, [width - x for x in xs], ys, [height - y for y in ys]) num_too_close = sum([x < border_thresh for x in dist_to_border]) return num_too_close >= 2 def isRoiAtBorder(roi, width, height, border_thresh_perc): border_thresh = (height + width) / 2 * border_thresh_perc logging.debug('border_thresh: %f' % border_thresh) return min(roi[0], roi[1], height + 1 - roi[2], width + 1 - roi[3]) < border_thresh if args.with_display: imreader = backendMedia.MediaReader(rootdir=args.rootdir) # For the reference. c.execute('SELECT COUNT(1) FROM objects') num_before = c.fetchone()[0] c.execute('SELECT imagefile FROM images') for imagefile, in progressbar(c.fetchall()): if args.with_display: image = imreader.imread(imagefile) c.execute('SELECT width,height FROM images WHERE imagefile=?', (imagefile, )) (imwidth, imheight) = c.fetchone() c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, )) object_entries = c.fetchall() logging.debug('%d objects found for %s' % (len(object_entries), imagefile)) for object_entry in object_entries: for_deletion = False # Get all necessary entries. objectid = backendDb.objectField(object_entry, 'objectid') roi = backendDb.objectField(object_entry, 'roi') c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, )) polygon_entries = c.fetchall() # Find if the polygon or the roi is at the border, # polygon has preference over roi. if len(polygon_entries) > 0: if isPolygonAtBorder(polygon_entries, imwidth, imheight, args.border_thresh): logging.debug('border polygon %s' % str(polygon)) for_deletion = True elif roi is not None: if isRoiAtBorder(roi, imwidth, imheight, args.border_thresh): logging.debug('border roi %s' % str(roi)) for_deletion = True else: logging.error( 'Neither polygon, nor bbox is available for objectid %d' % objectid) # Draw polygon or roi. if args.with_display: if len(polygon_entries) > 0: polygon = [(backendDb.polygonField(p, 'x'), backendDb.polygonField(p, 'y')) for p in polygon_entries] util.drawScoredPolygon(image, polygon, score=(0 if for_deletion else 1)) elif roi is not None: util.drawScoredRoi(image, roi, score=(0 if for_deletion else 1)) # Delete if necessary if for_deletion: backendDb.deleteObject(c, objectid) if args.with_display: cv2.imshow('filterObjectsAtBorder', image[:, :, ::-1]) key = cv2.waitKey(-1) if key == 27: args.with_display = False cv2.destroyWindow('filterObjectsAtBorder') # For the reference. c.execute('SELECT COUNT(1) FROM objects') num_after = c.fetchone()[0] logging.info('Deleted %d out of %d objects.' % (num_before - num_after, num_before))
def filterObjectsByIntersection(c, args): def getRoiIntesection(rioi1, roi2): dy = min(roi1[2], roi2[2]) - max(roi1[0], roi2[0]) dx = min(roi1[3], roi2[3]) - max(roi1[1], roi2[1]) if dy <= 0 or dx <= 0: return 0 return dy * dx if args.with_display: imreader = backendMedia.MediaReader(rootdir=args.rootdir) c.execute('SELECT imagefile FROM images') for imagefile, in progressbar(c.fetchall()): if args.with_display: image = imreader.imread(imagefile) c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, )) object_entries = c.fetchall() logging.debug('%d objects found for %s' % (len(object_entries), imagefile)) good_objects = np.ones(shape=len(object_entries), dtype=bool) for iobject1, object_entry1 in enumerate(object_entries): #roi1 = _expandCarBbox_(object_entry1, args) roi1 = backendDb.objectField(object_entry1, 'roi') if roi1 is None: logging.error( 'No roi for objectid %d, intersection on polygons ' 'not implemented.', iobject1) continue area1 = (roi1[2] - roi1[0]) * (roi1[3] - roi1[1]) if area1 == 0: logging.warning('An object in %s has area 0. Will delete.' % imagefile) good_objects[iobject1] = False break for iobject2, object_entry2 in enumerate(object_entries): if iobject2 == iobject1: continue roi2 = backendDb.objectField(object_entry2, 'roi') if roi2 is None: continue intersection = getRoiIntesection(roi1, roi2) / float(area1) if intersection > args.intersection_thresh: good_objects[iobject1] = False break if args.with_display: image = imreader.imread(imagefile) util.drawScoredRoi(image, roi1, score=(1 if good_objects[iobject1] else 0)) for iobject2, object_entry2 in enumerate(object_entries): if iobject1 == iobject2: continue roi2 = backendDb.objectField(object_entry2, 'roi') if roi2 is None: continue util.drawScoredRoi(image, roi2, score=0.5) cv2.imshow('filterObjectsByIntersection', image[:, :, ::-1]) key = cv2.waitKey(-1) if key == 27: cv2.destroyWindow('filterObjectsByIntersection') args.with_display = 0 for object_entry, is_object_good in zip(object_entries, good_objects): if not is_object_good: backendDb.deleteObject( c, backendDb.objectField(object_entry, 'objectid'))
def importBdd(c, args): if args.with_display: imreader = backendMedia.MediaReader(args.rootdir) image_paths = sorted(glob(op.join(args.images_dir, '*.jpg'))) logging.info('Found %d JPG images in %s' % (len(image_paths), args.images_dir)) if args.detection_json: if not op.exists(args.detection_json): raise FileNotFoundError('Annotation file not found at "%s".' % args.detection_json) logging.info( 'Loading the json with annotations. This may take a few seconds.') with open(args.detection_json) as f: detections = json.load(f) # Dict with image name as the key. detections = {d['name']: d for d in detections} for image_path in progressbar(image_paths): filename = op.splitext(op.basename(image_path))[0] logging.debug('Processing image: "%s"' % filename) # Add image to the database. imheight, imwidth = backendMedia.getPictureSize(image_path) imagefile = op.relpath(image_path, args.rootdir) c.execute('INSERT INTO images(imagefile,width,height) VALUES (?,?,?)', (imagefile, imwidth, imheight)) if args.with_display: img = imreader.imread(imagefile) # Detection annotations. if args.detection_json: imagename = op.basename(imagefile) if imagename not in detections: logging.error('Cant find image name "%s" in "%s"', args.detection_json, imagename) continue detections_for_image = detections[imagename] image_properties = detections_for_image['attributes'] for object_ in detections_for_image['labels']: object_bddid = object_['id'] object_name = object_['category'] object_properties = { key: value for key, value in object_['attributes'].items() if value != 'none' } object_properties.update(image_properties) # Skip 3d object. TODO: import it to properties. if 'box3d' in object_: logging.warning('Will skip 3D object %d.' % object_bddid) continue # Get the bbox if exists. x1 = y1 = width = height = None if 'box2d' in object_: box2d = object_['box2d'] x1 = int(float(box2d['x1'])) y1 = int(float(box2d['y1'])) width = int(float(box2d['x2']) - x1) height = int(float(box2d['y2']) - y1) if args.with_display: roi = utilBoxes.bbox2roi((x1, y1, width, height)) util.drawScoredRoi(img, roi, object_name) c.execute( 'INSERT INTO objects(imagefile,x1,y1,width,height,name) ' 'VALUES (?,?,?,?,?,?)', (imagefile, x1, y1, width, height, object_name)) objectid = c.lastrowid # Get the polygon if it exists. if 'poly2d' in object_: if len(object_['poly2d']) > 1: assert 0, len(object_['poly2d']) polygon = object_['poly2d'][0] polygon_name = None if polygon['closed'] else 'open_loop' for pt in polygon['vertices']: c.execute( 'INSERT INTO polygons(objectid,x,y,name) ' 'VALUES (?,?,?,?)', (objectid, pt[0], pt[1], polygon_name)) if args.with_display: util.drawScoredPolygon(img, [(int(x[0]), int(x[1])) for x in polygon['vertices']], object_name) # Insert image-level and object-level attributes into # "properties" table. for key, value in object_properties.items(): c.execute( 'INSERT INTO properties(objectid,key,value) VALUES (?,?,?)', (objectid, key, value)) # Segmentation annotations. if args.segmentation_dir: segmentation_path = op.join(args.segmentation_dir, '%s_train_id.png' % filename) if not op.exists(segmentation_path): raise FileNotFoundError('Annotation file not found at "%s".' % segmentation_path) # Add image to the database. maskfile = op.relpath(segmentation_path, args.rootdir) c.execute('UPDATE images SET maskfile=? WHERE imagefile=?', (maskfile, imagefile)) if args.with_display: mask = imreader.maskread(maskfile) img = util.drawMaskAside(img, mask, labelmap=None) # Maybe display. if args.with_display: cv2.imshow('importKitti', img[:, :, ::-1]) if cv2.waitKey(-1) == 27: args.with_display = False cv2.destroyWindow('importKitti')