def filterBadImages(c, args):
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
num_deleted_imagefiles = 0
c.execute('SELECT imagefile,maskfile FROM images')
image_entries = c.fetchall()
if (isinstance(imreader, backendMedia.PictureReader)
and not args.force_single_thread):
# Can make it parallel.
max_workers = multiprocessing.cpu_count() - 1
logging.info('Running filtering with %d workers.', max_workers)
with concurrent.futures.ThreadPoolExecutor(max_workers) as executor:
futures = []
for imagefile, maskfile in progressbar(image_entries):
logging.debug('Imagefile "%s"', imagefile)
futures.append(
executor.submit(isImageOk, imreader, imagefile, maskfile))
for future in concurrent.futures.as_completed(futures):
if not future.result():
backendDb.deleteImage(c, imagefile)
num_deleted_imagefiles += 1
else:
logging.info('Running filtering in a single thread.')
for imagefile, maskfile in progressbar(image_entries):
logging.debug('Imagefile "%s"', imagefile)
if not isImageOk(imreader, imagefile, maskfile):
backendDb.deleteImage(c, imagefile)
num_deleted_imagefiles += 1
logging.info("Deleted %d image(s).", num_deleted_imagefiles)
def __init__(self,
db_file,
rootdir='.',
where_image='TRUE',
where_object='TRUE',
mode='r',
copy_to_memory=True,
used_keys=None,
transform_group=None):
'''
Args:
db_file: (string) A path to an sqlite3 database file.
rootdir: (string) A root path, is pre-appended to "imagefile"
entries of "images" table in the sqlite3 database.
where_image: (string) The WHERE part of the SQL query on the
"images" table, as in:
"SELECT * FROM images WHERE ${where_image};"
Allows to query only needed images.
where_object: (string) The WHERE part of the SQL query on the
"objects" table, as in:
"SELECT * FROM objects WHERE ${where_object};"
Allows to query only needed objects for each image.
mode: ("r" or "w") The readonly or write-read mode to open
the database. The default is "r", use "w" only if
you plan to call addRecord().
copy_to_memory: (bool) Copies database into memory.
Only for mode="r". Should be used for python2.
used_keys: (None or list of str)
Originally __getitem__ returns a dict with keys:
'image', 'mask', 'objects', 'imagefile', 'name',
'score' (see the comments to this class above).
Argument `used_keys` determines which of these keys
are needed, and which can be disposed of.
transform_group: (a callable or a dict string -> callable)
Transform(s) to be applied on a sample.
If it is a callable, it is applied to the sample.
If it is a dict, each key is matched to a key in the
sample, and callables are called on the respective
elements.
'''
self.mode = mode
self.conn = utils.openConnection(db_file, mode, copy_to_memory)
self.c = self.conn.cursor()
utils.checkWhereImage(where_image)
self.c.execute('SELECT * FROM images WHERE %s ORDER BY imagefile' %
where_image)
self.image_entries = self.c.fetchall()
self.imreader = backendMedia.MediaReader(rootdir=rootdir)
utils.checkWhereObject(where_object)
self.where_object = where_object
_checkUsedKeys(used_keys)
self.used_keys = used_keys
utils.checkTransformGroup(transform_group)
self.transform_group = transform_group
def test_imreadPicture(self):
reader = backendMedia.MediaReader(rootdir='.')
img_gt = skimage.io.imread('testdata/cars/images/000000.jpg')
img = reader.imread('testdata/cars/images/000000.jpg')
self.assertEqual(img.shape, img_gt.shape)
self.assertTrue((img == img_gt).all())
# Second read.
img = reader.imread('testdata/cars/images/000000.jpg')
self.assertEqual(img.shape, img_gt.shape)
self.assertTrue((img == img_gt).all())
def test_maskreadPicture(self):
reader = backendMedia.MediaReader(rootdir='.')
mask_gt = skimage.io.imread('testdata/cars/masks/000000.png')
mask = reader.maskread('testdata/cars/masks/000000.png')
self.assertEqual(mask.shape, mask_gt.shape)
self.assertLess(_diff(mask, mask_gt), 0.02)
#self.assertTrue((mask == mask_gt).all())
# Second read.
mask = reader.imread('testdata/cars/masks/000000.png')
self.assertEqual(mask.shape, mask_gt.shape)
self.assertLess(_diff(mask, mask_gt), 0.02)
def _object_dataset_function(db_file,
rootdir='.',
where_object='TRUE',
used_keys=None,
transform_group=None):
'''
Args:
db_file: (string) A path to an sqlite3 database file.
rootdir: (string) A root path, is pre-appended to "imagefile"
entries of "images" table in the sqlite3 database.
where_object: (string) The WHERE part of the SQL query on the
"objects" table, as in:
"SELECT * FROM objects WHERE ${where_object};"
Allows to query only needed objects.
used_keys: (a list of strings or None) If specified, use only
these keys for every sample, and discard the rest.
transform_group: A dict {string: callable}. Each key of this dict
should match a key in each sample, and the callables
are applied to the respective sample dict values.
'''
conn = utils.openConnection(db_file, 'r', copy_to_memory=False)
c = conn.cursor()
utils.checkWhereObject(where_object)
c.execute('SELECT * FROM objects WHERE %s ORDER BY objectid' %
where_object)
object_entries = c.fetchall()
imreader = backendMedia.MediaReader(rootdir=rootdir)
_checkUsedKeys(used_keys)
utils.checkTransformGroup(transform_group)
samples = []
logging.info('Loading samples...')
for index in range(len(object_entries)):
object_entry = object_entries[index]
sample = utils.buildObjectSample(object_entry, c, imreader)
if sample is None:
continue
sample = _filterKeys(used_keys, sample)
sample = utils.applyTransformGroup(transform_group, sample)
samples.append(sample)
logging.info('Loaded %d samples.', len(object_entries))
return samples
def __getitem__(self, index):
'''
Get the i-th image entry with all its objects.
Args:
index: the index of the image entry in the database.
Returns:
dictionary with the following keys.
{
'image': Numpy array.
'mask': Numpy array.
'imagefile' String.
'maskfile' String or None.
'name' String (e.g. when predicted by a classification model).
'objects' Dictionary with the same keys as field names
in table "objects" of the Shuffler schema.
}
'''
if index >= len(self):
raise IndexError("Index %d is greater than the dataframe size %d" %
(index, len(self)))
self.cursor.execute("SELECT imagefile,maskfile,name FROM images")
# TODO: figure out how to not load the whole database.
image_entries = self.cursor.fetchall()
imagefile, maskfile, name = image_entries[index]
cols = backendDb.getColumnsInTable(self.cursor, 'objects')
self.cursor.execute(
"SELECT %s FROM objects WHERE imagefile=?" % ','.join(cols),
(imagefile, ))
objects = self.cursor.fetchall()
imreader = backendMedia.MediaReader(rootdir=self.rootdir)
if imagefile is not None:
image = imreader.imread(imagefile)
if maskfile is not None:
mask = imreader.maskread(imagefile)
return {
'image': image,
'mask': mask,
'objects': [zip(cols, o) for o in objects],
'imagefile': imagefile,
'maskfile': maskfile,
'name': name
}
def __init__(self,
db_file,
rootdir='.',
where_object='TRUE',
mode='r',
copy_to_memory=True,
used_keys=None,
transform_group=None,
preload_samples=False):
'''
Args:
db_file: (string) A path to an sqlite3 database file.
rootdir: (string) A root path, is pre-appended to "imagefile"
entries of "images" table in the sqlite3 database.
where_object: (string) The WHERE part of the SQL query on the
"objects" table, as in:
"SELECT * FROM objects WHERE ${where_object};"
Allows to query only needed objects.
mode: ("r" or "w") The readonly or write-read mode to open
the database. The default is "r", use "w" only if
you plan to call addRecord().
copy_to_memory: (bool) Copies database into memory.
Only for mode="r". Should be used for python2.
used_keys: (a list of strings or None) If specified, use only
these keys for every sample, and discard the rest.
transform_group: ((1) a callable, or (2) a list of callables,
or (3) a dict {string: callable})
Transform(s) to be applied on a sample.
(1) A callable: It is applied to the sample.
(2) A list of callables: Each callable is applied
to the sample sequentially.
(3) A dict {string: callable}: Each key of this dict
should match a key in each sample, and the callables
are applied to the respective sample dict values.
preload_samples: (bool) If true, will try to preload all samples
(including images) into memory in __init__.
'''
self.mode = mode
self.conn = utils.openConnection(db_file, mode, copy_to_memory)
self.c = self.conn.cursor()
utils.checkWhereObject(where_object)
self.c.execute('SELECT * FROM objects WHERE %s ORDER BY objectid' %
where_object)
self.object_entries = self.c.fetchall()
self.imreader = backendMedia.MediaReader(rootdir=rootdir)
_checkUsedKeys(used_keys)
self.used_keys = used_keys
utils.checkTransformGroup(transform_group)
self.transform_group = transform_group
if not preload_samples:
self.preloaded_samples = None
else:
self.preloaded_samples = []
logging.info('Loading samples...')
for index in range(len(self)):
object_entry = self.object_entries[index]
sample = utils.buildObjectSample(object_entry, self.c,
self.imreader)
if sample is None:
logging.warning('Skip bad sample %d', index)
else:
self.preloaded_samples.append(sample)
logging.info('Loaded %d samples.', len(self))
def displayImagesPlt(c, args):
c.execute('SELECT * FROM images WHERE (%s)' % args.where_image)
image_entries = c.fetchall()
logging.info('%d images found.', len(image_entries))
if len(image_entries) == 0:
logging.error('There are no images. Exiting.')
return
if args.shuffle:
np.random.shuffle(image_entries)
if len(image_entries) < args.limit:
image_entries = image_entries[:args.limit]
def _getNumRows(total, ncols):
nrows = int((total - 1) / ncols) + 1
logging.info('Grid: %dx%d from the total of %d', nrows, ncols, total)
return nrows
if args.limit < len(image_entries):
image_entries = image_entries[:args.limit]
nrows = _getNumRows(len(image_entries), args.ncols)
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For overlaying masks.
labelmap = literal_eval(
args.mask_mapping_dict) if args.mask_mapping_dict else None
logging.info('Parsed mask_mapping_dict to %s', pformat(labelmap))
for i_image, image_entry in enumerate(image_entries):
ax = plt.subplot(nrows, args.ncols, i_image + 1)
imagefile = backendDb.imageField(image_entry, 'imagefile')
maskfile = backendDb.imageField(image_entry, 'maskfile')
imname = backendDb.imageField(image_entry, 'name')
imscore = backendDb.imageField(image_entry, 'score')
logging.info('Imagefile "%s"', imagefile)
logging.debug('Image name="%s", score=%s', imname, imscore)
image = imreader.imread(imagefile)
# Overlay the mask.
if maskfile is not None:
mask = imreader.maskread(maskfile)
if args.mask_aside:
image = util.drawMaskAside(image, mask, labelmap=labelmap)
elif args.mask_alpha is not None:
image = util.drawMaskOnImage(image,
mask,
alpha=args.mask_alpha,
labelmap=labelmap)
else:
logging.info('No mask for this image.')
# Put the objects on top of the image.
if args.with_objects:
c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, ))
object_entries = c.fetchall()
logging.info('Found %d objects for image %s', len(object_entries),
imagefile)
for object_entry in object_entries:
objectid = backendDb.objectField(object_entry, 'objectid')
roi = backendDb.objectField(object_entry, 'roi')
score = backendDb.objectField(object_entry, 'score')
name = backendDb.objectField(object_entry, 'name')
logging.info('objectid: %d, roi: %s, score: %s, name: %s',
objectid, roi, score, name)
c.execute('SELECT * FROM polygons WHERE objectid=?',
(objectid, ))
polygon_entries = c.fetchall()
if len(polygon_entries) > 0:
logging.info('showing object with a polygon.')
polygon = [(int(backendDb.polygonField(p, 'x')),
int(backendDb.polygonField(p, 'y')))
for p in polygon_entries]
drawScoredPolygon(ax, polygon, label=name, score=score)
elif roi is not None:
logging.info('showing object with a bounding box.')
drawScoredRoi(ax, roi, label=name, score=score)
else:
logging.warning(
'Neither polygon, nor bbox is available for objectid %d',
objectid)
# Overlay imagefile.
title = ""
if args.with_imagefile:
title += '%s ' % op.basename(
backendMedia.normalizeSeparators(imagefile))
# Overlay score.
if args.with_score:
title += '%.3f ' % imscore if imscore is not None else ''
ax.set_title(title)
# Display
ax.imshow(image)
ax.axis('off')
plt.tight_layout()
plt.show()
def evaluateBinarySegmentation(c, args):
import pandas as pd
# Get corresponding maskfiles from predictions and ground truth.
c.execute('ATTACH ? AS "attached"', (args.gt_db_file, ))
c.execute('SELECT pr.imagefile,pr.maskfile,gt.maskfile '
'FROM images pr INNER JOIN attached.images gt '
'WHERE pr.imagefile=gt.imagefile '
'AND pr.maskfile IS NOT NULL '
'AND gt.maskfile IS NOT NULL '
'AND %s '
'ORDER BY pr.imagefile ASC' % args.where_image)
entries = c.fetchall()
logging.info(
'Total %d images in both the open and the ground truth databases.' %
len(entries))
logging.debug(pprint.pformat(entries))
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
TPs = np.zeros((256, ), dtype=int)
FPs = np.zeros((256, ), dtype=int)
FNs = np.zeros((256, ), dtype=int)
if args.display_images_roc:
fig = plt.figure()
plt.xlabel('recall')
plt.ylabel('precision')
plt.xlim(0, 1)
plt.ylim(0, 1)
for imagefile, maskfile_pr, maskfile_gt in progressbar.progressbar(
entries):
# Load masks and bring them to comparable form.
mask_gt = imreader.maskread(maskfile_gt)
mask_pr = imreader.maskread(maskfile_pr)
mask_pr = cv2.resize(mask_pr, (mask_gt.shape[1], mask_gt.shape[0]),
cv2.INTER_NEAREST)
# Some printputs.
gt_pos = np.count_nonzero(mask_gt == 255)
gt_neg = np.count_nonzero(mask_gt == 0)
gt_other = mask_gt.size - gt_pos - gt_neg
logging.debug('GT: positive: %d, negative: %d, others: %d.', gt_pos,
gt_neg, gt_other)
# If there is torch.
try:
import torch
# Use only relevant pixels (not the 'dontcare' class.)
relevant = np.bitwise_or(mask_gt == 0, mask_gt == 255)
mask_gt = mask_gt[relevant].flatten()
mask_pr = mask_pr[relevant].flatten()
mask_gt = torch.Tensor(mask_gt)
mask_pr = torch.Tensor(mask_pr)
try:
mask_gt = mask_gt.cuda()
mask_pr = mask_pr.cuda()
except RuntimeError:
pass
TP = np.zeros((256, ), dtype=int)
FP = np.zeros((256, ), dtype=int)
FN = np.zeros((256, ), dtype=int)
for val in range(256):
tp = torch.nonzero(torch.mul(mask_pr > val,
mask_gt == 255)).size()[0]
fp = torch.nonzero(torch.mul(mask_pr > val,
mask_gt != 255)).size()[0]
fn = torch.nonzero(torch.mul(mask_pr <= val,
mask_gt == 255)).size()[0]
tn = torch.nonzero(torch.mul(mask_pr <= val,
mask_gt != 255)).size()[0]
TP[val] = tp
FP[val] = fp
FN[val] = fn
TPs[val] += tp
FPs[val] += fp
FNs[val] += fn
ROC, area = getPrecRecall(TP, FP, FN)
logging.info('%s\t%.2f' % (op.basename(imagefile), area * 100.))
except ImportError:
# TODO: write the same without torch, on CPU
raise NotImplementedError(
'Non-torch implementation is still to be implemented.')
if args.display_images_roc:
plt.plot(ROC[:, 0], ROC[:, 1], 'go-', linewidth=2, markersize=4)
plt.pause(0.05)
fig.show()
# Accumulate into Precision-Recall curve.
ROC, area = getPrecRecall(TPs, FPs, FNs)
print(
"Average across image area under the Precision-Recall curve, perc: %.2f"
% (area * 100.))
if args.out_dir is not None:
if not op.exists(args.out_dir):
os.makedirs(args.out_dir)
fig = plt.figure()
plt.xlabel('recall')
plt.ylabel('precision')
plt.xlim(0, 1)
plt.ylim(0, 1)
plt.plot(ROC[:, 0], ROC[:, 1], 'bo-', linewidth=2, markersize=6)
out_plot_path = op.join(args.out_dir,
'%srecall-prec.png' % args.out_prefix)
fig.savefig(out_plot_path,
transparent=True,
bbox_inches='tight',
pad_inches=0,
dpi=300)
def importCityscapes(c, args):
if args.with_display:
imreader = backendMedia.MediaReader(args.rootdir)
logging.info('Will load splits: %s' % args.splits)
logging.info('Will load json type: %s' % args.type)
logging.info('Will load mask type: %s' % args.mask_type)
if not op.exists(args.cityscapes_dir):
raise Exception('Cityscape directory "%s" does not exist' %
args.cityscapes_dir)
# Directory accessed by label type and by split.
dirs_by_typesplit = {}
for type_ in [args.type, 'leftImg8bit']:
type_dir_template = op.join(args.cityscapes_dir, '%s*' % type_)
for type_dir in [x for x in glob(type_dir_template) if op.isdir(x)]:
logging.debug('Looking for splits in %s' % type_dir)
for split in args.splits:
typesplit_dir = op.join(type_dir, split)
if op.exists(typesplit_dir):
logging.debug('Found split %s in %s' % (split, type_dir))
# Add the info into the main dictionary "dirs_by_typesplit".
if split not in dirs_by_typesplit:
dirs_by_typesplit[split] = {}
dirs_by_typesplit[split][type_] = typesplit_dir
logging.info('Found the following directories: \n%s' %
pformat(dirs_by_typesplit))
for split in args.splits:
# List of cities.
assert 'leftImg8bit' in dirs_by_typesplit[split]
leftImg8bit_dir = dirs_by_typesplit[split]['leftImg8bit']
cities = os.listdir(leftImg8bit_dir)
cities = [x for x in cities if op.isdir(op.join(leftImg8bit_dir, x))]
logging.info('Found %d cities in %s' % (len(cities), leftImg8bit_dir))
for city in cities:
image_names = os.listdir(op.join(leftImg8bit_dir, city))
logging.info('In split "%s", city "%s" has %d images' %
(split, city, len(image_names)))
for image_name in image_names:
# Get the image path.
image_path = op.join(leftImg8bit_dir, city, image_name)
name_parts = op.splitext(image_name)[0].split('_')
if len(name_parts) != 4:
raise Exception(
'Expect to have 4 parts in the name of image "%s"' %
image_name)
if name_parts[0] != city:
raise Exception('The first part of name of image "%s" '
'is expected to be city "%s".' %
(image_name, city))
if name_parts[3] != 'leftImg8bit':
raise Exception('The last part of name of image "%s" '
'is expected to be city "leftImg8bit".' %
image_name)
imheight, imwidth = backendMedia.getPictureSize(image_path)
imagefile = op.relpath(image_path, args.rootdir)
c.execute(
'INSERT INTO images(imagefile,width,height) VALUES (?,?,?)',
(imagefile, imwidth, imheight))
# Get the json label.
if args.type in dirs_by_typesplit[split]:
city_dir = op.join(dirs_by_typesplit[split][args.type],
city)
if op.exists(city_dir):
json_name = '_'.join(name_parts[:3] +
[args.type, 'polygons']) + '.json'
json_path = op.join(city_dir, json_name)
if op.exists(json_path):
_importJson(c, json_path, imagefile, imheight,
imwidth)
if args.mask_type is not None:
mask_name = '_'.join(
name_parts[:3] +
[args.type, args.mask_type]) + '.png'
mask_path = op.join(city_dir, mask_name)
if op.exists(mask_path):
maskfile = op.relpath(mask_path, args.rootdir)
c.execute(
'UPDATE images SET maskfile=? WHERE imagefile=?',
(maskfile, imagefile))
if args.with_display:
img = imreader.imread(imagefile)
c.execute(
'SELECT objectid,name FROM objects WHERE imagefile=?',
(imagefile, ))
for objectid, name in c.fetchall():
# Draw polygon.
c.execute('SELECT x,y FROM polygons WHERE objectid=?',
(objectid, ))
polygon = c.fetchall()
util.drawScoredPolygon(img, [(int(pt[0]), int(pt[1]))
for pt in polygon], name)
cv2.imshow('importCityscapes', img[:, :, ::-1])
if cv2.waitKey(-1) == 27:
args.with_display = False
cv2.destroyWindow('importCityscapes')
# Statistics.
c.execute('SELECT COUNT(1) FROM images')
logging.info('Imported %d images' % c.fetchone()[0])
c.execute('SELECT COUNT(1) FROM images WHERE maskfile IS NOT NULL')
logging.info('Imported %d masks' % c.fetchone()[0])
c.execute('SELECT COUNT(DISTINCT(imagefile)) FROM objects')
logging.info('Objects are found in %d images' % c.fetchone()[0])
def importKitti(c, args):
if args.with_display:
imreader = backendMedia.MediaReader(args.rootdir)
image_paths = sorted(glob(op.join(args.images_dir, '*.png')))
logging.info('Found %d PNG images in %s' %
(len(image_paths), args.images_dir))
for image_path in progressbar(image_paths):
filename = op.splitext(op.basename(image_path))[0]
logging.debug('Processing image: "%s"' % filename)
# Add image to the database.
imheight, imwidth = backendMedia.getPictureSize(image_path)
imagefile = op.relpath(image_path, args.rootdir)
c.execute('INSERT INTO images(imagefile,width,height) VALUES (?,?,?)',
(imagefile, imwidth, imheight))
if args.with_display:
img = imreader.imread(imagefile)
# Detection annotations.
if args.detection_dir:
detection_path = op.join(args.detection_dir, '%s.txt' % filename)
if not op.exists(detection_path):
raise FileNotFoundError('Annotation file not found at "%s".' %
detection_path)
# Read annotation file.
with open(detection_path) as f:
lines = f.read().splitlines()
logging.debug('Read %d lines from detection file "%s"' %
(len(lines), detection_path))
for line in lines:
objectid = _parseObject(c, line, imagefile)
if args.with_display:
c.execute('SELECT * FROM objects WHERE objectid=?',
(objectid, ))
object_entry = c.fetchone()
name = backendDb.objectField(object_entry, 'name')
roi = backendDb.objectField(object_entry, 'roi')
score = backendDb.objectField(object_entry, 'score')
util.drawScoredRoi(img, roi, name, score=score)
# Segmentation annotations.
if args.segmentation_dir:
segmentation_path = op.join(args.segmentation_dir,
'%s.png' % filename)
if not op.exists(segmentation_path):
raise FileNotFoundError('Annotation file not found at "%s".' %
segmentation_path)
# Add image to the database.
maskfile = op.relpath(segmentation_path, args.rootdir)
c.execute('UPDATE images SET maskfile=? WHERE imagefile=?',
(maskfile, imagefile))
if args.with_display:
mask = imreader.maskread(maskfile)
img = util.drawMaskAside(img, mask, labelmap=None)
# Maybe display.
if args.with_display:
cv2.imshow('importKitti', img[:, :, ::-1])
if cv2.waitKey(-1) == 27:
args.with_display = False
cv2.destroyWindow('importKitti')
def evaluateSegmentationIoU(c, args):
import pandas as pd
import matplotlib.pyplot as plt
# Get corresponding maskfiles from predictions and ground truth.
logging.info('Opening ground truth dataset: %s', args.gt_db_file)
c.execute('ATTACH ? AS "attached"', (args.gt_db_file, ))
c.execute('SELECT pr.imagefile,pr.maskfile,gt.maskfile '
'FROM images pr INNER JOIN attached.images gt '
'WHERE pr.imagefile=gt.imagefile AND pr.maskfile IS NOT NULL '
'AND gt.maskfile IS NOT NULL '
'AND %s '
'ORDER BY pr.imagefile ASC' % args.where_image)
entries = c.fetchall()
logging.info(
'Total %d images in both the open and the ground truth databases.',
len(entries))
logging.debug(pprint.pformat(entries))
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
labelmap_gt, labelmap_pr, class_names = _label2classMapping(
args.gt_mapping_dict, args.pred_mapping_dict)
if args.class_to_record_iou is not None and not args.class_to_record_iou in class_names:
raise ValueError(
'class_to_record_iou=%s is not among values of gt_mapping_dict=%s'
% (args.class_to_record_iou, args.gt_mapping_dict))
hist_all = np.zeros((len(class_names), len(class_names)))
for imagefile, maskfile_pr, maskfile_gt in progressbar.progressbar(
entries):
# Load masks and bring them to comparable form.
mask_gt = util.applyLabelMappingToMask(imreader.maskread(maskfile_gt),
labelmap_gt)
mask_pr = util.applyLabelMappingToMask(imreader.maskread(maskfile_pr),
labelmap_pr)
mask_pr = cv2.resize(mask_pr, (mask_gt.shape[1], mask_gt.shape[0]),
interpolation=cv2.INTER_NEAREST)
# Evaluate one image pair.
careabout = ~np.isnan(mask_gt)
mask_gt = mask_gt[careabout][:].astype(int)
mask_pr = mask_pr[careabout][:].astype(int)
hist = fast_hist(mask_gt, mask_pr, len(class_names))
hist_all += hist
# Compute and record results by image.
iou_list = per_class_iu(hist)
if args.class_to_record_iou is None:
iou = iou_list.mean()
else:
iou = iou_list[class_names.index(args.class_to_record_iou)]
c.execute('UPDATE images SET score=? WHERE imagefile=?',
(iou, imagefile))
# Get label distribution.
pr_per_class = hist_all.sum(0)
gt_per_class = hist_all.sum(1)
iou_list = per_class_iu(hist_all)
fwIoU = calc_fw_iu(hist_all)
pixAcc = calc_pixel_accuracy(hist_all)
mAcc = calc_mean_accuracy(hist_all)
result_df = pd.DataFrame({
'class': class_names,
'IoU': iou_list,
"pr_distribution": pr_per_class,
"gt_distribution": gt_per_class,
})
result_df["IoU"] *= 100 # Changing to percent ratio.
result_df.set_index("class", inplace=True)
print("---- info per class -----")
print(result_df)
result_ser = pd.Series({
"pixAcc": pixAcc,
"mAcc": mAcc,
"fwIoU": fwIoU,
"mIoU": iou_list.mean()
})
result_ser = result_ser[["pixAcc", "mAcc", "fwIoU", "mIoU"]]
result_ser *= 100 # change to percent ratio
if args.out_dir is not None:
if not op.exists(args.out_dir):
os.makedirs(args.out_dir)
out_summary_path = op.join(args.out_dir, args.out_summary_file)
logging.info('Will add summary to: %s', out_summary_path)
with open(out_summary_path, 'a') as f:
f.write(args.out_prefix + '\t' +
'\t'.join(['%.2f' % x for x in result_df['IoU']]) + '\n')
# Save confusion matrix
fig = plt.figure()
normalized_hist = (hist.astype("float") /
hist.sum(axis=1)[:, np.newaxis])
plot_confusion_matrix(normalized_hist, classes=class_names)
outfigfn = op.join(args.out_dir, "%sconf_mat.pdf" % args.out_prefix)
fig.savefig(outfigfn,
transparent=True,
bbox_inches='tight',
pad_inches=0,
dpi=300)
print("Confusion matrix was saved to %s" % outfigfn)
outdffn = op.join(args.out_dir,
"%seval_result_df.csv" % args.out_prefix)
result_df.to_csv(outdffn)
print('Info per class was saved at %s !' % outdffn)
outserfn = op.join(args.out_dir,
"%seval_result_ser.csv" % args.out_prefix)
result_ser.to_csv(outserfn)
print('Total result is saved at %s !' % outserfn)
def test_closeUninitWithoutError(self):
reader = backendMedia.MediaReader(rootdir='.')
reader.close()
def test_closePictureWithoutError(self):
reader = backendMedia.MediaReader(rootdir='.')
reader.imread('testdata/cars/images/000000.jpg')
reader.close()
def labelObjects(c, args):
cv2.namedWindow("labelObjects")
c.execute('SELECT COUNT(*) FROM objects WHERE (%s) ' % args.where_object)
logging.info('Found %d objects in db.' % c.fetchone()[0])
c.execute('SELECT * FROM objects WHERE (%s)' % args.where_object)
object_entries = c.fetchall()
logging.info('Found %d objects in db.' % len(object_entries))
if len(object_entries) == 0:
return
if args.shuffle:
np.random.shuffle(object_entries)
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For parsing keys.
key_reader = KeyReader(args.key_dict)
button = 0
index_object = 0
another_object = True
while button != 27:
go_next_object = False
if another_object:
another_object = False
logging.info(' ')
logging.info('Object %d out of %d' %
(index_object, len(object_entries)))
object_entry = object_entries[index_object]
objectid = backendDb.objectField(object_entry, 'objectid')
bbox = backendDb.objectField(object_entry, 'bbox')
roi = backendDb.objectField(object_entry, 'roi')
imagefile = backendDb.objectField(object_entry, 'imagefile')
logging.info('imagefile: %s' % imagefile)
image = imreader.imread(imagefile)
# Display an image, wait for the key from user, and parse that key.
scale = float(args.winsize) / max(image.shape[0:2])
logging.debug('Will resize image and annotations with scale: %f' %
scale)
image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale)
logging.info('objectid: %d, roi: %s' % (objectid, roi))
c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, ))
polygon_entries = c.fetchall()
if len(polygon_entries) > 0:
logging.info('showing object with a polygon.')
polygon = [(backendDb.polygonField(p, 'x'),
backendDb.polygonField(p, 'y'))
for p in polygon_entries]
logging.debug('nonscaled polygon: %s' % pformat(polygon))
polygon = [(int(scale * p[0]), int(scale * p[1]))
for p in polygon]
logging.debug('scaled polygon: %s' % pformat(polygon))
util.drawScoredPolygon(image, polygon, label=None, score=None)
elif roi is not None:
logging.info('showing object with a bounding box.')
logging.debug('nonscaled roi: %s' % pformat(roi))
roi = [int(scale * r)
for r in roi] # For displaying the scaled image.
logging.debug('scaled roi: %s' % pformat(roi))
util.drawScoredRoi(image, roi, label=None, score=None)
else:
raise Exception(
'Neither polygon, nor bbox is available for objectid %d' %
objectid)
c.execute(
'SELECT key,value FROM properties WHERE objectid=? AND key=?',
(objectid, args.property))
# TODO: Multiple properties are possible because there is no
# contraint on uniqueness on table properties(objectid,key).
# Change when the uniqueness constraint is added to the
# database schema. On the other hand, it's a feature.
properties = c.fetchall()
if len(properties) > 1:
logging.warning(
'Multiple values for object %s and property %s. '
'If reassigned, both will be changed' %
(objectid, args.property))
for iproperty, (key, value) in enumerate(properties):
cv2.putText(image, '%s: %s' % (key, value),
(10, util.SCALE * (iproperty + 1)), util.FONT,
util.FONT_SIZE, (0, 0, 0), util.THICKNESS)
cv2.putText(image, '%s: %s' % (key, value),
(10, util.SCALE * (iproperty + 1)), util.FONT,
util.FONT_SIZE, (255, 255, 255),
util.THICKNESS - 1)
logging.info('objectid: %d. %s = %s.' % (objectid, key, value))
cv2.imshow('labelObjects', image[:, :, ::-1])
action = key_reader.parse(cv2.waitKey(-1))
if action == 'exit':
break
elif action == 'delete_label' and any_object_in_focus:
logging.info('Remove label from objectid "%s"' % objectid)
c.execute('DELETE FROM properties WHERE objectid=? AND key=?',
(objectid, args.property))
go_next_object = True
elif action is not None and action not in ['previous', 'next']:
# User pressed something else which has an assigned action,
# assume it is a new value.
logging.info('Setting label "%s" to objectid "%s"' %
(action, objectid))
if len(properties) > 0:
c.execute('DELETE FROM properties WHERE objectid=? AND key=?',
(objectid, args.property))
c.execute(
'INSERT INTO properties(objectid,key,value) VALUES (?,?,?)',
(objectid, args.property, str(action)))
go_next_object = True
# Navigation.
if action == 'previous':
logging.debug('previous object')
another_object = True
if index_object > 0:
index_object -= 1
else:
logging.warning('Already at the first object.')
elif action == 'next' or go_next_object == True:
logging.debug('next object')
another_object = True
if index_object < len(object_entries) - 1:
index_object += 1
else:
logging.warning(
'Already at the last object. Press Esc to save and exit.')
cv2.destroyWindow("labelObjects")
def labelMatches(c, args):
cv2.namedWindow("labelMatches")
cv2.setMouseCallback('labelMatches', _monitorPressRelease)
global mousePressed, mouseReleased, xpress, ypress
c.execute('SELECT imagefile FROM images WHERE %s' % args.where_image)
image_entries = c.fetchall()
logging.debug('Found %d images' % len(image_entries))
if len(image_entries) < 2:
logging.error('Found only %d images. Quit.' % len(image_entries))
return
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For parsing keys.
key_reader = KeyReader(args.key_dict)
index_image = 1
action = None
while action != 'exit' and index_image < len(image_entries):
(imagefile1, ) = image_entries[index_image - 1]
(imagefile2, ) = image_entries[index_image]
img1 = imreader.imread(imagefile1)
img2 = imreader.imread(imagefile2)
# offset of the 2nd image, when they are stacked
yoffset = img1.shape[0]
# Make sure images have the same width, so that we can stack them.
# That will happen when the video changes, so we'll skip that pair.
if img1.shape[1] != img2.shape[1]:
logging.warning('Skipping image pair "%s" and "%s" '
'since they are of different width.' %
(imagefile1, imagefile2))
index_image += 1
# get objects from both images
c.execute('SELECT * FROM objects WHERE imagefile=? ', (imagefile1, ))
objects1 = c.fetchall()
logging.info('%d objects found for %s' % (len(objects1), imagefile1))
c.execute('SELECT * FROM objects WHERE imagefile=? ', (imagefile2, ))
objects2 = c.fetchall()
logging.info('%d objects found for %s' % (len(objects2), imagefile2))
# draw cars in both images
for object_ in objects1:
util.drawScoredRoi(img1, backendDb.objectField(object_, 'roi'))
for object_ in objects2:
util.drawScoredRoi(img2, backendDb.objectField(object_, 'roi'))
i1 = i2 = None # Matches selected with a mouse.
selectedMatch = None
needRedraw = True
action = None
# Stay in side the loop until a key is pressed.
while action is None:
img_stack = np.vstack((img1, img2))
if needRedraw:
# find existing matches, and make a map
matchesOf1 = {}
matchesOf2 = {}
for j1 in range(len(objects1)):
object1 = objects1[j1]
for j2 in range(len(objects2)):
object2 = objects2[j2]
c.execute(
'SELECT match FROM matches WHERE objectid = ? INTERSECT '
'SELECT match FROM matches WHERE objectid = ?',
(backendDb.objectField(object1, 'objectid'),
backendDb.objectField(object2, 'objectid')))
matches = c.fetchall()
if len(matches) > 0:
assert len(matches) == 1 # No duplicate matches.
roi1 = backendDb.objectField(object1, 'roi')
roi2 = backendDb.objectField(object2, 'roi')
_drawMatch(img_stack, roi1, roi2, yoffset)
matchesOf1[j1] = matches[0][0]
matchesOf2[j2] = matches[0][0]
# draw image
scale = float(args.winsize) / max(img_stack.shape[0:2])
img_show = cv2.resize(img_stack,
dsize=(0, 0),
fx=scale,
fy=scale)
cv2.imshow('labelMatches', img_show[:, :, ::-1])
logging.info('Will draw %d matches found between the pair' %
len(matchesOf1))
needRedraw = False
# process mouse callback effect (button has been pressed)
if mousePressed:
i2 = None # reset after the last unsuccessful match
logging.debug('Pressed x=%d, y=%d' % (xpress, ypress))
xpress /= scale
ypress /= scale
i1 = _findPressedObject(xpress, ypress, objects1)
if i1 is not None:
logging.debug('Found pressed object: %d' % i1)
mousePressed = False
# process mouse callback effect (button has been released)
if mouseReleased:
logging.debug('released x=%d, y=%d' % (xpress, ypress))
xpress /= scale
ypress /= scale
i2 = _findPressedObject(xpress, ypress - yoffset, objects2)
if i2 is not None:
logging.debug('Found released object: %d' % i2)
mouseReleased = False
# If we could find pressed and released objects, add match
if i1 is not None and i2 is not None:
# If one of the objects is already matched in this image pair, discard.
if i1 in matchesOf1 or i2 in matchesOf2:
logging.warning(
'One or two connected objects is already matched')
i1 = i2 = None
else:
# Add the match to the list.
objectid1 = backendDb.objectField(objects1[i1], 'objectid')
objectid2 = backendDb.objectField(objects2[i2], 'objectid')
logging.debug('i1 = %d, i2 = %d' % (i1, i2))
# Check if this object already in the matches.
c.execute('SELECT match FROM matches WHERE objectid=?',
(objectid1, ))
matches1 = c.fetchall()
c.execute('SELECT match FROM matches WHERE objectid=?',
(objectid2, ))
matches2 = c.fetchall()
if len(matches1) > 1 or len(matches2) > 1:
logging.error(
'One of the objectids %d, %d is in several matches.'
% (objectid1, objectid2))
continue
elif len(matches1) == 1 and len(matches2) == 1:
logging.info(
'Will merge matches of objectids %d and %d' %
(objectid1, objectid2))
c.execute('UPDATE matches SET match=? WHERE match=?',
(matches1[0][0], matches2[0][0]))
elif len(matches1) == 1 and len(matches2) == 0:
logging.info('Add objectid %d to match %d' %
(objectid2, matches1[0][0]))
c.execute(
'INSERT INTO matches(match, objectid) VALUES (?,?)',
(matches1[0][0], objectid2))
elif len(matches1) == 0 and len(matches2) == 1:
logging.info('Add objectid %d to match %d' %
(objectid1, matches2[0][0]))
c.execute(
'INSERT INTO matches(match, objectid) VALUES (?,?)',
(matches2[0][0], objectid1))
elif len(matches1) == 0 and len(matches2) == 0:
logging.info(
'Add a new match between objectids %d and %d.' %
(objectid1, objectid2))
# Find a free match index
c.execute('SELECT MAX(match) FROM matches')
maxmatch = c.fetchone()[0]
match = int(
maxmatch) + 1 if maxmatch is not None else 1
c.execute(
'INSERT INTO matches(match, objectid) VALUES (?,?)',
(match, objectid1))
c.execute(
'INSERT INTO matches(match, objectid) VALUES (?,?)',
(match, objectid2))
else:
assert False
# Reset when a new match is made.
needRedraw = True
i1 = i2 = None
# Stay inside the loop inside one image pair until some button is pressed
action = key_reader.parse(cv2.waitKey(50))
# Process pressed key (all except exit)
if action == 'previous':
logging.info('Previous image pair')
if index_image == 1:
logging.warning('already the first image pair')
else:
index_image -= 1
elif action == 'next':
logging.info('Next image pair')
index_image += 1 # exit at last image pair from outer loop
elif action == 'delete_match':
# if any car was selected, and it is matched
if i1 is not None and i1 in matchesOf1:
match = matchesOf1[i1]
objectid1 = backendDb.objectField(objects1[i1], 'objectid')
logging.info('deleting match %d' % match)
c.execute(
'DELETE FROM matches WHERE match = ? AND objectid = ?',
(match, objectid1))
else:
logging.debug('delete is pressed, but no match is selected')
cv2.destroyWindow("labelMatches")
def examineImages(c, args):
cv2.namedWindow("examineImages")
c.execute('SELECT * FROM images WHERE (%s)' % args.where_image)
image_entries = c.fetchall()
logging.info('%d images found.' % len(image_entries))
if len(image_entries) == 0:
logging.error('There are no images. Exiting.')
return
if args.shuffle:
np.random.shuffle(image_entries)
if args.snapshot_dir and not op.exists(args.snapshot_dir):
os.makedirs(args.snapshot_dir)
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For parsing keys.
key_reader = KeyReader(args.key_dict)
# For overlaying masks.
labelmap = literal_eval(
args.mask_mapping_dict) if args.mask_mapping_dict else None
logging.info('Parsed mask_mapping_dict to %s' % pformat(labelmap))
index_image = 0
while True: # Until a user hits the key for the "exit" action.
image_entry = image_entries[index_image]
imagefile = backendDb.imageField(image_entry, 'imagefile')
maskfile = backendDb.imageField(image_entry, 'maskfile')
imname = backendDb.imageField(image_entry, 'name')
imscore = backendDb.imageField(image_entry, 'score')
logging.info('Imagefile "%s"' % imagefile)
logging.debug('Image name="%s", score=%s' % (imname, imscore))
image = imreader.imread(imagefile)
# Overlay the mask.
if maskfile is not None:
mask = imreader.maskread(maskfile)
if args.mask_aside:
image = util.drawMaskAside(image, mask, labelmap=labelmap)
elif args.mask_alpha is not None:
image = util.drawMaskOnImage(image,
mask,
alpha=args.mask_alpha,
labelmap=labelmap)
else:
logging.info('No mask for this image.')
# Put the objects on top of the image.
if args.with_objects:
c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, ))
object_entries = c.fetchall()
logging.info('Found %d objects for image %s' %
(len(object_entries), imagefile))
for object_entry in object_entries:
objectid = backendDb.objectField(object_entry, 'objectid')
roi = backendDb.objectField(object_entry, 'roi')
score = backendDb.objectField(object_entry, 'score')
name = backendDb.objectField(object_entry, 'name')
logging.info('objectid: %d, roi: %s, score: %s, name: %s' %
(objectid, roi, score, name))
c.execute('SELECT * FROM polygons WHERE objectid=?',
(objectid, ))
polygon_entries = c.fetchall()
if len(polygon_entries) > 0:
logging.info('showing object with a polygon.')
polygon = [(int(backendDb.polygonField(p, 'x')),
int(backendDb.polygonField(p, 'y')))
for p in polygon_entries]
util.drawScoredPolygon(image,
polygon,
label=name,
score=score)
elif roi is not None:
logging.info('showing object with a bounding box.')
util.drawScoredRoi(image, roi, label=name, score=score)
else:
logging.warning(
'Neither polygon, nor bbox is available for objectid %d'
% objectid)
# Display an image, wait for the key from user, and parse that key.
scale = float(args.winsize) / max(list(image.shape[0:2]))
scaled_image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale)
# Overlay imagefile.
if args.with_imagefile:
util.drawTextOnImage(
scaled_image,
op.basename(backendMedia.normalizeSeparators(imagefile)))
# Overlay score.
# TODO: add y offset, if necessary
if args.with_score:
util.drawTextOnImage(scaled_image, '%.3f' % imscore)
# Display
cv2.imshow('examineImages', scaled_image[:, :, ::-1])
action = key_reader.parse(cv2.waitKey(-1))
if action is None:
# User pressed something which does not have an action.
continue
elif action == 'snapshot':
if args.snapshot_dir:
snaphot_path = op.join(args.snapshot_dir,
'%08d.png' % index_image)
logging.info('Making a snapshot at path: %s' % snaphot_path)
imageio.imwrite(snaphot_path, image)
else:
logging.warning(
'The user pressed a snapshot key, but snapshot_dir is not '
'specified. Will not write a snapshot.')
elif action == 'delete':
backendDb.deleteImage(c, imagefile)
del image_entries[index_image]
if len(image_entries) == 0:
logging.warning('Deleted the last image.')
break
index_image += 1
elif action == 'exit':
break
elif action == 'previous':
index_image -= 1
elif action == 'next':
index_image += 1
else:
# User pressed something else which has an assigned action, assume it is a new name.
logging.info('Setting name "%s" to imagefile "%s"' %
(action, imagefile))
c.execute('UPDATE images SET name=? WHERE imagefile=?' (action,
imagefile))
index_image = index_image % len(image_entries)
cv2.destroyWindow("examineImages")
def test_maskreadPictureRootdir(self):
reader = backendMedia.MediaReader(rootdir='testdata/cars')
mask_gt = skimage.io.imread('testdata/cars/masks/000000.png')
mask = reader.maskread('masks/000000.png')
self.assertEqual(mask.shape, mask_gt.shape)
self.assertLess(_diff(mask, mask_gt), 0.02)
def labelAzimuth(c, args):
from lib.backendScenes import Window
from scenes.lib.cvScrollZoomWindow import Window
from scenes.lib.homography import getFrameFlattening, getHfromPose
from scenes.lib.cache import PoseCache, MapsCache
from scenes.lib.warp import transformPoint
os.environ['SCENES_PATH'] = args.scenes_dir
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
key_reader = dbGui.KeyReader(args.key_dict)
c.execute('SELECT * FROM objects WHERE (%s)' % args.where_object)
object_entries = c.fetchall()
logging.info('Found %d objects in db.' % len(object_entries))
if len(object_entries) == 0:
return
if args.shuffle:
np.random.shuffle(object_entries)
# Cached poses and azimuth maps.
topdown_azimuths = MapsCache('topdown_azimuth')
poses = PoseCache()
button = 0
index_object = 0
another_object = True
char_list = []
while button != 27:
go_next_object = False
update_yaw_in_db = False
if another_object:
another_object = False
logging.info(' ')
logging.info('Object %d out of %d' %
(index_object, len(object_entries)))
object_entry = object_entries[index_object]
objectid = backendDb.objectField(object_entry, 'objectid')
bbox = backendDb.objectField(object_entry, 'bbox')
roi = backendDb.objectField(object_entry, 'roi')
imagefile = backendDb.objectField(object_entry, 'imagefile')
# Update yaw inside the loop in case it was just assigned.
c.execute(
'SELECT value FROM properties WHERE objectid=? AND key="yaw"',
(objectid, ))
yaw = c.fetchone()
yaw = float(yaw[0]) if yaw is not None else None
y, x = roi[0] * 0.3 + roi[2] * 0.7, roi[1] * 0.5 + roi[3] * 0.5
try:
flattening = _getFlatteningFromImagefile(
poses, imagefile, y, x)
except:
# A hack that allowed me write all the images into one dir but
# keep info about imagefile.
c.execute('SELECT name FROM images WHERE imagefile=?',
(imagefile, ))
image_name = c.fetchone()[0]
flattening = _getFlatteningFromImagefile(
poses, image_name, y, x)
axis_x = np.linalg.norm(np.asarray(bbox[2:4]), ord=2)
axis_y = axis_x * flattening
display = imreader.imread(imagefile)
window = AzimuthWindow(display,
x,
y,
axis_x,
axis_y,
winsize=args.winsize)
if yaw is not None:
logging.info('Yaw is: %.0f' % yaw)
window.yaw = yaw
window.update_cached_zoomed_img()
window.redraw()
button = cv2.waitKey(50)
action = key_reader.parse(button) if button is not -1 else None
if action == 'delete':
c.execute('DELETE FROM properties WHERE objectid=? AND key="yaw"',
(objectid, ))
logging.info('Yaw is deleted.')
go_next_object = True
char_list = []
# Entry in keyboard.
if button >= ord('0') and button <= ord('9') or button == ord('.'):
char_list += chr(button)
logging.debug('Added %s character to number and got %s' %
(chr(button), ''.join(char_list)))
# Enter accepts GUI or keyboard entry.
elif button == 13:
if char_list: # After keyboard entry.
number_str = ''.join(char_list)
char_list = []
try:
logging.info('Accepting entry from the keyboard.')
yaw = float(number_str)
update_yaw_in_db = True
go_next_object = True
except ValueError:
logging.warning('Could not convert entered %s to number.' %
number_str)
continue
else: # Just navigation.
logging.info('A navigation Enter.')
go_next_object = True
# Entry in GUI.
elif window.selected == True:
logging.info('Accepting entry from GUI.')
yaw = window.yaw
update_yaw_in_db = True
go_next_object = True
# No entry:
else:
yaw = None
# Entry happened one way or the other.
# Update the yaw and go to the next object.
if update_yaw_in_db:
c.execute(
'SELECT COUNT(1) FROM properties WHERE objectid=? AND key="yaw"',
(objectid, ))
if c.fetchone()[0] == 0:
c.execute(
'INSERT INTO properties(value,key,objectid) VALUES (?,"yaw",?)',
(str(yaw), objectid))
else:
c.execute(
'UPDATE properties SET value=? WHERE objectid=? AND key="yaw"',
(str(yaw), objectid))
logging.info('Yaw is assigned to %.f' % yaw)
# Navigation.
if action == 'previous':
logging.debug('previous object')
if index_object > 0:
index_object -= 1
another_object = True
else:
logging.warning('Already at the first object.')
elif action == 'next' or go_next_object == True:
logging.debug('next object')
if index_object < len(object_entries) - 1:
index_object += 1
another_object = True
else:
logging.warning(
'Already at the last object. Press Esc to save and exit.')
def test_imreadNotPictureOrVideoRootdir(self):
reader = backendMedia.MediaReader(rootdir='testdata/cars')
with self.assertRaises(TypeError):
reader.imread('images/000005.txt')
def test_imreadNotExistingRootdir(self):
reader = backendMedia.MediaReader(rootdir='testdata/cars')
with self.assertRaises(TypeError):
reader.imread('noExisting.jpg')
def examineObjects(c, args):
cv2.namedWindow("examineObjects")
c.execute('SELECT COUNT(*) FROM objects WHERE (%s) ' % args.where_object)
logging.info('Found %d objects in db.' % c.fetchone()[0])
c.execute('SELECT DISTINCT imagefile FROM objects WHERE (%s) ' %
args.where_object)
image_entries = c.fetchall()
logging.info('%d images found.' % len(image_entries))
if len(image_entries) == 0:
logging.error('There are no images. Exiting.')
return
if args.shuffle:
np.random.shuffle(image_entries)
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For parsing keys.
key_reader = KeyReader(args.key_dict)
index_image = 0
index_object = 0
# Iterating over images, because for each image we want to show all objects.
while True: # Until a user hits the key for the "exit" action.
(imagefile, ) = image_entries[index_image]
logging.info('Imagefile "%s"' % imagefile)
image = imreader.imread(imagefile)
scale = float(args.winsize) / max(image.shape[0:2])
image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale)
c.execute(
'SELECT * FROM objects WHERE imagefile=? AND (%s)' %
args.where_object, (imagefile, ))
object_entries = c.fetchall()
logging.info('Found %d objects for image %s' %
(len(object_entries), imagefile))
# Put the objects on top of the image.
if len(object_entries) > 0:
assert index_object < len(object_entries)
object_entry = object_entries[index_object]
objectid = backendDb.objectField(object_entry, 'objectid')
roi = backendDb.objectField(object_entry, 'roi')
score = backendDb.objectField(object_entry, 'score')
name = backendDb.objectField(object_entry, 'name')
scaledroi = [int(scale * r)
for r in roi] # For displaying the scaled image.
logging.info('objectid: %d, roi: %s, score: %s, name: %s' %
(objectid, roi, score, name))
c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, ))
polygon_entries = c.fetchall()
if len(polygon_entries) > 0:
logging.info('showing object with a polygon.')
polygon = [(backendDb.polygonField(p, 'x'),
backendDb.polygonField(p, 'y'))
for p in polygon_entries]
logging.debug('nonscaled polygon: %s' % pformat(polygon))
polygon = [(int(scale * x), int(scale * y))
for x, y in polygon]
logging.debug('scaled polygon: %s' % pformat(polygon))
util.drawScoredPolygon(image, polygon, label=None, score=score)
elif roi is not None:
logging.info('showing object with a bounding box.')
util.drawScoredRoi(image, scaledroi, label=None, score=score)
else:
raise Exception(
'Neither polygon, nor bbox is available for objectid %d' %
objectid)
c.execute('SELECT key,value FROM properties WHERE objectid=?',
(objectid, ))
properties = c.fetchall()
if name is not None:
properties.append(('name', name))
if score is not None:
properties.append(('score', score))
for iproperty, (key, value) in enumerate(properties):
cv2.putText(image, '%s: %s' % (key, value),
(scaledroi[3] + 10,
scaledroi[0] - 10 + util.SCALE * (iproperty + 1)),
util.FONT, util.FONT_SIZE, (0, 0, 0),
util.THICKNESS)
cv2.putText(image, '%s: %s' % (key, value),
(scaledroi[3] + 10,
scaledroi[0] - 10 + util.SCALE * (iproperty + 1)),
util.FONT, util.FONT_SIZE, (255, 255, 255),
util.THICKNESS - 1)
logging.info('objectid: %d. %s = %s.' % (objectid, key, value))
# Display an image, wait for the key from user, and parse that key.
cv2.imshow('examineObjects', image[:, :, ::-1])
action = key_reader.parse(cv2.waitKey(-1))
if action is None:
# User pressed something which does not have an action.
continue
elif action == 'exit':
break
elif action == 'previous':
index_object -= 1
if index_object < 0:
index_image -= 1
index_object = 0
elif action == 'next':
index_object += 1
if index_object >= len(object_entries):
index_image += 1
index_object = 0
elif action == 'delete' and len(object_entries) > 0:
backendDb.deleteObject(c, objectid)
del object_entries[index_object]
if index_object >= len(object_entries):
index_image += 1
index_object = 0
elif action == 'unname' and len(object_entries) > 0:
logging.info('Remove the name from objectid "%s"' % objectid)
c.execute('UPDATE objects SET name=NULL WHERE objectid=?',
(objectid, ))
index_image = index_image % len(image_entries)
cv2.destroyWindow("examineObjects")
def __init__(self,
db_file,
rootdir='.',
where_image='TRUE',
where_object='TRUE',
mode='r',
copy_to_memory=True,
used_keys=None,
transform_group=None,
batch_size=1,
shuffle=False):
'''
Args:
db_file: (string) A path to an sqlite3 database file.
rootdir: (string) A root path, is pre-appended to "imagefile"
entries of "images" table in the sqlite3 database.
where_image: (string) The WHERE part of the SQL query on the
"images" table, as in:
"SELECT * FROM images WHERE ${where_image};"
Allows to query only needed images.
where_object: (string) The WHERE part of the SQL query on the
"objects" table, as in:
"SELECT * FROM objects WHERE ${where_object};"
Allows to query only needed objects for each image.
mode: ("r" or "w") The readonly or write-read mode to open
the database. The default is "r", use "w" only if
you plan to call addRecord().
copy_to_memory: (bool) Copies database into memory.
Only for mode="r". Should be used for python2.
used_keys: (None, list of str, tuple of str, or dict str -> str)
Originally __getitem__ returns a dict with keys:
'image', 'mask', 'objects', 'imagefile', 'name',
'score' (see the comments to this class above).
Argument `used_keys` determines which of these keys
are needed, and which can be disposed of.
Options for `used_keys`.
1) None. Each sample is unchanged dict.
2) List of str. Each str is a key.
__getitem__ returns a list.
3) Tuple of str. Same as above.
__getitem__ returns a tuple.
4) Dict str -> str. The key is the key in the
database, the value is the key in the output dict.
__getitem__ returns a dict.
transform_group: (a callable or a dict string -> callable)
Transform(s) to be applied on a sample.
1) If it is a callable, it is applied to the sample.
2) If it is a dict, each key is matched to a key in
the sample (after used dict), and callables are
called on the respective elements.
'''
self.batch_size = batch_size
self.shuffle = shuffle
if not op.exists(db_file):
raise ValueError('db_file does not exist: %s' % db_file)
self.mode = mode
self.conn = utils.openConnection(db_file, mode, copy_to_memory)
self.c = self.conn.cursor()
self.c.execute('SELECT * FROM images WHERE %s ORDER BY imagefile' %
where_image)
self.image_entries = self.c.fetchall()
self.imreader = backendMedia.MediaReader(rootdir=rootdir)
self.where_object = where_object
_checkUsedKeys(used_keys)
self.used_keys = used_keys
utils.checkTransformGroup(transform_group)
self.transform_group = transform_group
self.on_epoch_end()
def examineMatches(c, args):
cv2.namedWindow("examineMatches")
c.execute('SELECT DISTINCT(match) FROM matches')
match_entries = c.fetchall()
if args.shuffle:
np.random.shuffle(match_entries)
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For parsing keys.
key_reader = KeyReader(args.key_dict)
index_match = 0
# Iterating over images, because for each image we want to show all objects.
while True: # Until a user hits the key for the "exit" action.
(match, ) = match_entries[index_match]
c.execute(
'SELECT * FROM objects WHERE objectid IN '
'(SELECT objectid FROM matches WHERE match=?)', (match, ))
object_entries = c.fetchall()
logging.info('Found %d objects for match %d' %
(len(object_entries), match))
images = []
for object_entry in object_entries:
imagefile = backendDb.objectField(object_entry, 'imagefile')
objectid = backendDb.objectField(object_entry, 'objectid')
roi = backendDb.objectField(object_entry, 'roi')
score = backendDb.objectField(object_entry, 'score')
image = imreader.imread(imagefile)
util.drawScoredRoi(image, roi, score=score)
scale = float(args.winsize) / max(image.shape[0:2])
image = cv2.resize(image, dsize=(0, 0), fx=scale, fy=scale)
images.append(image)
# Assume all images have the same size for now.
# Assume there are not so many matches.
image = np.hstack(images)
# Display an image, wait for the key from user, and parse that key.
cv2.imshow('examineMatches', image[:, :, ::-1])
action = key_reader.parse(cv2.waitKey(-1))
if action is None:
# User pressed something which does not have an action.
continue
elif action == 'exit':
break
elif action == 'previous':
index_match -= 1
elif action == 'next':
index_match += 1
else:
# User pressed something else which has an assigned action,
# assume it is a new name.
logging.info('Setting name "%s" to imagefile "%s"' %
(action, imagefile))
c.execute('UPDATE images SET name=? WHERE imagefile=?' (action,
imagefile))
index_match = index_match % len(match_entries)
cv2.destroyWindow("examineMatches")
def filterObjectsAtBorder(c, args):
def isPolygonAtBorder(polygon_entries, width, height, border_thresh_perc):
xs = [backendDb.polygonField(p, 'x') for p in polygon_entries]
ys = [backendDb.polygonField(p, 'y') for p in polygon_entries]
border_thresh = (height + width) / 2 * border_thresh_perc
dist_to_border = min(xs, [width - x for x in xs], ys,
[height - y for y in ys])
num_too_close = sum([x < border_thresh for x in dist_to_border])
return num_too_close >= 2
def isRoiAtBorder(roi, width, height, border_thresh_perc):
border_thresh = (height + width) / 2 * border_thresh_perc
logging.debug('border_thresh: %f' % border_thresh)
return min(roi[0], roi[1], height + 1 - roi[2],
width + 1 - roi[3]) < border_thresh
if args.with_display:
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
# For the reference.
c.execute('SELECT COUNT(1) FROM objects')
num_before = c.fetchone()[0]
c.execute('SELECT imagefile FROM images')
for imagefile, in progressbar(c.fetchall()):
if args.with_display:
image = imreader.imread(imagefile)
c.execute('SELECT width,height FROM images WHERE imagefile=?',
(imagefile, ))
(imwidth, imheight) = c.fetchone()
c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, ))
object_entries = c.fetchall()
logging.debug('%d objects found for %s' %
(len(object_entries), imagefile))
for object_entry in object_entries:
for_deletion = False
# Get all necessary entries.
objectid = backendDb.objectField(object_entry, 'objectid')
roi = backendDb.objectField(object_entry, 'roi')
c.execute('SELECT * FROM polygons WHERE objectid=?', (objectid, ))
polygon_entries = c.fetchall()
# Find if the polygon or the roi is at the border,
# polygon has preference over roi.
if len(polygon_entries) > 0:
if isPolygonAtBorder(polygon_entries, imwidth, imheight,
args.border_thresh):
logging.debug('border polygon %s' % str(polygon))
for_deletion = True
elif roi is not None:
if isRoiAtBorder(roi, imwidth, imheight, args.border_thresh):
logging.debug('border roi %s' % str(roi))
for_deletion = True
else:
logging.error(
'Neither polygon, nor bbox is available for objectid %d' %
objectid)
# Draw polygon or roi.
if args.with_display:
if len(polygon_entries) > 0:
polygon = [(backendDb.polygonField(p, 'x'),
backendDb.polygonField(p, 'y'))
for p in polygon_entries]
util.drawScoredPolygon(image,
polygon,
score=(0 if for_deletion else 1))
elif roi is not None:
util.drawScoredRoi(image,
roi,
score=(0 if for_deletion else 1))
# Delete if necessary
if for_deletion:
backendDb.deleteObject(c, objectid)
if args.with_display:
cv2.imshow('filterObjectsAtBorder', image[:, :, ::-1])
key = cv2.waitKey(-1)
if key == 27:
args.with_display = False
cv2.destroyWindow('filterObjectsAtBorder')
# For the reference.
c.execute('SELECT COUNT(1) FROM objects')
num_after = c.fetchone()[0]
logging.info('Deleted %d out of %d objects.' %
(num_before - num_after, num_before))
def filterObjectsByIntersection(c, args):
def getRoiIntesection(rioi1, roi2):
dy = min(roi1[2], roi2[2]) - max(roi1[0], roi2[0])
dx = min(roi1[3], roi2[3]) - max(roi1[1], roi2[1])
if dy <= 0 or dx <= 0: return 0
return dy * dx
if args.with_display:
imreader = backendMedia.MediaReader(rootdir=args.rootdir)
c.execute('SELECT imagefile FROM images')
for imagefile, in progressbar(c.fetchall()):
if args.with_display:
image = imreader.imread(imagefile)
c.execute('SELECT * FROM objects WHERE imagefile=?', (imagefile, ))
object_entries = c.fetchall()
logging.debug('%d objects found for %s' %
(len(object_entries), imagefile))
good_objects = np.ones(shape=len(object_entries), dtype=bool)
for iobject1, object_entry1 in enumerate(object_entries):
#roi1 = _expandCarBbox_(object_entry1, args)
roi1 = backendDb.objectField(object_entry1, 'roi')
if roi1 is None:
logging.error(
'No roi for objectid %d, intersection on polygons '
'not implemented.', iobject1)
continue
area1 = (roi1[2] - roi1[0]) * (roi1[3] - roi1[1])
if area1 == 0:
logging.warning('An object in %s has area 0. Will delete.' %
imagefile)
good_objects[iobject1] = False
break
for iobject2, object_entry2 in enumerate(object_entries):
if iobject2 == iobject1:
continue
roi2 = backendDb.objectField(object_entry2, 'roi')
if roi2 is None:
continue
intersection = getRoiIntesection(roi1, roi2) / float(area1)
if intersection > args.intersection_thresh:
good_objects[iobject1] = False
break
if args.with_display:
image = imreader.imread(imagefile)
util.drawScoredRoi(image,
roi1,
score=(1 if good_objects[iobject1] else 0))
for iobject2, object_entry2 in enumerate(object_entries):
if iobject1 == iobject2:
continue
roi2 = backendDb.objectField(object_entry2, 'roi')
if roi2 is None:
continue
util.drawScoredRoi(image, roi2, score=0.5)
cv2.imshow('filterObjectsByIntersection', image[:, :, ::-1])
key = cv2.waitKey(-1)
if key == 27:
cv2.destroyWindow('filterObjectsByIntersection')
args.with_display = 0
for object_entry, is_object_good in zip(object_entries, good_objects):
if not is_object_good:
backendDb.deleteObject(
c, backendDb.objectField(object_entry, 'objectid'))
def importBdd(c, args):
if args.with_display:
imreader = backendMedia.MediaReader(args.rootdir)
image_paths = sorted(glob(op.join(args.images_dir, '*.jpg')))
logging.info('Found %d JPG images in %s' %
(len(image_paths), args.images_dir))
if args.detection_json:
if not op.exists(args.detection_json):
raise FileNotFoundError('Annotation file not found at "%s".' %
args.detection_json)
logging.info(
'Loading the json with annotations. This may take a few seconds.')
with open(args.detection_json) as f:
detections = json.load(f)
# Dict with image name as the key.
detections = {d['name']: d for d in detections}
for image_path in progressbar(image_paths):
filename = op.splitext(op.basename(image_path))[0]
logging.debug('Processing image: "%s"' % filename)
# Add image to the database.
imheight, imwidth = backendMedia.getPictureSize(image_path)
imagefile = op.relpath(image_path, args.rootdir)
c.execute('INSERT INTO images(imagefile,width,height) VALUES (?,?,?)',
(imagefile, imwidth, imheight))
if args.with_display:
img = imreader.imread(imagefile)
# Detection annotations.
if args.detection_json:
imagename = op.basename(imagefile)
if imagename not in detections:
logging.error('Cant find image name "%s" in "%s"',
args.detection_json, imagename)
continue
detections_for_image = detections[imagename]
image_properties = detections_for_image['attributes']
for object_ in detections_for_image['labels']:
object_bddid = object_['id']
object_name = object_['category']
object_properties = {
key: value
for key, value in object_['attributes'].items()
if value != 'none'
}
object_properties.update(image_properties)
# Skip 3d object. TODO: import it to properties.
if 'box3d' in object_:
logging.warning('Will skip 3D object %d.' % object_bddid)
continue
# Get the bbox if exists.
x1 = y1 = width = height = None
if 'box2d' in object_:
box2d = object_['box2d']
x1 = int(float(box2d['x1']))
y1 = int(float(box2d['y1']))
width = int(float(box2d['x2']) - x1)
height = int(float(box2d['y2']) - y1)
if args.with_display:
roi = utilBoxes.bbox2roi((x1, y1, width, height))
util.drawScoredRoi(img, roi, object_name)
c.execute(
'INSERT INTO objects(imagefile,x1,y1,width,height,name) '
'VALUES (?,?,?,?,?,?)',
(imagefile, x1, y1, width, height, object_name))
objectid = c.lastrowid
# Get the polygon if it exists.
if 'poly2d' in object_:
if len(object_['poly2d']) > 1:
assert 0, len(object_['poly2d'])
polygon = object_['poly2d'][0]
polygon_name = None if polygon['closed'] else 'open_loop'
for pt in polygon['vertices']:
c.execute(
'INSERT INTO polygons(objectid,x,y,name) '
'VALUES (?,?,?,?)',
(objectid, pt[0], pt[1], polygon_name))
if args.with_display:
util.drawScoredPolygon(img,
[(int(x[0]), int(x[1]))
for x in polygon['vertices']],
object_name)
# Insert image-level and object-level attributes into
# "properties" table.
for key, value in object_properties.items():
c.execute(
'INSERT INTO properties(objectid,key,value) VALUES (?,?,?)',
(objectid, key, value))
# Segmentation annotations.
if args.segmentation_dir:
segmentation_path = op.join(args.segmentation_dir,
'%s_train_id.png' % filename)
if not op.exists(segmentation_path):
raise FileNotFoundError('Annotation file not found at "%s".' %
segmentation_path)
# Add image to the database.
maskfile = op.relpath(segmentation_path, args.rootdir)
c.execute('UPDATE images SET maskfile=? WHERE imagefile=?',
(maskfile, imagefile))
if args.with_display:
mask = imreader.maskread(maskfile)
img = util.drawMaskAside(img, mask, labelmap=None)
# Maybe display.
if args.with_display:
cv2.imshow('importKitti', img[:, :, ::-1])
if cv2.waitKey(-1) == 27:
args.with_display = False
cv2.destroyWindow('importKitti')
