def nms(classes,
offsets,
anchors,
class_thresh=0.9,
iou_thresh=0.2,
is_soft=True):
# get all non-zero (non-background) objects
objects = np.argmax(classes, axis=1)
nonbg = np.nonzero(objects)[0]
#print("Candidate non bg: ", nonbg.size)
indexes = []
while True:
scores = np.zeros((classes.shape[0], ))
scores[nonbg] = np.amax(classes[nonbg], axis=1)
score_idx = np.argmax(scores, axis=0)
score_max = scores[score_idx]
# print(score_max)
nonbg = nonbg[nonbg != score_idx]
if score_max < class_thresh:
if nonbg.size == 0:
break
continue
indexes.append(score_idx)
score_anc = anchors[score_idx]
score_off = offsets[score_idx][0:4]
score_box = score_anc + score_off
score_box = np.expand_dims(score_box, axis=0)
nonbg_copy = np.copy(nonbg)
for idx in nonbg_copy:
anchor = anchors[idx]
offset = offsets[idx][0:4]
box = anchor + offset
box = np.expand_dims(box, axis=0)
iou = layer_utils.iou(box, score_box)[0][0]
if iou >= iou_thresh:
print(score_idx, "overlaps ", idx, "with iou ", iou)
if is_soft:
iou = iou * iou
iou /= 0.5
classes[idx] *= math.exp(-iou)
print("Scaling...")
else:
nonbg = nonbg[nonbg != idx]
print("Removing ...")
if nonbg.size == 0:
break
scores = np.zeros((classes.shape[0], ))
scores[indexes] = np.amax(classes[indexes], axis=1)
print("Validated non bg: ", len(indexes))
#if is_soft:
# print("Soft NMS")
return objects, indexes, scores
def __data_generation(self, keys):
"""Generate train data: images and
object detection ground truth labels
Arguments:
keys (array): Randomly sampled keys
(key is image filename)
Returns:
x (tensor): Batch images
y (tensor): Batch classes, offsets, and masks
"""
# train input data
x = np.zeros((self.args.batch_size, *self.input_shape))
dim = (self.args.batch_size, self.n_boxes, self.n_classes)
# class ground truth
gt_class = np.zeros(dim)
dim = (self.args.batch_size, self.n_boxes, 4)
# offsets ground truth
gt_offset = np.zeros(dim)
# masks of valid bounding boxes
gt_mask = np.zeros(dim)
for i, key in enumerate(keys):
# images are assumed to be stored in self.args.data_path
# key is the image filename
image_path = os.path.join(self.args.data_path, key)
image = skimage.img_as_float(imread(image_path))
# assign image to a batch index
x[i] = image
# a label entry is made of 4-dim bounding box coords
# and 1-dim class label
labels = self.dictionary[key]
labels = np.array(labels)
# 4 bounding box coords are 1st four items of labels
# last item is object class label
boxes = labels[:,0:-1]
for index, feature_shape in enumerate(self.feature_shapes):
# generate anchor boxes
anchors = anchor_boxes(feature_shape,
image.shape,
index=index,
n_layers=self.args.layers)
# each feature layer has a row of anchor boxes
anchors = np.reshape(anchors, [-1, 4])
# compute IoU of each anchor box
# with respect to each bounding boxes
iou = layer_utils.iou(anchors, boxes)
# generate ground truth class, offsets & mask
gt = get_gt_data(iou,
n_classes=self.n_classes,
anchors=anchors,
labels=labels,
normalize=self.args.normalize,
threshold=self.args.threshold)
gt_cls, gt_off, gt_msk = gt
if index == 0:
cls = np.array(gt_cls)
off = np.array(gt_off)
msk = np.array(gt_msk)
else:
cls = np.append(cls, gt_cls, axis=0)
off = np.append(off, gt_off, axis=0)
msk = np.append(msk, gt_msk, axis=0)
gt_class[i] = cls
gt_offset[i] = off
gt_mask[i] = msk
y = [gt_class, np.concatenate((gt_offset, gt_mask), axis=-1)]
return x, y
def __data_generation(self, keys):
data_path = config.params['data_path']
x = np.empty((self.batch_size, *self.input_shape))
dim = (self.batch_size, self.n_boxes, self.n_classes)
gt_class = np.empty(dim)
dim = (self.batch_size, self.n_boxes, 4)
gt_offset = np.empty(dim)
gt_mask = np.empty(dim)
for i, key in enumerate(keys):
# images are assumed to be stored in config data_path
# key is the image filename
image_path = os.path.join(data_path, key)
image = skimage.img_as_float(imread(image_path))
# if augment data is enabled
if self.aug_data:
image = self.apply_random_noise(image)
image = self.apply_random_intensity_rescale(image)
image = self.apply_random_exposure_adjust(image)
x[i] = image
labels = self.dictionary[key]
labels = np.array(labels)
# 4 boxes coords are 1st four items of labels
boxes = labels[:,0:-1]
for index, shape in enumerate(self.feature_shapes):
shape = (1, *shape)
# generate anchor boxes
anchors = anchor_boxes(shape,
image.shape,
index=index,
n_layers=self.n_layers)
anchors = np.reshape(anchors, [-1, 4])
# compute IoU of each anchor box
# with respect to each bounding boxes
iou = layer_utils.iou(anchors, boxes)
# generate ground truth class and offsets
ret = get_gt_data(iou,
n_classes=self.n_classes,
anchors=anchors,
labels=labels,
normalize=self.normalize)
gt_cls, gt_off, gt_msk = ret
if index == 0:
cls = np.array(gt_cls)
off = np.array(gt_off)
msk = np.array(gt_msk)
else:
cls = np.append(cls, gt_cls, axis=0)
off = np.append(off, gt_off, axis=0)
msk = np.append(msk, gt_msk, axis=0)
gt_class[i] = cls
gt_offset[i] = off
gt_mask[i] = msk
y = [gt_class, np.concatenate((gt_offset, gt_mask), axis=-1)]
return x, y
def nms(args, classes, offsets, anchors):
"""Perform NMS (Algorithm 11.12.1).
Arguments:
args : User-defined configurations
classes (tensor): Predicted classes
offsets (tensor): Predicted offsets
Returns:
objects (tensor): class predictions per anchor
indexes (tensor): indexes of detected objects
filtered by NMS
scores (tensor): array of detected objects scores
filtered by NMS
"""
# get all non-zero (non-background) objects
objects = np.argmax(classes, axis=1)
# non-zero indexes are not background
nonbg = np.nonzero(objects)[0]
# D and S indexes in Line 1
indexes = []
while True:
# list of zero probability values
scores = np.zeros((classes.shape[0], ))
# set probability values of non-background
scores[nonbg] = np.amax(classes[nonbg], axis=1)
# max probability given the list
# Lines 3 and 4
score_idx = np.argmax(scores, axis=0)
score_max = scores[score_idx]
# get all non max probability & set it as new nonbg
# Line 5
nonbg = nonbg[nonbg != score_idx]
# if max obj probability is less than threshold (def 0.8)
if score_max < args.class_threshold:
# we are done
break
# Line 5
indexes.append(score_idx)
score_anc = anchors[score_idx]
score_off = offsets[score_idx][0:4]
score_box = score_anc + score_off
score_box = np.expand_dims(score_box, axis=0)
nonbg_copy = np.copy(nonbg)
# get all overlapping predictions (Line 6)
# perform Non-Max Suppression (NMS)
for idx in nonbg_copy:
anchor = anchors[idx]
offset = offsets[idx][0:4]
box = anchor + offset
box = np.expand_dims(box, axis=0)
iou = layer_utils.iou(box, score_box)[0][0]
# if soft NMS is chosen (Line 7)
if args.soft_nms:
# adjust score: Line 8
iou = -2 * iou * iou
classes[idx] *= math.exp(iou)
# else NMS (Line 9), (iou threshold def 0.2)
elif iou >= args.iou_threshold:
# remove overlapping predictions with iou>threshold
# Line 10
nonbg = nonbg[nonbg != idx]
# Line 2, nothing else to process
if nonbg.size == 0:
break
# get the array of object scores
scores = np.zeros((classes.shape[0], ))
scores[indexes] = np.amax(classes[indexes], axis=1)
return objects, indexes, scores
def nms(classes, offsets, anchors):
class_thresh = config.params['class_thresh']
iou_thresh = config.params['iou_thresh']
is_soft = config.params['is_soft_nms']
# get all non-zero (non-background) objects
objects = np.argmax(classes, axis=1)
# non-zero indexes are not background
nonbg = np.nonzero(objects)[0]
#print("Candidate non bg: ", nonbg.size)
indexes = []
while True:
# list of zero probability values
scores = np.zeros((classes.shape[0], ))
# set probability values of non-background
scores[nonbg] = np.amax(classes[nonbg], axis=1)
# max probability given the list
score_idx = np.argmax(scores, axis=0)
score_max = scores[score_idx]
# print(score_max)
# get all non max probability & set it as new nonbg
nonbg = nonbg[nonbg != score_idx]
# if obj probability is less than threshold
if score_max < class_thresh:
# we are done
break
indexes.append(score_idx)
score_anc = anchors[score_idx]
score_off = offsets[score_idx][0:4]
score_box = score_anc + score_off
score_box = np.expand_dims(score_box, axis=0)
nonbg_copy = np.copy(nonbg)
# get all overlapping predictions
for idx in nonbg_copy:
anchor = anchors[idx]
offset = offsets[idx][0:4]
box = anchor + offset
box = np.expand_dims(box, axis=0)
iou = layer_utils.iou(box, score_box)[0][0]
if is_soft:
iou = -2 * iou * iou
classes[idx] *= math.exp(iou)
#print("Soft NMS scaling ...", idx)
elif iou >= iou_thresh:
#print(score_idx, "overlaps ", idx, "with iou ", iou)
nonbg = nonbg[nonbg != idx]
#print("NMS Removing ...", idx)
if nonbg.size == 0:
break
scores = np.zeros((classes.shape[0], ))
scores[indexes] = np.amax(classes[indexes], axis=1)
return objects, indexes, scores
def evaluate_test(self):
# test labels csv path
path = os.path.join(self.args.data_path,
self.args.test_labels)
# test dictionary
dictionary, _ = build_label_dictionary(path)
keys = np.array(list(dictionary.keys()))
# sum of precision
s_precision = 0
# sum of recall
s_recall = 0
# sum of IoUs
s_iou = 0
# evaluate per image
for key in keys:
# grounnd truth labels
labels = np.array(dictionary[key])
# 4 boxes coords are 1st four items of labels
gt_boxes = labels[:, 0:-1]
# last one is class
gt_class_ids = labels[:, -1]
# load image id by key
image_file = os.path.join(self.args.data_path, key)
image = skimage.img_as_float(imread(image_file))
image, classes, offsets = self.detect_objects(image)
# perform nms
_, _, class_ids, boxes = show_boxes(args,
image,
classes,
offsets,
self.feature_shapes,
show=False)
boxes = np.reshape(np.array(boxes), (-1,4))
# compute IoUs
iou = layer_utils.iou(gt_boxes, boxes)
# skip empty IoUs
if iou.size ==0:
continue
# the class of predicted box w/ max iou
maxiou_class = np.argmax(iou, axis=1)
# true positive
tp = 0
# false positiove
fp = 0
# sum of objects iou per image
s_image_iou = []
for n in range(iou.shape[0]):
# ground truth bbox has a label
if iou[n, maxiou_class[n]] > 0:
s_image_iou.append(iou[n, maxiou_class[n]])
# true positive has the same class and gt
if gt_class_ids[n] == class_ids[maxiou_class[n]]:
tp += 1
else:
fp += 1
# objects that we missed (false negative)
fn = abs(len(gt_class_ids) - tp)
s_iou += (np.sum(s_image_iou) / iou.shape[0])
s_precision += (tp/(tp + fp))
s_recall += (tp/(tp + fn))
n_test = len(keys)
print_log("mIoU: %f" % (s_iou/n_test),
self.args.verbose)
print_log("Precision: %f" % (s_precision/n_test),
self.args.verbose)
print_log("Recall : %f" % (s_recall/n_test),
self.args.verbose)
def evaluate_test(self):
csv_path = os.path.join(config.params['data_path'],
config.params['test_labels'])
print("CSV", csv_path)
dictionary, _ = build_label_dictionary(csv_path)
keys = np.array(list(dictionary.keys()))
n_iou = 0
s_iou = 0
i = 0
tp = 0
fp = 0
for key in keys:
labels = dictionary[key]
labels = np.array(labels)
# 4 boxes coords are 1st four items of labels
gt_boxes = labels[:, 0:-1]
gt_class_ids = labels[:, -1]
image_file = os.path.join(config.params['data_path'], key)
print("Image: ", image_file)
image = skimage.img_as_float(imread(image_file))
image = np.expand_dims(image, axis=0)
classes, offsets = self.ssd.predict(image)
image = np.squeeze(image, axis=0)
classes = np.squeeze(classes)
offsets = np.squeeze(offsets)
_, _, class_ids, boxes = show_boxes(image,
classes,
offsets,
self.feature_shapes,
show=False,
normalize=self.normalize)
boxes = np.reshape(np.array(boxes), (-1, 4))
iou = layer_utils.iou(gt_boxes, boxes)
if iou.size == 0:
continue
print("--------------")
print("gt:", gt_class_ids, gt_boxes)
print("iou w/ gt:", iou)
print("iou shape:", iou.shape)
maxiou_class = np.argmax(iou, axis=1)
print("classes: ", maxiou_class)
n = iou.shape[0]
n_iou += n
s = []
for j in range(n):
s.append(iou[j, maxiou_class[j]])
if gt_class_ids[j] == class_ids[maxiou_class[j]]:
tp += 1
else:
fp += 1
fp += abs(len(class_ids) - len(gt_class_ids))
print("max ious: ", s)
s = np.sum(s)
s_iou += s
print("pred:", class_ids, boxes)
print("--------------")
# i += 1
#if i==10:
# break
print("sum:", s_iou)
print("num:", n_iou)
print("mIoU:", s_iou / n_iou)
print("tp:", tp)
print("fp:", fp)
print("precision:", tp / (tp + fp))