def test(model, config, limit=None):
dataset_test = S3DDataset()
dataset_test.load_s3d('test')
dataset_test.prepare()
print("Start inferencing on %d images..." % len(dataset_test.image_info))
APs = []
import tqdm
for i in tqdm.tqdm(range(len(dataset_test.image_info))):
if limit is not None:
if i >= int(limit):
continue
image, meta, gt_class_ids, gt_bbox, gt_mask = modellib.load_image_gt(
dataset_test, config, i)
result = model.detect([image], verbose=0)[0]
bbox = result['rois']
mask = result['masks']
class_ids = result['class_ids']
scores = result['scores']
second_class_ids = result['second_class_ids']
second_scores = result['second_scores']
probs = result['probs'][0]
# print(class_ids)
# print(second_class_ids)
# print(scores)
# print(second_scores)
# print(bbox.shape, gt_bbox.shape)
# print(mask.shape, gt_mask.shape)
# print(class_ids.shape, gt_class_ids.shape)
# print(scores.shape)
# 基础的结果
basemAP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask)
delta = 0.0
# 计入概率次高分类之后的结果
for i in range(len(class_ids)):
ori = class_ids[i]
class_ids[i] = second_class_ids[i]
mAP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask)
class_ids[i] = ori
if mAP - basemAP > 0:
delta += mAP - basemAP
if mAP >= 0 and mAP <= 1:
APs.append(basemAP + delta)
print('%.3f' % np.mean(APs))
def evaluate_gdxray(model, dataset, eval_type="bbox", limit=0, image_ids=None):
"""Runs GDXRay evaluation.
dataset: A Dataset object with valiadtion data
eval_type: "bbox" or "segm" for bounding box or segmentation evaluation
limit: if not 0, it's the number of images to use for evaluation
"""
image_ids = image_ids or dataset.image_ids
random.shuffle(image_ids)
limit = int(limit)
num_processed = 0
APs = []
for image_id in image_ids:
# Load image
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
# Run object detection
results = model.detect([image], verbose=0)
# Compute AP
r = results[0]
AP, precisions, recalls, overlaps =\
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r['rois'], r['class_ids'], r['scores'], r['masks'])
print("Image", image_id, "AP:", AP)
num_processed += 1
APs.append(AP)
if limit and num_processed > limit:
break
return np.mean(APs)
def compute_mean_AP(model, config, dataset, n_images):
""" Compute VOC-Style mAP @ IoU=0.5
"""
image_ids = np.random.choice(dataset.image_ids, n_images)
APs = []
for image_id in image_ids:
# Load image and ground truth data
result = modellib.load_image_gt(dataset,
config,
image_id,
use_mini_mask=False)
if len(result) == 5:
image, image_meta, class_ids, gt_bbox, gt_mask = result
else:
image, image_meta, gt_bbox, gt_mask = result
class_ids = gt_bbox[:, 4]
gt_bbox = gt_bbox[:, :4]
molded_images = np.expand_dims(modellib.mold_image(image, config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, class_ids, r["rois"], r["class_ids"], r["scores"])
APs.append(AP)
return np.mean(APs)
def evaluate_model(dataset, model, cfg):
APs = list()
for image_id in dataset.image_ids:
# load image, bounding boxes and masks for the image id
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(
dataset, cfg, image_id, use_mini_mask=False)
# convert pixel values (e.g. center)
scaled_image = modellib.mold_image(image, cfg)
# convert image into one sample
sample = expand_dims(scaled_image, 0)
# make prediction
yhat = model.detect(sample, verbose=0)
# extract results for first sample
r = yhat[0]
try:
# calculate statistics, including AP
AP, _, _, _ = utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"],
r["scores"], r['masks'])
# store
APs.append(AP)
print("good image id : %d" % image_id)
print("AP : %.3f" % AP)
except:
print("bad image id : %d" % image_id)
# calculate the mean AP across all images
mAP = mean(APs)
return mAP
def basicResults():
basemAP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores,
mask)
if basemAP >= 0 and basemAP <= 1:
APs1.append(basemAP)
fout.write('%d %f\n' % (i, basemAP))
def secondClassResults():
# 基础的结果
basemAP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask)
delta = 0.0
# 计入概率次高分类之后的结果
for i in range(len(class_ids)):
ori = class_ids[i]
class_ids[i] = second_class_ids[i]
mAP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask)
class_ids[i] = ori
if mAP - basemAP > 0:
delta += mAP - basemAP
if basemAP >= 0 and basemAP <= 1:
APs2.append(basemAP + delta)
def compute_batch_ap(image_ids):
APs = []
for image_id in image_ids:
# Load image
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
# Run object detection
results = model.detect([image], verbose=0)
# Compute AP
r = results[0]
AP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_id, r['rois'], r['class_ids'], r['scores'])
APs.append(AP)
return APs
def compute_map(iou_th):
mean_aps_this_th = torch.zeros(len(dataloader_eval_covid),
dtype=torch.float)
for id, b in enumerate(dataloader_eval_covid):
X, y = b
if device == torch.device('cuda'):
X, y['labels'], y['boxes'], y['masks'] = X.to(
device), y['labels'].to(device), y['boxes'].to(
device), y['masks'].to(device)
images = [im for im in X]
lab = {}
# THIS IS IMPORTANT!!!!!
# get rid of the first dimension (batch)
# IF you have >1 images, make another loop
# REPEAT: DO NOT USE BATCH DIMENSION
lab['boxes'] = y['boxes'].squeeze_(0)
lab['labels'] = y['labels'].squeeze_(0)
lab['masks'] = y['masks'].squeeze_(0)
image = [X.squeeze_(0)] # remove the batch dimension
out = maskrcnn_model(image)
# scores + bounding boxes + labels + masks
scores = out[0]['scores']
bboxes = out[0]['boxes']
classes = out[0]['labels']
# remove the empty dimension,
# output_size x 512 x 512
predict_mask = out[0]['masks'].squeeze_(1) > 0.5
if len(scores) > 0 and len(lab['labels']) > 0:
# threshold for the masks:
mAP, _, _, _ = utils.compute_ap(lab['boxes'],
lab['labels'],
lab['masks'],
bboxes,
classes,
scores,
predict_mask,
iou_threshold=iou_th)
mean_aps_this_th[id] = mAP
elif not len(scores) and not len(lab['labels']):
mean_aps_this_th[id] = 1
elif not len(scores) and len(lab['labels']) > 0:
continue
elif len(scores) > 0 and not len(y['labels']):
continue
print("AP @{0:.2f}:{1:.2f}".format(iou_th.item(),
mean_aps_this_th.mean().item()))
return mean_aps_this_th, mean_aps_this_th.mean().item()
def evaluate_wpif(model, config, dataset, eval_type="bbox", limit=0):
"""
Evaluation on WPIF dataset, using VOC-Style mAP # IoU=0.5 for bbox
@TODO: add segment evaluation
:param model:
:param config:
:param dataset:
:param eval_type:
:param limit:
:return:
"""
image_ids = dataset.image_ids
if limit:
image_ids = np.random.choice(dataset.image_ids, limit)
t_prediction = 0
t_start = time.time()
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(image, config), 0)
# Run object detection
t = time.time()
results = model.detect([image], verbose=0)
r = results[0]
t_prediction += (time.time() - t)
# Compute AP
AP, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id,
r["rois"], r["class_ids"], r["scores"])
APs.append(AP)
print("Prediction time: {}. Average {}/image".format(
t_prediction, t_prediction / len(image_ids)))
print("Total time: ", time.time() - t_start)
print("mAP: ", np.mean(APs))
def evaluate_apc(model, config, dataset, eval_type="bbox", limit=0):
"""
Evaluation on APC dataset, using VOC-Style mAP # IoU=0.5 for bbox
@TODO: add segment evaluation
:param model:
:param config:
:param dataset:
:param eval_type:
:param limit:
:return:
"""
image_ids = dataset.image_ids
if limit:
image_ids = np.random.choice(dataset.image_ids, limit)
t_prediction = 0
t_start = time.time()
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(image, config), 0)
# Run object detection
t = time.time()
results = model.detect([image], verbose=0)
r = results[0]
t_prediction += (time.time() - t)
# Compute AP
AP, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id,
r["rois"], r["class_ids"], r["scores"])
APs.append(AP)
print("Prediction time: {}. Average {}/image".format(
t_prediction, t_prediction / len(image_ids)))
print("Total time: ", time.time() - t_start)
print("mAP: ", np.mean(APs))
def evaluate_maskrcnn(dataset_val,inference_config = InferenceConfig()):
image_ids = np.random.choice(dataset_val.image_ids, 10)
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precisions, recalls, overlaps =\
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"], r["scores"], r['masks'])
APs.append(AP)
print("mAP: ", np.mean(APs))
return APs
def compute_map(model, iou_th, dl, device, mask_th):
mean_aps_this_th = torch.zeros(len(dl), dtype=torch.float)
for v, b in enumerate(dl):
x, y = b
if device == torch.device('cuda'):
x, y['labels'], y['boxes'], y['masks'] = x.to(
device), y['labels'].to(device), y['boxes'].to(
device), y['masks'].to(device)
lab = {
'boxes': y['boxes'].squeeze_(0),
'labels': y['labels'].squeeze_(0),
'masks': y['masks'].squeeze_(0)
}
image = [x.squeeze_(0)] # remove the batch dimension
out = model(image)
# scores + bounding boxes + labels + masks
scores = out[0]['scores']
bboxes = out[0]['boxes']
classes = out[0]['labels']
# remove the empty dimension,
# output_size x 512 x 512
predict_mask = out[0]['masks'].squeeze_(1) > mask_th
if len(scores) > 0 and len(lab['labels']) > 0:
# threshold for the masks:
ap, _, _, _ = utils.compute_ap(lab['boxes'],
lab['labels'],
lab['masks'],
bboxes,
classes,
scores,
predict_mask,
iou_threshold=iou_th)
mean_aps_this_th[v] = ap
elif not len(scores) and not len(lab['labels']):
mean_aps_this_th[v] = 1
elif not len(scores) and len(lab['labels']) > 0:
continue
elif len(scores) > 0 and not len(y['labels']):
continue
return mean_aps_this_th.mean().item()
def evaluate_engine(model,
dataset,
inference_config,
eval_type="bbox",
limit=0,
image_ids=None):
"""Runs official COCO evaluation.
dataset: A Dataset object with valiadtion data
eval_type: "bbox" or "segm" for bounding box or segmentation evaluation
limit: if not 0, it's the number of images to use for evaluation
"""
# Pick images from the dataset
image_ids = np.random.choice(dataset.image_ids, 100)
# Limit to a subset
if limit:
image_ids = image_ids[:limit]
t_prediction = 0
t_start = time.time()
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset, inference_config,
image_id, use_mini_mask=False)
molded_images = np.expand_dims(
modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precisions, recalls, overlaps =\
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"], r["scores"], r['masks'])
APs.append(AP)
print("mAP: ", np.mean(APs))
print("Total time: ", time.time() - t_start)
def evaluate(self):
self.yolo.model = self.model
# gather all detections and annotations
all_detections = [[
None for i in range(self.generator.num_classes())
] for j in range(self.generator.size())]
all_annotations = [[
None for i in range(self.generator.num_classes())
] for j in range(self.generator.size())]
for i in range(self.generator.size()):
raw_image = self.generator.load_image(i)
raw_height, raw_width, raw_channels = raw_image.shape
# make the boxes and the labels
pred_boxes = self.yolo.predict(raw_image)
score = np.array([box.score for box in pred_boxes])
pred_labels = np.array([box.label for box in pred_boxes])
if len(pred_boxes) > 0:
pred_boxes = np.array([[
box.xmin * raw_width, box.ymin * raw_height,
box.xmax * raw_width, box.ymax * raw_height, box.score
] for box in pred_boxes])
else:
pred_boxes = np.array([[]])
# sort the boxes and the labels according to scores
score_sort = np.argsort(-score)
pred_labels = pred_labels[score_sort]
pred_boxes = pred_boxes[score_sort]
# copy detections to all_detections
for label in range(self.generator.num_classes()):
all_detections[i][label] = pred_boxes[pred_labels ==
label, :]
annotations = self.generator.load_annotation(i)
# copy detections to all_annotations
for label in range(self.generator.num_classes()):
all_annotations[i][label] = annotations[
annotations[:, 4] == label, :4].copy()
# compute mAP by comparing all detections and all annotations
average_precisions = {}
for label in range(self.generator.num_classes()):
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
for i in range(self.generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
if max_overlap >= self.iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
# no annotations -> AP for this class is 0 (is this correct?)
if num_annotations == 0:
average_precisions[label] = 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# compute average precision
average_precision = compute_ap(recall, precision)
average_precisions[label] = average_precision
return average_precisions
results = model.detect([original_image], verbose=1)
r = results[0]
visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'],
dataset_val.class_names, r['scores'], ax=get_ax(), target=VIS_DIR+"result_" + str(image_id) + ".jpg")
##########################################
###### EVALUATION ######
##########################################
# Compute VOC-Style mAP @ IoU=0.5
# Running on 10 images. Increase for better accuracy.
image_ids = np.random.choice(dataset_val.image_ids, 10)
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id,
r["rois"], r["class_ids"], r["scores"])
APs.append(AP)
print("mAP: ", np.mean(APs))
def val(self):
APs = []
for i, (images, targets) in enumerate(self.val_data_loader):
if i == 10:
break
images = self.to_var(images)
targets = self.to_var(targets)
with torch.no_grad():
output = self.net(images)
output = utils.non_max_suppression(output,
80,
conf_thres=self.conf_thres,
nms_thres=self.nms_thres)
# Compute average precision for each sample
for sample_i in range(targets.size(0)):
correct = []
# Get labels for sample where width is not zero (dummies)
annotations = targets[sample_i, targets[sample_i, :, 3] != 0]
# Extract detections
detections = output[sample_i]
if detections is None:
# If there are no detections but there are annotations mask as zero AP
if annotations.size(0) != 0:
APs.append(0)
continue
# Get detections sorted by decreasing confidence scores
detections = detections[np.argsort(-detections[:, 4])]
# If no annotations add number of detections as incorrect
if annotations.size(0) == 0:
correct.extend([0 for _ in range(len(detections))])
else:
# Extract target boxes as (x1, y1, x2, y2)
target_boxes = torch.FloatTensor(annotations[:, 1:].shape)
target_boxes[:, 0] = (annotations[:, 1] -
annotations[:, 3] / 2)
target_boxes[:, 1] = (annotations[:, 2] -
annotations[:, 4] / 2)
target_boxes[:, 2] = (annotations[:, 1] +
annotations[:, 3] / 2)
target_boxes[:, 3] = (annotations[:, 2] +
annotations[:, 4] / 2)
target_boxes *= self.image_size
detected = []
for *pred_bbox, conf, obj_conf, obj_pred in detections:
pred_bbox = torch.FloatTensor(pred_bbox).view(1, -1)
# Compute iou with target boxes
iou = utils.bbox_iou(pred_bbox, target_boxes)
# Extract index of largest overlap
best_i = np.argmax(iou)
# If overlap exceeds threshold and classification is correct mark as correct
if iou[best_i] > self.iou_thres and obj_pred == annotations[
best_i, 0] and best_i not in detected:
correct.append(1)
detected.append(best_i)
else:
correct.append(0)
# Extract true and false positives
true_positives = np.array(correct)
false_positives = 1 - true_positives
# Compute cumulative false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# Compute recall and precision at all ranks
recall = true_positives / annotations.size(
0) if annotations.size(0) else true_positives
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# Compute average precision
AP = utils.compute_ap(recall, precision)
APs.append(AP)
return np.mean(APs)
plt.imsave(str(img)+'.png', image)
mask,gt_class_ids = test_dataset.load_mask(img)
#print ('mask', mask.shape, image.shape)
results = model.detect([image], verbose=1)
r = results[0]
#print (r.keys())
plt.imsave('gt' + str(img)+'.png', mask[:,:,0])
#print ("get class ids")
#print (r['rois'], r['class_ids'], r['scores'])
# for the base model-keep only cow predictions
present_class = r['class_ids']
for idx, cls in enumerate(present_class):
if cls not in good_classes:
np.delete(r['rois'], idx, axis=0)
np.delete(r['class_ids'], idx, axis=0)
np.delete(r['scores'], idx, axis=0)
#print ('class', r['rois'].shape, r['scores'].shape, r['class_ids'].shape, r['masks'].shape, gt_bbox.shape, mask.shape)
AP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_ids, mask, r['rois'], r['class_ids'], r['scores'], r['masks'], iou_threshold = th)
print (AP, precisions, recalls)
AP_list.append(AP)
print (np.mean(AP_list), th)
all_AP_list.append(np.mean(AP_list))
print ("mAP", np.mean(AP_list))
all_AP_list.append(np.mean(all_AP_list))
print ('all_AP_list', all_AP_list)
#pickle.dump(str(all_AP_list), open("coco_cows_val_base_640.r","wb"))
original_image2, image_meta2, gt_class_id2, gt_bbox2, gt_mask2, gt_keypoint2 =\
modellib.load_image_gt_keypoints(val_dataset_keypoints, inference_config,
2*image_id+1, augment=False,use_mini_mask=inference_config.USE_MINI_MASK)
if(inference_config.USE_MINI_MASK):
gt_mask1 = utils.expand_mask(gt_bbox1,gt_mask1,original_image1.shape)
gt_mask2 = utils.expand_mask(gt_bbox2,gt_mask2,original_image2.shape)
results = model.detect_keypoint([original_image1, original_image2], verbose=0)
r1 = results[0]
r2 = results[1]
try:
AP.append(utils.compute_ap(gt_bbox1, gt_class_id1, gt_mask1, r1['rois'], r1['class_ids'], r1['scores'], r1['masks'])[0])
except ValueError:
print('predict no person')
AP.append(0)
try:
AP.append(utils.compute_ap(gt_bbox2, gt_class_id2, gt_mask2, r2['rois'], r2['class_ids'], r2['scores'], r2['masks'])[0])
except ValueError:
print('predict no person')
AP.append(0)
try:
OKS.append(OKS_algorithm(gt_bbox1, gt_keypoint1, r1))
except ValueError:
print('predict no person')
OKS.append(0)
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset_test, inference_config,
image_id, use_mini_mask=False)
molded_images = np.expand_dims(
modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
#IF PREDICT SOMETHING
if not r['masks'].shape[0] == 0:
# Compute AP
AP, precisions, recalls, overlaps, pred_match, gt_match = \
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"], r["scores"], r["masks"], iou_threshold=0.25)
ev_ids.append(r['class_ids'])
ev_scores.append(r['scores'])
ev_pred_match.append(pred_match)
ev_gt_match.append(gt_match)
# print((gt_match))
APs.append(AP)
if image_id % 500 == 1 or image_id == len(image_ids):
print(image_id)
with open('evaluationVarsM/class_ids.pkl', 'wb') as f:
pickle.dump(ev_ids, f)
with open('evaluationVarsM/scores.pkl', 'wb') as f:
pickle.dump(ev_scores, f)
iou_thresholds = np.arange(0.5,0.95,0.05)
for i in iou_thresholds:
APs_i = []
P_Kaggle_i = []
print('threshold: {}'.format(i))
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precision_kaggle, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"], r["scores"], r['masks'],
iou_threshold = i)
APs_i.append(AP)
P_Kaggle_i.append(precision_kaggle)
APs.append(APs_i)
P_Kaggles.append(P_Kaggle_i)
print("mAP: ", np.mean(np.array(APs),axis=1))
print("Precision Kaggle: ", np.mean(np.array(P_Kaggles),axis=1))
##
def rle_encoding(mask):
'''
x: numpy array of shape (height, width), 1 - mask, 0 - background
Returns run length as list
for image_id in tqdm(dataset_val.image_ids):
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(
dataset_val, InferenceConfig, image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(image, config),
0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox,
gt_class_id,
r["rois"],
r["class_ids"],
r["scores"],
iou_threshold=0.5)
APs.append(AP)
print("mAP: ", np.mean(APs))
elif args.command == "predict":
class InferenceConfig(NucleiConfig):
# Set batch size to 1 since we'll be running inference on
# one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
GPU_COUNT = 1
IMAGES_PER_GPU = 1
DETECTION_NMS_THRESHOLD = 0.3
dataset_val.class_names,
r['scores'],
ax=get_ax())
# ## Evaluation
# In[14]:
# Compute VOC-Style mAP @ IoU=0.5
# Running on 10 images. Increase for better accuracy.
image_ids = np.random.choice(dataset_val.image_ids, 10)
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(
dataset_val, inference_config, image_id, use_mini_mask=False)
molded_images = np.expand_dims(
modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"],
r['masks'])
APs.append(AP)
print("mAP: ", np.mean(APs))
# In[ ]:
def evaluate(self,
model,
imgs,
obj_threshold=0.3,
nms_threshold=0.3,
iou_threshold=0.5):
"""
# Arguments
model : The model to evaluate.
imgs : list of parsed test_img dictionaries.
obj_threshold : The score confidence threshold to use for detections.
nms_threshold : The threshold used to consider when a detection is positive or negative.
# Returns
A dict mapping class names to mAP scores.
"""
# gather all detections and annotations
test_size = len(imgs)
all_detections = [[None for i in range(self.n_class)]
for j in range(test_size)]
all_annotations = [[None for i in range(self.n_class)]
for j in range(test_size)]
ious = []
for i in range(test_size):
image_name = imgs[i]['filename']
if '.jpg' not in image_name and '.png' not in image_name:
image_name += '.jpg'
raw_image = cv2.imread(image_name)
raw_height, raw_width, raw_channels = raw_image.shape
# make the boxes and the labels
pred_boxes = self.predict(model, raw_image, obj_threshold,
nms_threshold)
score = np.array([box.score for box in pred_boxes])
pred_labels = np.array([box.label for box in pred_boxes])
if len(pred_boxes) > 0:
pred_boxes = np.array([[
box.xmin * raw_width, box.ymin * raw_height,
box.xmax * raw_width, box.ymax * raw_height, box.score
] for box in pred_boxes])
else:
pred_boxes = np.array([[]])
# sort the boxes and the labels according to scores
score_sort = np.argsort(-score)
pred_labels = pred_labels[score_sort]
pred_boxes = pred_boxes[score_sort]
# copy detections to all_detections
for label in range(self.n_class):
all_detections[i][label] = pred_boxes[pred_labels == label, :]
annotations = load_annotation(imgs, i, self.labels)
# copy detections to all_annotations
for label in range(self.n_class):
all_annotations[i][label] = annotations[annotations[:, 4] ==
label, :4].copy()
# compute mAP by comparing all detections and all annotations
average_precisions = {}
for label in range(self.n_class):
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
for i in range(test_size):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
ious.append(max_overlap)
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
# no annotations -> AP for this class is 0 (is this correct?)
if num_annotations == 0:
average_precisions[label] = 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# compute average precision
average_precision = compute_ap(recall, precision)
average_precisions[label] = average_precision
map_dict = {}
# print evaluation
for label, average_precision in average_precisions.items():
map_dict[self.labels[label]] = average_precision
print(self.labels[label], '{:.4f}'.format(average_precision))
print('mAP: {:.4f}'.format(
sum(average_precisions.values()) / len(average_precisions)))
print('average IOU: {:.4f}'.format(np.mean(ious)))
average_map = sum(
average_precisions.values()) / len(average_precisions)
return [average_map, map_dict, np.mean(ious)]
def segment_inference(**kwargs):
data_base_dir = kwargs['data_base_dir']
dataset_type = kwargs['dataset_type']
image_id = kwargs['image_id']
epochs = kwargs['epochs']
use_edgelist = kwargs['use_edgelist']
outputs_base_dir = 'outputs'
vis_result_save_dir = os.path.join(outputs_base_dir, 'visual_result',
dataset_type)
trained_model_dir = os.path.join(outputs_base_dir, 'snapshot')
edgelist_result_dir = os.path.join(outputs_base_dir, 'edgelist')
model_path = os.path.join(trained_model_dir,
'mask_rcnn_sketchyscene_' + epochs + '.h5')
os.makedirs(vis_result_save_dir, exist_ok=True)
config = SketchInferConfig()
config.display()
# val/test dataset
dataset_infer = SketchDataset(data_base_dir)
dataset_infer.load_sketches(dataset_type)
dataset_infer.prepare()
# Recreate the model in inference mode
model = modellib.MaskRCNN(mode="inference",
config=config,
model_dir='',
log_dir='')
# Load trained weights (fill in path to trained weights here)
assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
original_image, _, gt_class_id, gt_bbox, gt_mask, _ = \
modellib.load_image_gt(dataset_infer, config, image_id - 1, use_mini_mask=False)
log("original_image", original_image)
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)
gt_seg_path = os.path.join(vis_result_save_dir, str(image_id) + '_gt.png')
visualize.display_instances(original_image,
gt_bbox,
gt_mask,
gt_class_id,
dataset_infer.class_names,
title='Ground-Truth',
save_path=gt_seg_path,
fix_color=True)
## inference
results = model.detect([original_image], verbose=1)
r = results[0]
pred_boxes = r["rois"] # (nRoIs, (y1, x1, y2, x2))
pred_class_ids = r["class_ids"] # (nRoIs)
pred_scores = r["scores"] # (nRoIs)
pred_masks = r["masks"] # (768, 768, nRoIs)
log("pred_boxes", pred_boxes)
log("pred_masks", pred_masks)
if config.IGNORE_BG:
# Use original_image(768, 768, 3) {0, 255} to filter pred_masks
pred_masks = np.transpose(pred_masks, (2, 0, 1)) # (nRoIs, 768, 768)
bin_input = original_image[:, :, 0] == 255
pred_masks[:, bin_input[:, :]] = 0 # (nRoIs, 768, 768)
pred_masks = np.transpose(pred_masks, (1, 2, 0)) # (768, 768, nRoIs)
# refine pred_masks(768, 768, nRoIs) with edge-list
if use_edgelist:
refined_pred_masks = \
refine_mask_with_edgelist(image_id, dataset_type, data_base_dir, edgelist_result_dir,
pred_masks.copy(), pred_boxes)
# caculating AP
iou_thresholds = np.linspace(.5,
0.95,
np.round((0.95 - .5) / .05) + 1,
endpoint=True)
APs = np.zeros([len(iou_thresholds)], dtype=np.float32)
APs_edg = np.zeros([len(iou_thresholds)], dtype=np.float32)
for i in range(len(iou_thresholds)):
iouThr = iou_thresholds[i]
AP, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
pred_boxes, pred_class_ids, pred_scores, pred_masks,
iou_threshold=iouThr)
APs[i] = AP
if use_edgelist:
AP_edg, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
pred_boxes, pred_class_ids, pred_scores, refined_pred_masks,
iou_threshold=iouThr)
APs_edg[i] = AP_edg
mAP = np.mean(APs)
mAP_edg = np.mean(APs_edg)
print('APs', APs)
print('mAP', mAP)
print('APs_edg', APs_edg)
print('mAP_edg', mAP_edg)
# save visual results
visual_seg_path = os.path.join(vis_result_save_dir,
str(image_id) + '_seg.png')
visualize.display_instances(original_image,
pred_boxes,
pred_masks,
pred_class_ids,
dataset_infer.class_names,
pred_scores,
title='Normal result',
save_path=visual_seg_path,
fix_color=True)
if use_edgelist:
visual_seg_edg_path = os.path.join(vis_result_save_dir,
str(image_id) + '_seg_edgelist.png')
visualize.display_instances(original_image,
pred_boxes,
refined_pred_masks,
pred_class_ids,
dataset_infer.class_names,
pred_scores,
title='Result with edgelist',
save_path=visual_seg_edg_path,
fix_color=True)
r = results[0]
visualize.display_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
dataset.class_names,
r['scores'],
ax=ax,
title="Predictions")
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)
# Draw precision-recall curve
AP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_id,
gt_mask, r['rois'],
r['class_ids'],
r['scores'], r['masks'])
visualize.plot_precision_recall(AP, precisions, recalls)
# Grid of ground truth objects and their predictions
visualize.plot_overlaps(gt_class_id, r['class_ids'], r['scores'], overlaps,
dataset.class_names)
# Compute VOC-style Average Precision
def compute_batch_ap(image_ids):
APs = []
for image_id in image_ids:
# Load image
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset, config,
visualize.display_instances(original_image,
r['rois'],
r['masks'],
r['class_ids'],
dataset_val.class_names,
r['scores'],
ax=get_ax())
plt.show()
# Compute VOC-Style mAP @ IoU=0.5
# Running on 10 images. Increase for better accuracy.
image_ids = np.random.choice(dataset_val.image_ids, 10)
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask=False)
molded_images = np.expand_dims(
modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precisions, recalls, overlaps =\
utils.compute_ap(gt_bbox[:,:4], gt_bbox[:,4],
r["rois"], r["class_ids"], r["scores"])
APs.append(AP)
print("mAP: ", np.mean(APs))
r['rois'],
r['masks'],
r['class_ids'],
dataset_val.class_names,
r['scores'],
ax=get_ax())
# ## Evaluation
# In[14]:
# Compute VOC-Style mAP @ IoU=0.5
# Running on 10 images. Increase for better accuracy.
image_ids = np.random.choice(dataset_val.image_ids, 10)
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_bbox, gt_mask = modellib.load_image_gt(
dataset_val, inference_config, image_id, use_mini_mask=False)
molded_images = np.expand_dims(
modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox[:, :4], gt_bbox[:, 4], r["rois"], r["class_ids"], r["scores"])
APs.append(AP)
print("mAP: ", np.mean(APs))
def validation(model,
device,
valid_dataloader,
csv_log_file,
tensor_type=torch.cuda.FloatTensor,
num_classes=8):
# Set model to evaluation mode
model.train(False)
model.eval()
# Variables to store detections
all_detections = []
all_annotations = []
for batch_i, (_, images, labels) in enumerate(
tqdm.tqdm(valid_dataloader, desc="Detecting objects")):
images = Variable(images.type(tensor_type))
with torch.no_grad():
outputs = model(images)
outputs = non_max_suppression(outputs,
num_classes,
conf_thres=0.8,
nms_thres=0.4)
for output, annotations in zip(outputs, labels):
all_detections.append([np.array([]) for _ in range(num_classes)])
if output is not None:
# Get predicted boxes, confidence scores and labels
pred_boxes = output[:, :5].cpu().numpy()
scores = output[:, 4].cpu().numpy()
pred_labels = output[:, -1].cpu().numpy()
# Order by confidence
sort_i = np.argsort(scores)
pred_labels = pred_labels[sort_i]
pred_boxes = pred_boxes[sort_i]
for label in range(num_classes):
all_detections[-1][label] = pred_boxes[pred_labels ==
label]
all_annotations.append([np.array([]) for _ in range(num_classes)])
if any(annotations[:, -1] > 0):
annotation_labels = annotations[annotations[:, -1] > 0,
0].numpy()
_annotation_boxes = annotations[annotations[:, -1] > 0, 1:]
# Reformat to x1, y1, x2, y2 and rescale to image dimensions
annotation_boxes = np.empty_like(_annotation_boxes)
annotation_boxes[:,
0] = _annotation_boxes[:,
0] - _annotation_boxes[:,
2] / 2
annotation_boxes[:,
1] = _annotation_boxes[:,
1] - _annotation_boxes[:,
3] / 2
annotation_boxes[:,
2] = _annotation_boxes[:,
0] + _annotation_boxes[:,
2] / 2
annotation_boxes[:,
3] = _annotation_boxes[:,
1] + _annotation_boxes[:,
3] / 2
annotation_boxes *= 416
for label in range(num_classes):
all_annotations[-1][label] = annotation_boxes[
annotation_labels == label, :]
# Clear variables from gpu, collect garbage and clear gpu cache memory
del images, labels
gc.collect()
torch.cuda.empty_cache()
average_precisions = {}
for label in range(num_classes):
true_positives = []
scores = []
num_annotations = 0
for i in tqdm.tqdm(range(len(all_annotations)),
desc=f"Computing AP for class '{label}'"):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for *bbox, score in detections:
scores.append(score)
if annotations.shape[0] == 0:
true_positives.append(0)
continue
overlaps = bbox_iou_numpy(np.expand_dims(bbox, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
if max_overlap >= 0.5 and assigned_annotation not in detected_annotations:
true_positives.append(1)
detected_annotations.append(assigned_annotation)
else:
true_positives.append(0)
# no annotations -> AP for this class is 0
if num_annotations == 0:
average_precisions[label] = 0
continue
true_positives = np.array(true_positives)
false_positives = np.ones_like(true_positives) - true_positives
# sort by score
indices = np.argsort(-np.array(scores))
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# compute average precision
average_precision = compute_ap(recall, precision)
average_precisions[label] = average_precision
ap_list = []
for c, ap in average_precisions.items():
print(f"+ Class '{c}' - AP: {ap}")
# Write ap details to csv file
ap_list.append(ap)
mAP = np.mean(list(average_precisions.values()))
ap_list.append(mAP)
print(f"mAP: {mAP}")
#writer = csv.writer(csv_log_file)
csv_log_file.writerow(ap_list)
return model, mAP
results = model.detect([original_image], verbose=1)
# original_image = dataset_val.load_image(image_id)
r = results[0]
visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'],
dataset_val.class_names, r['scores'], ax=get_ax())
# Compute VOC-Style mAP @ IoU=0.5
# Running on 10 images. Increase for better accuracy.
image_ids = np.random.choice(dataset_val.image_ids, 10)
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"], r["scores"], r['masks'])
APs.append(AP)
print("mAP: ", np.mean(APs))
print(r['scores'])
'''
val mAP
'''
if 0:
ths = np.linspace(0.5, 0.95, 10)
image_ids = dataset_val.image_ids
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask=False)
molded_images = np.expand_dims(
modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP under different iou threshold
AP = []
for th in ths:
AP_th, precisions, recalls, overlaps =\
utils.compute_ap(gt_bbox, gt_class_id,
r["rois"], r["class_ids"], r["scores"],
iou_threshold=th)
AP.append(AP_th)
APs.append(AP)
print("mAP: ", np.mean(APs))
def evaluate(self,
generator,
iou_threshold=0.3,
score_threshold=0.3,
max_detections=100,
save_path=None):
""" Evaluate a given dataset using a given model.
code originally from https://github.com/fizyr/keras-retinanet
# Arguments
generator : The generator that represents the dataset to evaluate.
model : The model to evaluate.
iou_threshold : The threshold used to consider when a detection is positive or negative.
score_threshold : The score confidence threshold to use for detections.
max_detections : The maximum number of detections to use per image.
save_path : The path to save images with visualized detections to.
# Returns
A dict mapping class names to mAP scores.
"""
# gather all detections and annotations
all_detections = [[None for i in range(generator.num_classes())]
for j in range(generator.size())]
all_annotations = [[None for i in range(generator.num_classes())]
for j in range(generator.size())]
for i in range(generator.size()):
raw_image = generator.load_image(i)
raw_height, raw_width, raw_channels = raw_image.shape
# make the boxes and the labels
pred_boxes = self.predict(raw_image)
score = np.array([box.score for box in pred_boxes])
pred_labels = np.array([box.label for box in pred_boxes])
if len(pred_boxes) > 0:
pred_boxes = np.array([[
box.xmin * raw_width, box.ymin * raw_height,
box.xmax * raw_width, box.ymax * raw_height, box.score
] for box in pred_boxes])
else:
pred_boxes = np.array([[]])
# sort the boxes and the labels according to scores
score_sort = np.argsort(-score)
pred_labels = pred_labels[score_sort]
pred_boxes = pred_boxes[score_sort]
# copy detections to all_detections
for label in range(generator.num_classes()):
all_detections[i][label] = pred_boxes[pred_labels == label, :]
annotations = generator.load_annotation(i)
# copy detections to all_annotations
for label in range(generator.num_classes()):
all_annotations[i][label] = annotations[annotations[:, 4] ==
label, :4].copy()
# compute mAP by comparing all detections and all annotations
average_precisions = {}
for label in range(generator.num_classes()):
false_positives = np.zeros((0, ))
true_positives = np.zeros((0, ))
scores = np.zeros((0, ))
num_annotations = 0.0
for i in range(generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0),
annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
# no annotations -> AP for this class is 0 (is this correct?)
if num_annotations == 0:
average_precisions[label] = 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(
true_positives + false_positives,
np.finfo(np.float64).eps)
# compute average precision
average_precision = compute_ap(recall, precision)
average_precisions[label] = average_precision
return average_precisions
def evaluate(self,
generator,
iou_threshold=0.3,
score_threshold=0.3,
max_detections=100,
save_path=None):
""" Evaluate a given dataset using a given model.
code originally from https://github.com/fizyr/keras-retinanet
# Arguments
generator : The generator that represents the dataset to evaluate.
model : The model to evaluate.
iou_threshold : The threshold used to consider when a detection is positive or negative.
score_threshold : The score confidence threshold to use for detections.
max_detections : The maximum number of detections to use per image.
save_path : The path to save images with visualized detections to.
# Returns
A dict mapping class names to mAP scores.
"""
# gather all detections and annotations
all_detections = [[None for i in range(generator.num_classes())] for j in range(generator.size())]
all_annotations = [[None for i in range(generator.num_classes())] for j in range(generator.size())]
for i in range(generator.size()):
raw_image = generator.load_image(i)
raw_height, raw_width, raw_channels = raw_image.shape
# make the boxes and the labels
pred_boxes = self.predict(raw_image)
score = np.array([box.score for box in pred_boxes])
pred_labels = np.array([box.label for box in pred_boxes])
if len(pred_boxes) > 0:
pred_boxes = np.array([[box.xmin*raw_width, box.ymin*raw_height, box.xmax*raw_width, box.ymax*raw_height, box.score] for box in pred_boxes])
else:
pred_boxes = np.array([[]])
# sort the boxes and the labels according to scores
score_sort = np.argsort(-score)
pred_labels = pred_labels[score_sort]
pred_boxes = pred_boxes[score_sort]
# copy detections to all_detections
for label in range(generator.num_classes()):
all_detections[i][label] = pred_boxes[pred_labels == label, :]
annotations = generator.load_annotation(i)
# copy detections to all_annotations
for label in range(generator.num_classes()):
all_annotations[i][label] = annotations[annotations[:, 4] == label, :4].copy()
# compute mAP by comparing all detections and all annotations
average_precisions = {}
for label in range(generator.num_classes()):
false_positives = np.zeros((0,))
true_positives = np.zeros((0,))
scores = np.zeros((0,))
num_annotations = 0.0
for i in range(generator.size()):
detections = all_detections[i][label]
annotations = all_annotations[i][label]
num_annotations += annotations.shape[0]
detected_annotations = []
for d in detections:
scores = np.append(scores, d[4])
if annotations.shape[0] == 0:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
continue
overlaps = compute_overlap(np.expand_dims(d, axis=0), annotations)
assigned_annotation = np.argmax(overlaps, axis=1)
max_overlap = overlaps[0, assigned_annotation]
if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
false_positives = np.append(false_positives, 0)
true_positives = np.append(true_positives, 1)
detected_annotations.append(assigned_annotation)
else:
false_positives = np.append(false_positives, 1)
true_positives = np.append(true_positives, 0)
# no annotations -> AP for this class is 0 (is this correct?)
if num_annotations == 0:
average_precisions[label] = 0
continue
# sort by score
indices = np.argsort(-scores)
false_positives = false_positives[indices]
true_positives = true_positives[indices]
# compute false positives and true positives
false_positives = np.cumsum(false_positives)
true_positives = np.cumsum(true_positives)
# compute recall and precision
recall = true_positives / num_annotations
precision = true_positives / np.maximum(true_positives + false_positives, np.finfo(np.float64).eps)
# compute average precision
average_precision = compute_ap(recall, precision)
average_precisions[label] = average_precision
return average_precisions
