def add_detections(self, filename, boxes):
fileid = os.path.basename(filename)
fileid = ''.join(fileid.split('.')[:-1])
img = cv2.imread(filename)
img_size = Size(img.shape[1], img.shape[0])
for conf, box in boxes:
xmin, xmax, ymin, ymax = prop2abs(box.center, box.size, img_size)
if xmin < 0:
xmin = 0
if xmin >= img_size.w:
xmin = img_size.w - 1
if xmax < 0:
xmax = 0
if xmax >= img_size.w:
xmax = img_size.w - 1
if ymin < 0:
ymin = 0
if ymin >= img_size.h:
ymin = img_size.h - 1
if ymax < 0:
ymax = 0
if ymax >= img_size.h:
ymax = img_size.h - 1
det = Detection(fileid, conf, float(xmin + 1), float(ymin + 1),
float(xmax + 1), float(ymax + 1))
self.boxes[box.label].append(det)
def transform_box(box, orig_size, new_size, h_off, w_off):
#---------------------------------------------------------------------------
# Compute the new coordinates of the box
#---------------------------------------------------------------------------
xmin, xmax, ymin, ymax = prop2abs(box.center, box.size, orig_size)
xmin += w_off
xmax += w_off
ymin += h_off
ymax += h_off
#---------------------------------------------------------------------------
# Check if the center falls within the image
#---------------------------------------------------------------------------
width = xmax - xmin
height = ymax - ymin
new_cx = xmin + int(width / 2)
new_cy = ymin + int(height / 2)
if new_cx < 0 or new_cx >= new_size.w:
return None
if new_cy < 0 or new_cy >= new_size.h:
return None
center, size = abs2prop(xmin, xmax, ymin, ymax, new_size)
return Box(box.label, box.labelid, center, size)
def anchors2array(anchors, img_size):
"""
Computes a numpy array out of absolute anchor params (img_size is needed
as a reference)
"""
arr = np.zeros((len(anchors), 4))
for i in range(len(anchors)):
anchor = anchors[i]
xmin, xmax, ymin, ymax = prop2abs(anchor.center, anchor.size, img_size)
arr[i] = np.array([xmin, xmax, ymin, ymax])
return arr
def __call__(self, data, label, gt):
#-----------------------------------------------------------------------
# Check whether to sample or not
#-----------------------------------------------------------------------
if not self.sample:
return data, label, gt
#-----------------------------------------------------------------------
# Retry sampling a couple of times
#-----------------------------------------------------------------------
source_boxes = anchors2array(gt.boxes, gt.imgsize)
box = None
box_arr = None
for _ in range(self.max_trials):
#-------------------------------------------------------------------
# Sample a bounding box
#-------------------------------------------------------------------
scale = random.uniform(self.min_scale, self.max_scale)
aspect_ratio = random.uniform(self.min_aspect_ratio,
self.max_aspect_ratio)
# make sure width and height will not be larger than 1
aspect_ratio = max(aspect_ratio, scale**2)
aspect_ratio = min(aspect_ratio, 1 / (scale**2))
width = scale * sqrt(aspect_ratio)
height = scale / sqrt(aspect_ratio)
cx = 0.5 * width + random.uniform(0, 1 - width)
cy = 0.5 * height + random.uniform(0, 1 - height)
center = Point(cx, cy)
size = Size(width, height)
#-------------------------------------------------------------------
# Check if the box satisfies the jaccard overlap constraint
#-------------------------------------------------------------------
box_arr = np.array(prop2abs(center, size, gt.imgsize))
overlap = compute_overlap(box_arr, source_boxes, 0)
if overlap.best and overlap.best.score >= self.min_jaccard_overlap:
box = Box(None, None, center, size)
break
if box is None:
return None
#-----------------------------------------------------------------------
# Crop the box and adjust the ground truth
#-----------------------------------------------------------------------
new_size = Size(box_arr[1] - box_arr[0], box_arr[3] - box_arr[2])
w_off = -box_arr[0]
h_off = -box_arr[2]
data = data[box_arr[2]:box_arr[3], box_arr[0]:box_arr[1]]
gt = transform_gt(gt, new_size, h_off, w_off)
return data, label, gt
def add_detections(self, gt_boxes, boxes):
"""
Add new detections to the calculator.
:param gt_sample: ground truth sample
:param boxes: a list of (float, Box) tuples representing
detections and their confidences, the detections
must have a correctly set label
"""
sample_id = len(self.gt_boxes)
self.gt_boxes.append(gt_boxes)
for conf, box in boxes:
arr = np.array(prop2abs(box.center, box.size, IMG_SIZE))
self.det_params[box.label].append(arr)
self.det_confidence[box.label].append(conf)
self.det_sample_ids[box.label].append(sample_id)
def compute_aps(self):
"""
Compute the average precision per class
"""
# Split the ground truth samples by class and sample
counts = defaultdict(lambda: 0)
gt_map = defaultdict(dict)
for sample_id, boxes in enumerate(self.gt_boxes):
boxes_by_class = defaultdict(list)
for box in boxes:
counts[box.label] += 1
boxes_by_class[box.label].append(box)
for k, v in boxes_by_class.items():
arr = np.zeros((len(v), 4))
match = np.zeros((len(v)), dtype=np.bool)
for i, box in enumerate(v):
arr[i] = np.array(prop2abs(box.center, box.size, IMG_SIZE))
gt_map[k][sample_id] = (arr, match)
# Compare predictions to ground truth
aps = {}
for k in gt_map:
# Create numpy arrays of detection parameters and sort them
# in descending order
params = np.array(self.det_params[k], dtype=np.float32)
confs = np.array(self.det_confidence[k], dtype=np.float32)
sample_ids = np.array(self.det_sample_ids[k], dtype=np.int)
idxs_max = np.argsort(-confs)
params = params[idxs_max]
confs = confs[idxs_max]
sample_ids = sample_ids[idxs_max]
# Loop over the detections and count true and false positives
tps = np.zeros((params.shape[0])) # true positives
fps = np.zeros((params.shape[0])) # false positives
for i in range(params.shape[0]):
sample_id = sample_ids[i]
box = params[i]
# The image this detection comes from contains no objects of
# of this class
if not sample_id in gt_map[k]:
fps[i] = 1
continue
# Compute the jaccard overlap and see if it's over the threshold
# Note:
# gt:
# The 2D array containing all the boxes in sample with sample_id
# that belong to the class k
# matched:
# The boolean array of the same shape as gt.shape[0]
gt = gt_map[k][sample_id][0]
matched = gt_map[k][sample_id][1]
iou = jaccard_overlap(box, gt)
max_idx = np.argmax(iou)
if iou[max_idx] < self.minoverlap:
fps[i] = 1
continue
# Check if the max overlap ground truth box is already matched
# Note:
# If multiple detections of the same object are found, they will be
# considered as false positives.
if matched[max_idx]:
fps[i] = 1
continue
tps[i] = 1
matched[max_idx] = True
# Compute the precision, recall
fps = np.cumsum(fps)
tps = np.cumsum(tps)
# Note:
# This is the running precision of the predictions sorted according
# to confidence
# However, recall calculations are performed with respect to the total
# number of ground truth boxes. Hence, the recall value monotonically
# increases while the precision value can fluctuate
# To smooth out the precision curve, at each recall level, we consider
# the maximum precision level occuring at or after that recall level.
# This makes the precision vs recall curve monotonically decreasing.
recall = tps / counts[k]
prec = tps / (tps + fps)
ap = 0
for r_tilde in np.arange(0, 1.1, 0.1):
prec_rec = prec[recall >= r_tilde]
if len(prec_rec) > 0:
ap += np.amax(prec_rec)
ap /= 11.
aps[k] = ap
return aps
def non_maximum_suppression(boxes, overlap_threshold):
#---------------------------------------------------------------------------
# Convert to absolute coordinates and to a more convenient format
#---------------------------------------------------------------------------
xmin = []
xmax = []
ymin = []
ymax = []
conf = []
img_size = Size(1000, 1000)
for box in boxes:
params = prop2abs(box[1].center, box[1].size, img_size)
xmin.append(params[0])
xmax.append(params[1])
ymin.append(params[2])
ymax.append(params[3])
conf.append(box[0])
xmin = np.array(xmin)
xmax = np.array(xmax)
ymin = np.array(ymin)
ymax = np.array(ymax)
conf = np.array(conf)
#---------------------------------------------------------------------------
# Compute the area of each box and sort the indices by confidence level
# (lowest confidence first first).
#---------------------------------------------------------------------------
area = (xmax - xmin + 1) * (ymax - ymin + 1)
idxs = np.argsort(conf)
pick = []
#---------------------------------------------------------------------------
# Loop until we still have indices to process
#---------------------------------------------------------------------------
while len(idxs) > 0:
#-----------------------------------------------------------------------
# Grab the last index (ie. the most confident detection), remove it from
# the list of indices to process, and put it on the list of picks
#-----------------------------------------------------------------------
last = idxs.shape[0] - 1
i = idxs[last]
idxs = np.delete(idxs, last)
pick.append(i)
suppress = []
#-----------------------------------------------------------------------
# Figure out the intersection with the remaining windows
#-----------------------------------------------------------------------
xxmin = np.maximum(xmin[i], xmin[idxs])
xxmax = np.minimum(xmax[i], xmax[idxs])
yymin = np.maximum(ymin[i], ymin[idxs])
yymax = np.minimum(ymax[i], ymax[idxs])
w = np.maximum(0, xxmax - xxmin + 1)
h = np.maximum(0, yymax - yymin + 1)
intersection = w * h
#-----------------------------------------------------------------------
# Compute IOU and suppress indices with IOU higher than a threshold
#-----------------------------------------------------------------------
union = area[i] + area[idxs] - intersection
iou = intersection / union
overlap = iou > overlap_threshold
suppress = np.nonzero(overlap)[0]
idxs = np.delete(idxs, suppress)
#---------------------------------------------------------------------------
# Return the selected boxes
#---------------------------------------------------------------------------
selected = []
for i in pick:
selected.append(boxes[i])
return selected
def box2array(box, img_size):
xmin, xmax, ymin, ymax = prop2abs(box.center, box.size, img_size)
return np.array([xmin, xmax, ymin, ymax])
def __call__(self, data, label, gt):
#-----------------------------------------------------------------------
# Check whether to sample or not
#-----------------------------------------------------------------------
if not self.sample:
return data, label, gt
#-----------------------------------------------------------------------
# Retry sampling a couple of times
#-----------------------------------------------------------------------
source_boxes = anchors2array(
gt.boxes, gt.imgsize) # get abs value(xmin,xmax,ymin,ymax)
box = None
box_arr = None
for _ in range(self.max_trials):
#-------------------------------------------------------------------
# Sample a bounding box
#-------------------------------------------------------------------
# min_scale=0.3, max_scale=1.0,
# min_aspect_ratio=0.5, max_aspect_ratio=2.0,
scale = random.uniform(self.min_scale, self.max_scale)
aspect_ratio = random.uniform(self.min_aspect_ratio,
self.max_aspect_ratio)
# make sure width and height will not be larger than 1
aspect_ratio = max(aspect_ratio, scale**2)
aspect_ratio = min(aspect_ratio, 1 / (scale**2))
width = scale * sqrt(aspect_ratio)
height = scale / sqrt(aspect_ratio)
cx = 0.5 * width + random.uniform(0, 1 - width)
cy = 0.5 * height + random.uniform(0, 1 - height)
center = Point(cx, cy)
size = Size(width, height)
#-------------------------------------------------------------------
# Check if the box satisfies the jaccard overlap constraint
#-------------------------------------------------------------------
box_arr = np.array(prop2abs(center, size, gt.imgsize))
overlap = compute_overlap(box_arr, source_boxes, 0)
if overlap.best and overlap.best.score >= self.min_jaccard_overlap:
box = Box(None, None, center, size)
break
if box is None:
return None
#-----------------------------------------------------------------------
# Crop the box and adjust the ground truth
#-----------------------------------------------------------------------
new_size = Size(box_arr[1] - box_arr[0], box_arr[3] - box_arr[2])
w_off = -box_arr[0]
h_off = -box_arr[2]
data = data.crop(
(box_arr[0], box_arr[2], box_arr[1], box_arr[3])
) # gt와 gt근처의 가상의 anchor를 잡은뒤, iou 0.1, 0.3, 0.5, 0.7, 0.9 이상이면 그 가상의 anchor부분을 crop하고 gt 좌표도 같이 변경
# 즉 일부로 gt와 iou 연관있는 그림부분을 crop함
gt = transform_gt(gt, new_size, h_off, w_off)
return data, label, gt # change gt and image_size
def compute_aps(self):
"""
Compute the average precision per class as well as mAP.
"""
#-----------------------------------------------------------------------
# Split the ground truth samples by class and sample
#-----------------------------------------------------------------------
counts = defaultdict(lambda: 0)
gt_map = defaultdict(dict)
for sample_id, boxes in enumerate(self.gt_boxes):
boxes_by_class = defaultdict(list)
for box in boxes:
counts[box.label] += 1
boxes_by_class[box.label].append(box)
for k, v in boxes_by_class.items():
arr = np.zeros((len(v), 4))
match = np.zeros((len(v)), dtype=np.bool)
for i, box in enumerate(v):
arr[i] = np.array(prop2abs(box.center, box.size, IMG_SIZE))
gt_map[k][sample_id] = (arr, match)
#-----------------------------------------------------------------------
# Compare predictions to ground truth
#-----------------------------------------------------------------------
aps = {}
for k in gt_map:
#-------------------------------------------------------------------
# Create numpy arrays of detection parameters and sort them
# in descending order
#-------------------------------------------------------------------
params = np.array(self.det_params[k], dtype=np.float32)
confs = np.array(self.det_confidence[k], dtype=np.float32)
sample_ids = np.array(self.det_sample_ids[k], dtype=np.int)
idxs_max = np.argsort(-confs)
params = params[idxs_max]
confs = confs[idxs_max]
sample_ids = sample_ids[idxs_max]
#-------------------------------------------------------------------
# Loop over the detections and count true and false positives
#-------------------------------------------------------------------
tps = np.zeros((params.shape[0])) # true positives
fps = np.zeros((params.shape[0])) # false positives
for i in range(params.shape[0]):
sample_id = sample_ids[i]
box = params[i]
#---------------------------------------------------------------
# The image this detection comes from contains no objects of
# of this class
#---------------------------------------------------------------
if not sample_id in gt_map[k]:
fps[i] = 1
continue
#---------------------------------------------------------------
# Compute the jaccard overlap and see if it's over the threshold
#---------------------------------------------------------------
gt = gt_map[k][sample_id][0]
matched = gt_map[k][sample_id][1]
iou = jaccard_overlap(box, gt)
max_idx = np.argmax(iou)
if iou[max_idx] < self.minoverlap:
fps[i] = 1
continue
#---------------------------------------------------------------
# Check if the max overlap ground truth box is already matched
#---------------------------------------------------------------
if matched[max_idx]:
fps[i] = 1
continue
tps[i] = 1
matched[max_idx] = True
#-------------------------------------------------------------------
# Compute the precision, recall
#-------------------------------------------------------------------
fps = np.cumsum(fps)
tps = np.cumsum(tps)
recall = tps / counts[k]
prec = tps / (tps + fps)
ap = 0
for r_tilde in np.arange(0, 1.01, 0.01):
prec_rec = prec[recall >= r_tilde]
if len(prec_rec) > 0:
ap += np.amax(prec_rec)
ap /= 101.
aps[k] = ap
return aps
