def setup_raw(self, sqlite_path, image_dir, log_interval=10):
# Load raw AFLW dataset
self.dataset = _load_raw_aflw(sqlite_path, image_dir)
# Calculate selective search rectangles (This takes many minutes)
logger.info('Calculate selective search rectangles for AFLW')
for i, entry in enumerate(self.dataset):
# Logging
if i % log_interval == 0:
logger.info(' {}/{}'.format(i, len(self.dataset)))
# Load image
img = cv2.imread(entry['img_path'])
if img is None or img.size == 0:
# Empty elements
logger.warn('Failed to load image {}'.format(
entry['img_path']))
self.dataset[i]['ssrects'] = list()
self.dataset[i]['ssrect_overlaps'] = list()
else:
# Selective search
ssrects = common.selective_search_dlib(img)
ssrects = _extract_valid_rects(ssrects, img,
entry['others_landmark_pts'])
overlaps = [
common.rect_overlap_rate(ssrect, entry['rect'])
for ssrect in ssrects
]
self.dataset[i]['ssrects'] = ssrects
self.dataset[i]['ssrect_overlaps'] = overlaps
def __call__(self, img):
logger.info('Start analyzing image')
# ========== Iterative Region Proposals (IRP) ==========
img_rect = (0, 0, img.shape[1], img.shape[0])
detections, landmarks, visibilities = None, None, None
for stage_cnt in six.moves.xrange(3):
if stage_cnt == 0:
# Selective search, crop and normalize
ssrects = common.selective_search_dlib(img,
max_img_size=(999, 999),
kvals=(50, 200, 2),
min_size=1500,
check=False,
debug_window=False)
logger.info('Starting run ' + str(stage_cnt + 1) + ' of 3 with ' +
str(len(ssrects)) + ' initial proposed regions')
# Forward
detections, landmarks, visibilities, poses, genders = \
_forward_with_rects(self.model, img, ssrects, self.batchsize)
# Update ssrects using landmarks
new_ssrects = list()
for i in six.moves.xrange(len(ssrects)):
if detections[i] > 0.25: # TODO configure
new_ssrect = _proposal_region(landmarks[i],
visibilities[i], img_rect)
if new_ssrect[2] > 0 and new_ssrect[3] > 0:
new_ssrects.append(new_ssrect)
ssrects = new_ssrects
# [DEBUG] Draw IRP rectabgles
# for rect in ssrects:
# drawing._draw_rect(img, rect, (0, 1, 0))
# Extract detected entries
valid_idxs = [i for i, det in enumerate(detections) if det > 0.5]
landmarks = np.asarray(landmarks)[valid_idxs]
visibilities = np.asarray(visibilities)[valid_idxs]
poses = np.asarray(poses)[valid_idxs]
genders = np.asarray(genders)[valid_idxs]
# ========== Landmark-based NMS ==========
res_idx_sets = list()
precise_rects = [
_bounding_region(l, v) for l, v in zip(landmarks, visibilities)
]
areas = [common.rect_area(rect) for rect in precise_rects]
overlap_tls = 0.20 # TODO configure
scorebase_idxs = np.argsort(areas).tolist() # ascending order
while len(scorebase_idxs) > 0:
# Register new index set with the best rect index
best_rect_idx = scorebase_idxs.pop()
res_idx_sets.append([best_rect_idx])
# Register overlapped indices
best_rect = precise_rects[best_rect_idx]
# # [DEBUG] Draw L-NMS rectabgles
# drawing._draw_rect(img, best_rect, (0, 1, 0))
removal_scorebase_idxs = list()
for s_i, i in enumerate(scorebase_idxs):
overlap = common.rect_overlap_rate(best_rect, precise_rects[i])
if overlap > overlap_tls:
res_idx_sets[-1].append(i)
removal_scorebase_idxs.append(s_i)
# Remove registered indices (reverse)
for i in removal_scorebase_idxs[::-1]:
scorebase_idxs.pop(i)
# Extract middle value
res_landmarks = list()
res_visibilities = list()
res_poses = list()
res_genders = list()
res_rects = list()
for idx_set in res_idx_sets:
res_landmarks.append(np.median(landmarks[idx_set], axis=0))
res_visibilities.append(np.median(visibilities[idx_set], axis=0))
res_poses.append(np.median(poses[idx_set], axis=0))
res_genders.append(np.median(genders[idx_set], axis=0))
res_rects.append(
_proposal_region(res_landmarks[-1], res_visibilities[-1],
img_rect))
return (res_landmarks, res_visibilities, res_poses, res_genders,
res_rects)