def get_face_embedding(filename, arg_params, aux_params, sym, model):
img_orig = cv2.imread(filename)
img_orig = cv2.cvtColor(img_orig, cv2.COLOR_BGR2RGB)
img, scale = resize(img_orig.copy(), 600, 1000)
im_info = np.array([[img.shape[0], img.shape[1], scale]], dtype=np.float32) # (h, w, scale)
img = np.swapaxes(img, 0, 2)
img = np.swapaxes(img, 1, 2) # change to (c, h, w) order
img = img[np.newaxis, :] # extend to (n, c, h, w)
arg_params["data"] = mx.nd.array(img, ctx)
arg_params["im_info"] = mx.nd.array(im_info, ctx)
exe = sym.bind(ctx, arg_params, args_grad=None, grad_req="null", aux_states=aux_params)
exe.forward(is_train=False)
output_dict = {name: nd for name, nd in zip(sym.list_outputs(), exe.outputs)}
rois = output_dict['rpn_rois_output'].asnumpy()[:, 1:] # first column is index
scores = output_dict['cls_prob_reshape_output'].asnumpy()[0]
bbox_deltas = output_dict['bbox_pred_reshape_output'].asnumpy()[0]
pred_boxes = bbox_pred(rois, bbox_deltas)
pred_boxes = clip_boxes(pred_boxes, (im_info[0][0], im_info[0][1]))
cls_boxes = pred_boxes[:, 4:8]
cls_scores = scores[:, 1]
keep = np.where(cls_scores >0.6)[0]
cls_boxes = cls_boxes[keep, :]
cls_scores = cls_scores[keep]
dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = nms(dets.astype(np.float32), 0.3)
dets = dets[keep, :]
bbox = dets[0, :4]
roundfunc = lambda t: int(round(t/scale))
vfunc = np.vectorize(roundfunc)
bbox = vfunc(bbox)
f_vector, jpeg = model.get_feature(img_orig, bbox, None)
fT = f_vector.T
return fT
def process(self, img_color, img_dest_boxes):
tic = time.time()
img = cv2.cvtColor(img_color, cv2.COLOR_BGR2RGB)
img, scale = self.resize(img.copy(), self.app_args.scale,
self.app_args.max_scale)
im_info = np.array([[img.shape[0], img.shape[1], scale]],
dtype=np.float32) # (h, w, scale)
img = np.swapaxes(img, 0, 2)
img = np.swapaxes(img, 1, 2) # change to (c, h, w) order
img = img[np.newaxis, :] # extend to (n, c, h, w)
self.arg_params["data"] = mx.nd.array(img, self.ctx)
self.arg_params["im_info"] = mx.nd.array(im_info, self.ctx)
exe = self.sym.bind(self.ctx,
self.arg_params,
args_grad=None,
grad_req="null",
aux_states=self.aux_params)
exe.forward(is_train=False)
output_dict = {
name: nd
for name, nd in zip(self.sym.list_outputs(), exe.outputs)
}
rois = output_dict['rpn_rois_output'].asnumpy(
)[:, 1:] # first column is index
# everything below is slow...
bbox_deltas = output_dict['bbox_pred_reshape_output']
bbox_deltas = bbox_deltas.asnumpy()[0]
pred_boxes = bbox_pred(rois, bbox_deltas)
pred_boxes = clip_boxes(pred_boxes, (im_info[0][0], im_info[0][1]))
cls_boxes = pred_boxes[:, 4:8]
scores = output_dict['cls_prob_reshape_output']
scores = scores.asnumpy()[0]
cls_scores = scores[:, 1]
keep = np.where(cls_scores >= self.app_args.thresh)[0]
cls_boxes = cls_boxes[keep, :]
cls_scores = cls_scores[keep]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = nms(dets.astype(np.float32), self.app_args.nms_thresh)
dets = dets[keep, :]
toc = time.time()
print("time cost is:%4.4f s" % (toc - tic))
for i in range(dets.shape[0]):
bbox = dets[i, :4]
cv2.rectangle(
img_dest_boxes,
(int(round(bbox[0] / scale)), int(round(bbox[1] / scale))),
(int(round(bbox[2] / scale)), int(round(bbox[3] / scale))),
(0, 255, 0), 2)
return img_color
def main():
color = cv2.imread(args.img)
img = cv2.cvtColor(color, cv2.COLOR_BGR2RGB)
img, scale = resize(img.copy(), args.scale, args.max_scale)
im_info = np.array([[img.shape[0], img.shape[1], scale]],
dtype=np.float32) # (h, w, scale)
img = np.swapaxes(img, 0, 2)
img = np.swapaxes(img, 1, 2) # change to (c, h, w) order
img = img[np.newaxis, :] # extend to (n, c, h, w)
ctx = mx.gpu(args.gpu)
_, arg_params, aux_params = mx.model.load_checkpoint(
args.prefix, args.epoch)
arg_params, aux_params = ch_dev(arg_params, aux_params, ctx)
sym = resnet_50(num_class=2)
arg_params["data"] = mx.nd.array(img, ctx)
arg_params["im_info"] = mx.nd.array(im_info, ctx)
exe = sym.bind(ctx,
arg_params,
args_grad=None,
grad_req="null",
aux_states=aux_params)
tic = time.time()
exe.forward(is_train=False)
output_dict = {
name: nd
for name, nd in zip(sym.list_outputs(), exe.outputs)
}
rois = output_dict['rpn_rois_output'].asnumpy(
)[:, 1:] # first column is index
scores = output_dict['cls_prob_reshape_output'].asnumpy()[0]
bbox_deltas = output_dict['bbox_pred_reshape_output'].asnumpy()[0]
pred_boxes = bbox_pred(rois, bbox_deltas)
pred_boxes = clip_boxes(pred_boxes, (im_info[0][0], im_info[0][1]))
cls_boxes = pred_boxes[:, 4:8]
cls_scores = scores[:, 1]
keep = np.where(cls_scores >= args.thresh)[0]
cls_boxes = cls_boxes[keep, :]
cls_scores = cls_scores[keep]
dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = nms(dets.astype(np.float32), args.nms_thresh)
dets = dets[keep, :]
toc = time.time()
print("time cost is:{}s".format(toc - tic))
for i in range(dets.shape[0]):
bbox = dets[i, :4]
cv2.rectangle(
color, (int(round(bbox[0] / scale)), int(round(bbox[1] / scale))),
(int(round(bbox[2] / scale)), int(round(bbox[3] / scale))),
(0, 255, 0), 2)
cv2.imwrite("result.jpg", color)
def detect(nparr):
img_orig = cv2.imdecode(nparr, 1)
img, scale = resize(img_orig.copy(), 600, 1000)
im_info = np.array([[img.shape[0], img.shape[1], scale]],
dtype=np.float32) # (h, w, scale)
img = np.swapaxes(img, 0, 2)
img = np.swapaxes(img, 1, 2) # change to (c, h, w) order
img = img[np.newaxis, :] # extend to (n, c, h, w)
ctx = mx.gpu(0)
_, arg_params, aux_params = mx.model.load_checkpoint('mxnet-face-fr50', 0)
arg_params, aux_params = ch_dev(arg_params, aux_params, ctx)
sym = resnet_50(num_class=2)
arg_params["data"] = mx.nd.array(img, ctx)
arg_params["im_info"] = mx.nd.array(im_info, ctx)
exe = sym.bind(ctx,
arg_params,
args_grad=None,
grad_req="null",
aux_states=aux_params)
tic = time.time()
exe.forward(is_train=False)
output_dict = {
name: nd
for name, nd in zip(sym.list_outputs(), exe.outputs)
}
rois = output_dict['rpn_rois_output'].asnumpy(
)[:, 1:] # first column is index
scores = output_dict['cls_prob_reshape_output'].asnumpy()[0]
bbox_deltas = output_dict['bbox_pred_reshape_output'].asnumpy()[0]
pred_boxes = bbox_pred(rois, bbox_deltas)
pred_boxes = clip_boxes(pred_boxes, (im_info[0][0], im_info[0][1]))
cls_boxes = pred_boxes[:, 4:8]
cls_scores = scores[:, 1]
keep = np.where(cls_scores >= 0.8)[0]
cls_boxes = cls_boxes[keep, :]
cls_scores = cls_scores[keep]
dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = nms(dets.astype(np.float32), 0.3)
dets = dets[keep, :]
toc = time.time()
color = cv2.cvtColor(img_orig, cv2.COLOR_RGB2BGR)
print("time cost is:{}s".format(toc - tic))
for i in range(dets.shape[0]):
bbox = dets[i, :4]
cv2.rectangle(
color, (int(round(bbox[0] / scale)), int(round(bbox[1] / scale))),
(int(round(bbox[2] / scale)), int(round(bbox[3] / scale))),
(0, 255, 0), 2)
ret, jpeg = cv2.imencode('.png', color)
return jpeg.tobytes()
def getEmbedding(model, img_orig):
ctx = mx.cpu()
_, arg_params, aux_params = mx.model.load_checkpoint('mxnet-face-fr50', 0)
arg_params, aux_params = ch_dev(arg_params, aux_params, ctx)
sym = resnet_50(num_class=2)
img0, scale = resize(img_orig.copy(), 600, 1000)
im_info = np.array([[img0.shape[0], img0.shape[1], scale]],
dtype=np.float32) # (h, w, scale)
img = np.swapaxes(img0, 0, 2)
img = np.swapaxes(img, 1, 2) # change to (c, h, w) order
img = img[np.newaxis, :] # extend to (n, c, h, w)
arg_params["data"] = mx.nd.array(img, ctx)
arg_params["im_info"] = mx.nd.array(im_info, ctx)
exe = sym.bind(ctx,
arg_params,
args_grad=None,
grad_req="null",
aux_states=aux_params)
exe.forward(is_train=False)
output_dict = {
name: nd
for name, nd in zip(sym.list_outputs(), exe.outputs)
}
rois = output_dict['rpn_rois_output'].asnumpy(
)[:, 1:] # first column is index
scores = output_dict['cls_prob_reshape_output'].asnumpy()[0]
bbox_deltas = output_dict['bbox_pred_reshape_output'].asnumpy()[0]
pred_boxes = bbox_pred(rois, bbox_deltas)
pred_boxes = clip_boxes(pred_boxes, (im_info[0][0], im_info[0][1]))
cls_boxes = pred_boxes[:, 4:8]
cls_scores = scores[:, 1]
keep = np.where(cls_scores >= 0.6)[0]
cls_boxes = cls_boxes[keep, :]
cls_scores = cls_scores[keep]
dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = nms(dets.astype(np.float32), 0.3)
dets = dets[keep, :]
print(dets.shape[0])
i = 0
bbox = dets[i, :4]
roundfunc = lambda t: int(round(t / scale))
vfunc = np.vectorize(roundfunc)
bbox = vfunc(bbox)
f2, jpeg = model.get_feature(img_orig, bbox, None)
return f2
def main():
color = cv2.imread(args.img)
img = cv2.cvtColor(color, cv2.COLOR_BGR2RGB)
img, scale = resize(img.copy(), args.scale, args.max_scale)
im_info = np.array([[img.shape[0], img.shape[1], scale]], dtype=np.float32) # (h, w, scale)
img = np.swapaxes(img, 0, 2)
img = np.swapaxes(img, 1, 2) # change to (c, h, w) order
img = img[np.newaxis, :] # extend to (n, c, h, w)
ctx = mx.gpu(args.gpu)
_, arg_params, aux_params = mx.model.load_checkpoint(args.prefix, args.epoch)
arg_params, aux_params = ch_dev(arg_params, aux_params, ctx)
sym = resnet_50(num_class=2)
arg_params["data"] = mx.nd.array(img, ctx)
arg_params["im_info"] = mx.nd.array(im_info, ctx)
exe = sym.bind(ctx, arg_params, args_grad=None, grad_req="null", aux_states=aux_params)
tic = time.time()
exe.forward(is_train=False)
output_dict = {name: nd for name, nd in zip(sym.list_outputs(), exe.outputs)}
rois = output_dict['rpn_rois_output'].asnumpy()[:, 1:] # first column is index
scores = output_dict['cls_prob_reshape_output'].asnumpy()[0]
bbox_deltas = output_dict['bbox_pred_reshape_output'].asnumpy()[0]
pred_boxes = bbox_pred(rois, bbox_deltas)
pred_boxes = clip_boxes(pred_boxes, (im_info[0][0], im_info[0][1]))
cls_boxes = pred_boxes[:, 4:8]
cls_scores = scores[:, 1]
keep = np.where(cls_scores >= args.thresh)[0]
cls_boxes = cls_boxes[keep, :]
cls_scores = cls_scores[keep]
dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = nms(dets.astype(np.float32), args.nms_thresh)
dets = dets[keep, :]
toc = time.time()
print "time cost is:{}s".format(toc-tic)
for i in range(dets.shape[0]):
bbox = dets[i, :4]
cv2.rectangle(color, (int(round(bbox[0]/scale)), int(round(bbox[1]/scale))),
(int(round(bbox[2]/scale)), int(round(bbox[3]/scale))), (0, 255, 0), 2)
cv2.imwrite("result.jpg", color)
img = np.swapaxes(img, 0, 2)
img = np.swapaxes(img, 1, 2) # change to (c, h, w) order
img = img[np.newaxis, :] # extend to (n, c, h, w)
arg_params["data"] = mx.nd.array(img, ctx)
arg_params["im_info"] = mx.nd.array(im_info, ctx)
exe = sym.bind(ctx, arg_params, args_grad=None, grad_req="null", aux_states=aux_params)
tic = time.time()
exe.forward(is_train=False)
output_dict = {name: nd for name, nd in zip(sym.list_outputs(), exe.outputs)}
rois = output_dict['rpn_rois_output'].asnumpy()[:, 1:] # first column is index
scores = output_dict['cls_prob_reshape_output'].asnumpy()[0]
bbox_deltas = output_dict['bbox_pred_reshape_output'].asnumpy()[0]
pred_boxes = bbox_pred(rois, bbox_deltas)
pred_boxes = clip_boxes(pred_boxes, (im_info[0][0], im_info[0][1]))
cls_boxes = pred_boxes[:, 4:8]
cls_scores = scores[:, 1]
keep = np.where(cls_scores >= 0.6)[0]
cls_boxes = cls_boxes[keep, :]
cls_scores = cls_scores[keep]
dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = nms(dets.astype(np.float32), 0.3)
dets = dets[keep, :]
print("video position (ms): "+msg.key())
print(dets.shape[0])
toc = time.time()
img_final = img_orig.copy()
# color = cv2.cvtColor(img_orig, cv2.COLOR_RGB2BGR)
print("time cost is:{}s".format(toc-tic))
def forward(self, is_train, req, in_data, out_data, aux):
# for each (H, W) location i
# generate A anchor boxes centered on cell i
# apply predicted bbox deltas at cell i to each of the A anchors
# clip predicted boxes to image
# remove predicted boxes with either height or width < threshold
# sort all (proposal, score) pairs by score from highest to lowest
# take top pre_nms_topN proposals before NMS
# apply NMS with threshold 0.7 to remaining proposals
# take after_nms_topN proposals after NMS
# return the top proposals (-> RoIs top, scores top)
pre_nms_topN = config[self.cfg_key].RPN_PRE_NMS_TOP_N
post_nms_topN = config[self.cfg_key].RPN_POST_NMS_TOP_N
nms_thresh = config[self.cfg_key].RPN_NMS_THRESH
min_size = config[self.cfg_key].RPN_MIN_SIZE
# the first set of anchors are background probabilities
# keep the second part
scores = in_data[0].asnumpy()[:, self._num_anchors:, :, :]
if np.isnan(scores).any():
raise ValueError("there is nan in input scores")
bbox_deltas = in_data[1].asnumpy()
if np.isnan(bbox_deltas).any():
raise ValueError("there is nan in input bbox_deltas")
im_info = in_data[2].asnumpy()[0, :]
# 1. Generate proposals from bbox_deltas and shifted anchors
height, width = scores.shape[-2:]
if self.cfg_key == 'TRAIN':
height, width = int(im_info[0] / self._feat_stride), int(
im_info[1] / self._feat_stride)
# Enumerate all shifts
shift_x = np.arange(0, width) * self._feat_stride
shift_y = np.arange(0, height) * self._feat_stride
shift_x, shift_y = np.meshgrid(shift_x, shift_y)
shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(),
shift_y.ravel())).transpose()
# Enumerate all shifted anchors:
#
# add A anchors (1, A, 4) to
# cell K shifts (K, 1, 4) to get
# shift anchors (K, A, 4)
# reshape to (K*A, 4) shifted anchors
A = self._num_anchors
K = shifts.shape[0]
anchors = self._anchors.reshape((1, A, 4)) + shifts.reshape(
(1, K, 4)).transpose((1, 0, 2))
anchors = anchors.reshape((K * A, 4))
# Transpose and reshape predicted bbox transformations to get them
# into the same order as the anchors:
#
# bbox deltas will be (1, 4 * A, H, W) format
# transpose to (1, H, W, 4 * A)
# reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a)
# in slowest to fastest order
bbox_deltas = clip_pad(bbox_deltas, (height, width))
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4))
# Same story for the scores:
#
# scores are (1, A, H, W) format
# transpose to (1, H, W, A)
# reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a)
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
# Convert anchors into proposals via bbox transformations
proposals = bbox_pred(anchors, bbox_deltas)
# 2. clip predicted boxes to image
proposals = clip_boxes(proposals, im_info[:2])
# 3. remove predicted boxes with either height or width < threshold
# (NOTE: convert min_size to input image scale stored in im_info[2])
keep = ProposalOperator._filter_boxes(proposals, min_size * im_info[2])
proposals = proposals[keep, :]
scores = scores[keep]
# 4. sort all (proposal, score) pairs by score from highest to lowest
# 5. take top pre_nms_topN (e.g. 6000)
order = scores.ravel().argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
# 6. apply nms (e.g. threshold = 0.7)
# 7. take after_nms_topN (e.g. 300)
# 8. return the top proposals (-> RoIs top)
keep = nms(np.hstack((proposals, scores)), nms_thresh)
if post_nms_topN > 0:
keep = keep[:post_nms_topN]
# pad to ensure output size remains unchanged
if len(keep) < post_nms_topN:
pad = npr.choice(keep, size=post_nms_topN - len(keep))
keep = np.hstack((keep, pad))
proposals = proposals[keep, :]
scores = scores[keep]
# Output rois array
# Our RPN implementation only supports a single input image, so all
# batch inds are 0
batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32)
blob = np.hstack((batch_inds, proposals.astype(np.float32,
copy=False)))
self.assign(out_data[0], req[0], blob)
if self._output_score:
self.assign(out_data[1], req[1],
scores.astype(np.float32, copy=False))
