from utils.graph_optimize import attach_quantize_node
worker_data_shape = dict([(name, tuple(shape)) for name, shape in data_shape])
# print(worker_data_shape)
# raise NotImplementedError
_, out_shape, _ = sym.get_internals().infer_shape(**worker_data_shape)
out_shape_dictoinary = dict(zip(sym.get_internals().list_outputs(), out_shape))
sym = attach_quantize_node(sym, out_shape_dictoinary, pQuant.WeightQuantizeParam,
pQuant.ActQuantizeParam, pQuant.quantized_op)
sym.save(pTest.model.prefix + "_infer_speed.json")
ctx = mx.gpu(gpu)
mod = DetModule(sym, data_names=data_names, context=ctx)
mod.bind(data_shapes=data_shape, for_training=False)
mod.set_params({}, {}, True)
# let AUTOTUNE run for once
mod.forward(data_batch, is_train=False)
for output in mod.get_outputs():
output.wait_to_read()
tic = time.time()
for _ in range(count):
mod.forward(data_batch, is_train=False)
for output in mod.get_outputs():
output.wait_to_read()
toc = time.time()
print((toc - tic) / count * 1000)
padded_image = padded_image.transpose((2, 0, 1))
img_array = []
img_array.append(padded_image)
iminfo_array = []
iminfo_array.append(im_info)
im_id = mx.nd.array([1])
rec_id = mx.nd.array([1])
data = [mx.nd.array(img_array)]
data.append(mx.nd.array(iminfo_array))
data.append(im_id)
data.append(rec_id)
mbatch = mx.io.DataBatch(data=data, provide_data=provide_data)
start_t = time.time()
mod.forward(mbatch, is_train=False)
outs = [x.asnumpy() for x in mod.get_outputs()]
im_info = outs[2] # h_raw, w_raw, scale
cls_score = outs[3]
bbox_xyxy = outs[4]
if cls_score.ndim == 3:
cls_score = cls_score[0]
bbox_xyxy = bbox_xyxy[0]
bbox_xyxy = bbox_xyxy / scale # scale to original image scale
cls_score = cls_score[:, 1:] # remove background score
# TODO: the output shape of class_agnostic box is [n, 4], while class_aware box is [n, 4 * (1 + class)]
bbox_xyxy = bbox_xyxy[:, 4:] if bbox_xyxy.shape[1] != 4 else bbox_xyxy
final_dets = {}
for cid in range(cls_score.shape[1]):
score = cls_score[:, cid]
if bbox_xyxy.shape[1] != 4:
class TDNDetector:
def __init__(self, configFn, ctx, outFolder, threshold):
os.environ["MXNET_CUDNN_AUTOTUNE_DEFAULT"] = "0"
config = importlib.import_module(configFn.replace('.py', '').replace('/', '.'))
_,_,_,_,_,_, self.__pModel,_, self.__pTest, self.transform,_,_,_ = config.get_config(is_train=False)
self.__pModel = patch_config_as_nothrow(self.__pModel)
self.__pTest = patch_config_as_nothrow(self.__pTest)
self.resizeParam = (800, 1200)
if callable(self.__pTest.nms.type):
self.__nms = self.__pTest.nms.type(self.__pTest.nms.thr)
else:
from operator_py.nms import py_nms_wrapper
self.__nms = py_nms_wrapper(self.__pTest.nms.thr)
arg_params, aux_params = load_checkpoint(self.__pTest.model.prefix, self.__pTest.model.epoch)
sym = self.__pModel.test_symbol
from utils.graph_optimize import merge_bn
sym, arg_params, aux_params = merge_bn(sym, arg_params, aux_params)
self.__mod = DetModule(sym, data_names=['data','im_info','im_id','rec_id'], context=ctx)
self.__mod.bind(data_shapes=[('data', (1, 3, self.resizeParam[0], self.resizeParam[1])),
('im_info', (1, 3)),
('im_id', (1,)),
('rec_id', (1,))], for_training=False)
self.__mod.set_params(arg_params, aux_params, allow_extra=False)
self.__saveSymbol(sym, outFolder, self.__pTest.model.prefix.split('/')[-1])
self.__threshold = threshold
def __call__(self, imgFilename): # detect onto image
roi_record, scale = self.__readImg(imgFilename)
h, w = roi_record['data'][0].shape
im_c1 = roi_record['data'][0].reshape(1,1,h,w)
im_c2 = roi_record['data'][1].reshape(1,1,h,w)
im_c3 = roi_record['data'][2].reshape(1,1,h,w)
im_data = np.concatenate((im_c1, im_c2, im_c3), axis=1)
im_info, im_id, rec_id = [(h, w, scale)], [1], [1]
data = mx.io.DataBatch(data=[mx.nd.array(im_data),
mx.nd.array(im_info),
mx.nd.array(im_id),
mx.nd.array(rec_id)])
self.__mod.forward(data, is_train=False)
# extract results
outputs = self.__mod.get_outputs(merge_multi_context=False)
rid, id, info, cls, box = [x[0].asnumpy() for x in outputs]
rid, id, info, cls, box = rid.squeeze(), id.squeeze(), info.squeeze(), cls.squeeze(), box.squeeze()
cls = cls[:, 1:] # remove background
box = box / scale
output_record = dict(rec_id=rid, im_id=id, im_info=info, bbox_xyxy=box, cls_score=cls)
output_record = self.__pTest.process_output([output_record], None)[0]
final_result = self.__do_nms(output_record)
# obtain representable output
detections = []
for cid ,bbox in final_result.items():
idx = np.where(bbox[:,-1] > self.__threshold)[0]
for i in idx:
final_box = bbox[i][:4]
score = bbox[i][-1]
detections.append({'cls':cid, 'box':final_box, 'score':score})
return detections,None
def __do_nms(self, all_output):
box = all_output['bbox_xyxy']
score = all_output['cls_score']
final_dets = {}
for cid in range(score.shape[1]):
score_cls = score[:, cid]
valid_inds = np.where(score_cls > self.__threshold)[0]
box_cls = box[valid_inds]
score_cls = score_cls[valid_inds]
if valid_inds.shape[0]==0:
continue
det = np.concatenate((box_cls, score_cls.reshape(-1, 1)), axis=1).astype(np.float32)
det = self.__nms(det)
cls = coco[cid]
final_dets[cls] = det
return final_dets
def __readImg(self, imgFilename):
img = cv2.imread(imgFilename, cv2.IMREAD_COLOR)
height, width, channels = img.shape
roi_record = {'gt_bbox': np.array([[0., 0., 0., 0.]]),'gt_class': np.array([0])}
roi_record['image_url'] = imgFilename
roi_record['h'] = height
roi_record['w'] = width
for trans in self.transform:
trans.apply(roi_record)
img_shape = [roi_record['h'], roi_record['w']]
shorts, longs = min(img_shape), max(img_shape)
scale = min(self.resizeParam[0] / shorts, self.resizeParam[1] / longs)
return roi_record, scale
def __saveSymbol(self, sym, outFolder, fnPrefix):
if not os.path.exists(outFolder): os.makedirs(outFolder)
resFilename = os.path.join(outFolder, fnPrefix + "_symbol_test.json")
sym.save(resFilename)
class predictor(object):
def __init__(self, config, batch_size, gpu_id, thresh):
self.config = config
self.batch_size = batch_size
self.thresh = thresh
# Parse the parameter file of model
pGen, pKv, pRpn, pRoi, pBbox, pDataset, pModel, pOpt, pTest, \
transform, data_name, label_name, metric_list = config.get_config(is_train=False)
self.data_name = data_name
self.label_name = label_name
self.p_long, self.p_short = transform[1].p.long, transform[1].p.short
# Define NMS type
if callable(pTest.nms.type):
self.do_nms = pTest.nms.type(pTest.nms.thr)
else:
from operator_py.nms import py_nms_wrapper
self.do_nms = py_nms_wrapper(pTest.nms.thr)
sym = pModel.test_symbol
sym.save(pTest.model.prefix + "_test.json")
ctx = mx.gpu(gpu_id)
data_shape = [
('data', (batch_size, 3, 800, 1200)),
("im_info", (1, 3)),
("im_id", (1, )),
("rec_id", (1, )),
]
# Load network
arg_params, aux_params = load_checkpoint(pTest.model.prefix,
pTest.model.epoch)
self.mod = DetModule(sym, data_names=data_name, context=ctx)
self.mod.bind(data_shapes=data_shape, for_training=False)
self.mod.set_params(arg_params, aux_params, allow_extra=False)
def preprocess_image(self, input_img):
image = input_img[:, :, ::-1] # BGR -> RGB
short = min(image.shape[:2])
long = max(image.shape[:2])
scale = min(self.p_short / short, self.p_long / long)
h, w = image.shape[:2]
im_info = (round(h * scale), round(w * scale), scale)
image = cv2.resize(image,
None,
None,
scale,
scale,
interpolation=cv2.INTER_LINEAR)
image = image.transpose((2, 0, 1)) # HWC -> CHW
return image, im_info
def run_image(self, img_path):
image = cv2.imread(img_path, cv2.IMREAD_COLOR)
image, im_info = self.preprocess_image(image)
input_data = {
'data': [image],
'im_info': [im_info],
'im_id': [0],
'rec_id': [0],
}
data = [mx.nd.array(input_data[name]) for name in self.data_name]
label = []
provide_data = [(k, v.shape) for k, v in zip(self.data_name, data)]
provide_label = [(k, v.shape) for k, v in zip(self.label_name, label)]
data_batch = mx.io.DataBatch(data=data,
label=label,
provide_data=provide_data,
provide_label=provide_label)
self.mod.forward(data_batch, is_train=False)
out = [x.asnumpy() for x in self.mod.get_outputs()]
cls_score = out[3]
bboxes = out[4]
result = {}
for cid in range(cls_score.shape[1]):
if cid == 0: # Ignore the background
continue
score = cls_score[:, cid]
if bboxes.shape[1] != 4:
cls_box = bboxes[:, cid * 4:(cid + 1) * 4]
else:
cls_box = bboxes
valid_inds = np.where(score >= self.thresh)[0]
box = cls_box[valid_inds]
score = score[valid_inds]
det = np.concatenate((box, score.reshape(-1, 1)),
axis=1).astype(np.float32)
det = self.do_nms(det)
if len(det) > 0:
det[:, :4] = det[:, :4] / im_info[
2] # Restore to the original size
result[CATEGORIES[cid]] = det
return result
class TDNDetector:
def __init__(self, configFn, ctx, outFolder, threshold):
os.environ["MXNET_CUDNN_AUTOTUNE_DEFAULT"] = "0"
config = importlib.import_module(
configFn.replace('.py', '').replace('/', '.'))
_, _, _, _, _, _, self.__pModel, _, self.__pTest, self.transform, _, _, _ = config.get_config(
is_train=False)
self.__pModel = patch_config_as_nothrow(self.__pModel)
self.__pTest = patch_config_as_nothrow(self.__pTest)
self.resizeParam = (800, 1200)
if callable(self.__pTest.nms.type):
self.__nms = self.__pTest.nms.type(self.__pTest.nms.thr)
else:
from operator_py.nms import py_nms_wrapper
self.__nms = py_nms_wrapper(self.__pTest.nms.thr)
arg_params, aux_params = load_checkpoint(self.__pTest.model.prefix,
self.__pTest.model.epoch)
sym = self.__pModel.test_symbol
from utils.graph_optimize import merge_bn
sym, arg_params, aux_params = merge_bn(sym, arg_params, aux_params)
self.__mod = DetModule(
sym,
data_names=['data', 'im_info', 'im_id', 'rec_id'],
context=ctx)
self.__mod.bind(data_shapes=[('data', (1, 3, self.resizeParam[0],
self.resizeParam[1])),
('im_info', (1, 3)), ('im_id', (1, )),
('rec_id', (1, ))],
for_training=False)
self.__mod.set_params(arg_params, aux_params, allow_extra=False)
self.__saveSymbol(sym, outFolder,
self.__pTest.model.prefix.split('/')[-1])
self.__threshold = threshold
self.outFolder = outFolder
def __call__(self, imgFilename): # detect onto image
roi_record, scale, img = self.__readImg(imgFilename)
h, w = roi_record['data'][0].shape
im_c1 = roi_record['data'][0].reshape(1, 1, h, w)
im_c2 = roi_record['data'][1].reshape(1, 1, h, w)
im_c3 = roi_record['data'][2].reshape(1, 1, h, w)
im_data = np.concatenate((im_c1, im_c2, im_c3), axis=1)
im_info, im_id, rec_id = [(h, w, scale)], [1], [1]
data = mx.io.DataBatch(data=[
mx.nd.array(im_data),
mx.nd.array(im_info),
mx.nd.array(im_id),
mx.nd.array(rec_id)
])
self.__mod.forward(data, is_train=False)
# extract results
outputs = self.__mod.get_outputs(merge_multi_context=False)
rid, id, info, cls, box = [x[0].asnumpy() for x in outputs]
rid, id, info, cls, box = rid.squeeze(), id.squeeze(), info.squeeze(
), cls.squeeze(), box.squeeze()
cls = cls[:, 1:] # remove background
box = box / scale
output_record = dict(rec_id=rid,
im_id=id,
im_info=info,
bbox_xyxy=box,
cls_score=cls)
output_record = self.__pTest.process_output([output_record], None)[0]
final_result = self.__do_nms(output_record)
# obtain representable output
detections = []
for cid, bbox in final_result.items():
idx = np.where(bbox[:, -1] > self.__threshold)[0]
for i in idx:
final_box = bbox[i][:4]
score = bbox[i][-1]
detections.append({
'cls': cid,
'box': final_box,
'score': score
})
img_vis = self.__vis_detections(detections, img)
cv2.imwrite(os.path.join(self.outFolder, imgFilename), img_vis)
#print(os.path.join(self.outFolder,imgFilename))
return detections, None
def __vis_detections(self, dets, img):
font = cv2.FONT_HERSHEY_SIMPLEX
for d in dets:
box = d['box']
clsID = d['cls']
score = d['score']
img = cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]),
(255, 0, 0), 4)
img = cv2.putText(img,
str(clsID) + ': ' + str(round(score, 2)),
(box[0], box[1]), font, 1, (255, 0, 0), 2,
cv2.LINE_AA)
return img
def __do_nms(self, all_output):
box = all_output['bbox_xyxy']
score = all_output['cls_score']
final_dets = {}
for cid in range(score.shape[1]):
score_cls = score[:, cid]
valid_inds = np.where(score_cls > self.__threshold)[0]
box_cls = box[valid_inds]
score_cls = score_cls[valid_inds]
if valid_inds.shape[0] == 0:
continue
det = np.concatenate((box_cls, score_cls.reshape(-1, 1)),
axis=1).astype(np.float32)
det = self.__nms(det)
#cls = coco[cid]
final_dets[cid] = det
return final_dets
def __readImg(self, imgFilename):
img = cv2.imread(imgFilename, cv2.IMREAD_COLOR)
height, width, channels = img.shape
roi_record = {
'gt_bbox': np.array([[0., 0., 0., 0.]]),
'gt_class': np.array([0])
}
roi_record['image_url'] = imgFilename
roi_record['h'] = height
roi_record['w'] = width
for trans in self.transform:
trans.apply(roi_record)
img_shape = [roi_record['h'], roi_record['w']]
shorts, longs = min(img_shape), max(img_shape)
scale = min(self.resizeParam[0] / shorts, self.resizeParam[1] / longs)
return roi_record, scale, img
def __saveSymbol(self, sym, outFolder, fnPrefix):
if not os.path.exists(outFolder): os.makedirs(outFolder)
resFilename = os.path.join(outFolder, fnPrefix + "_symbol_test.json")
sym.save(resFilename)
#import mxnet as mx
#import argparse
#from infer import TDNDetector
#def parse_args():
# parser = argparse.ArgumentParser(description='Test Detection')
# parser.add_argument('--config', type=str, default='config/faster_r101v2c4_c5_256roi_1x.py', help='config file path')
# parser.add_argument('--ctx', type=int, default=0, help='GPU index. Set negative value to use CPU')
# #parser.add_argument('--inputs', type=str, nargs='+', required=True, default='', help='File(-s) to test')
# parser.add_argument('--output', type=str, default='results', help='Where to store results')
# parser.add_argument('--threshold', type=float, default=0.5, help='Detector threshold')
# return parser.parse_args()
#if __name__ == "__main__":
# args = parse_args()
# ctx = mx.gpu(args.ctx) if args.ctx>=0 else args.cpu()
# #imgFilenames = args.inputs
# imgFilenames = ['car.jpg', 'COCO_val2014_000000581929.jpg']
# detector = TDNDetector(args.config, ctx, args.output, args.threshold)
# for i, imgFilename in enumerate(imgFilenames):
# print(imgFilename)
# dets,_= detector(imgFilename)
# print(dets)
