time_list_key.append(time_key_feat + time_key_rpn)
print "Process key frame {}, Feature extraction: {:.4f}s, Task network: {:.4f}s, Totally: {:.4f}s.".format(
im_name, time_key_feat, time_key_rpn, time_key)
else:
time_cur_rpn = toc()
time_list_cur_rpn.append(time_cur_rpn)
time_cur = time_cur_flow + time_cur_prop + time_cur_rpn
time_list_cur.append(time_cur_flow + time_cur_prop + time_cur_rpn)
print "Process non-key frame {}, flow: {:.4f}s, prop: {:.4f}s, rpn: {:.4f}s, Totally: {:.4f}s.".format(
im_name, time_cur_flow, time_cur_prop, time_cur_rpn, time_cur)
# visualize
im = cv2.imread(im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
# show_boxes(im, dets_nms, classes, 1)
out_im = draw_boxes(im, dets_nms, classes, 1)
_, filename = os.path.split(im_name)
output_dir = "/home/yizhao/Code/Deep-Feature-Flow/demo/rfcn_dff_components/"
cv2.imwrite(output_dir + filename, out_im)
print "Average running time of key frame, feat: {:.4f}s, rpn: {:.4f}s, total {:.4f}s".format(
np.average(time_list_key_feat), np.average(time_list_key_rpn),
np.average(time_list_key))
print "Average running time of non-key frame, flow: {:.4f}s, prop: {:.4f}s, rpn: {:.4f}s, total: {:.4f}s.".format(
np.average(time_list_cur_flow), np.average(time_list_cur_prop),
np.average(time_list_cur_rpn), np.average(time_list_cur))
exit(0)
# data_batch_key = mx.io.DataBatch(data=[mx.nd.array(data[idx]['data']), mx.nd.array(data[idx]['im_info']), mx.nd.array(data[idx]['data_key']), mx.nd.array(data[idx]['feat_key'])], label=[], pad=0, index=0,provide_data=[mx.io.DataDesc('data', (1, 3, 562, 1000)), mx.io.DataDesc('im_info', (1, 3)), mx.io.DataDesc('data_key', (1, 3, 562, 1000)), mx.io.DataDesc('feat_key', (1, 1024, 36, 63))])
# mod_key.forward(data_batch_key)
# print 'As Whole: ', mod_key.get_outputs()
# print inter_conv_feat.shape
def main():
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_rfcn'
model = '/../model/rfcn_vid'
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_test_symbol(config)
# set up class names
num_classes = 31
classes = ['airplane', 'antelope', 'bear', 'bicycle',
'bird', 'bus', 'car', 'cattle',
'dog', 'domestic_cat', 'elephant', 'fox',
'giant_panda', 'hamster', 'horse', 'lion',
'lizard', 'monkey', 'motorcycle', 'rabbit',
'red_panda', 'sheep', 'snake', 'squirrel',
'tiger', 'train', 'turtle', 'watercraft',
'whale', 'zebra']
# load demo data
image_names = glob.glob(cur_path + '/../demo/ILSVRC2015_val_00007010/*.JPEG')
output_dir = cur_path + '/../demo/rfcn/'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
#
data = []
for im_name in image_names:
assert os.path.exists(im_name), ('%s does not exist'.format(im_name))
im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(cur_path + model, 0, process=True)
predictor = Predictor(sym, data_names, label_names,
context=[mx.gpu(0)], max_data_shapes=max_data_shape,
provide_data=provide_data, provide_label=provide_label,
arg_params=arg_params, aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# warm up
for j in xrange(2):
data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config)
# test
time = 0
count = 0
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
tic()
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config)
time += toc()
count += 1
print 'testing {} {:.4f}s'.format(im_name, time/count)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
# visualize
im = cv2.imread(im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
# show_boxes(im, dets_nms, classes, 1)
out_im = draw_boxes(im, dets_nms, classes, 1)
_, filename = os.path.split(im_name)
cv2.imwrite(output_dir + filename,out_im)
print 'done'
def main():
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_flownet_rfcn'
model = '/../model/rfcn_dff_flownet_vid'
sym_instance = eval(config.symbol + '.' + config.symbol)()
key_sym = sym_instance.get_key_test_symbol(config)
cur_sym = sym_instance.get_cur_test_symbol(config)
# set up class names
num_classes = 31
classes = [
'airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car',
'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda',
'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit',
'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle',
'watercraft', 'whale', 'zebra'
]
# load demo data
image_names = glob.glob(cur_path +
'/../demo/ILSVRC2015_val_00007010/*.JPEG')
output_dir = cur_path + '/../demo/rfcn_dff/'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
key_frame_interval = 10
#
data = []
key_im_tensor = None
for idx, im_name in enumerate(image_names):
assert os.path.exists(im_name), ('%s does not exist'.format(im_name))
im = cv2.imread(im_name,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
if idx % key_frame_interval == 0:
key_im_tensor = im_tensor
data.append({
'data':
im_tensor,
'im_info':
im_info,
'data_key':
key_im_tensor,
'feat_key':
np.zeros((1, config.network.DFF_FEAT_DIM, 1, 1))
})
# get predictor
data_names = ['data', 'im_info', 'data_key', 'feat_key']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in range(len(data))]
max_data_shape = [[
('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES]))),
('data_key', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES]))),
]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in range(len(data))]
provide_label = [None for i in range(len(data))]
arg_params, aux_params = load_param(cur_path + model, 0, process=True)
key_predictor = Predictor(key_sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
cur_predictor = Predictor(cur_sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# warm up
for j in range(2):
data_batch = mx.io.DataBatch(data=[data[j]],
label=[],
pad=0,
index=0,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data[j])
]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in range(len(data_batch.data))
]
if j % key_frame_interval == 0:
scores, boxes, data_dict, feat = im_detect(key_predictor,
data_batch, data_names,
scales, config)
else:
data_batch.data[0][-1] = feat
data_batch.provide_data[0][-1] = ('feat_key', feat.shape)
scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch,
data_names, scales, config)
print("warmup done")
# test
time = 0
count = 0
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in range(len(data_batch.data))
]
tic()
if idx % key_frame_interval == 0:
scores, boxes, data_dict, feat = im_detect(key_predictor,
data_batch, data_names,
scales, config)
else:
data_batch.data[0][-1] = feat
data_batch.provide_data[0][-1] = ('feat_key', feat.shape)
scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch,
data_names, scales, config)
time += toc()
count += 1
print('testing {} {:.4f}s'.format(im_name, time / count))
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if config.CLASS_AGNOSTIC else boxes[:, j *
4:(j +
1) *
4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
# visualize
im = cv2.imread(im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
# show_boxes(im, dets_nms, classes, 1)
out_im = draw_boxes(im, dets_nms, classes, 1)
_, filename = os.path.split(im_name)
cv2.imwrite(output_dir + filename, out_im)
print('done')
def main():
# get symbol
pprint.pprint(config)
config.symbol = 'impression_network_dynamic_offset_sparse'
model = '/../local_run_output/impression_dynamic_offset-lr-10000-times-neighbor-4-dense-4'
first_sym_instance = eval(config.symbol + '.' + config.symbol)()
key_sym_instance = eval(config.symbol + '.' + config.symbol)()
cur_sym_instance = eval(config.symbol + '.' + config.symbol)()
first_sym = first_sym_instance.get_first_test_symbol_impression(config)
key_sym = key_sym_instance.get_key_test_symbol_impression(config)
cur_sym = cur_sym_instance.get_cur_test_symbol_impression(config)
# set up class names
num_classes = 31
classes = [
'airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car',
'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda',
'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit',
'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle',
'watercraft', 'whale', 'zebra'
]
# load demo data
image_names = glob.glob(cur_path +
'/../demo/ILSVRC2015_val_00011005/*.JPEG')
output_dir = cur_path + '/../demo/motion-prior-output-00011005/'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
key_frame_interval = 10
image_names.sort()
data = []
for idx, im_name in enumerate(image_names):
assert os.path.exists(im_name), ('%s does not exist'.format(im_name))
im = cv2.imread(im_name,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
if idx % key_frame_interval == 0:
if idx == 0:
data_oldkey = im_tensor.copy()
data_newkey = im_tensor.copy()
data_cur = im_tensor.copy()
else:
data_oldkey = data_newkey.copy()
data_newkey = im_tensor
else:
data_cur = im_tensor
shape = im_tensor.shape
infer_height = int(np.ceil(shape[2] / 16.0))
infer_width = int(np.ceil(shape[3] / 16.0))
data.append({
'data_oldkey':
data_oldkey,
'data_newkey':
data_newkey,
'data_cur':
data_cur,
'im_info':
im_info,
'impression':
np.zeros(
(1, config.network.DFF_FEAT_DIM, infer_height, infer_width)),
'key_feat_task':
np.zeros(
(1, config.network.DFF_FEAT_DIM, infer_height, infer_width))
})
# get predictor
data_names = [
'data_oldkey', 'data_cur', 'data_newkey', 'im_info', 'impression',
'key_feat_task'
]
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in xrange(len(data))]
max_data_shape = [[
('data_oldkey', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES]))),
('data_newkey', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES]))),
('data_cur', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES]))),
('impression', (1, 1024, 38, 63)), ('key_feat_task', (1, 1024, 38, 63))
]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(cur_path + model, 4, process=True)
first_predictor = Predictor(first_sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
key_predictor = Predictor(key_sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
cur_predictor = Predictor(cur_sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# warm up
for j in xrange(2):
data_batch = mx.io.DataBatch(data=[data[j]],
label=[],
pad=0,
index=0,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data[j])
]],
provide_label=[None])
scales = [
data_batch.data[i][3].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
if j % key_frame_interval == 0: # keyframe
if j == 0: # first frame
scores, boxes, data_dict, conv_feat, _, _, _ = im_detect_impression_online(
first_predictor, data_batch, data_names, scales, config)
feat_task = conv_feat
impression = conv_feat
else: # keyframe
data_batch.data[0][-2] = impression
data_batch.provide_data[0][-2] = ('impression',
impression.shape)
scores, boxes, data_dict, conv_feat, impression, feat_task = im_detect_impression_online(
key_predictor, data_batch, data_names, scales, config)
else: # current frame
data_batch.data[0][-1] = feat_task
data_batch.provide_data[0][-1] = ('key_feat_task', feat_task.shape)
scores, boxes, data_dict, _, _, _, _ = im_detect_impression_online(
cur_predictor, data_batch, data_names, scales, config)
print "warmup done"
# test
time = 0
count = 0
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][3].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
tic()
print(idx)
if idx % key_frame_interval == 0: # keyframe
if idx == 0: # first frame
scores, boxes, data_dict, conv_feat, _, _, _ = im_detect_impression_online(
first_predictor, data_batch, data_names, scales, config)
feat_task = conv_feat
impression = conv_feat
feat_task_numpy = feat_task.asnumpy()
np.save("features/impression_%s.npy" % (idx), feat_task_numpy)
else: # keyframe
data_batch.data[0][-2] = impression
data_batch.provide_data[0][-2] = ('impression',
impression.shape)
scores, boxes, data_dict, conv_feat, impression, feat_task, _ = im_detect_impression_online(
key_predictor, data_batch, data_names, scales, config)
feat_task_key_numpy = feat_task.asnumpy()
np.save("features/impression_%s.npy" % (idx),
feat_task_key_numpy)
else: # current frame
data_batch.data[0][-1] = feat_task
data_batch.provide_data[0][-1] = ('key_feat_task', feat_task.shape)
scores, boxes, data_dict, _, _, _, feat_task_cur = im_detect_impression_online(
cur_predictor, data_batch, data_names, scales, config)
if idx >= 1:
feat_task_cur_numpy = feat_task_cur.asnumpy()
np.save("features/impression_%s.npy" % (idx),
feat_task_cur_numpy)
#import pdb;pdb.set_trace()
time += toc()
count += 1
print 'testing {} {:.4f}s'.format(im_name, time / count)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if config.CLASS_AGNOSTIC else boxes[:, j *
4:(j +
1) *
4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
# visualize
im = cv2.imread(im_name)
#im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
# show_boxes(im, dets_nms, classes, 1)
out_im = draw_boxes(im, dets_nms, classes, 1)
_, filename = os.path.split(im_name)
cv2.imwrite(output_dir + filename, out_im)
print 'done'
