time_list_key.append(time_key_feat + time_key_rpn) print "Process key frame {}, Feature extraction: {:.4f}s, Task network: {:.4f}s, Totally: {:.4f}s.".format( im_name, time_key_feat, time_key_rpn, time_key) else: time_cur_rpn = toc() time_list_cur_rpn.append(time_cur_rpn) time_cur = time_cur_flow + time_cur_prop + time_cur_rpn time_list_cur.append(time_cur_flow + time_cur_prop + time_cur_rpn) print "Process non-key frame {}, flow: {:.4f}s, prop: {:.4f}s, rpn: {:.4f}s, Totally: {:.4f}s.".format( im_name, time_cur_flow, time_cur_prop, time_cur_rpn, time_cur) # visualize im = cv2.imread(im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # show_boxes(im, dets_nms, classes, 1) out_im = draw_boxes(im, dets_nms, classes, 1) _, filename = os.path.split(im_name) output_dir = "/home/yizhao/Code/Deep-Feature-Flow/demo/rfcn_dff_components/" cv2.imwrite(output_dir + filename, out_im) print "Average running time of key frame, feat: {:.4f}s, rpn: {:.4f}s, total {:.4f}s".format( np.average(time_list_key_feat), np.average(time_list_key_rpn), np.average(time_list_key)) print "Average running time of non-key frame, flow: {:.4f}s, prop: {:.4f}s, rpn: {:.4f}s, total: {:.4f}s.".format( np.average(time_list_cur_flow), np.average(time_list_cur_prop), np.average(time_list_cur_rpn), np.average(time_list_cur)) exit(0) # data_batch_key = mx.io.DataBatch(data=[mx.nd.array(data[idx]['data']), mx.nd.array(data[idx]['im_info']), mx.nd.array(data[idx]['data_key']), mx.nd.array(data[idx]['feat_key'])], label=[], pad=0, index=0,provide_data=[mx.io.DataDesc('data', (1, 3, 562, 1000)), mx.io.DataDesc('im_info', (1, 3)), mx.io.DataDesc('data_key', (1, 3, 562, 1000)), mx.io.DataDesc('feat_key', (1, 1024, 36, 63))]) # mod_key.forward(data_batch_key) # print 'As Whole: ', mod_key.get_outputs() # print inter_conv_feat.shape
def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_rfcn' model = '/../model/rfcn_vid' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_test_symbol(config) # set up class names num_classes = 31 classes = ['airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', 'watercraft', 'whale', 'zebra'] # load demo data image_names = glob.glob(cur_path + '/../demo/ILSVRC2015_val_00007010/*.JPEG') output_dir = cur_path + '/../demo/rfcn/' if not os.path.exists(output_dir): os.makedirs(output_dir) # data = [] for im_name in image_names: assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + model, 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) # test time = 0 count = 0 for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] tic() scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) time += toc() count += 1 print 'testing {} {:.4f}s'.format(im_name, time/count) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) # visualize im = cv2.imread(im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # show_boxes(im, dets_nms, classes, 1) out_im = draw_boxes(im, dets_nms, classes, 1) _, filename = os.path.split(im_name) cv2.imwrite(output_dir + filename,out_im) print 'done'
def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_flownet_rfcn' model = '/../model/rfcn_dff_flownet_vid' sym_instance = eval(config.symbol + '.' + config.symbol)() key_sym = sym_instance.get_key_test_symbol(config) cur_sym = sym_instance.get_cur_test_symbol(config) # set up class names num_classes = 31 classes = [ 'airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', 'watercraft', 'whale', 'zebra' ] # load demo data image_names = glob.glob(cur_path + '/../demo/ILSVRC2015_val_00007010/*.JPEG') output_dir = cur_path + '/../demo/rfcn_dff/' if not os.path.exists(output_dir): os.makedirs(output_dir) key_frame_interval = 10 # data = [] key_im_tensor = None for idx, im_name in enumerate(image_names): assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) if idx % key_frame_interval == 0: key_im_tensor = im_tensor data.append({ 'data': im_tensor, 'im_info': im_info, 'data_key': key_im_tensor, 'feat_key': np.zeros((1, config.network.DFF_FEAT_DIM, 1, 1)) }) # get predictor data_names = ['data', 'im_info', 'data_key', 'feat_key'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in range(len(data))] max_data_shape = [[ ('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ('data_key', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in range(len(data))] provide_label = [None for i in range(len(data))] arg_params, aux_params = load_param(cur_path + model, 0, process=True) key_predictor = Predictor(key_sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) cur_predictor = Predictor(cur_sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in range(2): data_batch = mx.io.DataBatch(data=[data[j]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[j]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data)) ] if j % key_frame_interval == 0: scores, boxes, data_dict, feat = im_detect(key_predictor, data_batch, data_names, scales, config) else: data_batch.data[0][-1] = feat data_batch.provide_data[0][-1] = ('feat_key', feat.shape) scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch, data_names, scales, config) print("warmup done") # test time = 0 count = 0 for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data)) ] tic() if idx % key_frame_interval == 0: scores, boxes, data_dict, feat = im_detect(key_predictor, data_batch, data_names, scales, config) else: data_batch.data[0][-1] = feat data_batch.provide_data[0][-1] = ('feat_key', feat.shape) scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch, data_names, scales, config) time += toc() count += 1 print('testing {} {:.4f}s'.format(im_name, time / count)) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) # visualize im = cv2.imread(im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # show_boxes(im, dets_nms, classes, 1) out_im = draw_boxes(im, dets_nms, classes, 1) _, filename = os.path.split(im_name) cv2.imwrite(output_dir + filename, out_im) print('done')
def main(): # get symbol pprint.pprint(config) config.symbol = 'impression_network_dynamic_offset_sparse' model = '/../local_run_output/impression_dynamic_offset-lr-10000-times-neighbor-4-dense-4' first_sym_instance = eval(config.symbol + '.' + config.symbol)() key_sym_instance = eval(config.symbol + '.' + config.symbol)() cur_sym_instance = eval(config.symbol + '.' + config.symbol)() first_sym = first_sym_instance.get_first_test_symbol_impression(config) key_sym = key_sym_instance.get_key_test_symbol_impression(config) cur_sym = cur_sym_instance.get_cur_test_symbol_impression(config) # set up class names num_classes = 31 classes = [ 'airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', 'watercraft', 'whale', 'zebra' ] # load demo data image_names = glob.glob(cur_path + '/../demo/ILSVRC2015_val_00011005/*.JPEG') output_dir = cur_path + '/../demo/motion-prior-output-00011005/' if not os.path.exists(output_dir): os.makedirs(output_dir) key_frame_interval = 10 image_names.sort() data = [] for idx, im_name in enumerate(image_names): assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) if idx % key_frame_interval == 0: if idx == 0: data_oldkey = im_tensor.copy() data_newkey = im_tensor.copy() data_cur = im_tensor.copy() else: data_oldkey = data_newkey.copy() data_newkey = im_tensor else: data_cur = im_tensor shape = im_tensor.shape infer_height = int(np.ceil(shape[2] / 16.0)) infer_width = int(np.ceil(shape[3] / 16.0)) data.append({ 'data_oldkey': data_oldkey, 'data_newkey': data_newkey, 'data_cur': data_cur, 'im_info': im_info, 'impression': np.zeros( (1, config.network.DFF_FEAT_DIM, infer_height, infer_width)), 'key_feat_task': np.zeros( (1, config.network.DFF_FEAT_DIM, infer_height, infer_width)) }) # get predictor data_names = [ 'data_oldkey', 'data_cur', 'data_newkey', 'im_info', 'impression', 'key_feat_task' ] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[ ('data_oldkey', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ('data_newkey', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ('data_cur', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ('impression', (1, 1024, 38, 63)), ('key_feat_task', (1, 1024, 38, 63)) ]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + model, 4, process=True) first_predictor = Predictor(first_sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) key_predictor = Predictor(key_sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) cur_predictor = Predictor(cur_sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in xrange(2): data_batch = mx.io.DataBatch(data=[data[j]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[j]) ]], provide_label=[None]) scales = [ data_batch.data[i][3].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] if j % key_frame_interval == 0: # keyframe if j == 0: # first frame scores, boxes, data_dict, conv_feat, _, _, _ = im_detect_impression_online( first_predictor, data_batch, data_names, scales, config) feat_task = conv_feat impression = conv_feat else: # keyframe data_batch.data[0][-2] = impression data_batch.provide_data[0][-2] = ('impression', impression.shape) scores, boxes, data_dict, conv_feat, impression, feat_task = im_detect_impression_online( key_predictor, data_batch, data_names, scales, config) else: # current frame data_batch.data[0][-1] = feat_task data_batch.provide_data[0][-1] = ('key_feat_task', feat_task.shape) scores, boxes, data_dict, _, _, _, _ = im_detect_impression_online( cur_predictor, data_batch, data_names, scales, config) print "warmup done" # test time = 0 count = 0 for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][3].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() print(idx) if idx % key_frame_interval == 0: # keyframe if idx == 0: # first frame scores, boxes, data_dict, conv_feat, _, _, _ = im_detect_impression_online( first_predictor, data_batch, data_names, scales, config) feat_task = conv_feat impression = conv_feat feat_task_numpy = feat_task.asnumpy() np.save("features/impression_%s.npy" % (idx), feat_task_numpy) else: # keyframe data_batch.data[0][-2] = impression data_batch.provide_data[0][-2] = ('impression', impression.shape) scores, boxes, data_dict, conv_feat, impression, feat_task, _ = im_detect_impression_online( key_predictor, data_batch, data_names, scales, config) feat_task_key_numpy = feat_task.asnumpy() np.save("features/impression_%s.npy" % (idx), feat_task_key_numpy) else: # current frame data_batch.data[0][-1] = feat_task data_batch.provide_data[0][-1] = ('key_feat_task', feat_task.shape) scores, boxes, data_dict, _, _, _, feat_task_cur = im_detect_impression_online( cur_predictor, data_batch, data_names, scales, config) if idx >= 1: feat_task_cur_numpy = feat_task_cur.asnumpy() np.save("features/impression_%s.npy" % (idx), feat_task_cur_numpy) #import pdb;pdb.set_trace() time += toc() count += 1 print 'testing {} {:.4f}s'.format(im_name, time / count) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) # visualize im = cv2.imread(im_name) #im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # show_boxes(im, dets_nms, classes, 1) out_im = draw_boxes(im, dets_nms, classes, 1) _, filename = os.path.split(im_name) cv2.imwrite(output_dir + filename, out_im) print 'done'