def gt_sdsdb(self): """ :return: """ cache_file = os.path.join(self.cache_path, self.name + '_gt_sdsdb.pkl') """ if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: sdsdb = cPickle.load(fid) print '{} gt sdsdb loaded from {}'.format(self.name, cache_file) return sdsdb """ # for internal useage tic() gt_sdsdb_temp = [ self.load_coco_sds_annotation(index) for index in self.image_set_index ] gt_sdsdb = [x[0] for x in gt_sdsdb_temp] print 'prepare gt_sdsdb using', toc(), 'seconds' #objs = [x[1] for x in gt_sdsdb_temp] tic() generate_cache_seg_inst_kernel(gt_sdsdb_temp[0]) pool = mp.Pool(mp.cpu_count()) pool.map(generate_cache_seg_inst_kernel, gt_sdsdb_temp) pool.close() pool.join() print 'generate cache_seg_inst using', toc(), 'seconds' """ with open(cache_file, 'wb') as fid: cPickle.dump(gt_sdsdb, fid, cPickle.HIGHEST_PROTOCOL) """ # for future release usage # need to implement load sbd data return gt_sdsdb
def read_files(dtype): tic() files = sorted(os.listdir('../out')) files = [f for f in files if f.startswith('features-') and f.endswith('-%s.csv' % dtype)] names, X = zip(*map(read_file, files)) toc('read %s data' % dtype) return names, np.concatenate(X, 1)
def read_files(dtype): tic() files = sorted(os.listdir('../out')) files = [ f for f in files if f.startswith('features-') and f.endswith('-%s.csv' % dtype) ] names, X = zip(*map(read_file, files)) toc('read %s data' % dtype) return names, np.concatenate(X, 1)
def inference(self, image_names): # get data data = self.load_data_and_get_predictor(image_names) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[ [(k, v.shape) for k, v in zip(self.data_names, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] tic() scores, boxes, data_dict = im_detect( self.predictor, data_batch, self.data_names, scales, config) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = self.nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.6, :] dets_nms.append(cls_dets) print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize #im = cv2.imread(cur_path + '/../demo/' + im_name) im = cv2.imread(im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) self.show_save_boxes(im, dets_nms, self.classes, im_name, 1) #show_boxes(im, dets_nms, self.classes, 1) pass
def next(self): tic() if self.iter_next(): logging.debug("loading the data time in Prefetching: {}".format( toc())) return self.current_batch else: raise StopIteration
def inference(self, im, debug=True): tic() dets, masks = self.net.forward(im, conf_thresh=self.min_score) if debug: print('inference time: {:.4f}s'.format(toc())) data = reformat_data(dets, masks, self.classes) return data
def main(im_name, frame, score): data = [] # only resize input image to target size and return scale im, im_scale = resize(frame, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config) im_shapes = [data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data))] masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') # print (im_height, im_width) boxes = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = cpu_mask_voting(masks, boxes, scores[0], num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH) dets = [result_dets[j] for j in range(1, num_classes)] masks = [result_masks[j][:, 0, :, :] for j in range(1, num_classes)] print 'testing {} {:.4f}s'.format(im_name, toc()) min_confidence = score # visualize for i in xrange(len(dets)): keep = np.where(dets[i][:,-1] > min_confidence) dets[i] = dets[i][keep] masks[i] = masks[i][keep] ''' dets: [ numpy.ndarray([[x1 y1 x2 y2 score]]) for j in classes ] [x1 y1] - upper left corner of object [x2 y2] - lower right corner of object masks: [ numpy.ndarray([21, 21]) for j in classes ] confidence that pixel belongs to object ''' for i in range(len(dets)): if len(dets[i]) > 0: for j in range(len(dets[i])): print('{name}: {score} ({loc})'.format(name = classes[i], score = dets[i][j][-1], loc = dets[i][j][:-1].tolist()))
def next(self): tic() if self.iter_next(): self.current_batch = self.queue.get() logging.debug("loading the data time in Group Inter: {}".format( toc())) return self.current_batch else: raise StopIteration
def sync_extract(module, csv, params): create = not os.path.exists(csv) if not create: m1 = os.path.getmtime(csv) m2 = os.path.getmtime('features/' + module + '.py') create = m2 > m1 if create: tic() i = importlib.import_module('features.' + module) X, names = i.fn(*params) toc(module[8:]) if len(X): if len(X[0].shape) == 1: X = [x[:, np.newaxis] for x in X] X = np.concatenate(X, 1) assert X.shape[1] == len(names) #names = ['"' + name + '"' for name in names] header = ','.join(names) fmt = '%d' if X.dtype == int else '%.6f' np.savetxt(csv, X, fmt, delimiter=',', header=header, comments='')
def extract(info_filename, pairs_filename, mode): info_filename = os.path.join('../data', info_filename) pairs_filename = os.path.join('../data', pairs_filename) tic() info_df = pd.read_csv( info_filename, dtype={'itemID': int, 'categoryID': int, 'price': float}, usecols=(0, 1, 6, 7, 8, 9, 10), index_col=0) info_df['line'] = np.arange(len(info_df), dtype=int) toc('info file') info_reader = MyCSVReader(info_filename) toc('info reader') cols = (0, 1) if mode == 'train' else (1, 2) pairs = np.genfromtxt(pairs_filename, int, delimiter=',', skip_header=1, usecols=cols) toc('pairs file') # transforma ItemID em linhas do ficheiro CSV e da matriz info a = info_df.ix[pairs[:, 0]]['line'] b = info_df.ix[pairs[:, 1]]['line'] pairs_lines = np.c_[a, b] toc('pairs lines') params = (info_filename, info_reader, info_df, pairs_lines) modules = [module[:-3] for module in sorted(os.listdir('features')) if module.startswith('extract-')] csvs = ['../out/features-%s-%s.csv' % (module[8:], mode) for module in modules] # create features from modules that have been created or changed #pool = multiprocessing.Pool(multiprocessing.cpu_count()/2) #res = [] for module, csv in itertools.izip(modules, csvs): #res.append(pool.apply_async(sync_extract, (module, csv, params))) sync_extract(module, csv, params) #for r in res: # r.get() # remove whatever has been created by extiguish modules vestiges = [os.path.join('../out', f) for f in os.listdir('../out') if f.startswith('features-') and f.endswith('-%s.csv' % mode)] for v in vestiges: if v not in csvs: print 'removing old %s...' % v os.remove(v)
def main(tempFileList, fileOp): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_rfcn_dcn' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) out_dir = os.path.join( cur_path, 'demo/output/terror-det-rg-data-output/terror-det-v0.9-test/JPEGImages' ) if not os.path.exists(out_dir): os.makedirs(out_dir) # set up class names num_classes = 7 classes = [ 'tibetan flag', 'guns', 'knives', 'not terror', 'islamic flag', 'isis flag' ] # load demo data image_names = tempFileList data = [] for im_name in image_names: im_file = im_name print(im_file) im = cv2.imread(im_file, cv2.IMREAD_COLOR) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + '/demo/models/' + ('rfcn_voc'), 10, process=True) #modify by zxt #mx.model.save_checkpoint('f1/final', 10, sym, arg_params, aux_params) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[0]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) # test # fileOp = open(os.path.join(cur_path, 'terror-det-rg-test-result.txt'), 'w') fileOp = fileOp for idx, im_name in enumerate(image_names): print("begining process %s" % (im_name)) data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize im = cv2.imread(im_name) #im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) im_result = show_boxes(fileOp, im_name, im, dets_nms, classes, 1) cv2.imwrite(out_dir + im_name.split('/')[-1], im_result) print 'done'
time_list_cur_prop = [] time_list_cur_rpn = [] time_list_cur = [] for idx, im_name in enumerate(image_names): im_shape = data[idx]['data'].shape if idx % key_frame_interval == 0: tic() data_batch_key_feat = mx.io.DataBatch( data=[mx.nd.array(data[idx][data_name]) for data_name in ['data']], provide_data=[mx.io.DataDesc('data', (1, 3, 562, 1000))]) mod_key_feat.forward(data_batch_key_feat) conv_feat = mod_key_feat.get_outputs()[0] if conv_feat[0][0][0][0] == 0.3: nothing = 0 time_key_feat = toc() time_list_key_feat.append(time_key_feat) # print type(conv_feat) # print 'As Parts: conv_feat: ', mod_key_feat.get_outputs() tic() data_batch_key_rpn = mx.io.DataBatch( data=[conv_feat, mx.nd.array(data[idx]['im_info'])], provide_data=[ mx.io.DataDesc('conv_feat', (1, 1024, 36, 63)), mx.io.DataDesc('im_info', (1, 3)) ]) mod_key_rpn.forward(data_batch_key_rpn) rois_output, cls_prob, bbox_pred_c = mod_key_rpn.get_outputs() feat_key = conv_feat else:
def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_rfcn_dcn' if not args.rfcn_only else 'resnet_v1_101_rfcn' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 81 classes = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'] # load demo data image_names = ['COCO_test2015_000000000891.jpg', 'COCO_test2015_000000001669.jpg'] data = [] for im_name in image_names: assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + '/../model/' + ('rfcn_dcn_coco' if not args.rfcn_only else 'rfcn_coco'), 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] tic() scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize im = cv2.imread(cur_path + '/../demo/' + im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) show_boxes(im, dets_nms, classes, 1) print 'done'
def main(): # get symbol ctx_id = [int(i) for i in config.gpus.split(',')] pprint.pprint(config) sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names # set up class names num_classes = 81 classes = [ 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush' ] # load demo data image_names = [] names_dirs = os.listdir(cur_path + '/../' + test_dir) for im_name in names_dirs: if im_name[-4:] == '.jpg' or im_name[-4:] == '.png': image_names.append(im_name) data = [] for im_name in image_names: assert os.path.exists(cur_path + '/../' + test_dir + im_name), ( '%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../' + test_dir + im_name, cv2.IMREAD_COLOR | long(128)) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] #print "before scale: " #print im.shape im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) #print "after scale: " #print im.shape #im_scale = 1.0 #print "scale ratio: " #print im_scale im_tensor = transform(im, config.network.PIXEL_MEANS) #print im_tensor.shape im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + '/../' + model_dir, 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(ctx_id[0])], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # warm up for i in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[0]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] _, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, [1.0], config) im_shapes = [ data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data)) ] #print im_shapes if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(num_classes)] all_masks = [[] for _ in xrange(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: masks = masks[0][:, 1:, :, :] result_masks, result_dets = gpu_mask_voting( masks, boxes[0], scores[0], num_classes, 100, im_shapes[0][1], im_shapes[0][0], config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0]) dets = [result_dets[j] for j in range(1, num_classes)] masks = [ result_masks[j][:, 0, :, :] for j in range(1, num_classes) ] print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize for i in xrange(len(dets)): keep = np.where(dets[i][:, -1] > 0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] im = cv2.imread(cur_path + '/../' + test_dir + im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) show_masks(im, dets, masks, classes, config, 1.0 / scales[0], False) # Save img cv2.imwrite(cur_path + '/../' + result_dir + im_name, cv2.cvtColor(im, cv2.COLOR_BGR2RGB)) print 'done'
def batch_extract(self, multiple=True, gt_dir=None, epoch=0): """ :param multiple: :param gt_dir: :return: """ if len(self.img_list) % self.batch_size != 0: batch = len(self.img_list) / self.batch_size + 1 else: batch = len(self.img_list) / self.batch_size for i in xrange(batch): if i < batch - 1: self.batch_list = self.img_list[i * self.batch_size:(i + 1) * self.batch_size] else: self.batch_list = self.img_list[i * self.batch_size:] print '\nMini-batch %d\t' % (i + 1) tmp_data = [] target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] tic() for img in self.batch_list: assert os.path.exists(img), ('%s does not exist.'.format(img)) im = cv2.imread( img, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) # im_info: height, width, scale im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) tmp_data.append({ self.data_names[0]: im_tensor, self.data_names[1]: im_info }) self.ctx = [int(i) for i in config.gpus.split(',')] self.data = [[ mx.nd.array(tmp_data[i][name], mx.gpu(self.ctx[0])) for name in self.data_names ] for i in xrange(len(tmp_data))] max_data_shape = [[(self.data_names[0], (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(self.data_names, self.data[i])] for i in xrange(len(self.data))] provide_label = [None for i in xrange(len(self.data))] arg_params, aux_params = load_param(self.model_dir, epoch, process=True) self.predictor = Predictor(self.sym, self.data_names, self.label_name, context=[mx.gpu(self.ctx[0])], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) print 'preparation: %.4fs' % toc() if i == 0: self.warmup() self.forward(multiple=multiple, gt_dir=gt_dir) self.cleaner()
def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_rfcn' model = '/../model/rfcn_vid' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_test_symbol(config) # set up class names num_classes = 31 classes = ['airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', 'watercraft', 'whale', 'zebra'] # load demo data image_names = glob.glob(cur_path + '/../demo/ILSVRC2015_val_00007010/*.JPEG') output_dir = cur_path + '/../demo/rfcn/' if not os.path.exists(output_dir): os.makedirs(output_dir) # data = [] for im_name in image_names: assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + model, 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) # test time = 0 count = 0 for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] tic() scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) time += toc() count += 1 print 'testing {} {:.4f}s'.format(im_name, time/count) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) # visualize im = cv2.imread(im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # show_boxes(im, dets_nms, classes, 1) out_im = draw_boxes(im, dets_nms, classes, 1) _, filename = os.path.split(im_name) cv2.imwrite(output_dir + filename,out_im) print 'done'
def main(): # get symbol ctx_id = [int(i) for i in config.gpus.split(',')] pprint.pprint(config) sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 81 classes = [ 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush' ] # load demo data image_names = [ 'COCO_test2015_000000000275.jpg', 'COCO_test2015_000000001412.jpg', 'COCO_test2015_000000073428.jpg', 'COCO_test2015_000000393281.jpg' ] data = [] for im_name in image_names: assert os.path.exists(cur_path + '/../demo/' + im_name), ( '%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + '/../model/fcis_coco', convert=True, 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.cpu()], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config) im_shapes = [ data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data)) ] if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(num_classes)] all_masks = [[] for _ in xrange(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') print(im_height, im_width) boxes = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = cpu_mask_voting( masks, boxes, scores[0], num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH) dets = [result_dets[j] for j in range(1, num_classes)] masks = [ result_masks[j][:, 0, :, :] for j in range(1, num_classes) ] print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize for i in xrange(len(dets)): keep = np.where(dets[i][:, -1] > 0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] for i in range(len(dets)): if len(dets[i]) > 0: for j in range(len(dets[i])): print('{name}: {score} ({loc})'.format( name=classes[i], score=dets[i][j][-1], loc=dets[i][j][:-1].tolist())) print 'done'
def process_video_frame(raw_frame_queue, bbox_frame_queue): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_rfcn' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) arg_params, aux_params = load_param( './output/rfcn/road_obj/road_train_all/all/' + 'rfcn_road', 19, process=True) # set up class names; Don't count the background in, even we are treat the background as label '0' num_classes = 4 classes = ['vehicle', 'pedestrian', 'cyclist', 'traffic lights'] target_size = config.SCALES[0][1] max_size = config.SCALES[0][1] while True: tic() i = 0 data = [] frame_list = [] while len(data) < 15: frame = raw_frame_queue.get() if frame is None: continue if i < 2: i += 1 frame, im_scale = resize(frame, target_size, max_size, stride=config.network.IMAGE_STRIDE) bbox_frame_queue.put(frame) continue frame, im_scale = resize(frame, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(frame, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) frame_list.append(frame) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] # print('Debug: [data] shape: {}, cont: {}'.format(type(data), data)) max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] # print('Debug: [max_data_shape] shape: {}, cont: {}'.format(type(max_data_shape), max_data_shape)) provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] # print('Debug: [provide_data] shape: {}, cont: {}'.format(type(provide_data), provide_data)) provide_label = [None for i in xrange(len(data))] # print('Debug: [provide_label] shape: {}, cont: {}'.format(type(provide_label), provide_label)) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # Process video frame # image_names = ['frame'] # for idx, frame in enumerate(frame_list): data_batch = mx.io.DataBatch(data=data, label=[], pad=0, provide_data=provide_data, provide_label=provide_label) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] # print("length: {}".format(len(data_batch.data))) # print('Debug: [scales] cont: {}'.format(scales)) scores_all, boxes_all, data_dict_all = im_detect( predictor, data_batch, data_names, scales, config) # print('scores_all: Type: {}, Values: {}, Length: {}'.format(type(scores_all), scores_all, len(scores_all))) # print('boxes_all: Type: {}, Values: {}, Length: {}'.format(type(boxes_all), boxes_all, len(boxes_all))) # print('data_dict_all: Type: {}, Values: {}, length: {}'.format(type(data_dict_all), data_dict_all, len(data_dict_all))) # print('frame_list: Type: {}, Values: {}, Length: {}'.format(type(frame_list), frame_list, len(frame_list))) # print('scores_all: Type: {}, Length: {}, Values: {}'.format(type(scores_all[0]), len(scores_all[0]), scores_all[0])) # print(scores_all[0].shape) # print('boxes_all: Type: {}, Length: {}'.format(type(boxes_all), len(boxes_all))) # print(boxes_all[0].shape) # print('data_dict_all: Type: {}, length: {}'.format(type(data_dict_all), len(data_dict_all))) # print('frame_list: Type: {}, Length: {}'.format(type(frame_list), len(frame_list))) for idx, frame in enumerate(frame_list): # print('index: {}'.format(str(idx))) boxes = boxes_all[0].astype('f') scores = scores_all[0].astype('f') dets_nms = [] # print(scores.shape) for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4: (j + 1 ) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) bbox_frame_queue.put( draw_bbox_on_frame(frame, dets_nms, classes, scale=scales[idx])) print(toc())
def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_flownet_rfcn' model = '/../model/rfcn_dff_flownet_vid' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_batch_test_symbol(config) sym.save('dff_rfcn.json') #print config.network.get_internals() #mx.visualization.plot_network(sym).view() #print sym.get_intervals() #x = input() # set up class names num_classes = 31 classes = [ 'airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', 'watercraft', 'whale', 'zebra' ] # load demo data image_names = glob.glob(cur_path + '/../demo/sample/*.JPEG') output_dir = cur_path + '/../demo/rfcn_dff_batch/' if not os.path.exists(output_dir): os.makedirs(output_dir) key_frame_interval = 10 # data = [] key_im_tensor = None cur_im_tensor = [] im_info_tensor = [] image_names_list = [] image_names_batch = [] for idx, im_name in enumerate(image_names): assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread( im_name) #, cv2.IMREAD_COLOR)# | cv2.IMREAD_IGNORE_ORIENTATION) #im = cv2.resize(im, (176,176,3)) #height, width, channel = img.shape #gray = im = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) #im = np.zeros(height * width * channel).reshape((height, width, channel)) #im[:,:,0] = gray #im[:,:,1] = gray #im[:,:,2] = gray target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) #print im.shape #print im_scale.shape im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) if idx % key_frame_interval == 0: key_im_tensor = im_tensor else: cur_im_tensor.append(im_tensor) im_info_tensor.append(im_info) image_names_batch.append(im_name) if (idx + 1) % key_frame_interval == 0 or idx == len(image_names) - 1: data.append({ 'data_other': np.concatenate(cur_im_tensor), 'im_info': np.concatenate(im_info_tensor), 'data_key': key_im_tensor }) key_im_tensor = None cur_im_tensor = [] im_info_tensor = [] image_names_list.append(image_names_batch) image_names_batch = [] # get predictor data_names = ['data_other', 'im_info', 'data_key'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[ ('data_other', (key_frame_interval - 1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ('data_key', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + model, 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) #print predictor nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in xrange(1): data_batch = mx.io.DataBatch(data=[data[j]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[j]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[:, 2] for i in xrange(len(data_batch.data)) ] print scales[0].shape scores_all, boxes_all, data_dict = im_batch_detect( predictor, data_batch, data_names, scales, config) print "warmup done" # test time = 0 count = 0 for idx, im_names in enumerate(image_names_list): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[:, 2] for i in xrange(len(data_batch.data)) ] tic() scores_all, boxes_all, data_dict = im_batch_detect( predictor, data_batch, data_names, scales, config) time += toc() count += len(scores_all) print 'testing {} {:.4f}s x {:d}'.format(im_names[0], time / count, len(scores_all)) ''' for batch_idx in xrange(len(scores_all)): boxes = boxes_all[batch_idx].astype('f') scores = scores_all[batch_idx].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) # visualize im = cv2.imread(im_names[batch_idx]) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # show_boxes(im, dets_nms, classes, 1) out_im = draw_boxes(im, dets_nms, classes, 1) _, filename = os.path.split(im_names[batch_idx]) cv2.imwrite(output_dir + filename,out_im) ''' print 'done'
def main(): # get symbol pprint.pprint(config) config.symbol = "resnet_v1_101_fpn_dcn_rcnn" if not args.rfcn_only else "resnet_v1_101_fpn_rcnn" sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 81 classes = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'] # test # find all videos video_path = "../../tmp"#"../../aic2018/track1/track1_videos" video_files = sorted([ x for x in os.listdir(video_path) if x.endswith(".mp4")]) save_path = "../../tmp/output"#"../../aic2018/track1/output" if not os.path.isdir(save_path): os.makedirs(save_path) print("processing {} videos...".format(len(video_files))) pbar = tqdm(total=len(video_files)) for vf in video_files: vid = imageio.get_reader(os.path.join(video_path, vf),'ffmpeg') data = [] for idx, im in enumerate(vid): if idx == 0: #assert os.path.exists(im_path + im_name), ('%s does not exist'.format(im_path + im_name)) #im = cv2.imread(im_path + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) else: break #data.append({'data': None, 'im_info': None}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] print("hhhhh") print(provide_data, provide_label) print("hhhhh") arg_params, aux_params = load_param(cur_path + '/../model/demo_model/' + ('fpn_dcn_coco' if not args.rfcn_only else 'fpn_coco'), 0, process=True) #print(type(arg_params), type(aux_params)) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) print("successfully load model") vout = [] # write to video writer = skvideo.io.FFmpegWriter(os.path.join(save_path, vf.replace(".mp4","_out.mp4")), outputdict={'-vcodec': 'libx264', '-b': '300000000'}) for frame_idx, im in enumerate(vid): #im = cv2.imread(im_path + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) im_original = im.copy() target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data_idx = [{"data": im_tensor, "im_info": im_info}] data_idx = [[mx.nd.array(data_idx[i][name]) for name in data_names] for i in xrange(len(data_idx))] data_batch = mx.io.DataBatch(data=[data_idx[0]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data_idx[0])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] tic() scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] num_dets = 0 for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.65, :] dets_nms.append(cls_dets) num_dets += cls_dets.shape[0] print 'testing {} the {} th frame at {:.4f}s, detections {}'.format(vf, frame_idx, toc(), num_dets) # save results #im = cv2.imread(im_path + im_name) #im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) #im_bbox = show_boxes(im, dets_nms, classes, 1) #cv2.imwrite(im_path + im_name.replace(".jpg", "_bbox.jpg"), im_bbox) save_im, outputs = show_boxes(im_original, dets_nms, classes, 1, False) #cv2.imwrite(os.path.join(save_path, "{}_{}.jpg".format(vf.replace(".mp4", ""), str(frame_idx).zfill(5))), save_im) writer.writeFrame(save_im) for out in outputs: vout.append([frame_idx] + out) # save the whole video detection into pickle file writer.close() with open(os.path.join(save_path, vf.replace(".mp4", "_detect.pkl")), "wb") as f: pickle.dump(vout, f, protocol=2) pbar.update(1) pbar.close() print 'done'
def main(): # get symbol pprint.pprint(config) #config.symbol = "resnet_v1_101_fpn_dcn_rcnn" if not args.rfcn_only else "resnet_v1_101_fpn_rcnn" config.symbol = "resnet_v1_101_fpn_dcn_rcnn" sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 5 classes = ["car", "bus", "van", "others"] # load demo videos im_path = '../../aic2018/track1/images/' image_names = [ x for x in os.listdir('../../aic2018/track1/images/') if (x.endswith(".jpg") and (x.startswith("9_1") or x.startswith("9_1")) ) and not x.endswith("_bbox.jpg") ] data = [] for idx, im_name in enumerate(image_names[:1]): if idx == 0: assert os.path.exists(im_path + im_name), ( '%s does not exist'.format(im_path + im_name)) im = cv2.imread(im_path + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) else: data.append({'data': None, 'im_info': None}) print(data) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[0][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] # what does provide_data and provide_label work for? provide_data = [[(k, v.shape) for k, v in zip(data_names, data[0])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] ## load parameters arg_params, aux_params = load_param(cur_path + '/../model/' + 'fpn_detrac', 1, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) print("successfully load model") # find all videos video_path = "../../tmp" video_files = [x for x in os.listdir(video_path) if x.endswith(".mp4")] save_path = "../../tmp/output" if not os.path.isdir(save_path): os.makedirs(save_path) print("processing {} videos...".format(len(video_files))) pbar = tqdm(total=len(video_files)) for vf in video_files: vid = imageio.get_reader(os.path.join(video_path, vf), 'ffmpeg') vout = [] for frame_idx, im in enumerate(vid): #im = cv2.imread(im_path + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data_idx = [{"data": im_tensor, "im_info": im_info}] data_idx = [[ mx.nd.array(data_idx[i][name]) for name in data_names ] for i in xrange(len(data_idx))] data_batch = mx.io.DataBatch( data=[data_idx[0]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data_idx[0])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4: (j + 1 ) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) print 'testing {} the {} th frame at {:.4f}s, detections {}'.format( vf, frame_idx, toc(), len(dets_nms)) # save results #im = cv2.imread(im_path + im_name) #im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) #im_bbox = show_boxes(im, dets_nms, classes, 1) #cv2.imwrite(im_path + im_name.replace(".jpg", "_bbox.jpg"), im_bbox) save_im, outputs = show_boxes(im, dets_nms, classes, 1) #cv2.imwrite(os.path.join(save_path, "{}_{}.jpg".format(vf.replace(".mp4", ""), str(frame_idx).zfill(5))), save_im) for out in outputs: vout.append([frame_idx] + out) # save the whole video detection into pickle file with open(os.path.join(save_path, vf.replace(".mp4", ".pkl")), "wb") as f: pickle.dump(vout, f, protocol=2) pbar.update(1) pbar.close() print 'done'
# -*- coding: utf-8 -*- import sys sys.dont_write_bytecode = True import os from utils.tictoc import tic, toc import pickle import numpy as np import pandas as pd from scipy import stats print 'load items info...' tic() Xinfo = pd.read_csv('../../data/ItemInfo_train.csv', index_col=0, usecols=[0, 1, 6, 7, 8, 9, 10]) toc() print 'load items pairs...' tic() Xpair = pd.read_csv('../../data/ItemPairs_train.csv', usecols=[0, 1, 2]) toc() # idxmap is an efficient mapping between item-id and row index # we could also use Xinfo.ix[indices], but this approach seems # slightly faster tic() print 'load items mapping...' if os.path.exists('idxmap.pickle'): with open('idxmap.pickle', 'rb') as f: idxmap = pickle.load(f) else: lastid = Xinfo.index[-1]
def main(): # get symbol ctx_id = [int(i) for i in config.gpus.split(',')] pprint.pprint(config) sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 6 classes = [ '__background__', # always index 0 '1', '2', '3', '4', '5' ] # classes = ['__background__', # always index 0 # 'CubeBody', 'CubeFace', 'CylinderBody', 'CylinderFace', 'Grip'] # load demo data image_names = [] names_dirs = os.listdir(cur_path + '/../' + test_dir) for im_name in names_dirs: if im_name[-4:] == '.jpg': image_names.append(im_name) data = [] # next [3001: len(image_names)] for im_name in image_names[9001:len(image_names)]: assert os.path.exists(cur_path + '/../' + test_dir + im_name), ( '%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../' + test_dir + im_name, cv2.IMREAD_COLOR | long(128)) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] # print "before scale: " # print im.shape im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) # print "after scale: " # print im.shape # im_scale = 1.0 # print "scale ratio: " # print im_scale im_tensor = transform(im, config.network.PIXEL_MEANS) # print im_tensor.shape im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] model_path = cur_path + '/../' + config.output_path + '/' + config.config_name + '/' + config.dataset.image_set + '/' + config.TRAIN.model_prefix arg_params, aux_params = load_param(model_path, config.TEST.test_epoch, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(ctx_id[0])], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # warm up for i in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[0]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] _, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config) # 12342_mask Log # LogTxt = open('LogRecorder.txt', 'w') # test for idx, im_name in enumerate(image_names[9001:len(image_names)]): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, [1.0], config) im_shapes = [ data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data)) ] # print im_shapes if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(num_classes)] all_masks = [[] for _ in xrange(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: masks = masks[0][:, 1:, :, :] result_masks, result_dets = gpu_mask_voting( masks, boxes[0], scores[0], num_classes, 100, im_shapes[0][1], im_shapes[0][0], config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0]) dets = [result_dets[j] for j in range(1, num_classes)] masks = [ result_masks[j][:, 0, :, :] for j in range(1, num_classes) ] print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize for i in xrange(len(dets)): keep = np.where(dets[i][:, -1] > 0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] im = cv2.imread(cur_path + '/../' + test_dir + im_name) # im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) ims = show_masks(im, dets, masks, classes, config, 1.0 / scales[0], False) # label = cv2.imread(cur_path + '/../' + label_dir + im_name[0:len(im_name)-4] + '.png') # label = cv2.cvtColor(label, cv2.COLOR_BGR2RGB) # label_flag = checkLabel(label) if dets[0].size: cv2.imwrite( cur_path + '/../' + result_dir + im_name[0:-4] + '_CuBo.png', ims[0]) if dets[1].size: cv2.imwrite( cur_path + '/../' + result_dir + im_name[0:-4] + '_CuFa.png', ims[1]) if dets[2].size: cv2.imwrite( cur_path + '/../' + result_dir + im_name[0:-4] + '_CyBo.png', ims[2]) if dets[3].size: cv2.imwrite( cur_path + '/../' + result_dir + im_name[0:-4] + '_CyFa.png', ims[3]) if dets[4].size: cv2.imwrite( cur_path + '/../' + result_dir + im_name[0:-4] + '_Grip.png', ims[4]) # if np.unique(ims[0]).shape[0] >= 2 and (1 in label_flag): # cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_CuBo.png', ims[0]) # if np.unique(ims[1]).shape[0] >= 2 and (2 in label_flag): # cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_CuFa.png', ims[1]) # if np.unique(ims[2]).shape[0] >= 2 and (3 in label_flag): # cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_CyBo.png', ims[2]) # if np.unique(ims[3]).shape[0] >= 2 and (4 in label_flag): # cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_CyFa.png', ims[3]) # if np.unique(ims[4]).shape[0] >= 2 and (5 in label_flag): # cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_Grip.png', ims[4]) print 'done'
def main(): # get symbol ctx_id = [int(i) for i in config.gpus.split(',')] pprint.pprint(config) sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 31 classes =['alphabet_of_the_magi', 'anglo-saxon_futhorc', 'arcadian', 'armenian', 'asomtavruli_(georgian)', 'balinese', 'bengali', 'blackfoot_(canadian_aboriginal_syllabics)', 'braille', 'burmese_(myanmar)', 'cyrillic', 'early_aramaic', 'futurama', 'grantha', 'greek', 'gujarati', 'hebrew', 'inuktitut_(canadian_aboriginal_syllabics)', 'japanese_(hiragana)', 'japanese_(katakana)', 'korean', 'latin', 'malay_(jawi_-_arabic)', 'mkhedruli_(georgian)', 'n_ko', 'ojibwe_(canadian_aboriginal_syllabics)', 'sanskrit', 'syriac_(estrangelo)', 'tagalog', 'tifinagh'] # load demo data image_names = ['train_000000.jpg', 'train_000001.jpg', 'train_000002.jpg', 'val_000000.jpg', 'val_000001.jpg', 'val_000002.jpg'] data = [] for im_name in image_names: assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + '/../model/fcis_voc', 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(ctx_id[0])], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # warm up for i in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] _, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config) im_shapes = [data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data))] if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(num_classes)] all_masks = [[] for _ in xrange(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') print (im_height, im_width) boxes = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = gpu_mask_voting(masks, boxes, scores[0], num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0]) dets = [result_dets[j] for j in range(1, num_classes)] masks = [result_masks[j][:, 0, :, :] for j in range(1, num_classes)] print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize for i in xrange(len(dets)): keep = np.where(dets[i][:,-1]>0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] im = cv2.imread(cur_path + '/../demo/' + im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) show_masks(im, dets, masks, classes, config) print 'done'
def inference_rcnn_UADETRAC(cfg, dataset, image_set, root_path, dataset_path, ctx, prefix, epoch, vis, ignore_cache, shuffle, has_rpn, proposal, thresh, logger=None, output_path=None): if not logger: assert False, 'require a logger' # print cfg pprint.pprint(cfg) logger.info('testing cfg:{}\n'.format(pprint.pformat(cfg))) # load symbol and testing data if has_rpn: sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() sym = sym_instance.get_symbol(cfg, is_train=False) imdb = eval(dataset)(image_set, root_path, dataset_path, result_path=output_path) #roidb = imdb.gt_roidb_Shuo() roidb = imdb.gt_roidb() else: sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() sym = sym_instance.get_symbol_rfcn(cfg, is_train=False) imdb = eval(dataset)(image_set, root_path, dataset_path, result_path=output_path) gt_roidb = imdb.gt_roidb_Shuo() roidb = eval('imdb.' + proposal + '_roidb')(gt_roidb) print 'len(roidb):', len(roidb) # get test data iter test_data = TestLoader(roidb, cfg, batch_size=len(ctx), shuffle=shuffle, has_rpn=has_rpn) # load model arg_params, aux_params = load_param(prefix, epoch, process=True) print 'inferring: ', prefix, ' epoch: ', epoch """# write parameters to file print 'type(arg_params):',type(arg_params) print 'type(aux_params):',type(aux_params) thefile1 = open('/raid10/home_ext/Deformable-ConvNets/data/data_Shuo/UADETRAC/arg_params.txt','w') thefile2 = open('/raid10/home_ext/Deformable-ConvNets/data/data_Shuo/UADETRAC/aux_params.txt','w') for item_arg in arg_params.items(): thefile1.write(item_arg[0] + str(type(item_arg[1])) + str(item_arg[1].shape)+'\n') for item_aux in aux_params.items(): thefile2.write(item_aux[0] + str(type(item_aux[1])) + str(item_aux[1].shape)+'\n') """ # infer shape data_shape_dict = dict(test_data.provide_data_single) sym_instance.infer_shape(data_shape_dict) sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict, is_train=False) # decide maximum shape data_names = [k[0] for k in test_data.provide_data_single] label_names = None max_data_shape = [[('data', (1, 3, max([v[0] for v in cfg.SCALES]), max([v[1] for v in cfg.SCALES])))]] if not has_rpn: max_data_shape.append( ('rois', (cfg.TEST.PROPOSAL_POST_NMS_TOP_N + 30, 5))) # create predictor predictor = Predictor(sym, data_names, label_names, context=ctx, max_data_shapes=max_data_shape, provide_data=test_data.provide_data, provide_label=test_data.provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(cfg.TEST.NMS, 0) # start detection # pred_eval(predictor, test_data, imdb, cfg, vis=vis, ignore_cache=ignore_cache, thresh=thresh, logger=logger) print 'test_data.size', test_data.size print 'test_data:', test_data print 'data_names:', data_names print 'test_data.provide_data:', test_data.provide_data print 'test_data.provide_label:', test_data.provide_label nnn = 0 classes = ['__background', 'vehicle'] #num_classes = 10 #classes = ['__DontCare__','Car','Suv','SmallTruck','MediumTruck','LargeTruck','Pedestrian','Bus','Van','GroupofPeople'] for im_info, data_batch in test_data: print nnn #print 'roidb[nnn]:',roidb[nnn]['image'] image_name = roidb[nnn]['image'] tic() scales = [iim_info[0, 2] for iim_info in im_info] scores_all, boxes_all, data_dict_all = im_detect( predictor, data_batch, data_names, scales, cfg) boxes = boxes_all[0].astype('f') scores = scores_all[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if cfg.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] #cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) print 'testing {} {:.4f}s'.format(image_name, toc()) # visualize im = cv2.imread(image_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) #print 'cls_dets:',cls_dets #show_boxes(im, dets_nms, classes, 1) nnn = nnn + 1 image_name_length = len(image_name.split('/')) sequence_name = image_name.split('/')[image_name_length - 2] output_file = os.path.join( '/raid10/home_ext/Deformable-ConvNets/data/data_Shuo/UADETRAC', 'Outputs', sequence_name + '_Det_DFCN.txt') frame_id = int(image_name.split('/')[image_name_length - 1][3:8]) thefile = open(output_file, 'a') det_id = 0 for x_small, y_small, x_large, y_large, prob in dets_nms[0]: det_id += 1 thefile.write( str(frame_id) + ',' + str(det_id) + ',' + str(x_small) + ',' + str(y_small) + ',' + str(max(x_large - x_small, 0.001)) + ',' + str(max(y_large - y_small, 0.001)) + ',' + str(prob) + '\n')
def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_flownet_rfcn' model = '/../model/rfcn_dff_flownet_vid' sym_instance = eval(config.symbol + '.' + config.symbol)() key_sym = sym_instance.get_key_test_symbol(config) cur_sym = sym_instance.get_cur_test_symbol(config) # set up class names num_classes = 31 classes = [ 'airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', 'watercraft', 'whale', 'zebra' ] # load demo data image_names = glob.glob(cur_path + '/../demo/ILSVRC2015_val_00007010/*.JPEG') output_dir = cur_path + '/../demo/rfcn_dff/' if not os.path.exists(output_dir): os.makedirs(output_dir) key_frame_interval = 10 # data = [] key_im_tensor = None for idx, im_name in enumerate(image_names): assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) if idx % key_frame_interval == 0: key_im_tensor = im_tensor data.append({ 'data': im_tensor, 'im_info': im_info, 'data_key': key_im_tensor, 'feat_key': np.zeros((1, config.network.DFF_FEAT_DIM, 1, 1)) }) # get predictor data_names = ['data', 'im_info', 'data_key', 'feat_key'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in range(len(data))] max_data_shape = [[ ('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ('data_key', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in range(len(data))] provide_label = [None for i in range(len(data))] arg_params, aux_params = load_param(cur_path + model, 0, process=True) key_predictor = Predictor(key_sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) cur_predictor = Predictor(cur_sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in range(2): data_batch = mx.io.DataBatch(data=[data[j]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[j]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data)) ] if j % key_frame_interval == 0: scores, boxes, data_dict, feat = im_detect(key_predictor, data_batch, data_names, scales, config) else: data_batch.data[0][-1] = feat data_batch.provide_data[0][-1] = ('feat_key', feat.shape) scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch, data_names, scales, config) print("warmup done") # test time = 0 count = 0 for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data)) ] tic() if idx % key_frame_interval == 0: scores, boxes, data_dict, feat = im_detect(key_predictor, data_batch, data_names, scales, config) else: data_batch.data[0][-1] = feat data_batch.provide_data[0][-1] = ('feat_key', feat.shape) scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch, data_names, scales, config) time += toc() count += 1 print('testing {} {:.4f}s'.format(im_name, time / count)) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) # visualize im = cv2.imread(im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # show_boxes(im, dets_nms, classes, 1) out_im = draw_boxes(im, dets_nms, classes, 1) _, filename = os.path.split(im_name) cv2.imwrite(output_dir + filename, out_im) print('done')
def process_one_batch_images_fun(isUrlFlag=False, one_batch_images_list=None, init_model_param=None, fileOp=None, vis=False): # init_model_param list : [sym, arg_params, aux_params] num_classes = 11 # 0 is background, # classes = ['tibetan flag', 'guns', 'knives', # 'not terror', 'islamic flag', 'isis flag'] classes = [ 'islamic flag', 'isis flag', 'tibetan flag', 'knives_true', 'guns_true', 'knives_false', 'knives_kitchen', 'guns_anime', 'guns_tools', 'not terror' ] image_names = one_batch_images_list if len(image_names) <= 0: return all_can_read_image = [] data = [] for im_name in image_names: #print("process : %s"%(im_name)) im = readImage_fun(isUrlFlag=isUrlFlag, imagePath=im_name) # 判断 这个图片是否可读 if np.shape(im) == (): print("ReadImageError : %s" % (im_name)) continue all_can_read_image.append(im_name) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] predictor = Predictor(init_model_param[0], data_names, label_names, context=[mx.gpu(int(args.gpuId))], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=init_model_param[1], aux_params=init_model_param[2]) nms = gpu_nms_wrapper(config.TEST.NMS, 0) for idx, im_name in enumerate(all_can_read_image): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > args.threshold, :] dets_nms.append(cls_dets) print('testing {} {:.4f}s'.format(im_name, toc())) show_boxes(isUrlFlag=isUrlFlag, im_name=im_name, dets=dets_nms, classes=classes, scale=1, vis=vis, fileOp=fileOp, flag=args.outputFileFlag) print('process one batch images done') pass
def forward(self, multiple=True, gt_dir=None): """ :param multiple: :param gt_dir: :return: """ self.multiple = multiple # if multiple is False, gt_dir must be provided if not self.multiple: assert gt_dir is not None for idx, itm in enumerate(self.batch_list): itm = itm.split('/')[-1] data_batch = mx.io.DataBatch( data=[self.data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(self.data_names, self.data[idx]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] im_shapes = [ data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, masks, convs, data_dict = conv_detect( self.predictor, data_batch, self.data_names, scales, config) im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') # return # (1) mask merge if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(self.num_classes)] all_masks = [[] for _ in xrange(self.num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, self.num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, self.num_classes)] masks = [all_masks[j] for j in range(1, self.num_classes)] else: masks = masks[0][:, 1:, :, :] boxes = clip_boxes(boxes[0], (im_height, im_width)) # gpu mask voting if not self.cpu_mask_vote: result_masks, result_dets = gpu_mask_voting( masks, boxes, scores[0], self.num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, self.ctx[0]) # cpu mask voting else: result_masks, result_dets = cpu_mask_voting( masks, boxes, scores[0], self.num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH) # dets represent coordinates of bounding-boxes(up left, bottom right) dets = [result_dets[j] for j in range(1, self.num_classes)] masks = [ result_masks[j][:, 0, :, :] for j in range(1, self.num_classes) ] # (2) filter the result whose detection probability is under 0.7 for i in xrange(len(dets)): keep = np.where(dets[i][:, -1] > 0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] # (3) prepare for roi-pooling roi = [] # scaled bounding box coordinates(up left, bottom right) info = [ ] # class label, name of instance, probability, bounding box coordinates(up left, bottom right) if self.multiple: for k in xrange(0, self.num_classes - 1): nums = len(dets[k]) if nums > 0: for j in xrange(nums): roi.append(dets[k][j][0:-1] * scales[0] ) # note that the input image is scaled info.append((k, itm[:-4] + '-' + self.classes[k] + '_' + str(j + 1), dets[k][j][-1], np.array(np.round(dets[k][j][0:-1])))) else: # Method 1 dist = [] temp_roi = [] temp_info = [] # Instance fn = itm[:-4].split('_')[0] # Instre # fn = itm[:-4] with open(os.path.join(gt_dir, fn + '.txt'), 'r') as strm: cod = strm.readline() if cod.split('\t')[-1] != "\n": # Instance cod = [int(ele) for ele in cod.split('\t')[1:]] else: # Instre cod = [int(ele) for ele in cod.split('\t')[1:-1]] cod_center_x = cod[0] + cod[2] / 2 cod_center_y = cod[1] + cod[3] / 2 for k in xrange(0, self.num_classes - 1): nums = len(dets[k]) if nums > 0: for j in range(nums): det_center_x = ( (dets[k][j][2] - dets[k][j][0]) / 2 + dets[k][j][0]) det_center_y = ( (dets[k][j][3] - dets[k][j][1]) / 2 + dets[k][j][1]) x = (det_center_x - cod_center_x) * (det_center_x - cod_center_x) y = (det_center_y - cod_center_y) * (det_center_y - cod_center_y) dist.append(np.sqrt(x + y)) temp_roi.append( dets[k][j][0:-1] * scales[0] ) # note that the input image is scaled temp_info.append( (k, itm[:-4] + '-' + self.classes[k] + '_' + str(j + 1), dets[k][j][-1], np.array(np.round(dets[k][j][0:-1])))) npdist = np.array(dist) order = np.argsort(npdist) if len(order) > 0: roi.append(temp_roi[order[0]]) info.append(temp_info[order[0]]) # Method 2 # cod[2] += cod[0] # cod[3] += cod[1] # cords = np.array(cod) # roi.append(cords * scales[0]) # info.append((0, itm[:-4] + '-' + self.classes[0] + '_' + str(0 + 1), 0, np.array(cod))) # (4) perform roi-pooling & output features = pooling_delegator(convs, roi, self.pool_size, self.ctx[0], config) """ pca """ if features is not None and not self.mapping: features = self.pca.transform(features) features = normalize(features, norm='l2', axis=1) if self.img is not None and self.ftr is not None and features is not None: self.output(info, features) print 'testing {} {:.4f}s'.format(itm, toc()) # (5) visualize & save """""" if self.dst_dir: pass # im = cv2.imread(self.src_dir + itm) # im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # im = show_masks(im, dets, masks, self.classes, config) # im = save_results(self.dst_dir + itm.replace('.jpg', '.dtc.jpg'), im, dets, masks, self.classes, config) # cv2.imwrite(self.dst_dir + itm.replace('.jpg', '.msk.jpg'), im) """"""
def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_deeplab_dcn' if not args.deeplab_only else 'resnet_v1_101_deeplab' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 19 # load demo data image_names = ['frankfurt_000001_073088_leftImg8bit.png', 'lindau_000024_000019_leftImg8bit.png'] data = [] for im_name in image_names: assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data'] label_names = ['softmax_label'] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param(cur_path + '/../model/' + ('deeplab_dcn_cityscapes' if not args.deeplab_only else 'deeplab_cityscapes'), 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # warm up for j in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]], provide_label=[None]) output_all = predictor.predict(data_batch) output_all = [mx.ndarray.argmax(output['softmax_output'], axis=1).asnumpy() for output in output_all] # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) tic() output_all = predictor.predict(data_batch) output_all = [mx.ndarray.argmax(output['softmax_output'], axis=1).asnumpy() for output in output_all] pallete = getpallete(256) segmentation_result = np.uint8(np.squeeze(output_all)) segmentation_result = Image.fromarray(segmentation_result) segmentation_result.putpalette(pallete) print 'testing {} {:.4f}s'.format(im_name, toc()) pure_im_name, ext_im_name = os.path.splitext(im_name) segmentation_result.save(cur_path + '/../demo/seg_' + pure_im_name + '.png') # visualize im_raw = cv2.imread(cur_path + '/../demo/' + im_name) seg_res = cv2.imread(cur_path + '/../demo/seg_' + pure_im_name + '.png') cv2.imshow('Raw Image', im_raw) cv2.imshow('segmentation_result', seg_res) cv2.waitKey(0) print 'done'
def generate_detections(data, data_names, predictor, config, nms, image_list, detection_num): global classes ret_boxes = [] ret_scores = [] ret_classes = [] k = 0 conversion_dict = { 0: "11", 1: "12", 2: "13", 3: "15", 4: "17", 5: "18", 6: "19", 7: "20", 8: "21", 9: "23", 10: "24", 11: "25", 12: "26", 13: "27", 14: "28", 15: "29", 16: "32", 17: "33", 18: "34", 19: "35", 20: "36", 21: "37", 22: "38", 23: "40", 24: "41", 25: "42", 26: "44", 27: "45", 28: "47", 29: "49", 30: "50", 31: "51", 32: "52", 33: "53", 34: "54", 35: "55", 36: "56", 37: "57", 38: "59", 39: "60", 40: "61", 41: "62", 42: "63", 43: "64", 44: "65", 45: "66", 46: "71", 47: "72", 48: "73", 49: "74", 50: "76", 51: "77", 52: "79", 53: "83", 54: "84", 55: "86", 56: "89", 57: "91", 58: "93", 59: "94" } for idx, im in enumerate(image_list): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.08, :] dets_nms.append(cls_dets) # dets_nms format: # a length 60 array, with each element being a dict representing each class # the coordinates are in (xmin, ymin, xmax, ymax, confidence) format. The coordinates are not normalized # one sample is: [290.09448 439.60617 333.31235 461.8115 0.94750994] # below iterates class by class image_detection_num = 0 for index_class in range(len(dets_nms)): # for each class single_class_nms = dets_nms[index_class] image_detection_num += len(single_class_nms) if len(single_class_nms) != 0: print("detecting", single_class_nms.size, "number of objects", len(single_class_nms)) # print(single_class_nms) for index_single_class_nms in range( min(len(single_class_nms), detection_num)): # print("index_class,index_single_class_nms", index_class,index_single_class_nms, ) ret_boxes.append( dets_nms[index_class][index_single_class_nms] [:-1]) #get all the element other than the last one ret_scores.append( dets_nms[index_class][index_single_class_nms] [-1]) #last element ret_classes.append(conversion_dict[index_class]) # pad zeros # print("1st: len(ret_boxes), image_detection_num", len(ret_boxes), image_detection_num) if image_detection_num <= detection_num: for index_element in range(int(detection_num - image_detection_num)): ret_boxes.append(np.zeros( (4, ), dtype=np.float32 )) #get all the element other than the last one ret_scores.append(0) #last element ret_classes.append(0) else: print("~~~~~ too many predictions ~~~~~~~~~~~~~~~") print("len(ret_boxes), image_detection_num", len(ret_boxes), image_detection_num) print('testing image {} {:.4f}s, detection number {}'.format( idx + 1, toc(), image_detection_num)) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # visualize # don't show the final images. # show_boxes(im, dets_nms, classes, 1, show_image = False, img_save_name = str(idx) + ".png") ret_boxes = np.squeeze(np.array(ret_boxes)) ret_scores = np.squeeze(np.array(ret_scores)) ret_classes = np.squeeze(np.array(ret_classes)) return ret_boxes, ret_scores, ret_classes
def main(): # get symbol pprint.pprint(config) # config.symbol = 'resnet_v1_101_rfcn_dcn' if not args.rfcn_only else 'resnet_v1_101_rfcn' config.symbol = 'resnet_v1_101_fpn_rcnn_rotbox_light_head_RoITransformer' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 15 classes = ['__background__', # always index 0 'plane', 'baseball-diamond', 'bridge', 'ground-track-field', 'small-vehicle', 'large-vehicle', 'ship', 'tennis-court', 'basketball-court', 'storage-tank', 'soccer-ball-field', 'roundabout', 'harbor', 'swimming-pool', 'helicopter'] # load demo data image_names = ['P0004__1__0___0.png', 'P0053__1__0___0.png', 'P0060__1__1648___824.png'] data = [] for im_name in image_names: # pdb.set_trace() assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] # arg_params, aux_params = load_param(cur_path + '/../model/' + ('rfcn_dcn_coco' if not args.rfcn_only else 'rfcn_coco'), 0, process=True) # TODO: change this path arg_params, aux_params = load_param(r'/home/dj/code/Deformable_FPN_DOTA/output/fpn/DOTA/resnet_v1_101_dota_rotbox_light_head_Rroi_v6_trainval_fpn_end2end/train/fpn_DOTA_oriented', config.TEST.test_epoch, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] scores, boxes, data_dict = im_detect_rotbox_Rroi(predictor, data_batch, data_names, scales, config) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] tic() scores, boxes, data_dict = im_detect_rotbox_Rroi(predictor, data_batch, data_names, scales, config) # boxes = boxes[0].astype('f') # scores = scores[0].astype('f') boxes = boxes[0].astype('float64') scores = scores[0].astype('float64') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] # cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_boxes = boxes[:, 8:16] if config.CLASS_AGNOSTIC else boxes[:, j * 8:(j + 1) * 8] cls_quadrangle_dets = np.hstack((cls_boxes, cls_scores)) # keep = nms(cls_dets) keep = py_cpu_nms_poly(cls_quadrangle_dets, 0.3) cls_quadrangle_dets = cls_quadrangle_dets[keep, :] cls_quadrangle_dets = cls_quadrangle_dets[cls_quadrangle_dets[:, -1] > 0.7, :] dets_nms.append(cls_quadrangle_dets) print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize # im = cv2.imread(cur_path + '/../demo/' + im_name) # im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) # pdb.set_trace() im = draw_all_poly_detection(data_dict[0]['data'].asnumpy(), dets_nms, classes[1:], data[idx][1].asnumpy()[0][2], config, threshold=0.2) cv2.imwrite(cur_path + '/../demo/' + 'results' + im_name, im) # show_boxes(im, dets_nms, classes, 1) print 'done'
def run_detection(im_root, result_root, conf_threshold): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_rfcn_dcn' if not args.rfcn_only else 'resnet_v1_101_rfcn' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 81 classes = [ 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush' ] print('detection in {}'.format(im_root)) im_names = sorted(os.listdir(im_root)) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [] for idx, im_name in enumerate(im_names[:2]): im_file = os.path.join(im_root, im_name) im = cv2.imread(im_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param( cur_path + '/../model/' + ('rfcn_dcn_coco' if not args.rfcn_only else 'rfcn_coco'), config.TEST.test_epoch, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # nms = gpu_nms_wrapper(config.TEST.NMS, 0) # nms = soft_nms_wrapper(config.TEST.NMS, method=2) nms = gpu_soft_nms_wrapper(config.TEST.NMS, method=2, device_id=0) nms_t = Timer() for idx, im_name in enumerate(im_names): im_file = os.path.join(im_root, im_name) im = cv2.imread(im_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) origin_im = im.copy() target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) # input data = [mx.nd.array(im_tensor), mx.nd.array(im_info)] data_batch = mx.io.DataBatch(data=[data], label=[], pad=0, index=idx, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) nms_t.tic() keep = nms(cls_dets) nms_t.toc() cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) print 'testing {} {:.2f}ms'.format(im_name, toc() * 1000) print 'nms: {:.2f}ms'.format(nms_t.total_time * 1000) nms_t.clear() # save results person_dets = dets_nms[0] with open(os.path.join(result_root, '{:04d}.txt'.format(idx)), 'w') as f: f.write('{}\n'.format(len(person_dets))) for det in person_dets: x1, y1, x2, y2, s = det w = x2 - x1 h = y2 - y1 f.write('0 {} {} {} {} {}\n'.format(s, w, h, x1, y1)) # visualize im = origin_im # im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) im = show_boxes_cv2(im, dets_nms, classes, 1) cv2.imshow('det', im) cv2.waitKey(1)
def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_deeplab_dcn' if not args.deeplab_only else 'resnet_v1_101_deeplab' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 19 # load demo data image_names = [ 'frankfurt_000001_073088_leftImg8bit.png', 'lindau_000024_000019_leftImg8bit.png' ] data = [] for im_name in image_names: assert os.path.exists(cur_path + '/../demo/' + im_name), ( '%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data'] label_names = ['softmax_label'] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param( cur_path + '/../model/' + ('deeplab_dcn_cityscapes' if not args.deeplab_only else 'deeplab_cityscapes'), 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # warm up for j in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[0]) ]], provide_label=[None]) output_all = predictor.predict(data_batch) output_all = [ mx.ndarray.argmax(output['softmax_output'], axis=1).asnumpy() for output in output_all ] # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) tic() output_all = predictor.predict(data_batch) output_all = [ mx.ndarray.argmax(output['softmax_output'], axis=1).asnumpy() for output in output_all ] pallete = getpallete(256) segmentation_result = np.uint8(np.squeeze(output_all)) segmentation_result = Image.fromarray(segmentation_result) segmentation_result.putpalette(pallete) print 'testing {} {:.4f}s'.format(im_name, toc()) pure_im_name, ext_im_name = os.path.splitext(im_name) segmentation_result.save(cur_path + '/../demo/seg_' + pure_im_name + '.png') # visualize im_raw = cv2.imread(cur_path + '/../demo/' + im_name) seg_res = cv2.imread(cur_path + '/../demo/seg_' + pure_im_name + '.png') cv2.imshow('Raw Image', im_raw) cv2.imshow('segmentation_result', seg_res) cv2.waitKey(0) print 'done'
def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_flownet_deeplab' model1 = '/../model/rfcn_dff_flownet_vid' model2 = '/../model/deeplab_dcn_cityscapes' sym_instance = eval(config.symbol + '.' + config.symbol)() key_sym = sym_instance.get_key_test_symbol(config) cur_sym = sym_instance.get_cur_test_symbol(config) # settings num_classes = 19 interv = args.interval num_ex = args.num_ex # load demo data image_names = sorted( glob.glob(cur_path + '/../demo/cityscapes_data/cityscapes_frankfurt_all_i' + str(interv) + '/*.png')) image_names = image_names[:interv * num_ex] label_files = sorted( glob.glob( cur_path + '/../demo/cityscapes_data/cityscapes_frankfurt_labels_all/*.png')) output_dir = cur_path + '/../demo/deeplab_dff/' if not os.path.exists(output_dir): os.makedirs(output_dir) key_frame_interval = interv # data = [] key_im_tensor = None for idx, im_name in enumerate(image_names): assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) if idx % key_frame_interval == 0: key_im_tensor = im_tensor data.append({ 'data': im_tensor, 'im_info': im_info, 'data_key': key_im_tensor, 'feat_key': np.zeros((1, config.network.DFF_FEAT_DIM, 1, 1)) }) # get predictor data_names = ['data', 'data_key', 'feat_key'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[ ('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ('data_key', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]))), ]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] # models: rfcn_dff_flownet_vid, deeplab_cityscapes arg_params, aux_params = load_param_multi(cur_path + model1, cur_path + model2, 0, process=True) key_predictor = Predictor(key_sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) cur_predictor = Predictor(cur_sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in xrange(2): data_batch = mx.io.DataBatch(data=[data[j]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[j]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] if j % key_frame_interval == 0: # scores, boxes, data_dict, feat = im_detect(key_predictor, data_batch, data_names, scales, config) output_all, feat = im_segment(key_predictor, data_batch) output_all = [ mx.ndarray.argmax(output['croped_score_output'], axis=1).asnumpy() for output in output_all ] else: data_batch.data[0][-1] = feat data_batch.provide_data[0][-1] = ('feat_key', feat.shape) # scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch, data_names, scales, config) output_all, _ = im_segment(cur_predictor, data_batch) output_all = [ mx.ndarray.argmax(output['croped_score_output'], axis=1).asnumpy() for output in output_all ] print "warmup done" # test time = 0 count = 0 hist = np.zeros((num_classes, num_classes)) lb_idx = 0 for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() if idx % key_frame_interval == 0: print '\nframe {} (key)'.format(idx) # scores, boxes, data_dict, feat = im_detect(key_predictor, data_batch, data_names, scales, config) output_all, feat = im_segment(key_predictor, data_batch) output_all = [ mx.ndarray.argmax(output['croped_score_output'], axis=1).asnumpy() for output in output_all ] else: print '\nframe {} (intermediate)'.format(idx) data_batch.data[0][-1] = feat data_batch.provide_data[0][-1] = ('feat_key', feat.shape) # scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch, data_names, scales, config) output_all, _ = im_segment(cur_predictor, data_batch) output_all = [ mx.ndarray.argmax(output['croped_score_output'], axis=1).asnumpy() for output in output_all ] elapsed = toc() time += elapsed count += 1 print 'testing {} {:.4f}s [{:.4f}s]'.format(im_name, elapsed, time / count) pred = np.uint8(np.squeeze(output_all)) segmentation_result = Image.fromarray(pred) pallete = getpallete(256) segmentation_result.putpalette(pallete) _, im_filename = os.path.split(im_name) segmentation_result.save(output_dir + '/seg_' + im_filename) label = None _, lb_filename = os.path.split(label_files[lb_idx]) im_comps = im_filename.split('_') lb_comps = lb_filename.split('_') # if annotation available for frame if im_comps[1] == lb_comps[1] and im_comps[2] == lb_comps[2]: print 'label {}'.format(lb_filename) label = np.asarray(Image.open(label_files[lb_idx])) if lb_idx < len(label_files) - 1: lb_idx += 1 if label is not None: curr_hist = fast_hist(pred.flatten(), label.flatten(), num_classes) hist += curr_hist print 'mIoU {mIoU:.3f}'.format( mIoU=round(np.nanmean(per_class_iu(curr_hist)) * 100, 2)) print '(cum) mIoU {mIoU:.3f}'.format( mIoU=round(np.nanmean(per_class_iu(hist)) * 100, 2)) ious = per_class_iu(hist) * 100 print ' '.join('{:.03f}'.format(i) for i in ious) print '===> final mIoU {mIoU:.3f}'.format(mIoU=round(np.nanmean(ious), 2)) print 'done'
def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_rfcn_dcn' if not args.rfcn_only else 'resnet_v1_101_rfcn' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 81 classes = [ 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush' ] # load demo data image_names = [ 'COCO_test2015_000000000891.jpg', 'COCO_test2015_000000001669.jpg' ] data = [] for im_name in image_names: assert os.path.exists(cur_path + '/../demo/' + im_name), ( '%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in range(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in range(len(data))] provide_label = [None for i in range(len(data))] arg_params, aux_params = load_param( cur_path + '/../model/' + ('rfcn_dcn_coco' if not args.rfcn_only else 'rfcn_coco'), 0, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # warm up for j in range(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[0]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data)) ] scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data)) ] tic() scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) print('testing {} {:.4f}s'.format(im_name, toc())) # visualize im = cv2.imread(cur_path + '/../demo/' + im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) show_boxes(im, dets_nms, classes, 1) print('done')
def main(): # get symbol pprint.pprint(config) config.symbol = 'resnet_v1_101_rfcn' sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names; Don't count the background in, even we are treat the background as label '0' num_classes = 4 classes = ['vehicle', 'pedestrian', 'cyclist', 'traffic lights'] # load demo data image_path = './data/RoadImages/test/' image_names = glob.glob(image_path + '*.jpg') print("Image amount {}".format(len(image_names))) data = [] for im_name in image_names: assert os.path.exists(im_name), ('%s does not exist'.format(im_name)) im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][1] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))] provide_label = [None for i in xrange(len(data))] arg_params, aux_params = load_param( './output/rfcn/road_obj/road_train_all/all/' + 'rfcn_road', 19, process=True) predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(0)], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) nms = gpu_nms_wrapper(config.TEST.NMS, 0) # test notation_dict = {} for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data)) ] tic() scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config) boxes = boxes[0].astype('f') scores = scores[0].astype('f') dets_nms = [] for j in range(1, scores.shape[1]): cls_scores = scores[:, j, np.newaxis] cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) cls_dets = cls_dets[keep, :] cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :] dets_nms.append(cls_dets) print 'testing {} {:.4f}s'.format(im_name, toc()) # notation_list.append(get_notation(im_name, dets_nms, classes, scale=1.0, gen_bbox_pic=True)) notation_dict.update( get_notation(im_name, dets_nms, classes, scale=1.0, gen_bbox_pic=True)) save_notation_file(notation_dict) print 'done'
learning_rate=learning_rate, gamma=gamma, min_child_weight=min_child_weight, colsample_bytree=colsample, subsample=subsample) m.fit(Xtr, ytr) pp = m.predict_proba(Xts)[:, 1] if FINAL_SUBMISSION: import datetime timestamp = datetime.datetime.now().strftime( '%Y-%m-%d-%H:%M') scores = np.c_[np.arange(len(pp)), pp] np.savetxt('../out/vilab-submission-%s.csv' % timestamp, scores, '%d,%.8f', ',', header='id,probability', comments='') toc() else: toc('cs=%.2f md=%2d lr=%.2f mcw=%1d g=%d score=%.4f' % ( colsample, max_depth, learning_rate, min_child_weight, gamma, roc_auc_score(yts, pp))) sys.stdout.flush() import matplotlib.pyplot as plt plt.ioff() xgb.plot_importance(m, tick_label=names) plt.savefig('xgb-features.pdf') plt.show() ''' xgb.plot_tree(m) plt.savefig('xgb-tree.pdf', dpi=900)