def gt_sdsdb(self):
"""
:return:
"""
cache_file = os.path.join(self.cache_path, self.name + '_gt_sdsdb.pkl')
"""
if os.path.exists(cache_file):
with open(cache_file, 'rb') as fid:
sdsdb = cPickle.load(fid)
print '{} gt sdsdb loaded from {}'.format(self.name, cache_file)
return sdsdb
"""
# for internal useage
tic()
gt_sdsdb_temp = [
self.load_coco_sds_annotation(index)
for index in self.image_set_index
]
gt_sdsdb = [x[0] for x in gt_sdsdb_temp]
print 'prepare gt_sdsdb using', toc(), 'seconds'
#objs = [x[1] for x in gt_sdsdb_temp]
tic()
generate_cache_seg_inst_kernel(gt_sdsdb_temp[0])
pool = mp.Pool(mp.cpu_count())
pool.map(generate_cache_seg_inst_kernel, gt_sdsdb_temp)
pool.close()
pool.join()
print 'generate cache_seg_inst using', toc(), 'seconds'
"""
with open(cache_file, 'wb') as fid:
cPickle.dump(gt_sdsdb, fid, cPickle.HIGHEST_PROTOCOL)
"""
# for future release usage
# need to implement load sbd data
return gt_sdsdb
def read_files(dtype):
tic()
files = sorted(os.listdir('../out'))
files = [f for f in files
if f.startswith('features-') and f.endswith('-%s.csv' % dtype)]
names, X = zip(*map(read_file, files))
toc('read %s data' % dtype)
return names, np.concatenate(X, 1)
def read_files(dtype):
tic()
files = sorted(os.listdir('../out'))
files = [
f for f in files
if f.startswith('features-') and f.endswith('-%s.csv' % dtype)
]
names, X = zip(*map(read_file, files))
toc('read %s data' % dtype)
return names, np.concatenate(X, 1)
def inference(self, image_names):
# get data
data = self.load_data_and_get_predictor(image_names)
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx,
provide_data=[
[(k, v.shape) for k, v in zip(self.data_names, data[idx])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))]
tic()
scores, boxes, data_dict = im_detect(
self.predictor, data_batch, self.data_names, scales, config)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = self.nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.6, :]
dets_nms.append(cls_dets)
print 'testing {} {:.4f}s'.format(im_name, toc())
# visualize
#im = cv2.imread(cur_path + '/../demo/' + im_name)
im = cv2.imread(im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
self.show_save_boxes(im, dets_nms, self.classes, im_name, 1)
#show_boxes(im, dets_nms, self.classes, 1)
pass
def next(self):
tic()
if self.iter_next():
logging.debug("loading the data time in Prefetching: {}".format(
toc()))
return self.current_batch
else:
raise StopIteration
def inference(self, im, debug=True):
tic()
dets, masks = self.net.forward(im, conf_thresh=self.min_score)
if debug:
print('inference time: {:.4f}s'.format(toc()))
data = reformat_data(dets, masks, self.classes)
return data
def main(im_name, frame, score):
data = []
# only resize input image to target size and return scale
im, im_scale = resize(frame, target_size, max_size, stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))]
data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
tic()
scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config)
im_shapes = [data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data))]
masks = masks[0][:, 1:, :, :]
im_height = np.round(im_shapes[0][0] / scales[0]).astype('int')
im_width = np.round(im_shapes[0][1] / scales[0]).astype('int')
# print (im_height, im_width)
boxes = clip_boxes(boxes[0], (im_height, im_width))
result_masks, result_dets = cpu_mask_voting(masks, boxes, scores[0], num_classes,
100, im_width, im_height,
config.TEST.NMS, config.TEST.MASK_MERGE_THRESH,
config.BINARY_THRESH)
dets = [result_dets[j] for j in range(1, num_classes)]
masks = [result_masks[j][:, 0, :, :] for j in range(1, num_classes)]
print 'testing {} {:.4f}s'.format(im_name, toc())
min_confidence = score
# visualize
for i in xrange(len(dets)):
keep = np.where(dets[i][:,-1] > min_confidence)
dets[i] = dets[i][keep]
masks[i] = masks[i][keep]
'''
dets: [ numpy.ndarray([[x1 y1 x2 y2 score]]) for j in classes ]
[x1 y1] - upper left corner of object
[x2 y2] - lower right corner of object
masks: [ numpy.ndarray([21, 21]) for j in classes ]
confidence that pixel belongs to object
'''
for i in range(len(dets)):
if len(dets[i]) > 0:
for j in range(len(dets[i])):
print('{name}: {score} ({loc})'.format(name = classes[i], score = dets[i][j][-1], loc = dets[i][j][:-1].tolist()))
def next(self):
tic()
if self.iter_next():
self.current_batch = self.queue.get()
logging.debug("loading the data time in Group Inter: {}".format(
toc()))
return self.current_batch
else:
raise StopIteration
def sync_extract(module, csv, params):
create = not os.path.exists(csv)
if not create:
m1 = os.path.getmtime(csv)
m2 = os.path.getmtime('features/' + module + '.py')
create = m2 > m1
if create:
tic()
i = importlib.import_module('features.' + module)
X, names = i.fn(*params)
toc(module[8:])
if len(X):
if len(X[0].shape) == 1:
X = [x[:, np.newaxis] for x in X]
X = np.concatenate(X, 1)
assert X.shape[1] == len(names)
#names = ['"' + name + '"' for name in names]
header = ','.join(names)
fmt = '%d' if X.dtype == int else '%.6f'
np.savetxt(csv, X, fmt, delimiter=',', header=header, comments='')
def extract(info_filename, pairs_filename, mode):
info_filename = os.path.join('../data', info_filename)
pairs_filename = os.path.join('../data', pairs_filename)
tic()
info_df = pd.read_csv(
info_filename,
dtype={'itemID': int, 'categoryID': int, 'price': float},
usecols=(0, 1, 6, 7, 8, 9, 10), index_col=0)
info_df['line'] = np.arange(len(info_df), dtype=int)
toc('info file')
info_reader = MyCSVReader(info_filename)
toc('info reader')
cols = (0, 1) if mode == 'train' else (1, 2)
pairs = np.genfromtxt(pairs_filename, int, delimiter=',', skip_header=1,
usecols=cols)
toc('pairs file')
# transforma ItemID em linhas do ficheiro CSV e da matriz info
a = info_df.ix[pairs[:, 0]]['line']
b = info_df.ix[pairs[:, 1]]['line']
pairs_lines = np.c_[a, b]
toc('pairs lines')
params = (info_filename, info_reader, info_df, pairs_lines)
modules = [module[:-3] for module in sorted(os.listdir('features'))
if module.startswith('extract-')]
csvs = ['../out/features-%s-%s.csv' % (module[8:], mode)
for module in modules]
# create features from modules that have been created or changed
#pool = multiprocessing.Pool(multiprocessing.cpu_count()/2)
#res = []
for module, csv in itertools.izip(modules, csvs):
#res.append(pool.apply_async(sync_extract, (module, csv, params)))
sync_extract(module, csv, params)
#for r in res:
# r.get()
# remove whatever has been created by extiguish modules
vestiges = [os.path.join('../out', f) for f in os.listdir('../out')
if f.startswith('features-') and f.endswith('-%s.csv' % mode)]
for v in vestiges:
if v not in csvs:
print 'removing old %s...' % v
os.remove(v)
def main(tempFileList, fileOp):
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_rfcn_dcn'
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
out_dir = os.path.join(
cur_path,
'demo/output/terror-det-rg-data-output/terror-det-v0.9-test/JPEGImages'
)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# set up class names
num_classes = 7
classes = [
'tibetan flag', 'guns', 'knives', 'not terror', 'islamic flag',
'isis flag'
]
# load demo data
image_names = tempFileList
data = []
for im_name in image_names:
im_file = im_name
print(im_file)
im = cv2.imread(im_file, cv2.IMREAD_COLOR)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(cur_path + '/demo/models/' +
('rfcn_voc'),
10,
process=True)
#modify by zxt
#mx.model.save_checkpoint('f1/final', 10, sym, arg_params, aux_params)
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# warm up
for j in xrange(2):
data_batch = mx.io.DataBatch(data=[data[0]],
label=[],
pad=0,
index=0,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data[0])
]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names,
scales, config)
# test
# fileOp = open(os.path.join(cur_path, 'terror-det-rg-test-result.txt'), 'w')
fileOp = fileOp
for idx, im_name in enumerate(image_names):
print("begining process %s" % (im_name))
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
tic()
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names,
scales, config)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if config.CLASS_AGNOSTIC else boxes[:, j *
4:(j +
1) *
4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
print 'testing {} {:.4f}s'.format(im_name, toc())
# visualize
im = cv2.imread(im_name)
#im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
im_result = show_boxes(fileOp, im_name, im, dets_nms, classes, 1)
cv2.imwrite(out_dir + im_name.split('/')[-1], im_result)
print 'done'
time_list_cur_prop = []
time_list_cur_rpn = []
time_list_cur = []
for idx, im_name in enumerate(image_names):
im_shape = data[idx]['data'].shape
if idx % key_frame_interval == 0:
tic()
data_batch_key_feat = mx.io.DataBatch(
data=[mx.nd.array(data[idx][data_name]) for data_name in ['data']],
provide_data=[mx.io.DataDesc('data', (1, 3, 562, 1000))])
mod_key_feat.forward(data_batch_key_feat)
conv_feat = mod_key_feat.get_outputs()[0]
if conv_feat[0][0][0][0] == 0.3:
nothing = 0
time_key_feat = toc()
time_list_key_feat.append(time_key_feat)
# print type(conv_feat)
# print 'As Parts: conv_feat: ', mod_key_feat.get_outputs()
tic()
data_batch_key_rpn = mx.io.DataBatch(
data=[conv_feat, mx.nd.array(data[idx]['im_info'])],
provide_data=[
mx.io.DataDesc('conv_feat', (1, 1024, 36, 63)),
mx.io.DataDesc('im_info', (1, 3))
])
mod_key_rpn.forward(data_batch_key_rpn)
rois_output, cls_prob, bbox_pred_c = mod_key_rpn.get_outputs()
feat_key = conv_feat
else:
def main():
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_rfcn_dcn' if not args.rfcn_only else 'resnet_v1_101_rfcn'
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 81
classes = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat',
'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse',
'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie',
'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove',
'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon',
'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut',
'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse',
'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book',
'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush']
# load demo data
image_names = ['COCO_test2015_000000000891.jpg', 'COCO_test2015_000000001669.jpg']
data = []
for im_name in image_names:
assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name))
im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(cur_path + '/../model/' + ('rfcn_dcn_coco' if not args.rfcn_only else 'rfcn_coco'), 0, process=True)
predictor = Predictor(sym, data_names, label_names,
context=[mx.gpu(0)], max_data_shapes=max_data_shape,
provide_data=provide_data, provide_label=provide_label,
arg_params=arg_params, aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# warm up
for j in xrange(2):
data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config)
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
tic()
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
print 'testing {} {:.4f}s'.format(im_name, toc())
# visualize
im = cv2.imread(cur_path + '/../demo/' + im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
show_boxes(im, dets_nms, classes, 1)
print 'done'
def main():
# get symbol
ctx_id = [int(i) for i in config.gpus.split(',')]
pprint.pprint(config)
sym_instance = eval(config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
# set up class names
num_classes = 81
classes = [
'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train',
'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign',
'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep',
'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella',
'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard',
'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard',
'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork',
'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange',
'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair',
'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv',
'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave',
'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase',
'scissors', 'teddy bear', 'hair drier', 'toothbrush'
]
# load demo data
image_names = []
names_dirs = os.listdir(cur_path + '/../' + test_dir)
for im_name in names_dirs:
if im_name[-4:] == '.jpg' or im_name[-4:] == '.png':
image_names.append(im_name)
data = []
for im_name in image_names:
assert os.path.exists(cur_path + '/../' + test_dir + im_name), (
'%s does not exist'.format('../demo/' + im_name))
im = cv2.imread(cur_path + '/../' + test_dir + im_name,
cv2.IMREAD_COLOR | long(128))
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
#print "before scale: "
#print im.shape
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
#print "after scale: "
#print im.shape
#im_scale = 1.0
#print "scale ratio: "
#print im_scale
im_tensor = transform(im, config.network.PIXEL_MEANS)
#print im_tensor.shape
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(cur_path + '/../' + model_dir,
0,
process=True)
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.gpu(ctx_id[0])],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
# warm up
for i in xrange(2):
data_batch = mx.io.DataBatch(data=[data[0]],
label=[],
pad=0,
index=0,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data[0])
]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
_, _, _, _ = im_detect(predictor, data_batch, data_names, scales,
config)
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
tic()
scores, boxes, masks, data_dict = im_detect(predictor, data_batch,
data_names, [1.0], config)
im_shapes = [
data_batch.data[i][0].shape[2:4]
for i in xrange(len(data_batch.data))
]
#print im_shapes
if not config.TEST.USE_MASK_MERGE:
all_boxes = [[] for _ in xrange(num_classes)]
all_masks = [[] for _ in xrange(num_classes)]
nms = py_nms_wrapper(config.TEST.NMS)
for j in range(1, num_classes):
indexes = np.where(scores[0][:, j] > 0.7)[0]
cls_scores = scores[0][indexes, j, np.newaxis]
cls_masks = masks[0][indexes, 1, :, :]
try:
if config.CLASS_AGNOSTIC:
cls_boxes = boxes[0][indexes, :]
else:
raise Exception()
except:
cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j] = cls_dets[keep, :]
all_masks[j] = cls_masks[keep, :]
dets = [all_boxes[j] for j in range(1, num_classes)]
masks = [all_masks[j] for j in range(1, num_classes)]
else:
masks = masks[0][:, 1:, :, :]
result_masks, result_dets = gpu_mask_voting(
masks, boxes[0], scores[0], num_classes, 100, im_shapes[0][1],
im_shapes[0][0], config.TEST.NMS,
config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0])
dets = [result_dets[j] for j in range(1, num_classes)]
masks = [
result_masks[j][:, 0, :, :] for j in range(1, num_classes)
]
print 'testing {} {:.4f}s'.format(im_name, toc())
# visualize
for i in xrange(len(dets)):
keep = np.where(dets[i][:, -1] > 0.7)
dets[i] = dets[i][keep]
masks[i] = masks[i][keep]
im = cv2.imread(cur_path + '/../' + test_dir + im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
show_masks(im, dets, masks, classes, config, 1.0 / scales[0], False)
# Save img
cv2.imwrite(cur_path + '/../' + result_dir + im_name,
cv2.cvtColor(im, cv2.COLOR_BGR2RGB))
print 'done'
def batch_extract(self, multiple=True, gt_dir=None, epoch=0):
"""
:param multiple:
:param gt_dir:
:return:
"""
if len(self.img_list) % self.batch_size != 0:
batch = len(self.img_list) / self.batch_size + 1
else:
batch = len(self.img_list) / self.batch_size
for i in xrange(batch):
if i < batch - 1:
self.batch_list = self.img_list[i * self.batch_size:(i + 1) *
self.batch_size]
else:
self.batch_list = self.img_list[i * self.batch_size:]
print '\nMini-batch %d\t' % (i + 1)
tmp_data = []
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
tic()
for img in self.batch_list:
assert os.path.exists(img), ('%s does not exist.'.format(img))
im = cv2.imread(
img, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
# im_info: height, width, scale
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
tmp_data.append({
self.data_names[0]: im_tensor,
self.data_names[1]: im_info
})
self.ctx = [int(i) for i in config.gpus.split(',')]
self.data = [[
mx.nd.array(tmp_data[i][name], mx.gpu(self.ctx[0]))
for name in self.data_names
] for i in xrange(len(tmp_data))]
max_data_shape = [[(self.data_names[0],
(1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape)
for k, v in zip(self.data_names, self.data[i])]
for i in xrange(len(self.data))]
provide_label = [None for i in xrange(len(self.data))]
arg_params, aux_params = load_param(self.model_dir,
epoch,
process=True)
self.predictor = Predictor(self.sym,
self.data_names,
self.label_name,
context=[mx.gpu(self.ctx[0])],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
print 'preparation: %.4fs' % toc()
if i == 0:
self.warmup()
self.forward(multiple=multiple, gt_dir=gt_dir)
self.cleaner()
def main():
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_rfcn'
model = '/../model/rfcn_vid'
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_test_symbol(config)
# set up class names
num_classes = 31
classes = ['airplane', 'antelope', 'bear', 'bicycle',
'bird', 'bus', 'car', 'cattle',
'dog', 'domestic_cat', 'elephant', 'fox',
'giant_panda', 'hamster', 'horse', 'lion',
'lizard', 'monkey', 'motorcycle', 'rabbit',
'red_panda', 'sheep', 'snake', 'squirrel',
'tiger', 'train', 'turtle', 'watercraft',
'whale', 'zebra']
# load demo data
image_names = glob.glob(cur_path + '/../demo/ILSVRC2015_val_00007010/*.JPEG')
output_dir = cur_path + '/../demo/rfcn/'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
#
data = []
for im_name in image_names:
assert os.path.exists(im_name), ('%s does not exist'.format(im_name))
im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(cur_path + model, 0, process=True)
predictor = Predictor(sym, data_names, label_names,
context=[mx.gpu(0)], max_data_shapes=max_data_shape,
provide_data=provide_data, provide_label=provide_label,
arg_params=arg_params, aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# warm up
for j in xrange(2):
data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config)
# test
time = 0
count = 0
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
tic()
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config)
time += toc()
count += 1
print 'testing {} {:.4f}s'.format(im_name, time/count)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
# visualize
im = cv2.imread(im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
# show_boxes(im, dets_nms, classes, 1)
out_im = draw_boxes(im, dets_nms, classes, 1)
_, filename = os.path.split(im_name)
cv2.imwrite(output_dir + filename,out_im)
print 'done'
def main():
# get symbol
ctx_id = [int(i) for i in config.gpus.split(',')]
pprint.pprint(config)
sym_instance = eval(config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 81
classes = [
'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train',
'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign',
'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep',
'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella',
'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard',
'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard',
'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork',
'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange',
'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair',
'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv',
'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave',
'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase',
'scissors', 'teddy bear', 'hair drier', 'toothbrush'
]
# load demo data
image_names = [
'COCO_test2015_000000000275.jpg', 'COCO_test2015_000000001412.jpg',
'COCO_test2015_000000073428.jpg', 'COCO_test2015_000000393281.jpg'
]
data = []
for im_name in image_names:
assert os.path.exists(cur_path + '/../demo/' + im_name), (
'%s does not exist'.format('../demo/' + im_name))
im = cv2.imread(cur_path + '/../demo/' + im_name,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(cur_path + '/../model/fcis_coco',
convert=True,
0,
process=True)
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.cpu()],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
tic()
scores, boxes, masks, data_dict = im_detect(predictor, data_batch,
data_names, scales, config)
im_shapes = [
data_batch.data[i][0].shape[2:4]
for i in xrange(len(data_batch.data))
]
if not config.TEST.USE_MASK_MERGE:
all_boxes = [[] for _ in xrange(num_classes)]
all_masks = [[] for _ in xrange(num_classes)]
nms = py_nms_wrapper(config.TEST.NMS)
for j in range(1, num_classes):
indexes = np.where(scores[0][:, j] > 0.7)[0]
cls_scores = scores[0][indexes, j, np.newaxis]
cls_masks = masks[0][indexes, 1, :, :]
try:
if config.CLASS_AGNOSTIC:
cls_boxes = boxes[0][indexes, :]
else:
raise Exception()
except:
cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j] = cls_dets[keep, :]
all_masks[j] = cls_masks[keep, :]
dets = [all_boxes[j] for j in range(1, num_classes)]
masks = [all_masks[j] for j in range(1, num_classes)]
else:
masks = masks[0][:, 1:, :, :]
im_height = np.round(im_shapes[0][0] / scales[0]).astype('int')
im_width = np.round(im_shapes[0][1] / scales[0]).astype('int')
print(im_height, im_width)
boxes = clip_boxes(boxes[0], (im_height, im_width))
result_masks, result_dets = cpu_mask_voting(
masks, boxes, scores[0], num_classes, 100, im_width, im_height,
config.TEST.NMS, config.TEST.MASK_MERGE_THRESH,
config.BINARY_THRESH)
dets = [result_dets[j] for j in range(1, num_classes)]
masks = [
result_masks[j][:, 0, :, :] for j in range(1, num_classes)
]
print 'testing {} {:.4f}s'.format(im_name, toc())
# visualize
for i in xrange(len(dets)):
keep = np.where(dets[i][:, -1] > 0.7)
dets[i] = dets[i][keep]
masks[i] = masks[i][keep]
for i in range(len(dets)):
if len(dets[i]) > 0:
for j in range(len(dets[i])):
print('{name}: {score} ({loc})'.format(
name=classes[i],
score=dets[i][j][-1],
loc=dets[i][j][:-1].tolist()))
print 'done'
def process_video_frame(raw_frame_queue, bbox_frame_queue):
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_rfcn'
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
arg_params, aux_params = load_param(
'./output/rfcn/road_obj/road_train_all/all/' + 'rfcn_road',
19,
process=True)
# set up class names; Don't count the background in, even we are treat the background as label '0'
num_classes = 4
classes = ['vehicle', 'pedestrian', 'cyclist', 'traffic lights']
target_size = config.SCALES[0][1]
max_size = config.SCALES[0][1]
while True:
tic()
i = 0
data = []
frame_list = []
while len(data) < 15:
frame = raw_frame_queue.get()
if frame is None:
continue
if i < 2:
i += 1
frame, im_scale = resize(frame,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
bbox_frame_queue.put(frame)
continue
frame, im_scale = resize(frame,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(frame, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
frame_list.append(frame)
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in xrange(len(data))]
# print('Debug: [data] shape: {}, cont: {}'.format(type(data), data))
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
# print('Debug: [max_data_shape] shape: {}, cont: {}'.format(type(max_data_shape), max_data_shape))
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in xrange(len(data))]
# print('Debug: [provide_data] shape: {}, cont: {}'.format(type(provide_data), provide_data))
provide_label = [None for i in xrange(len(data))]
# print('Debug: [provide_label] shape: {}, cont: {}'.format(type(provide_label), provide_label))
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# Process video frame
# image_names = ['frame']
# for idx, frame in enumerate(frame_list):
data_batch = mx.io.DataBatch(data=data,
label=[],
pad=0,
provide_data=provide_data,
provide_label=provide_label)
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
# print("length: {}".format(len(data_batch.data)))
# print('Debug: [scales] cont: {}'.format(scales))
scores_all, boxes_all, data_dict_all = im_detect(
predictor, data_batch, data_names, scales, config)
# print('scores_all: Type: {}, Values: {}, Length: {}'.format(type(scores_all), scores_all, len(scores_all)))
# print('boxes_all: Type: {}, Values: {}, Length: {}'.format(type(boxes_all), boxes_all, len(boxes_all)))
# print('data_dict_all: Type: {}, Values: {}, length: {}'.format(type(data_dict_all), data_dict_all, len(data_dict_all)))
# print('frame_list: Type: {}, Values: {}, Length: {}'.format(type(frame_list), frame_list, len(frame_list)))
# print('scores_all: Type: {}, Length: {}, Values: {}'.format(type(scores_all[0]), len(scores_all[0]), scores_all[0]))
# print(scores_all[0].shape)
# print('boxes_all: Type: {}, Length: {}'.format(type(boxes_all), len(boxes_all)))
# print(boxes_all[0].shape)
# print('data_dict_all: Type: {}, length: {}'.format(type(data_dict_all), len(data_dict_all)))
# print('frame_list: Type: {}, Length: {}'.format(type(frame_list), len(frame_list)))
for idx, frame in enumerate(frame_list):
# print('index: {}'.format(str(idx)))
boxes = boxes_all[0].astype('f')
scores = scores_all[0].astype('f')
dets_nms = []
# print(scores.shape)
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if config.CLASS_AGNOSTIC else boxes[:,
j *
4:
(j +
1
) *
4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
bbox_frame_queue.put(
draw_bbox_on_frame(frame, dets_nms, classes,
scale=scales[idx]))
print(toc())
def main():
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_flownet_rfcn'
model = '/../model/rfcn_dff_flownet_vid'
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_batch_test_symbol(config)
sym.save('dff_rfcn.json')
#print config.network.get_internals()
#mx.visualization.plot_network(sym).view()
#print sym.get_intervals()
#x = input()
# set up class names
num_classes = 31
classes = [
'airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car',
'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda',
'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit',
'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle',
'watercraft', 'whale', 'zebra'
]
# load demo data
image_names = glob.glob(cur_path + '/../demo/sample/*.JPEG')
output_dir = cur_path + '/../demo/rfcn_dff_batch/'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
key_frame_interval = 10
#
data = []
key_im_tensor = None
cur_im_tensor = []
im_info_tensor = []
image_names_list = []
image_names_batch = []
for idx, im_name in enumerate(image_names):
assert os.path.exists(im_name), ('%s does not exist'.format(im_name))
im = cv2.imread(
im_name) #, cv2.IMREAD_COLOR)# | cv2.IMREAD_IGNORE_ORIENTATION)
#im = cv2.resize(im, (176,176,3))
#height, width, channel = img.shape
#gray = im = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
#im = np.zeros(height * width * channel).reshape((height, width, channel))
#im[:,:,0] = gray
#im[:,:,1] = gray
#im[:,:,2] = gray
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
#print im.shape
#print im_scale.shape
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
if idx % key_frame_interval == 0:
key_im_tensor = im_tensor
else:
cur_im_tensor.append(im_tensor)
im_info_tensor.append(im_info)
image_names_batch.append(im_name)
if (idx + 1) % key_frame_interval == 0 or idx == len(image_names) - 1:
data.append({
'data_other': np.concatenate(cur_im_tensor),
'im_info': np.concatenate(im_info_tensor),
'data_key': key_im_tensor
})
key_im_tensor = None
cur_im_tensor = []
im_info_tensor = []
image_names_list.append(image_names_batch)
image_names_batch = []
# get predictor
data_names = ['data_other', 'im_info', 'data_key']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in xrange(len(data))]
max_data_shape = [[
('data_other', (key_frame_interval - 1, 3,
max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES]))),
('data_key', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES]))),
]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(cur_path + model, 0, process=True)
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
#print predictor
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# warm up
for j in xrange(1):
data_batch = mx.io.DataBatch(data=[data[j]],
label=[],
pad=0,
index=0,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data[j])
]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[:, 2]
for i in xrange(len(data_batch.data))
]
print scales[0].shape
scores_all, boxes_all, data_dict = im_batch_detect(
predictor, data_batch, data_names, scales, config)
print "warmup done"
# test
time = 0
count = 0
for idx, im_names in enumerate(image_names_list):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[:, 2]
for i in xrange(len(data_batch.data))
]
tic()
scores_all, boxes_all, data_dict = im_batch_detect(
predictor, data_batch, data_names, scales, config)
time += toc()
count += len(scores_all)
print 'testing {} {:.4f}s x {:d}'.format(im_names[0], time / count,
len(scores_all))
'''
for batch_idx in xrange(len(scores_all)):
boxes = boxes_all[batch_idx].astype('f')
scores = scores_all[batch_idx].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
# visualize
im = cv2.imread(im_names[batch_idx])
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
# show_boxes(im, dets_nms, classes, 1)
out_im = draw_boxes(im, dets_nms, classes, 1)
_, filename = os.path.split(im_names[batch_idx])
cv2.imwrite(output_dir + filename,out_im)
'''
print 'done'
def main():
# get symbol
pprint.pprint(config)
config.symbol = "resnet_v1_101_fpn_dcn_rcnn" if not args.rfcn_only else "resnet_v1_101_fpn_rcnn"
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 81
classes = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat',
'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse',
'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie',
'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove',
'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon',
'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut',
'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse',
'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book',
'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush']
# test
# find all videos
video_path = "../../tmp"#"../../aic2018/track1/track1_videos"
video_files = sorted([ x for x in os.listdir(video_path) if x.endswith(".mp4")])
save_path = "../../tmp/output"#"../../aic2018/track1/output"
if not os.path.isdir(save_path):
os.makedirs(save_path)
print("processing {} videos...".format(len(video_files)))
pbar = tqdm(total=len(video_files))
for vf in video_files:
vid = imageio.get_reader(os.path.join(video_path, vf),'ffmpeg')
data = []
for idx, im in enumerate(vid):
if idx == 0:
#assert os.path.exists(im_path + im_name), ('%s does not exist'.format(im_path + im_name))
#im = cv2.imread(im_path + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
else:
break
#data.append({'data': None, 'im_info': None})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
print("hhhhh")
print(provide_data, provide_label)
print("hhhhh")
arg_params, aux_params = load_param(cur_path + '/../model/demo_model/' + ('fpn_dcn_coco' if not args.rfcn_only else 'fpn_coco'), 0, process=True)
#print(type(arg_params), type(aux_params))
predictor = Predictor(sym, data_names, label_names,
context=[mx.gpu(0)], max_data_shapes=max_data_shape,
provide_data=provide_data, provide_label=provide_label,
arg_params=arg_params, aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
print("successfully load model")
vout = []
# write to video
writer = skvideo.io.FFmpegWriter(os.path.join(save_path, vf.replace(".mp4","_out.mp4")), outputdict={'-vcodec': 'libx264', '-b': '300000000'})
for frame_idx, im in enumerate(vid):
#im = cv2.imread(im_path + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
im_original = im.copy()
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32)
data_idx = [{"data": im_tensor, "im_info": im_info}]
data_idx = [[mx.nd.array(data_idx[i][name]) for name in data_names] for i in xrange(len(data_idx))]
data_batch = mx.io.DataBatch(data=[data_idx[0]], label=[], pad=0, index=idx,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data_idx[0])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
tic()
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scales, config)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
num_dets = 0
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.65, :]
dets_nms.append(cls_dets)
num_dets += cls_dets.shape[0]
print 'testing {} the {} th frame at {:.4f}s, detections {}'.format(vf, frame_idx, toc(), num_dets)
# save results
#im = cv2.imread(im_path + im_name)
#im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
#im_bbox = show_boxes(im, dets_nms, classes, 1)
#cv2.imwrite(im_path + im_name.replace(".jpg", "_bbox.jpg"), im_bbox)
save_im, outputs = show_boxes(im_original, dets_nms, classes, 1, False)
#cv2.imwrite(os.path.join(save_path, "{}_{}.jpg".format(vf.replace(".mp4", ""), str(frame_idx).zfill(5))), save_im)
writer.writeFrame(save_im)
for out in outputs:
vout.append([frame_idx] + out)
# save the whole video detection into pickle file
writer.close()
with open(os.path.join(save_path, vf.replace(".mp4", "_detect.pkl")), "wb") as f:
pickle.dump(vout, f, protocol=2)
pbar.update(1)
pbar.close()
print 'done'
def main():
# get symbol
pprint.pprint(config)
#config.symbol = "resnet_v1_101_fpn_dcn_rcnn" if not args.rfcn_only else "resnet_v1_101_fpn_rcnn"
config.symbol = "resnet_v1_101_fpn_dcn_rcnn"
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 5
classes = ["car", "bus", "van", "others"]
# load demo videos
im_path = '../../aic2018/track1/images/'
image_names = [
x for x in os.listdir('../../aic2018/track1/images/')
if (x.endswith(".jpg") and (x.startswith("9_1") or x.startswith("9_1"))
) and not x.endswith("_bbox.jpg")
]
data = []
for idx, im_name in enumerate(image_names[:1]):
if idx == 0:
assert os.path.exists(im_path + im_name), (
'%s does not exist'.format(im_path + im_name))
im = cv2.imread(im_path + im_name,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
else:
data.append({'data': None, 'im_info': None})
print(data)
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[0][name]) for name in data_names]
for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
# what does provide_data and provide_label work for?
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[0])]
for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
## load parameters
arg_params, aux_params = load_param(cur_path + '/../model/' + 'fpn_detrac',
1,
process=True)
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
print("successfully load model")
# find all videos
video_path = "../../tmp"
video_files = [x for x in os.listdir(video_path) if x.endswith(".mp4")]
save_path = "../../tmp/output"
if not os.path.isdir(save_path):
os.makedirs(save_path)
print("processing {} videos...".format(len(video_files)))
pbar = tqdm(total=len(video_files))
for vf in video_files:
vid = imageio.get_reader(os.path.join(video_path, vf), 'ffmpeg')
vout = []
for frame_idx, im in enumerate(vid):
#im = cv2.imread(im_path + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data_idx = [{"data": im_tensor, "im_info": im_info}]
data_idx = [[
mx.nd.array(data_idx[i][name]) for name in data_names
] for i in xrange(len(data_idx))]
data_batch = mx.io.DataBatch(
data=[data_idx[0]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data_idx[0])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
tic()
scores, boxes, data_dict = im_detect(predictor, data_batch,
data_names, scales, config)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if config.CLASS_AGNOSTIC else boxes[:,
j *
4:
(j +
1
) *
4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
print 'testing {} the {} th frame at {:.4f}s, detections {}'.format(
vf, frame_idx, toc(), len(dets_nms))
# save results
#im = cv2.imread(im_path + im_name)
#im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
#im_bbox = show_boxes(im, dets_nms, classes, 1)
#cv2.imwrite(im_path + im_name.replace(".jpg", "_bbox.jpg"), im_bbox)
save_im, outputs = show_boxes(im, dets_nms, classes, 1)
#cv2.imwrite(os.path.join(save_path, "{}_{}.jpg".format(vf.replace(".mp4", ""), str(frame_idx).zfill(5))), save_im)
for out in outputs:
vout.append([frame_idx] + out)
# save the whole video detection into pickle file
with open(os.path.join(save_path, vf.replace(".mp4", ".pkl")),
"wb") as f:
pickle.dump(vout, f, protocol=2)
pbar.update(1)
pbar.close()
print 'done'
# -*- coding: utf-8 -*-
import sys
sys.dont_write_bytecode = True
import os
from utils.tictoc import tic, toc
import pickle
import numpy as np
import pandas as pd
from scipy import stats
print 'load items info...'
tic()
Xinfo = pd.read_csv('../../data/ItemInfo_train.csv', index_col=0,
usecols=[0, 1, 6, 7, 8, 9, 10])
toc()
print 'load items pairs...'
tic()
Xpair = pd.read_csv('../../data/ItemPairs_train.csv', usecols=[0, 1, 2])
toc()
# idxmap is an efficient mapping between item-id and row index
# we could also use Xinfo.ix[indices], but this approach seems
# slightly faster
tic()
print 'load items mapping...'
if os.path.exists('idxmap.pickle'):
with open('idxmap.pickle', 'rb') as f:
idxmap = pickle.load(f)
else:
lastid = Xinfo.index[-1]
def main():
# get symbol
ctx_id = [int(i) for i in config.gpus.split(',')]
pprint.pprint(config)
sym_instance = eval(config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 6
classes = [
'__background__', # always index 0
'1',
'2',
'3',
'4',
'5'
]
# classes = ['__background__', # always index 0
# 'CubeBody', 'CubeFace', 'CylinderBody', 'CylinderFace', 'Grip']
# load demo data
image_names = []
names_dirs = os.listdir(cur_path + '/../' + test_dir)
for im_name in names_dirs:
if im_name[-4:] == '.jpg':
image_names.append(im_name)
data = []
# next [3001: len(image_names)]
for im_name in image_names[9001:len(image_names)]:
assert os.path.exists(cur_path + '/../' + test_dir + im_name), (
'%s does not exist'.format('../demo/' + im_name))
im = cv2.imread(cur_path + '/../' + test_dir + im_name,
cv2.IMREAD_COLOR | long(128))
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
# print "before scale: "
# print im.shape
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
# print "after scale: "
# print im.shape
# im_scale = 1.0
# print "scale ratio: "
# print im_scale
im_tensor = transform(im, config.network.PIXEL_MEANS)
# print im_tensor.shape
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
model_path = cur_path + '/../' + config.output_path + '/' + config.config_name + '/' + config.dataset.image_set + '/' + config.TRAIN.model_prefix
arg_params, aux_params = load_param(model_path,
config.TEST.test_epoch,
process=True)
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.gpu(ctx_id[0])],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
# warm up
for i in xrange(2):
data_batch = mx.io.DataBatch(data=[data[0]],
label=[],
pad=0,
index=0,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data[0])
]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
_, _, _, _ = im_detect(predictor, data_batch, data_names, scales,
config)
# 12342_mask Log
# LogTxt = open('LogRecorder.txt', 'w')
# test
for idx, im_name in enumerate(image_names[9001:len(image_names)]):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
tic()
scores, boxes, masks, data_dict = im_detect(predictor, data_batch,
data_names, [1.0], config)
im_shapes = [
data_batch.data[i][0].shape[2:4]
for i in xrange(len(data_batch.data))
]
# print im_shapes
if not config.TEST.USE_MASK_MERGE:
all_boxes = [[] for _ in xrange(num_classes)]
all_masks = [[] for _ in xrange(num_classes)]
nms = py_nms_wrapper(config.TEST.NMS)
for j in range(1, num_classes):
indexes = np.where(scores[0][:, j] > 0.7)[0]
cls_scores = scores[0][indexes, j, np.newaxis]
cls_masks = masks[0][indexes, 1, :, :]
try:
if config.CLASS_AGNOSTIC:
cls_boxes = boxes[0][indexes, :]
else:
raise Exception()
except:
cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j] = cls_dets[keep, :]
all_masks[j] = cls_masks[keep, :]
dets = [all_boxes[j] for j in range(1, num_classes)]
masks = [all_masks[j] for j in range(1, num_classes)]
else:
masks = masks[0][:, 1:, :, :]
result_masks, result_dets = gpu_mask_voting(
masks, boxes[0], scores[0], num_classes, 100, im_shapes[0][1],
im_shapes[0][0], config.TEST.NMS,
config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0])
dets = [result_dets[j] for j in range(1, num_classes)]
masks = [
result_masks[j][:, 0, :, :] for j in range(1, num_classes)
]
print 'testing {} {:.4f}s'.format(im_name, toc())
# visualize
for i in xrange(len(dets)):
keep = np.where(dets[i][:, -1] > 0.7)
dets[i] = dets[i][keep]
masks[i] = masks[i][keep]
im = cv2.imread(cur_path + '/../' + test_dir + im_name)
# im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
ims = show_masks(im, dets, masks, classes, config, 1.0 / scales[0],
False)
# label = cv2.imread(cur_path + '/../' + label_dir + im_name[0:len(im_name)-4] + '.png')
# label = cv2.cvtColor(label, cv2.COLOR_BGR2RGB)
# label_flag = checkLabel(label)
if dets[0].size:
cv2.imwrite(
cur_path + '/../' + result_dir + im_name[0:-4] + '_CuBo.png',
ims[0])
if dets[1].size:
cv2.imwrite(
cur_path + '/../' + result_dir + im_name[0:-4] + '_CuFa.png',
ims[1])
if dets[2].size:
cv2.imwrite(
cur_path + '/../' + result_dir + im_name[0:-4] + '_CyBo.png',
ims[2])
if dets[3].size:
cv2.imwrite(
cur_path + '/../' + result_dir + im_name[0:-4] + '_CyFa.png',
ims[3])
if dets[4].size:
cv2.imwrite(
cur_path + '/../' + result_dir + im_name[0:-4] + '_Grip.png',
ims[4])
# if np.unique(ims[0]).shape[0] >= 2 and (1 in label_flag):
# cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_CuBo.png', ims[0])
# if np.unique(ims[1]).shape[0] >= 2 and (2 in label_flag):
# cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_CuFa.png', ims[1])
# if np.unique(ims[2]).shape[0] >= 2 and (3 in label_flag):
# cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_CyBo.png', ims[2])
# if np.unique(ims[3]).shape[0] >= 2 and (4 in label_flag):
# cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_CyFa.png', ims[3])
# if np.unique(ims[4]).shape[0] >= 2 and (5 in label_flag):
# cv2.imwrite(cur_path + '/../' + result_dir + im_name[0:-4] + '_Grip.png', ims[4])
print 'done'
def main():
# get symbol
ctx_id = [int(i) for i in config.gpus.split(',')]
pprint.pprint(config)
sym_instance = eval(config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 31
classes =['alphabet_of_the_magi',
'anglo-saxon_futhorc',
'arcadian',
'armenian',
'asomtavruli_(georgian)',
'balinese',
'bengali',
'blackfoot_(canadian_aboriginal_syllabics)',
'braille',
'burmese_(myanmar)',
'cyrillic',
'early_aramaic',
'futurama',
'grantha',
'greek',
'gujarati',
'hebrew',
'inuktitut_(canadian_aboriginal_syllabics)',
'japanese_(hiragana)',
'japanese_(katakana)',
'korean',
'latin',
'malay_(jawi_-_arabic)',
'mkhedruli_(georgian)',
'n_ko',
'ojibwe_(canadian_aboriginal_syllabics)',
'sanskrit',
'syriac_(estrangelo)',
'tagalog',
'tifinagh']
# load demo data
image_names = ['train_000000.jpg', 'train_000001.jpg', 'train_000002.jpg', 'val_000000.jpg', 'val_000001.jpg', 'val_000002.jpg']
data = []
for im_name in image_names:
assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name))
im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(cur_path + '/../model/fcis_voc', 0, process=True)
predictor = Predictor(sym, data_names, label_names,
context=[mx.gpu(ctx_id[0])], max_data_shapes=max_data_shape,
provide_data=provide_data, provide_label=provide_label,
arg_params=arg_params, aux_params=aux_params)
# warm up
for i in xrange(2):
data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
_, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config)
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
tic()
scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config)
im_shapes = [data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data))]
if not config.TEST.USE_MASK_MERGE:
all_boxes = [[] for _ in xrange(num_classes)]
all_masks = [[] for _ in xrange(num_classes)]
nms = py_nms_wrapper(config.TEST.NMS)
for j in range(1, num_classes):
indexes = np.where(scores[0][:, j] > 0.7)[0]
cls_scores = scores[0][indexes, j, np.newaxis]
cls_masks = masks[0][indexes, 1, :, :]
try:
if config.CLASS_AGNOSTIC:
cls_boxes = boxes[0][indexes, :]
else:
raise Exception()
except:
cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j] = cls_dets[keep, :]
all_masks[j] = cls_masks[keep, :]
dets = [all_boxes[j] for j in range(1, num_classes)]
masks = [all_masks[j] for j in range(1, num_classes)]
else:
masks = masks[0][:, 1:, :, :]
im_height = np.round(im_shapes[0][0] / scales[0]).astype('int')
im_width = np.round(im_shapes[0][1] / scales[0]).astype('int')
print (im_height, im_width)
boxes = clip_boxes(boxes[0], (im_height, im_width))
result_masks, result_dets = gpu_mask_voting(masks, boxes, scores[0], num_classes,
100, im_width, im_height,
config.TEST.NMS, config.TEST.MASK_MERGE_THRESH,
config.BINARY_THRESH, ctx_id[0])
dets = [result_dets[j] for j in range(1, num_classes)]
masks = [result_masks[j][:, 0, :, :] for j in range(1, num_classes)]
print 'testing {} {:.4f}s'.format(im_name, toc())
# visualize
for i in xrange(len(dets)):
keep = np.where(dets[i][:,-1]>0.7)
dets[i] = dets[i][keep]
masks[i] = masks[i][keep]
im = cv2.imread(cur_path + '/../demo/' + im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
show_masks(im, dets, masks, classes, config)
print 'done'
def inference_rcnn_UADETRAC(cfg,
dataset,
image_set,
root_path,
dataset_path,
ctx,
prefix,
epoch,
vis,
ignore_cache,
shuffle,
has_rpn,
proposal,
thresh,
logger=None,
output_path=None):
if not logger:
assert False, 'require a logger'
# print cfg
pprint.pprint(cfg)
logger.info('testing cfg:{}\n'.format(pprint.pformat(cfg)))
# load symbol and testing data
if has_rpn:
sym_instance = eval(cfg.symbol + '.' + cfg.symbol)()
sym = sym_instance.get_symbol(cfg, is_train=False)
imdb = eval(dataset)(image_set,
root_path,
dataset_path,
result_path=output_path)
#roidb = imdb.gt_roidb_Shuo()
roidb = imdb.gt_roidb()
else:
sym_instance = eval(cfg.symbol + '.' + cfg.symbol)()
sym = sym_instance.get_symbol_rfcn(cfg, is_train=False)
imdb = eval(dataset)(image_set,
root_path,
dataset_path,
result_path=output_path)
gt_roidb = imdb.gt_roidb_Shuo()
roidb = eval('imdb.' + proposal + '_roidb')(gt_roidb)
print 'len(roidb):', len(roidb)
# get test data iter
test_data = TestLoader(roidb,
cfg,
batch_size=len(ctx),
shuffle=shuffle,
has_rpn=has_rpn)
# load model
arg_params, aux_params = load_param(prefix, epoch, process=True)
print 'inferring: ', prefix, ' epoch: ', epoch
"""# write parameters to file
print 'type(arg_params):',type(arg_params)
print 'type(aux_params):',type(aux_params)
thefile1 = open('/raid10/home_ext/Deformable-ConvNets/data/data_Shuo/UADETRAC/arg_params.txt','w')
thefile2 = open('/raid10/home_ext/Deformable-ConvNets/data/data_Shuo/UADETRAC/aux_params.txt','w')
for item_arg in arg_params.items():
thefile1.write(item_arg[0] + str(type(item_arg[1])) + str(item_arg[1].shape)+'\n')
for item_aux in aux_params.items():
thefile2.write(item_aux[0] + str(type(item_aux[1])) + str(item_aux[1].shape)+'\n')
"""
# infer shape
data_shape_dict = dict(test_data.provide_data_single)
sym_instance.infer_shape(data_shape_dict)
sym_instance.check_parameter_shapes(arg_params,
aux_params,
data_shape_dict,
is_train=False)
# decide maximum shape
data_names = [k[0] for k in test_data.provide_data_single]
label_names = None
max_data_shape = [[('data', (1, 3, max([v[0] for v in cfg.SCALES]),
max([v[1] for v in cfg.SCALES])))]]
if not has_rpn:
max_data_shape.append(
('rois', (cfg.TEST.PROPOSAL_POST_NMS_TOP_N + 30, 5)))
# create predictor
predictor = Predictor(sym,
data_names,
label_names,
context=ctx,
max_data_shapes=max_data_shape,
provide_data=test_data.provide_data,
provide_label=test_data.provide_label,
arg_params=arg_params,
aux_params=aux_params)
nms = gpu_nms_wrapper(cfg.TEST.NMS, 0)
# start detection
# pred_eval(predictor, test_data, imdb, cfg, vis=vis, ignore_cache=ignore_cache, thresh=thresh, logger=logger)
print 'test_data.size', test_data.size
print 'test_data:', test_data
print 'data_names:', data_names
print 'test_data.provide_data:', test_data.provide_data
print 'test_data.provide_label:', test_data.provide_label
nnn = 0
classes = ['__background', 'vehicle']
#num_classes = 10
#classes = ['__DontCare__','Car','Suv','SmallTruck','MediumTruck','LargeTruck','Pedestrian','Bus','Van','GroupofPeople']
for im_info, data_batch in test_data:
print nnn
#print 'roidb[nnn]:',roidb[nnn]['image']
image_name = roidb[nnn]['image']
tic()
scales = [iim_info[0, 2] for iim_info in im_info]
scores_all, boxes_all, data_dict_all = im_detect(
predictor, data_batch, data_names, scales, cfg)
boxes = boxes_all[0].astype('f')
scores = scores_all[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if cfg.CLASS_AGNOSTIC else boxes[:, j *
4:(j + 1) *
4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
#cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
print 'testing {} {:.4f}s'.format(image_name, toc())
# visualize
im = cv2.imread(image_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
#print 'cls_dets:',cls_dets
#show_boxes(im, dets_nms, classes, 1)
nnn = nnn + 1
image_name_length = len(image_name.split('/'))
sequence_name = image_name.split('/')[image_name_length - 2]
output_file = os.path.join(
'/raid10/home_ext/Deformable-ConvNets/data/data_Shuo/UADETRAC',
'Outputs', sequence_name + '_Det_DFCN.txt')
frame_id = int(image_name.split('/')[image_name_length - 1][3:8])
thefile = open(output_file, 'a')
det_id = 0
for x_small, y_small, x_large, y_large, prob in dets_nms[0]:
det_id += 1
thefile.write(
str(frame_id) + ',' + str(det_id) + ',' + str(x_small) + ',' +
str(y_small) + ',' + str(max(x_large - x_small, 0.001)) + ',' +
str(max(y_large - y_small, 0.001)) + ',' + str(prob) + '\n')
def main():
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_flownet_rfcn'
model = '/../model/rfcn_dff_flownet_vid'
sym_instance = eval(config.symbol + '.' + config.symbol)()
key_sym = sym_instance.get_key_test_symbol(config)
cur_sym = sym_instance.get_cur_test_symbol(config)
# set up class names
num_classes = 31
classes = [
'airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car',
'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda',
'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit',
'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle',
'watercraft', 'whale', 'zebra'
]
# load demo data
image_names = glob.glob(cur_path +
'/../demo/ILSVRC2015_val_00007010/*.JPEG')
output_dir = cur_path + '/../demo/rfcn_dff/'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
key_frame_interval = 10
#
data = []
key_im_tensor = None
for idx, im_name in enumerate(image_names):
assert os.path.exists(im_name), ('%s does not exist'.format(im_name))
im = cv2.imread(im_name,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
if idx % key_frame_interval == 0:
key_im_tensor = im_tensor
data.append({
'data':
im_tensor,
'im_info':
im_info,
'data_key':
key_im_tensor,
'feat_key':
np.zeros((1, config.network.DFF_FEAT_DIM, 1, 1))
})
# get predictor
data_names = ['data', 'im_info', 'data_key', 'feat_key']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in range(len(data))]
max_data_shape = [[
('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES]))),
('data_key', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES]))),
]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in range(len(data))]
provide_label = [None for i in range(len(data))]
arg_params, aux_params = load_param(cur_path + model, 0, process=True)
key_predictor = Predictor(key_sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
cur_predictor = Predictor(cur_sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# warm up
for j in range(2):
data_batch = mx.io.DataBatch(data=[data[j]],
label=[],
pad=0,
index=0,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data[j])
]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in range(len(data_batch.data))
]
if j % key_frame_interval == 0:
scores, boxes, data_dict, feat = im_detect(key_predictor,
data_batch, data_names,
scales, config)
else:
data_batch.data[0][-1] = feat
data_batch.provide_data[0][-1] = ('feat_key', feat.shape)
scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch,
data_names, scales, config)
print("warmup done")
# test
time = 0
count = 0
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in range(len(data_batch.data))
]
tic()
if idx % key_frame_interval == 0:
scores, boxes, data_dict, feat = im_detect(key_predictor,
data_batch, data_names,
scales, config)
else:
data_batch.data[0][-1] = feat
data_batch.provide_data[0][-1] = ('feat_key', feat.shape)
scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch,
data_names, scales, config)
time += toc()
count += 1
print('testing {} {:.4f}s'.format(im_name, time / count))
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if config.CLASS_AGNOSTIC else boxes[:, j *
4:(j +
1) *
4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
# visualize
im = cv2.imread(im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
# show_boxes(im, dets_nms, classes, 1)
out_im = draw_boxes(im, dets_nms, classes, 1)
_, filename = os.path.split(im_name)
cv2.imwrite(output_dir + filename, out_im)
print('done')
def process_one_batch_images_fun(isUrlFlag=False,
one_batch_images_list=None,
init_model_param=None,
fileOp=None,
vis=False):
# init_model_param list : [sym, arg_params, aux_params]
num_classes = 11 # 0 is background,
# classes = ['tibetan flag', 'guns', 'knives',
# 'not terror', 'islamic flag', 'isis flag']
classes = [
'islamic flag', 'isis flag', 'tibetan flag', 'knives_true',
'guns_true', 'knives_false', 'knives_kitchen', 'guns_anime',
'guns_tools', 'not terror'
]
image_names = one_batch_images_list
if len(image_names) <= 0:
return
all_can_read_image = []
data = []
for im_name in image_names:
#print("process : %s"%(im_name))
im = readImage_fun(isUrlFlag=isUrlFlag, imagePath=im_name)
# 判断 这个图片是否可读
if np.shape(im) == ():
print("ReadImageError : %s" % (im_name))
continue
all_can_read_image.append(im_name)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
predictor = Predictor(init_model_param[0],
data_names,
label_names,
context=[mx.gpu(int(args.gpuId))],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=init_model_param[1],
aux_params=init_model_param[2])
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
for idx, im_name in enumerate(all_can_read_image):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
tic()
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names,
scales, config)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if config.CLASS_AGNOSTIC else boxes[:, j *
4:(j +
1) *
4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > args.threshold, :]
dets_nms.append(cls_dets)
print('testing {} {:.4f}s'.format(im_name, toc()))
show_boxes(isUrlFlag=isUrlFlag,
im_name=im_name,
dets=dets_nms,
classes=classes,
scale=1,
vis=vis,
fileOp=fileOp,
flag=args.outputFileFlag)
print('process one batch images done')
pass
def forward(self, multiple=True, gt_dir=None):
"""
:param multiple:
:param gt_dir:
:return:
"""
self.multiple = multiple # if multiple is False, gt_dir must be provided
if not self.multiple:
assert gt_dir is not None
for idx, itm in enumerate(self.batch_list):
itm = itm.split('/')[-1]
data_batch = mx.io.DataBatch(
data=[self.data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(self.data_names, self.data[idx])
]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
im_shapes = [
data_batch.data[i][0].shape[2:4]
for i in xrange(len(data_batch.data))
]
tic()
scores, boxes, masks, convs, data_dict = conv_detect(
self.predictor, data_batch, self.data_names, scales, config)
im_height = np.round(im_shapes[0][0] / scales[0]).astype('int')
im_width = np.round(im_shapes[0][1] / scales[0]).astype('int')
# return
# (1) mask merge
if not config.TEST.USE_MASK_MERGE:
all_boxes = [[] for _ in xrange(self.num_classes)]
all_masks = [[] for _ in xrange(self.num_classes)]
nms = py_nms_wrapper(config.TEST.NMS)
for j in range(1, self.num_classes):
indexes = np.where(scores[0][:, j] > 0.7)[0]
cls_scores = scores[0][indexes, j, np.newaxis]
cls_masks = masks[0][indexes, 1, :, :]
try:
if config.CLASS_AGNOSTIC:
cls_boxes = boxes[0][indexes, :]
else:
raise Exception()
except:
cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j] = cls_dets[keep, :]
all_masks[j] = cls_masks[keep, :]
dets = [all_boxes[j] for j in range(1, self.num_classes)]
masks = [all_masks[j] for j in range(1, self.num_classes)]
else:
masks = masks[0][:, 1:, :, :]
boxes = clip_boxes(boxes[0], (im_height, im_width))
# gpu mask voting
if not self.cpu_mask_vote:
result_masks, result_dets = gpu_mask_voting(
masks, boxes, scores[0], self.num_classes, 100,
im_width, im_height, config.TEST.NMS,
config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH,
self.ctx[0])
# cpu mask voting
else:
result_masks, result_dets = cpu_mask_voting(
masks, boxes, scores[0], self.num_classes, 100,
im_width, im_height, config.TEST.NMS,
config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH)
# dets represent coordinates of bounding-boxes(up left, bottom right)
dets = [result_dets[j] for j in range(1, self.num_classes)]
masks = [
result_masks[j][:, 0, :, :]
for j in range(1, self.num_classes)
]
# (2) filter the result whose detection probability is under 0.7
for i in xrange(len(dets)):
keep = np.where(dets[i][:, -1] > 0.7)
dets[i] = dets[i][keep]
masks[i] = masks[i][keep]
# (3) prepare for roi-pooling
roi = [] # scaled bounding box coordinates(up left, bottom right)
info = [
] # class label, name of instance, probability, bounding box coordinates(up left, bottom right)
if self.multiple:
for k in xrange(0, self.num_classes - 1):
nums = len(dets[k])
if nums > 0:
for j in xrange(nums):
roi.append(dets[k][j][0:-1] * scales[0]
) # note that the input image is scaled
info.append((k, itm[:-4] + '-' + self.classes[k] +
'_' + str(j + 1), dets[k][j][-1],
np.array(np.round(dets[k][j][0:-1]))))
else:
# Method 1
dist = []
temp_roi = []
temp_info = []
# Instance
fn = itm[:-4].split('_')[0]
# Instre
# fn = itm[:-4]
with open(os.path.join(gt_dir, fn + '.txt'), 'r') as strm:
cod = strm.readline()
if cod.split('\t')[-1] != "\n":
# Instance
cod = [int(ele) for ele in cod.split('\t')[1:]]
else:
# Instre
cod = [int(ele) for ele in cod.split('\t')[1:-1]]
cod_center_x = cod[0] + cod[2] / 2
cod_center_y = cod[1] + cod[3] / 2
for k in xrange(0, self.num_classes - 1):
nums = len(dets[k])
if nums > 0:
for j in range(nums):
det_center_x = (
(dets[k][j][2] - dets[k][j][0]) / 2 +
dets[k][j][0])
det_center_y = (
(dets[k][j][3] - dets[k][j][1]) / 2 +
dets[k][j][1])
x = (det_center_x - cod_center_x) * (det_center_x -
cod_center_x)
y = (det_center_y - cod_center_y) * (det_center_y -
cod_center_y)
dist.append(np.sqrt(x + y))
temp_roi.append(
dets[k][j][0:-1] * scales[0]
) # note that the input image is scaled
temp_info.append(
(k, itm[:-4] + '-' + self.classes[k] + '_' +
str(j + 1), dets[k][j][-1],
np.array(np.round(dets[k][j][0:-1]))))
npdist = np.array(dist)
order = np.argsort(npdist)
if len(order) > 0:
roi.append(temp_roi[order[0]])
info.append(temp_info[order[0]])
# Method 2
# cod[2] += cod[0]
# cod[3] += cod[1]
# cords = np.array(cod)
# roi.append(cords * scales[0])
# info.append((0, itm[:-4] + '-' + self.classes[0] + '_' + str(0 + 1), 0, np.array(cod)))
# (4) perform roi-pooling & output
features = pooling_delegator(convs, roi, self.pool_size,
self.ctx[0], config)
""" pca """
if features is not None and not self.mapping:
features = self.pca.transform(features)
features = normalize(features, norm='l2', axis=1)
if self.img is not None and self.ftr is not None and features is not None:
self.output(info, features)
print 'testing {} {:.4f}s'.format(itm, toc())
# (5) visualize & save
""""""
if self.dst_dir:
pass
# im = cv2.imread(self.src_dir + itm)
# im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
# im = show_masks(im, dets, masks, self.classes, config)
# im = save_results(self.dst_dir + itm.replace('.jpg', '.dtc.jpg'), im, dets, masks, self.classes, config)
# cv2.imwrite(self.dst_dir + itm.replace('.jpg', '.msk.jpg'), im)
""""""
def main():
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_deeplab_dcn' if not args.deeplab_only else 'resnet_v1_101_deeplab'
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 19
# load demo data
image_names = ['frankfurt_000001_073088_leftImg8bit.png', 'lindau_000024_000019_leftImg8bit.png']
data = []
for im_name in image_names:
assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name))
im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data']
label_names = ['softmax_label']
data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(cur_path + '/../model/' + ('deeplab_dcn_cityscapes' if not args.deeplab_only else 'deeplab_cityscapes'), 0, process=True)
predictor = Predictor(sym, data_names, label_names,
context=[mx.gpu(0)], max_data_shapes=max_data_shape,
provide_data=provide_data, provide_label=provide_label,
arg_params=arg_params, aux_params=aux_params)
# warm up
for j in xrange(2):
data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]],
provide_label=[None])
output_all = predictor.predict(data_batch)
output_all = [mx.ndarray.argmax(output['softmax_output'], axis=1).asnumpy() for output in output_all]
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]],
provide_label=[None])
tic()
output_all = predictor.predict(data_batch)
output_all = [mx.ndarray.argmax(output['softmax_output'], axis=1).asnumpy() for output in output_all]
pallete = getpallete(256)
segmentation_result = np.uint8(np.squeeze(output_all))
segmentation_result = Image.fromarray(segmentation_result)
segmentation_result.putpalette(pallete)
print 'testing {} {:.4f}s'.format(im_name, toc())
pure_im_name, ext_im_name = os.path.splitext(im_name)
segmentation_result.save(cur_path + '/../demo/seg_' + pure_im_name + '.png')
# visualize
im_raw = cv2.imread(cur_path + '/../demo/' + im_name)
seg_res = cv2.imread(cur_path + '/../demo/seg_' + pure_im_name + '.png')
cv2.imshow('Raw Image', im_raw)
cv2.imshow('segmentation_result', seg_res)
cv2.waitKey(0)
print 'done'
def generate_detections(data, data_names, predictor, config, nms, image_list,
detection_num):
global classes
ret_boxes = []
ret_scores = []
ret_classes = []
k = 0
conversion_dict = {
0: "11",
1: "12",
2: "13",
3: "15",
4: "17",
5: "18",
6: "19",
7: "20",
8: "21",
9: "23",
10: "24",
11: "25",
12: "26",
13: "27",
14: "28",
15: "29",
16: "32",
17: "33",
18: "34",
19: "35",
20: "36",
21: "37",
22: "38",
23: "40",
24: "41",
25: "42",
26: "44",
27: "45",
28: "47",
29: "49",
30: "50",
31: "51",
32: "52",
33: "53",
34: "54",
35: "55",
36: "56",
37: "57",
38: "59",
39: "60",
40: "61",
41: "62",
42: "63",
43: "64",
44: "65",
45: "66",
46: "71",
47: "72",
48: "73",
49: "74",
50: "76",
51: "77",
52: "79",
53: "83",
54: "84",
55: "86",
56: "89",
57: "91",
58: "93",
59: "94"
}
for idx, im in enumerate(image_list):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
tic()
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names,
scales, config)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if config.CLASS_AGNOSTIC else boxes[:, j *
4:(j +
1) *
4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.08, :]
dets_nms.append(cls_dets)
# dets_nms format:
# a length 60 array, with each element being a dict representing each class
# the coordinates are in (xmin, ymin, xmax, ymax, confidence) format. The coordinates are not normalized
# one sample is: [290.09448 439.60617 333.31235 461.8115 0.94750994]
# below iterates class by class
image_detection_num = 0
for index_class in range(len(dets_nms)):
# for each class
single_class_nms = dets_nms[index_class]
image_detection_num += len(single_class_nms)
if len(single_class_nms) != 0:
print("detecting", single_class_nms.size, "number of objects",
len(single_class_nms))
# print(single_class_nms)
for index_single_class_nms in range(
min(len(single_class_nms), detection_num)):
# print("index_class,index_single_class_nms", index_class,index_single_class_nms, )
ret_boxes.append(
dets_nms[index_class][index_single_class_nms]
[:-1]) #get all the element other than the last one
ret_scores.append(
dets_nms[index_class][index_single_class_nms]
[-1]) #last element
ret_classes.append(conversion_dict[index_class])
# pad zeros
# print("1st: len(ret_boxes), image_detection_num", len(ret_boxes), image_detection_num)
if image_detection_num <= detection_num:
for index_element in range(int(detection_num -
image_detection_num)):
ret_boxes.append(np.zeros(
(4, ), dtype=np.float32
)) #get all the element other than the last one
ret_scores.append(0) #last element
ret_classes.append(0)
else:
print("~~~~~ too many predictions ~~~~~~~~~~~~~~~")
print("len(ret_boxes), image_detection_num", len(ret_boxes),
image_detection_num)
print('testing image {} {:.4f}s, detection number {}'.format(
idx + 1, toc(), image_detection_num))
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
# visualize
# don't show the final images.
# show_boxes(im, dets_nms, classes, 1, show_image = False, img_save_name = str(idx) + ".png")
ret_boxes = np.squeeze(np.array(ret_boxes))
ret_scores = np.squeeze(np.array(ret_scores))
ret_classes = np.squeeze(np.array(ret_classes))
return ret_boxes, ret_scores, ret_classes
def main():
# get symbol
pprint.pprint(config)
# config.symbol = 'resnet_v1_101_rfcn_dcn' if not args.rfcn_only else 'resnet_v1_101_rfcn'
config.symbol = 'resnet_v1_101_fpn_rcnn_rotbox_light_head_RoITransformer'
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 15
classes = ['__background__', # always index 0
'plane', 'baseball-diamond',
'bridge', 'ground-track-field',
'small-vehicle', 'large-vehicle',
'ship', 'tennis-court',
'basketball-court', 'storage-tank',
'soccer-ball-field', 'roundabout',
'harbor', 'swimming-pool',
'helicopter']
# load demo data
image_names = ['P0004__1__0___0.png', 'P0053__1__0___0.png', 'P0060__1__1648___824.png']
data = []
for im_name in image_names:
# pdb.set_trace()
assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name))
im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names] for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
# arg_params, aux_params = load_param(cur_path + '/../model/' + ('rfcn_dcn_coco' if not args.rfcn_only else 'rfcn_coco'), 0, process=True)
# TODO: change this path
arg_params, aux_params = load_param(r'/home/dj/code/Deformable_FPN_DOTA/output/fpn/DOTA/resnet_v1_101_dota_rotbox_light_head_Rroi_v6_trainval_fpn_end2end/train/fpn_DOTA_oriented',
config.TEST.test_epoch, process=True)
predictor = Predictor(sym, data_names, label_names,
context=[mx.gpu(0)], max_data_shapes=max_data_shape,
provide_data=provide_data, provide_label=provide_label,
arg_params=arg_params, aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# warm up
for j in xrange(2):
data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
scores, boxes, data_dict = im_detect_rotbox_Rroi(predictor, data_batch, data_names, scales, config)
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
tic()
scores, boxes, data_dict = im_detect_rotbox_Rroi(predictor, data_batch, data_names, scales, config)
# boxes = boxes[0].astype('f')
# scores = scores[0].astype('f')
boxes = boxes[0].astype('float64')
scores = scores[0].astype('float64')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
# cls_boxes = boxes[:, 4:8] if config.CLASS_AGNOSTIC else boxes[:, j * 4:(j + 1) * 4]
cls_boxes = boxes[:, 8:16] if config.CLASS_AGNOSTIC else boxes[:, j * 8:(j + 1) * 8]
cls_quadrangle_dets = np.hstack((cls_boxes, cls_scores))
# keep = nms(cls_dets)
keep = py_cpu_nms_poly(cls_quadrangle_dets, 0.3)
cls_quadrangle_dets = cls_quadrangle_dets[keep, :]
cls_quadrangle_dets = cls_quadrangle_dets[cls_quadrangle_dets[:, -1] > 0.7, :]
dets_nms.append(cls_quadrangle_dets)
print 'testing {} {:.4f}s'.format(im_name, toc())
# visualize
# im = cv2.imread(cur_path + '/../demo/' + im_name)
# im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
# pdb.set_trace()
im = draw_all_poly_detection(data_dict[0]['data'].asnumpy(), dets_nms, classes[1:], data[idx][1].asnumpy()[0][2], config,
threshold=0.2)
cv2.imwrite(cur_path + '/../demo/' + 'results' + im_name, im)
# show_boxes(im, dets_nms, classes, 1)
print 'done'
def run_detection(im_root, result_root, conf_threshold):
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_rfcn_dcn' if not args.rfcn_only else 'resnet_v1_101_rfcn'
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 81
classes = [
'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train',
'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign',
'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep',
'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella',
'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard',
'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard',
'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork',
'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange',
'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair',
'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv',
'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave',
'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase',
'scissors', 'teddy bear', 'hair drier', 'toothbrush'
]
print('detection in {}'.format(im_root))
im_names = sorted(os.listdir(im_root))
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = []
for idx, im_name in enumerate(im_names[:2]):
im_file = os.path.join(im_root, im_name)
im = cv2.imread(im_file,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(
cur_path + '/../model/' +
('rfcn_dcn_coco' if not args.rfcn_only else 'rfcn_coco'),
config.TEST.test_epoch,
process=True)
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
# nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# nms = soft_nms_wrapper(config.TEST.NMS, method=2)
nms = gpu_soft_nms_wrapper(config.TEST.NMS, method=2, device_id=0)
nms_t = Timer()
for idx, im_name in enumerate(im_names):
im_file = os.path.join(im_root, im_name)
im = cv2.imread(im_file,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
origin_im = im.copy()
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
# input
data = [mx.nd.array(im_tensor), mx.nd.array(im_info)]
data_batch = mx.io.DataBatch(data=[data],
label=[],
pad=0,
index=idx,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data)
]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
tic()
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names,
scales, config)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if config.CLASS_AGNOSTIC else boxes[:, j *
4:(j +
1) *
4]
cls_dets = np.hstack((cls_boxes, cls_scores))
nms_t.tic()
keep = nms(cls_dets)
nms_t.toc()
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
print 'testing {} {:.2f}ms'.format(im_name, toc() * 1000)
print 'nms: {:.2f}ms'.format(nms_t.total_time * 1000)
nms_t.clear()
# save results
person_dets = dets_nms[0]
with open(os.path.join(result_root, '{:04d}.txt'.format(idx)),
'w') as f:
f.write('{}\n'.format(len(person_dets)))
for det in person_dets:
x1, y1, x2, y2, s = det
w = x2 - x1
h = y2 - y1
f.write('0 {} {} {} {} {}\n'.format(s, w, h, x1, y1))
# visualize
im = origin_im
# im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
im = show_boxes_cv2(im, dets_nms, classes, 1)
cv2.imshow('det', im)
cv2.waitKey(1)
def main():
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_deeplab_dcn' if not args.deeplab_only else 'resnet_v1_101_deeplab'
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 19
# load demo data
image_names = [
'frankfurt_000001_073088_leftImg8bit.png',
'lindau_000024_000019_leftImg8bit.png'
]
data = []
for im_name in image_names:
assert os.path.exists(cur_path + '/../demo/' + im_name), (
'%s does not exist'.format('../demo/' + im_name))
im = cv2.imread(cur_path + '/../demo/' + im_name,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data']
label_names = ['softmax_label']
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(
cur_path + '/../model/' +
('deeplab_dcn_cityscapes'
if not args.deeplab_only else 'deeplab_cityscapes'),
0,
process=True)
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
# warm up
for j in xrange(2):
data_batch = mx.io.DataBatch(data=[data[0]],
label=[],
pad=0,
index=0,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data[0])
]],
provide_label=[None])
output_all = predictor.predict(data_batch)
output_all = [
mx.ndarray.argmax(output['softmax_output'], axis=1).asnumpy()
for output in output_all
]
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
tic()
output_all = predictor.predict(data_batch)
output_all = [
mx.ndarray.argmax(output['softmax_output'], axis=1).asnumpy()
for output in output_all
]
pallete = getpallete(256)
segmentation_result = np.uint8(np.squeeze(output_all))
segmentation_result = Image.fromarray(segmentation_result)
segmentation_result.putpalette(pallete)
print 'testing {} {:.4f}s'.format(im_name, toc())
pure_im_name, ext_im_name = os.path.splitext(im_name)
segmentation_result.save(cur_path + '/../demo/seg_' + pure_im_name +
'.png')
# visualize
im_raw = cv2.imread(cur_path + '/../demo/' + im_name)
seg_res = cv2.imread(cur_path + '/../demo/seg_' + pure_im_name +
'.png')
cv2.imshow('Raw Image', im_raw)
cv2.imshow('segmentation_result', seg_res)
cv2.waitKey(0)
print 'done'
def main():
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_flownet_deeplab'
model1 = '/../model/rfcn_dff_flownet_vid'
model2 = '/../model/deeplab_dcn_cityscapes'
sym_instance = eval(config.symbol + '.' + config.symbol)()
key_sym = sym_instance.get_key_test_symbol(config)
cur_sym = sym_instance.get_cur_test_symbol(config)
# settings
num_classes = 19
interv = args.interval
num_ex = args.num_ex
# load demo data
image_names = sorted(
glob.glob(cur_path +
'/../demo/cityscapes_data/cityscapes_frankfurt_all_i' +
str(interv) + '/*.png'))
image_names = image_names[:interv * num_ex]
label_files = sorted(
glob.glob(
cur_path +
'/../demo/cityscapes_data/cityscapes_frankfurt_labels_all/*.png'))
output_dir = cur_path + '/../demo/deeplab_dff/'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
key_frame_interval = interv
#
data = []
key_im_tensor = None
for idx, im_name in enumerate(image_names):
assert os.path.exists(im_name), ('%s does not exist'.format(im_name))
im = cv2.imread(im_name,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
if idx % key_frame_interval == 0:
key_im_tensor = im_tensor
data.append({
'data':
im_tensor,
'im_info':
im_info,
'data_key':
key_im_tensor,
'feat_key':
np.zeros((1, config.network.DFF_FEAT_DIM, 1, 1))
})
# get predictor
data_names = ['data', 'data_key', 'feat_key']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in xrange(len(data))]
max_data_shape = [[
('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES]))),
('data_key', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES]))),
]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
# models: rfcn_dff_flownet_vid, deeplab_cityscapes
arg_params, aux_params = load_param_multi(cur_path + model1,
cur_path + model2,
0,
process=True)
key_predictor = Predictor(key_sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
cur_predictor = Predictor(cur_sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# warm up
for j in xrange(2):
data_batch = mx.io.DataBatch(data=[data[j]],
label=[],
pad=0,
index=0,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data[j])
]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
if j % key_frame_interval == 0:
# scores, boxes, data_dict, feat = im_detect(key_predictor, data_batch, data_names, scales, config)
output_all, feat = im_segment(key_predictor, data_batch)
output_all = [
mx.ndarray.argmax(output['croped_score_output'],
axis=1).asnumpy() for output in output_all
]
else:
data_batch.data[0][-1] = feat
data_batch.provide_data[0][-1] = ('feat_key', feat.shape)
# scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch, data_names, scales, config)
output_all, _ = im_segment(cur_predictor, data_batch)
output_all = [
mx.ndarray.argmax(output['croped_score_output'],
axis=1).asnumpy() for output in output_all
]
print "warmup done"
# test
time = 0
count = 0
hist = np.zeros((num_classes, num_classes))
lb_idx = 0
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
tic()
if idx % key_frame_interval == 0:
print '\nframe {} (key)'.format(idx)
# scores, boxes, data_dict, feat = im_detect(key_predictor, data_batch, data_names, scales, config)
output_all, feat = im_segment(key_predictor, data_batch)
output_all = [
mx.ndarray.argmax(output['croped_score_output'],
axis=1).asnumpy() for output in output_all
]
else:
print '\nframe {} (intermediate)'.format(idx)
data_batch.data[0][-1] = feat
data_batch.provide_data[0][-1] = ('feat_key', feat.shape)
# scores, boxes, data_dict, _ = im_detect(cur_predictor, data_batch, data_names, scales, config)
output_all, _ = im_segment(cur_predictor, data_batch)
output_all = [
mx.ndarray.argmax(output['croped_score_output'],
axis=1).asnumpy() for output in output_all
]
elapsed = toc()
time += elapsed
count += 1
print 'testing {} {:.4f}s [{:.4f}s]'.format(im_name, elapsed,
time / count)
pred = np.uint8(np.squeeze(output_all))
segmentation_result = Image.fromarray(pred)
pallete = getpallete(256)
segmentation_result.putpalette(pallete)
_, im_filename = os.path.split(im_name)
segmentation_result.save(output_dir + '/seg_' + im_filename)
label = None
_, lb_filename = os.path.split(label_files[lb_idx])
im_comps = im_filename.split('_')
lb_comps = lb_filename.split('_')
# if annotation available for frame
if im_comps[1] == lb_comps[1] and im_comps[2] == lb_comps[2]:
print 'label {}'.format(lb_filename)
label = np.asarray(Image.open(label_files[lb_idx]))
if lb_idx < len(label_files) - 1:
lb_idx += 1
if label is not None:
curr_hist = fast_hist(pred.flatten(), label.flatten(), num_classes)
hist += curr_hist
print 'mIoU {mIoU:.3f}'.format(
mIoU=round(np.nanmean(per_class_iu(curr_hist)) * 100, 2))
print '(cum) mIoU {mIoU:.3f}'.format(
mIoU=round(np.nanmean(per_class_iu(hist)) * 100, 2))
ious = per_class_iu(hist) * 100
print ' '.join('{:.03f}'.format(i) for i in ious)
print '===> final mIoU {mIoU:.3f}'.format(mIoU=round(np.nanmean(ious), 2))
print 'done'
def main():
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_rfcn_dcn' if not args.rfcn_only else 'resnet_v1_101_rfcn'
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 81
classes = [
'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train',
'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign',
'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep',
'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella',
'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard',
'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard',
'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork',
'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange',
'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair',
'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv',
'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave',
'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase',
'scissors', 'teddy bear', 'hair drier', 'toothbrush'
]
# load demo data
image_names = [
'COCO_test2015_000000000891.jpg', 'COCO_test2015_000000001669.jpg'
]
data = []
for im_name in image_names:
assert os.path.exists(cur_path + '/../demo/' + im_name), (
'%s does not exist'.format('../demo/' + im_name))
im = cv2.imread(cur_path + '/../demo/' + im_name,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in range(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in range(len(data))]
provide_label = [None for i in range(len(data))]
arg_params, aux_params = load_param(
cur_path + '/../model/' +
('rfcn_dcn_coco' if not args.rfcn_only else 'rfcn_coco'),
0,
process=True)
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# warm up
for j in range(2):
data_batch = mx.io.DataBatch(data=[data[0]],
label=[],
pad=0,
index=0,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data[0])
]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in range(len(data_batch.data))
]
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names,
scales, config)
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in range(len(data_batch.data))
]
tic()
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names,
scales, config)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if config.CLASS_AGNOSTIC else boxes[:, j *
4:(j +
1) *
4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
print('testing {} {:.4f}s'.format(im_name, toc()))
# visualize
im = cv2.imread(cur_path + '/../demo/' + im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
show_boxes(im, dets_nms, classes, 1)
print('done')
def main():
# get symbol
pprint.pprint(config)
config.symbol = 'resnet_v1_101_rfcn'
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names; Don't count the background in, even we are treat the background as label '0'
num_classes = 4
classes = ['vehicle', 'pedestrian', 'cyclist', 'traffic lights']
# load demo data
image_path = './data/RoadImages/test/'
image_names = glob.glob(image_path + '*.jpg')
print("Image amount {}".format(len(image_names)))
data = []
for im_name in image_names:
assert os.path.exists(im_name), ('%s does not exist'.format(im_name))
im = cv2.imread(im_name,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][1]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in xrange(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in xrange(len(data))]
provide_label = [None for i in xrange(len(data))]
arg_params, aux_params = load_param(
'./output/rfcn/road_obj/road_train_all/all/' + 'rfcn_road',
19,
process=True)
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.gpu(0)],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
nms = gpu_nms_wrapper(config.TEST.NMS, 0)
# test
notation_dict = {}
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
tic()
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names,
scales, config)
boxes = boxes[0].astype('f')
scores = scores[0].astype('f')
dets_nms = []
for j in range(1, scores.shape[1]):
cls_scores = scores[:, j, np.newaxis]
cls_boxes = boxes[:,
4:8] if config.CLASS_AGNOSTIC else boxes[:, j *
4:(j +
1) *
4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
cls_dets = cls_dets[keep, :]
cls_dets = cls_dets[cls_dets[:, -1] > 0.7, :]
dets_nms.append(cls_dets)
print 'testing {} {:.4f}s'.format(im_name, toc())
# notation_list.append(get_notation(im_name, dets_nms, classes, scale=1.0, gen_bbox_pic=True))
notation_dict.update(
get_notation(im_name,
dets_nms,
classes,
scale=1.0,
gen_bbox_pic=True))
save_notation_file(notation_dict)
print 'done'
learning_rate=learning_rate,
gamma=gamma,
min_child_weight=min_child_weight,
colsample_bytree=colsample,
subsample=subsample)
m.fit(Xtr, ytr)
pp = m.predict_proba(Xts)[:, 1]
if FINAL_SUBMISSION:
import datetime
timestamp = datetime.datetime.now().strftime(
'%Y-%m-%d-%H:%M')
scores = np.c_[np.arange(len(pp)), pp]
np.savetxt('../out/vilab-submission-%s.csv' % timestamp,
scores, '%d,%.8f', ',', header='id,probability',
comments='')
toc()
else:
toc('cs=%.2f md=%2d lr=%.2f mcw=%1d g=%d score=%.4f' % (
colsample, max_depth, learning_rate, min_child_weight,
gamma, roc_auc_score(yts, pp)))
sys.stdout.flush()
import matplotlib.pyplot as plt
plt.ioff()
xgb.plot_importance(m, tick_label=names)
plt.savefig('xgb-features.pdf')
plt.show()
'''
xgb.plot_tree(m)
plt.savefig('xgb-tree.pdf', dpi=900)
