def pred_demo_mask(predictor, test_data, imdb, roidb, result_path, vis=False, thresh=1e-1):
"""
wrapper for calculating offline validation for faster data analysis
in this example, all threshold are set by hand
:param predictor: Predictor
:param test_data: data iterator, must be non-shuffle
:param imdb: image database
:param vis: controls visualization
:param thresh: valid detection threshold
:return:
"""
assert vis or not test_data.shuffle
data_names = [k[0] for k in test_data.provide_data]
nms = py_nms_wrapper(config.TEST.NMS)
# limit detections to max_per_image over all classes
max_per_image = -1
num_images = imdb.num_images
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
i = 0
for im_info, data_batch in test_data:
roi_rec = roidb[i]
scale = im_info[0, 2]
scores, boxes, data_dict, mask_output = im_detect_mask(predictor, data_batch, data_names)
CLASSES = imdb.classes
all_boxes = [[[] for _ in range(num_images)]
for _ in range(imdb.num_classes)]
all_masks = [[[] for _ in range(num_images)]
for _ in range(imdb.num_classes)]
label = np.argmax(scores, axis=1)
label = label[:, np.newaxis]
for cls in CLASSES:
cls_ind = CLASSES.index(cls)
cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)]
cls_masks = mask_output[:, cls_ind, :, :]
cls_scores = scores[:, cls_ind, np.newaxis]
#print cls_scores.shape, label.shape
keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0]
cls_masks = cls_masks[keep, :, :]
dets = np.hstack((cls_boxes, cls_scores)).astype(np.float32)[keep, :]
keep = nms(dets)
#print dets.shape, cls_masks.shape
all_boxes[cls_ind] = dets[keep, :]
all_masks[cls_ind] = cls_masks[keep, :, :]
boxes_this_image = [[]] + [all_boxes[j] for j in range(1, len(CLASSES))]
masks_this_image = [[]] + [all_masks[j] for j in range(1, len(CLASSES))]
filename = roi_rec['image'].split("/")[-1]
filename = result_path + '/' + filename.replace('.png', '') + '.jpg'
data_dict = dict(zip(data_names, data_batch.data))
draw_detection_mask(data_dict['data'], boxes_this_image, masks_this_image, scale, filename)
i += 1
def pred_eval(predictor, test_data, roidb, vis=False, thresh=1e-3):
"""
wrapper for calculating offline validation for faster data analysis
in this example, all threshold are set by hand
:param predictor: Predictor
:param test_data: data iterator, must be non-shuffle
:param imdb: image database
:param vis: controls visualization
:param thresh: valid detection threshold
:return:
"""
data_root = os.getcwd()
det_file = os.path.join(data_root, 'data/cache/dection.pkl')
if not os.path.exists(det_file):
assert vis or not test_data.shuffle
data_names = [k[0] for k in test_data.provide_data]
nms = py_nms_wrapper(config.TEST.NMS)
num_samples = len(roidb)
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
all_boxes = [[] for _ in range(num_samples)]
i = 0
t = time.time()
for im_info, data_batch in test_data:
t1 = time.time() - t
t = time.time()
scale = im_info[0, 2]
scores, boxes, data_dict = im_detect(predictor, data_batch,
data_names, scale)
#print(scores)
t2 = time.time() - t
t = time.time()
#onlt class 1 is what we wanted
indexes = np.where(scores[:, 1] > thresh)[0]
cls_scores = scores[indexes, 1, np.newaxis]
cls_boxes = boxes[indexes, 1 * 4:2 * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[i] = cls_dets[keep, :]
t3 = time.time() - t
t = time.time()
print('testing {}/{} data {:.4f}s net {:.4f}s post {:.4f}s'.format(
i, num_samples, t1, t2, t3))
i += 1
with open(det_file, 'wb') as f:
cPickle.dump(all_boxes, f, protocol=cPickle.HIGHEST_PROTOCOL)
else:
with open(det_file, 'rb') as f:
all_boxes = cPickle.load(f)
evalutate_detections(all_boxes, roidb)
vis_all_detection(all_boxes, roidb)
def pred_demo_mask(predictor, test_data, imdb, roidb, result_path, vis=False, thresh=1e-1):
"""
wrapper for calculating offline validation for faster data analysis
in this example, all threshold are set by hand
:param predictor: Predictor
:param test_data: data iterator, must be non-shuffle
:param imdb: image database
:param vis: controls visualization
:param thresh: valid detection threshold
:return:
"""
assert vis or not test_data.shuffle
data_names = [k[0] for k in test_data.provide_data]
nms = py_nms_wrapper(config.TEST.NMS)
# limit detections to max_per_image over all classes
max_per_image = -1
num_images = imdb.num_images
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
i = 0
for im_info, data_batch in test_data:
roi_rec = roidb[i]
scale = im_info[0, 2]
scores, boxes, data_dict, mask_output = im_detect_mask(predictor, data_batch, data_names)
CLASSES = imdb.classes
all_boxes = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
all_masks = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
label = np.argmax(scores, axis=1)
label = label[:, np.newaxis]
for cls in CLASSES:
cls_ind = CLASSES.index(cls)
cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)]
cls_masks = mask_output[:, cls_ind, :, :]
cls_scores = scores[:, cls_ind, np.newaxis]
#print cls_scores.shape, label.shape
keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0]
cls_masks = cls_masks[keep, :, :]
dets = np.hstack((cls_boxes, cls_scores)).astype(np.float32)[keep, :]
keep = nms(dets)
#print dets.shape, cls_masks.shape
all_boxes[cls_ind] = dets[keep, :]
all_masks[cls_ind] = cls_masks[keep, :, :]
boxes_this_image = [[]] + [all_boxes[j] for j in range(1, len(CLASSES))]
masks_this_image = [[]] + [all_masks[j] for j in range(1, len(CLASSES))]
filename = roi_rec['image'].split("/")[-1]
filename = result_path + '/' + filename.replace('.png', '') + '.jpg'
data_dict = dict(zip(data_names, data_batch.data))
draw_detection_mask(data_dict['data'], boxes_this_image, masks_this_image, scale, filename)
i += 1
def demo_net(predictor, image_name, vis=True):
"""
generate data_batch -> im_detect -> post process
:param predictor: Predictor
:param image_name: image name
:param vis: will save as a new image if not visualized
:return: None
"""
BOOL = 0
assert os.path.exists(image_name), image_name + ' not found'
im = cv2.imread(image_name)
data_batch, data_names, im_scale = generate_batch(im)
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names,
im_scale)
all_boxes = [[] for _ in Global.CLASSES]
for cls in Global.CLASSES:
cls_ind = Global.CLASSES.index(cls)
cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)]
cls_scores = scores[:, cls_ind, np.newaxis]
keep = np.where(cls_scores >= Global.conf_thresh_value)[0]
dets = np.hstack((cls_boxes, cls_scores)).astype(np.float32)[keep, :]
keep = py_nms_wrapper(Global.nms_thresh_value)(dets)
all_boxes[cls_ind] = dets[keep, :]
boxes_this_image = [[]] + [
all_boxes[j] for j in range(1, len(Global.CLASSES))
]
for ind, boxes in enumerate(boxes_this_image):
if len(boxes) > 0:
BOOL = 1
logger.info('---%s---' % Global.CLASSES[ind])
logger.info('%s' % boxes)
result_file = image_name.replace(str(Global.open_img_dir),
str(Global.save_path))
print result_file
logger.info('results saved to %s' % result_file)
im, CLASS, SCORE = draw_all_detection(data_dict['data'].asnumpy(),
boxes_this_image, Global.CLASSES,
im_scale)
cv2.imwrite(result_file, im)
Global.PICTURE_INFO[0].append(result_file)
Global.PICTURE_INFO[1].append(CLASS)
Global.PICTURE_INFO[2].append(SCORE)
return CLASS, SCORE, BOOL
def pred_eval(predictor, test_data, imdb, vis=False, max_box=-1, thresh=1e-3):
"""
wrapper for calculating offline validation for faster data analysis
in this example, all threshold are set by hand
:param predictor: Predictor
:param test_data: data iterator, must be non-shuffle
:param imdb: image database
:param vis: controls visualization
:param max_box: maximum number of boxes detected in each image
:param thresh: valid detection threshold
:return:
"""
# assert vis or not test_data.shuffle
data_names = [k[0] for k in test_data.provide_data]
nms = py_nms_wrapper(config.TEST.NMS)
# limit detections to max_per_image over all classes
max_per_image = max_box
num_images = imdb.num_images
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
all_boxes = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
kept_boxes = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
all_gts = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
all_iminfos = []
all_imnames = []
all_crops = []
i = 0
_t = {'data': Timer(), 'im_detect' : Timer(), 'misc' : Timer()}
_t['data'].tic()
num_image = config.NUM_IMAGES_3DCE
key_idx = (num_image - 1) / 2 # adjust image for 3DCE
for im_info, imname, crop, data_batch in test_data:
_t['data'].toc()
_t['im_detect'].tic()
all_iminfos.append(im_info)
all_imnames.append(imname)
all_crops.append(crop)
# scale = im_info[0, 2]
scale = 1. # we have scaled the label in get_image(), so no need to scale the pred_box
gt_boxes = data_batch.label[0].asnumpy()[key_idx, :, :]
data_batch.label = None
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scale)
_t['im_detect'].toc()
_t['misc'].tic()
for j in range(1, imdb.num_classes):
indexes = np.where(scores[:, j] > thresh)[0]
cls_scores = scores[indexes, j, np.newaxis]
cls_boxes = boxes[indexes, j * 4:(j + 1) * 4]
cls_boxes = map_box_back(cls_boxes, crop[2], crop[0], im_info[0,2])
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j][i] = cls_dets[keep, :]
all_gts[j][i] = map_box_back(gt_boxes, crop[2], crop[0], im_info[0,2])
if max_per_image > 0:
image_scores = np.hstack([all_boxes[j][i][:, -1]
for j in range(1, imdb.num_classes)])
if len(image_scores) > max_per_image:
image_thresh = np.sort(image_scores)[-max_per_image]
for j in range(1, imdb.num_classes):
keep = np.where(all_boxes[j][i][:, -1] >= image_thresh)[0]
kept_boxes[j][i] = all_boxes[j][i][keep, :]
if vis:
boxes_this_image = [[]] + [kept_boxes[j][i] for j in range(1, imdb.num_classes)]
vis_all_detection(data_dict['data'].asnumpy(), boxes_this_image, imdb.classes, scale)
_t['misc'].toc()
if i % 200 == 0:
if i <= 400:
logger.info('im_detect: {:d}/{:d} data {:.3f}s im_detect {:.3f}s misc {:.3f}s'
.format(i, imdb.num_images, _t['data'].average_time, _t['im_detect'].average_time,
_t['misc'].average_time))
else:
print i,
sys.stdout.flush()
# logger.info('testing %d/%d data %.4fs net %.4fs post %.4fs' % (i, imdb.num_images, t1, t2, t3))
i += 1
_t['data'].tic()
print
sys.stdout.flush()
det_file = os.path.join(imdb.cache_path, imdb.name + '_detections.pkl')
with open(det_file, 'wb') as f:
cPickle.dump(kept_boxes, f, protocol=cPickle.HIGHEST_PROTOCOL)
default.res_dict = {'imname': all_imnames, 'boxes': all_boxes[1], 'gts': all_gts[1]}
# default.res_dict = {'imname': all_imnames, 'im_info': all_iminfos, 'crops': all_crops, 'boxes': all_boxes[1], 'gts': all_gts[1]}
acc = my_evaluate_detections(all_boxes, all_gts)
sys.stdout.flush()
return acc
'bottle', 'bus', 'car', 'cat', 'chair',
'cow', 'diningtable', 'dog', 'horse',
'motorbike', 'person', 'pottedplant',
'sheep', 'sofa', 'train', 'tvmonitor')
config.TEST.HAS_RPN = True
SHORT_SIDE = config.SCALES[0][0]
LONG_SIDE = config.SCALES[0][1]
PIXEL_MEANS = config.PIXEL_MEANS
DATA_NAMES = ['data', 'im_info']
LABEL_NAMES = None
DATA_SHAPES = [('data', (1, 3, LONG_SIDE, SHORT_SIDE)), ('im_info', (1, 3))]
LABEL_SHAPES = None
# visualization
CONF_THRESH = 0.7
NMS_THRESH = 0.3
nms = py_nms_wrapper(NMS_THRESH)
def get_net(symbol, prefix, epoch, ctx):
arg_params, aux_params = load_param(prefix, epoch, convert=True, ctx=ctx, process=True)
# infer shape
data_shape_dict = dict(DATA_SHAPES)
arg_names, aux_names = symbol.list_arguments(), symbol.list_auxiliary_states()
arg_shape, _, aux_shape = symbol.infer_shape(**data_shape_dict)
arg_shape_dict = dict(zip(arg_names, arg_shape))
aux_shape_dict = dict(zip(aux_names, aux_shape))
# check shapes
for k in symbol.list_arguments():
if k in data_shape_dict or 'label' in k:
def post_processing(pklfileIn,
pklfileOut,
dataset,
image_set,
root_path,
dataset_path,
thresh=1e-2,
_use_nms=False,
_use_box_voting=False):
imdb = eval(dataset)(image_set, root_path, dataset_path)
num_classes = imdb.num_classes
num_images = imdb.num_images
print("imdb: num of classes:{}, num of images:{}".format(
num_classes, num_images))
if _use_nms:
nms = py_nms_wrapper(config.TEST.NMS)
if _use_box_voting:
box_voting = py_box_voting_wrapper(config.TEST.BOX_VOTING_IOU_THRESH,
config.TEST.BOX_VOTING_SCORE_THRESH,
with_nms=_use_nms)
pklfileIn = pklfileIn.split(',')
recs = []
for detfile in pklfileIn:
with open(detfile.strip(), 'r') as f:
rec = cPickle.load(f)
assert len(
rec
) == num_classes, "num of classes in {}:{} != imdb.num_classes: {}".format(
detfile.strip(), len(rec), num_classes)
assert len(
rec[0]
) == num_images, "num of images in {}:{} != imdb.num_classes: {}".format(
detfile.strip(), len(rec[0]), num_images)
recs.append(rec)
all_boxes = recs[0]
# stack det result from different models
for rec_idx in xrange(1, len(recs)):
rec_per_model = recs[rec_idx]
print("num_cls:{},num_imgs:{}".format(len(rec_per_model),
len(rec_per_model[0])))
for cls_idx in xrange(num_classes):
for img_idx in xrange(num_images):
print("processing {}-th file:{} {}".format(
rec_idx, cls_idx, img_idx))
#print(all_boxes[cls_idx][img_idx], rec_per_model[cls_idx][img_idx])
if len(rec_per_model[cls_idx][img_idx]) == 0:
continue
elif len(all_boxes[cls_idx][img_idx]) == 0:
all_boxes[cls_idx][img_idx] = rec_per_model[cls_idx][
img_idx]
else:
all_boxes[cls_idx][img_idx]\
= np.concatenate((all_boxes[cls_idx][img_idx], rec_per_model[cls_idx][img_idx]), axis=0)
# do nms/box voting
for i in xrange(num_images):
for j in xrange(1, num_classes):
cls_dets = all_boxes[j][i]
if cls_dets.size == 0:
continue
indexes = np.where(cls_dets[:, -1] > thresh)[0]
cls_dets = cls_dets[indexes, :]
if _use_nms:
keep = nms(cls_dets)
if _use_box_voting:
nms_cls_dets = cls_dets[keep, :]
#print(nms_cls_dets)
all_boxes[j][i] = box_voting(nms_cls_dets, cls_dets)
else:
all_boxes[j][i] = cls_dets[keep, :]
else:
if _use_box_voting:
all_boxes[j][i] = box_voting(cls_dets)
# else: do nothing
print("saving all_boxes to : {}".format(pklfileOut))
with open(pklfileOut, 'wb') as f:
cPickle.dump(all_boxes, f, protocol=cPickle.HIGHEST_PROTOCOL)
if image_set != 'test':
imdb.evaluate_detections(all_boxes)
return all_boxes
def demo_maskrcnn(network, ctx, prefix, epoch,img_path,
vis= True, has_rpn = True, thresh = 0.001):
assert has_rpn,"Only has_rpn==True has been supported."
sym = eval('get_' + network + '_mask_test')(num_classes=config.NUM_CLASSES, num_anchors=config.NUM_ANCHORS)
arg_params, aux_params = load_param(prefix, epoch, convert=True, ctx=ctx, process=True)
max_image_shape = (1,3,1024,1024)
max_data_shapes = [("data",max_image_shape),("im_info",(1,3))]
mod = MutableModule(symbol = sym, data_names = ["data","im_info"], label_names= None,
max_data_shapes = max_data_shapes,
context=ctx)
mod.bind(data_shapes = max_data_shapes, label_shapes = None, for_training=False)
mod.init_params(arg_params=arg_params, aux_params=aux_params)
class OneDataBatch():
def __init__(self,img):
im_info = mx.nd.array([[img.shape[0],img.shape[1],1.0]])
img = np.transpose(img,(2,0,1))
img = img[np.newaxis,(2,1,0)]
self.data = [mx.nd.array(img),im_info]
self.label = None
self.provide_label = None
self.provide_data = [("data",(1,3,img.shape[2],img.shape[3])),("im_info",(1,3))]
img_ori = cv2.imread(img_path)
batch = OneDataBatch(img_ori)
mod.forward(batch, False)
results = mod.get_outputs()
output = dict(zip(mod.output_names, results))
rois = output['rois_output'].asnumpy()[:, 1:]
scores = output['cls_prob_reshape_output'].asnumpy()[0]
bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0]
mask_output = output['mask_prob_output'].asnumpy()
pred_boxes = bbox_pred(rois, bbox_deltas)
pred_boxes = clip_boxes(pred_boxes, [img_ori.shape[0],img_ori.shape[1]])
nms = py_nms_wrapper(config.TEST.NMS)
boxes= pred_boxes
CLASSES = ('__background__', 'person', 'rider', 'car', 'truck', 'bus', 'train', 'mcycle', 'bicycle')
all_boxes = [[[] for _ in xrange(1)]
for _ in xrange(len(CLASSES))]
all_masks = [[[] for _ in xrange(1)]
for _ in xrange(len(CLASSES))]
label = np.argmax(scores, axis=1)
label = label[:, np.newaxis]
for cls in CLASSES:
cls_ind = CLASSES.index(cls)
cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)]
cls_masks = mask_output[:, cls_ind, :, :]
cls_scores = scores[:, cls_ind, np.newaxis]
#print cls_scores.shape, label.shape
keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0]
cls_masks = cls_masks[keep, :, :]
dets = np.hstack((cls_boxes, cls_scores)).astype(np.float32)[keep, :]
keep = nms(dets)
#print dets.shape, cls_masks.shape
all_boxes[cls_ind] = dets[keep, :]
all_masks[cls_ind] = cls_masks[keep, :, :]
boxes_this_image = [[]] + [all_boxes[j] for j in range(1, len(CLASSES))]
masks_this_image = [[]] + [all_masks[j] for j in range(1, len(CLASSES))]
import copy
import random
class_names = CLASSES
color_white = (255, 255, 255)
scale = 1.0
im = copy.copy(img_ori)
for j, name in enumerate(class_names):
if name == '__background__':
continue
color = (random.randint(0, 256), random.randint(0, 256), random.randint(0, 256)) # generate a random color
dets = boxes_this_image[j]
masks = masks_this_image[j]
for i in range(len(dets)):
bbox = dets[i, :4] * scale
if bbox[2] == bbox[0] or bbox[3] == bbox[1] or bbox[0] == bbox[1] or bbox[2] == bbox[3] :
continue
score = dets[i, -1]
bbox = map(int, bbox)
cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color=color, thickness=2)
cv2.putText(im, '%s %.3f' % (class_names[j], score), (bbox[0], bbox[1] + 10),
color=color_white, fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.5)
mask = masks[i, :, :]
mask = cv2.resize(mask, (bbox[2] - bbox[0], (bbox[3] - bbox[1])), interpolation=cv2.INTER_LINEAR)
mask[mask > 0.5] = 1
mask[mask <= 0.5] = 0
mask_color = random.randint(0, 255)
c = random.randint(0, 2)
target = im[bbox[1]: bbox[3], bbox[0]: bbox[2], c] + mask_color * mask
target[target >= 255] = 255
im[bbox[1]: bbox[3], bbox[0]: bbox[2], c] = target
im = im[:,:,(2,1,0)]
plt.imshow(im)
if vis:
plt.show()
else:
plt.savefig("figures/test_result.jpg")
#CLASSES = ('__background__',
# 'bird_nest','good_circle1','bad_circle1','good_circle2',
# 'good_earthquake_hammer','bad_earthquake_hammer','spacer_bar')
config.TEST.HAS_RPN = True
SHORT_SIDE = config.SCALES[0][0]
LONG_SIDE = config.SCALES[0][1]
PIXEL_MEANS = config.PIXEL_MEANS
DATA_NAMES = ['data', 'im_info']
LABEL_NAMES = None
DATA_SHAPES = [('data', (1, 3, LONG_SIDE, SHORT_SIDE)), ('im_info', (1, 3))]
LABEL_SHAPES = None
# visualization
# CONF_THRESH = Global.conf_thresh_value
# NMS_THRESH = Global.nms_thresh_value
nms = py_nms_wrapper(Global.nms_thresh_value)
def get_net(symbol, prefix, ctx):
arg_params, aux_params = load_param(prefix,
convert=True,
ctx=ctx,
process=True)
# infer shape
data_shape_dict = dict(DATA_SHAPES)
arg_names, aux_names = symbol.list_arguments(
), symbol.list_auxiliary_states()
arg_shape, _, aux_shape = symbol.infer_shape(**data_shape_dict)
arg_shape_dict = dict(zip(arg_names, arg_shape))
aux_shape_dict = dict(zip(aux_names, aux_shape))
from rcnn.utils.load_model import load_param
from rcnn.processing.nms import py_nms_wrapper, cpu_nms_wrapper, gpu_nms_wrapper
CLASSES = ('__background__', 'uav')
config.TEST.HAS_RPN = True
SHORT_SIDE = config.SCALES[0][0]
LONG_SIDE = config.SCALES[0][1]
PIXEL_MEANS = config.PIXEL_MEANS
DATA_NAMES = ['data', 'im_info']
LABEL_NAMES = None
DATA_SHAPES = [('data', (1, 3, LONG_SIDE, SHORT_SIDE)), ('im_info', (1, 3))]
LABEL_SHAPES = None
# visualization
CONF_THRESH = 0.7
NMS_THRESH = 0.3
nms = py_nms_wrapper(NMS_THRESH)
def get_net(symbol, prefix, epoch, ctx):
arg_params, aux_params = load_param(prefix,
epoch,
convert=True,
ctx=ctx,
process=True)
# infer shape
data_shape_dict = dict(DATA_SHAPES)
arg_names, aux_names = symbol.list_arguments(
), symbol.list_auxiliary_states()
arg_shape, _, aux_shape = symbol.infer_shape(**data_shape_dict)
arg_shape_dict = dict(zip(arg_names, arg_shape))
def pred_eval(predictor, test_data, imdb, vis=False, thresh=1e-3):
"""
wrapper for calculating offline validation for faster data analysis
in this example, all threshold are set by hand
:param predictor: Predictor
:param test_data: data iterator, must be non-shuffle
:param imdb: image database
:param vis: controls visualization
:param thresh: valid detection threshold
:return:
"""
assert vis or not test_data.shuffle
data_names = [k[0] for k in test_data.provide_data]
nms = py_nms_wrapper(config.TEST.NMS)
# limit detections to max_per_image over all classes
max_per_image = -1
num_images = imdb.num_images
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
all_boxes = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
i = 0
t = time.time()
for im_info, data_batch in test_data:
t1 = time.time() - t
t = time.time()
scale = im_info[0, 2]
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names,
scale)
t2 = time.time() - t
t = time.time()
for j in range(1, imdb.num_classes):
indexes = np.where(scores[:, j] > thresh)[0]
cls_scores = scores[indexes, j, np.newaxis]
cls_boxes = boxes[indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j][i] = cls_dets[keep, :]
if max_per_image > 0:
image_scores = np.hstack(
[all_boxes[j][i][:, -1] for j in range(1, imdb.num_classes)])
if len(image_scores) > max_per_image:
image_thresh = np.sort(image_scores)[-max_per_image]
for j in range(1, imdb.num_classes):
keep = np.where(all_boxes[j][i][:, -1] >= image_thresh)[0]
all_boxes[j][i] = all_boxes[j][i][keep, :]
if vis:
boxes_this_image = [[]] + [
all_boxes[j][i] for j in range(1, imdb.num_classes)
]
vis_all_detection(data_dict['data'].asnumpy(), boxes_this_image,
imdb.classes, scale)
t3 = time.time() - t
t = time.time()
logger.info('testing %d/%d data %.4fs net %.4fs post %.4fs' %
(i, imdb.num_images, t1, t2, t3))
i += 1
det_file = os.path.join(imdb.cache_path, imdb.name + '_detections.pkl')
with open(det_file, 'wb') as f:
cPickle.dump(all_boxes, f, protocol=cPickle.HIGHEST_PROTOCOL)
imdb.evaluate_detections(all_boxes)
def pred_eval_mask(predictor,
test_data,
imdb,
roidb,
result_path,
vis=False,
thresh=1e-1):
"""
wrapper for calculating offline validation for faster data analysis
in this example, all threshold are set by hand
:param predictor: Predictor
:param test_data: data iterator, must be non-shuffle
:param imdb: image database
:param vis: controls visualization
:param thresh: valid detection threshold
:return:
"""
assert vis or not test_data.shuffle
data_names = [k[0] for k in test_data.provide_data]
nms = py_nms_wrapper(config.TEST.NMS)
num_images = imdb.num_images
i = 0
t = time.time()
results_list = []
all_boxes = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
all_masks = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
for im_info, data_batch in test_data:
roi_rec = roidb[i]
t1 = time.time() - t
t = time.time()
scores, boxes, data_dict, mask_output = im_detect_mask(
predictor, data_batch, data_names)
t2 = time.time() - t
t = time.time()
CLASSES = imdb.classes
label = np.argmax(scores, axis=1)
label = label[:, np.newaxis]
for cls in CLASSES:
cls_ind = CLASSES.index(cls)
cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)]
cls_masks = mask_output[:, cls_ind, :, :]
cls_scores = scores[:, cls_ind, np.newaxis]
keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0]
cls_masks = cls_masks[keep, :, :]
dets = np.hstack(
(cls_boxes, cls_scores)).astype(np.float32)[keep, :]
keep = nms(dets)
all_boxes[cls_ind][i] = dets[keep, :]
all_masks[cls_ind][i] = cls_masks[keep, :]
boxes_this_image = [[]] + [
all_boxes[cls_ind][i] for cls_ind in range(1, imdb.num_classes)
]
masks_this_image = [[]] + [
all_masks[cls_ind][i] for cls_ind in range(1, imdb.num_classes)
]
results_list.append({
'image': roi_rec['image'],
'im_info': im_info,
'boxes': boxes_this_image,
'masks': masks_this_image
})
t3 = time.time() - t
t = time.time()
print 'testing {}/{} data {:.4f}s net {:.4f}s post {:.4f}s'.format(
i, imdb.num_images, t1, t2, t3)
i += 1
results_pack = {
'all_boxes': all_boxes,
'all_masks': all_masks,
'results_list': results_list
}
imdb.evaluate_mask(results_pack)
def demo_maskrcnn(network,
ctx,
prefix,
epoch,
vis=True,
has_rpn=True,
thresh=0.001):
assert has_rpn, "Only has_rpn==True has been supported."
sym = eval('get_' + network + '_mask_test')(num_classes=config.NUM_CLASSES,
num_anchors=config.NUM_ANCHORS)
arg_params, aux_params = load_param(prefix,
epoch,
convert=True,
ctx=ctx,
process=True)
split = False
max_image_shape = (1, 3, 1024, 1024)
max_data_shapes = [("data", max_image_shape), ("im_info", (1, 3))]
mod = MutableModule(symbol=sym,
data_names=["data", "im_info"],
label_names=None,
max_data_shapes=max_data_shapes,
context=ctx)
mod.bind(data_shapes=max_data_shapes,
label_shapes=None,
for_training=False)
mod.init_params(arg_params=arg_params, aux_params=aux_params)
class OneDataBatch():
def __init__(self, img):
im_info = mx.nd.array([[img.shape[0], img.shape[1], 1.0]])
img = np.transpose(img, (2, 0, 1))
img = img[np.newaxis, (2, 1, 0)]
self.data = [mx.nd.array(img), im_info]
self.label = None
self.provide_label = None
self.provide_data = [("data", (1, 3, img.shape[2], img.shape[3])),
("im_info", (1, 3))]
imglist_file = os.path.join(default.dataset_path, 'imglists', 'test.lst')
#print(default.dataset_path)
assert os.path.exists(imglist_file), 'Path does not exist: {}'.format(
imglist_file)
imgfiles_list = []
with open(imglist_file, 'r') as f:
for line in f:
file_list = dict()
label = line.strip().split('\t')
file_list['img_path'] = label[1]
imgfiles_list.append(file_list)
roidb = []
index = 0
submit_dir = os.path.join(default.dataset_path, 'submit')
if not os.path.exists(submit_dir):
os.makedirs(submit_dir)
img_dir = os.path.join(default.dataset_path, 'test_result_img')
if not os.path.exists(img_dir):
os.makedirs(img_dir)
for im in range(len(imgfiles_list)):
index = im + 1
img_path = os.path.join(default.dataset_path, 'ch4_test_images',
'img_' + str(index) + '.jpg')
img_ori = cv2.imread(img_path)
batch = OneDataBatch(img_ori)
mod.forward(batch, False)
results = mod.get_outputs()
output = dict(zip(mod.output_names, results))
rois = output['rois_output'].asnumpy()[:, 1:]
scores = output['cls_prob_reshape_output'].asnumpy()[0]
bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0]
mask_output = output['mask_prob_output'].asnumpy()
pred_boxes = bbox_pred(rois, bbox_deltas)
pred_boxes = clip_boxes(pred_boxes,
[img_ori.shape[0], img_ori.shape[1]])
nms = py_nms_wrapper(config.TEST.NMS)
boxes = pred_boxes
CLASSES = ('__background__', 'text')
all_boxes = [[[] for _ in xrange(1)] for _ in xrange(len(CLASSES))]
all_masks = [[[] for _ in xrange(1)] for _ in xrange(len(CLASSES))]
label = np.argmax(scores, axis=1)
label = label[:, np.newaxis]
for cls in CLASSES:
cls_ind = CLASSES.index(cls)
cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)]
cls_masks = mask_output[:, cls_ind, :, :]
cls_scores = scores[:, cls_ind, np.newaxis]
#print cls_scores.shape, label.shape
keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0]
cls_masks = cls_masks[keep, :, :]
dets = np.hstack(
(cls_boxes, cls_scores)).astype(np.float32)[keep, :]
keep_la = nms(dets)
print('------------------------keep_la', keep_la)
all_boxes[cls_ind] = dets[keep_la, :]
all_masks[cls_ind] = cls_masks[keep_la, :, :]
boxes_this_image = [[]
] + [all_boxes[j] for j in range(1, len(CLASSES))]
masks_this_image = [[]
] + [all_masks[j] for j in range(1, len(CLASSES))]
import copy
import random
class_names = CLASSES
color_white = (255, 255, 255)
scale = 1.0
im = copy.copy(img_ori)
num_box = 1
num_boxes = 0
mini_box = np.zeros((4, 2))
mini_box = np.int32(mini_box)
if (len(dets) == 0):
submit_path = os.path.join(submit_dir,
'res_img_{}.txt'.format(index))
result_txt = open(submit_path, 'a')
for i in range(0, 4):
result_txt.write(str(mini_box[i][0]))
result_txt.write(',')
result_txt.write(str(mini_box[i][1]))
if i < 3:
result_txt.write(',')
result_txt.write('\r\n')
result_txt.close()
for k, name in enumerate(class_names):
if name == '__background__':
continue
color = (random.randint(0, 256), random.randint(0, 256),
random.randint(0, 256)) # generate a random color
dets = boxes_this_image[k]
masks = masks_this_image[k]
#im_binary_merge = np.zeros(im[:,:,0].shape)
print('------------------------len(dets)', len(dets))
for i in range(len(dets)):
bbox_i = dets[i, :4] * scale
#if bbox[2] == bbox[0] or bbox[3] == bbox[1] or bbox[0] == bbox[1] or bbox[2] == bbox[3] :
if bbox_i[2] == bbox_i[0] or bbox_i[3] == bbox_i[1]:
continue
score_i = dets[i, -1]
bbox_i = map(int, bbox_i)
mask_i = masks[i, :, :]
mask_i = masks[i, :, :]
mask_i = cv2.resize(mask_i, (bbox_i[2] - bbox_i[0],
(bbox_i[3] - bbox_i[1])),
interpolation=cv2.INTER_LINEAR)
mask_i[mask_i > 0.3] = 1
mask_i[mask_i <= 0.3] = 0
im_binary_i = np.zeros(im[:, :, 0].shape)
im_binary_i[bbox_i[1]:bbox_i[3],
bbox_i[0]:bbox_i[2]] = im_binary_i[
bbox_i[1]:bbox_i[3],
bbox_i[0]:bbox_i[2]] + mask_i
#print("len(dets is )-------------------------",len(dets))
overlap = []
overlap_other = []
for j in range(len(dets)):
if i == j:
continue
bbox_j = dets[j, :4] * scale
#if bbox[2] == bbox[0] or bbox[3] == bbox[1] or bbox[0] == bbox[1] or bbox[2] == bbox[3] :
if bbox_j[2] == bbox_j[0] or bbox_j[3] == bbox_j[1]:
continue
num_box += 1
score_j = dets[j, -1]
bbox_j = map(int, bbox_j)
mask_j = masks[j, :, :]
mask_j = masks[j, :, :]
mask_j = cv2.resize(mask_j, (bbox_j[2] - bbox_j[0],
(bbox_j[3] - bbox_j[1])),
interpolation=cv2.INTER_LINEAR)
#print("mask_j,score_j,img_path------------------------",mask_j,score_j,img_path)
mask_j[mask_j > 0.3] = 1
mask_j[mask_j <= 0.3] = 0
im_binary_j = np.zeros(im[:, :, 0].shape)
im_binary_j[bbox_j[1]:bbox_j[3],
bbox_j[0]:bbox_j[2]] = im_binary_j[
bbox_j[1]:bbox_j[3],
bbox_j[0]:bbox_j[2]] + mask_j
im_binary = im_binary_i + im_binary_j
#mask_inter = mask_i+mask_j
ni = np.sum(im_binary_i == 1)
nj = np.sum(im_binary_j == 1)
nij = np.sum(im_binary == 2)
IOU_ratio = float(nij) / (ni + nj - nij)
overlap.append(IOU_ratio)
#if np.sum(im_binary_i == 1) == 0:
# continue
#if np.sum(im_binary_j == 1) == 0:
# continue
IOU_ratio_self = float(
np.sum(im_binary == 2)) / np.sum(im_binary_i == 1)
overlap_other.append(IOU_ratio_self)
#IOU_ratio_other = float(np.sum(im_binary == 2)) / np.sum(im_binary_j == 1)
#overlap_other.append(IOU_ratio_other)
if num_box == 1:
overlap.append(0)
overlap_other.append(0)
if np.max(overlap) < 0.6 and split == False and np.max(
overlap_other) < 0.9:
num_boxes += 1
#cv2.rectangle(im, (bbox_i[0], bbox_i[1]), (bbox_i[2], bbox_i[3]), color=color, thickness=2)
cv2.putText(im,
'%s %.3f' % (class_names[k], score_i),
(bbox_i[0], bbox_i[1] + 10),
color=color_white,
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.5)
px = np.where(mask_i == 1)
x_min = np.min(px[1])
y_min = np.min(px[0])
x_max = np.max(px[1])
y_max = np.max(px[0])
if x_max - x_min <= 1 or y_max - y_min <= 1:
continue
mask_color = random.randint(0, 255)
c = random.randint(0, 2)
mini_boxt = np.zeros((4, 2))
target = im[bbox_i[1]:bbox_i[3], bbox_i[0]:bbox_i[2],
c] + mask_color * mask_i
target[target >= 255] = 255
im[bbox_i[1]:bbox_i[3], bbox_i[0]:bbox_i[2], c] = target
mini_box = minimum_bounding_rectangle(im_binary_i)
mini_boxt[0][0] = mini_box[0][1]
mini_boxt[0][1] = mini_box[0][0]
mini_boxt[1][0] = mini_box[1][1]
mini_boxt[1][1] = mini_box[1][0]
mini_boxt[2][0] = mini_box[2][1]
mini_boxt[2][1] = mini_box[2][0]
mini_boxt[3][0] = mini_box[3][1]
mini_boxt[3][1] = mini_box[3][0]
mini_box = mini_boxt
mini_box = np.int32(mini_box)
#print("---------------",mini_box)
cv2.polylines(im, [mini_box], 1, (255, 255, 255))
submit_path = os.path.join(submit_dir,
'res_img_{}.txt'.format(index))
result_txt = open(submit_path, 'a')
for i in range(0, 4):
result_txt.write(str(mini_box[i][0]))
result_txt.write(',')
result_txt.write(str(mini_box[i][1]))
if i < 3:
result_txt.write(',')
result_txt.write('\r\n')
result_txt.close()
if split == True:
if np.max(overlap_other) > 0.6:
W = bbox_j[2] - bbox_j[0]
H = bbox_j[3] - bbox_j[1]
bbox_i[2] = bbox_i[2] - W
bbox_i[3] = bbox_i[3] - H
num_boxes += 1
cv2.rectangle(im, (bbox_i[0], bbox_i[1]),
(bbox_i[2], bbox_i[3]),
color=color,
thickness=2)
cv2.putText(im,
'%s %.3f' % (class_names[k], score_i),
(bbox_i[0], bbox_i[1] + 10),
color=color_white,
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.5)
px = np.where(mask_i == 1)
x_min = np.min(px[1])
y_min = np.min(px[0])
x_max = np.max(px[1])
y_max = np.max(px[0])
if x_max - x_min <= 1 or y_max - y_min <= 1:
continue
mask_color = random.randint(0, 255)
c = random.randint(0, 2)
target = im[bbox_i[1]:bbox_i[3], bbox_i[0]:bbox_i[2],
c] + mask_color * mask_i
target[target >= 255] = 255
im[bbox_i[1]:bbox_i[3], bbox_i[0]:bbox_i[2], c] = target
#inst_path = os.path.join(inst_dir,'result_{}_{}.mat'.format(index,num_boxes))
#io.savemat(inst_path, {'Segmentation': im_binary_i})
#numbox = open('data/boxnum.txt','a')
#numbox.write(str(num_boxes)+'\n')
#numbox.close()
result_img_path = os.path.join(img_dir,
'result_{}.jpg'.format(index))
cv2.imwrite(result_img_path, im)
#zip_submit_dir = 'script_test_ch4'
zip_file = os.path.join('script_test_ch4', 'submit.zip')
createZip(submit_dir, zip_file)
os.system(
"python ./script_test_ch4/script.py -g=./script_test_ch4/gt.zip -s=./script_test_ch4/submit.zip"
)
def generate_proposals(predictor, test_data, imdb, vis=False, thresh=0.):
"""
Generate detections results using RPN.
:param predictor: Predictor
:param test_data: data iterator, must be non-shuffled
:param imdb: image database
:param vis: controls visualization
:param thresh: thresh for valid detections
:return: list of detected boxes
"""
assert vis or not test_data.shuffle
nms = py_nms_wrapper(config.TEST.NMS)
data_names = [k[0] for k in test_data.provide_data]
i = 0
t = time.time()
imdb_boxes = list()
original_boxes = list()
for im_info, data_batch in test_data:
t1 = time.time() - t
t = time.time()
scale = im_info[0, 2]
scores, boxes, data_dict = im_proposal(predictor, data_batch,
data_names, scale)
t2 = time.time() - t
t = time.time()
# assemble proposals
dets = np.hstack((boxes, scores))
original_boxes.append(dets)
# filter proposals
keep = np.where(dets[:, 4:] > 0.8)[0]
dets = dets[keep, :]
nms_keep = nms(dets)
dets = dets[nms_keep, :]
imdb_boxes.append(dets)
if vis:
vis_all_detection(data_dict['data'].asnumpy(), [dets], ['obj'],
scale)
logger.info('generating %d/%d ' % (i + 1, imdb.num_images) +
'proposal %d ' % (dets.shape[0]) + 'data %.4fs net %.4fs' %
(t1, t2))
i += 1
assert len(imdb_boxes) == imdb.num_images, 'calculations not complete'
# save results
rpn_folder = os.path.join(imdb.root_path, 'rpn_data')
if not os.path.exists(rpn_folder):
os.mkdir(rpn_folder)
rpn_file = os.path.join(rpn_folder, imdb.name + '_rpn.pkl')
with open(rpn_file, 'wb') as f:
pickle.dump(imdb_boxes, f, pickle.HIGHEST_PROTOCOL)
if thresh > 0:
full_rpn_file = os.path.join(rpn_folder, imdb.name + '_full_rpn.pkl')
with open(full_rpn_file, 'wb') as f:
pickle.dump(original_boxes, f, pickle.HIGHEST_PROTOCOL)
logger.info('wrote rpn proposals to %s' % rpn_file)
return imdb_boxes
def demo_net(predictor, data, image_names, im_scales):
data = [[mx.nd.array(data[i][name]) for name in DATA_NAMES] for i in xrange(len(data))]
# warm up
for i in xrange(2):
data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0,
provide_data=[[(k, v.shape) for k, v in zip(DATA_NAMES, data[0])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
_, _, _, _, _= im_detect(predictor, data_batch, DATA_NAMES, scales)
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx,
provide_data=[[(k, v.shape) for k, v in zip(DATA_NAMES, data[idx])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))]
tic()
scores, boxes, boxes2, masks, data_dict = im_detect(predictor, data_batch, DATA_NAMES, scales)
im_shapes = [data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data))]
# mask output
if not config.TEST.USE_MASK_MERGE:
all_boxes = [[] for _ in xrange(config.NUM_CLASSES)]
all_masks = [[] for _ in xrange(config.NUM_CLASSES)]
nms = py_nms_wrapper(config.TEST.NMS)
for j in range(1, config.NUM_CLASSES):
indexes = np.where(scores[0][:, j] > 0.7)[0]
cls_scores = scores[0][indexes, j, np.newaxis]
cls_masks = masks[0][indexes, 1, :, :]
try:
if config.CLASS_AGNOSTIC:
cls_boxes = boxes[0][indexes, :]
else:
raise Exception()
except:
cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j] = cls_dets[keep, :]
all_masks[j] = cls_masks[keep, :]
dets = [all_boxes[j] for j in range(1, config.NUM_CLASSES)]
masks = [all_masks[j] for j in range(1, config.NUM_CLASSES)]
else:
masks = masks[0][:, 1:, :, :]
im_height = np.round(im_shapes[0][0] / scales[0]).astype('int')
im_width = np.round(im_shapes[0][1] / scales[0]).astype('int')
print (im_height, im_width)
boxes_ = clip_boxes(boxes[0], (im_height, im_width))
result_masks, result_dets = gpu_mask_voting(masks, boxes_, scores[0], config.NUM_CLASSES,
100, im_width, im_height,
config.TEST.NMS, config.TEST.MASK_MERGE_THRESH,
config.BINARY_THRESH, 0)
dets = [result_dets[j] for j in range(1, config.NUM_CLASSES)]
masks = [result_masks[j][:, 0, :, :] for j in range(1, config.NUM_CLASSES)]
print 'testing {} {:.4f}s'.format(im_name, toc())
# visualize
for i in xrange(len(dets)):
keep = np.where(dets[i][:,-1]>0.7)
dets[i] = dets[i][keep]
masks[i] = masks[i][keep]
im = cv2.imread('../data/demo/' + im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
show_masks(im, dets, masks, CLASSES)
# debug
'''
for ii in range(scores[0].shape[0]):
for jj in range(1, scores[0].shape[1]):
if scores[0][ii][jj]>0.7:
print ii, jj, scores[0][ii][jj]
'''
# bounding box output
all_boxes = [[] for _ in CLASSES]
nms = py_nms_wrapper(NMS_THRESH)
for cls in CLASSES:
cls_ind = CLASSES.index(cls)+1
cls_boxes = boxes2[0][:, 4 * cls_ind:4 * (cls_ind + 1)]
cls_scores = scores[0][:, cls_ind, np.newaxis]
keep = np.where(cls_scores >= CONF_THRESH)[0]
#print cls, keep
dets = np.hstack((cls_boxes, cls_scores)).astype(np.float32)[keep, :]
keep = nms(dets)
all_boxes[cls_ind-1] = dets[keep, :]
boxes_this_image = [all_boxes[j] for j in range(len(CLASSES))]
vis_all_detection(data_dict[0]['data'].asnumpy(), boxes_this_image, CLASSES, im_scales[idx])
print 'done'
def pred_eval(predictor, test_data, imdb, vis=False, thresh=1e-3):
"""
wrapper for calculating offline validation for faster data analysis
in this example, all threshold are set by hand
:param predictor: Predictor
:param test_data: data iterator, must be non-shuffle
:param imdb: image database
:param vis: controls visualization
:param thresh: valid detection threshold
:return:
"""
assert vis or not test_data.shuffle
data_names = [k[0] for k in test_data.provide_data]
nms = py_nms_wrapper(config.TEST.NMS)
# limit detections to max_per_image over all classes
max_per_image = -1
num_images = imdb.num_images
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
all_boxes = [[[] for _ in range(num_images)]
for _ in range(imdb.num_classes)]
i = 0
t = time.time()
for im_info, data_batch in test_data:
t1 = time.time() - t
t = time.time()
scale = im_info[0, 2]
scores, boxes, data_dict = im_detect(predictor, data_batch, data_names, scale)
t2 = time.time() - t
t = time.time()
for j in range(1, imdb.num_classes):
indexes = np.where(scores[:, j] > thresh)[0]
cls_scores = scores[indexes, j, np.newaxis]
cls_boxes = boxes[indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j][i] = cls_dets[keep, :]
if max_per_image > 0:
image_scores = np.hstack([all_boxes[j][i][:, -1]
for j in range(1, imdb.num_classes)])
if len(image_scores) > max_per_image:
image_thresh = np.sort(image_scores)[-max_per_image]
for j in range(1, imdb.num_classes):
keep = np.where(all_boxes[j][i][:, -1] >= image_thresh)[0]
all_boxes[j][i] = all_boxes[j][i][keep, :]
if vis:
boxes_this_image = [[]] + [all_boxes[j][i] for j in range(1, imdb.num_classes)]
vis_all_detection(data_dict['data'].asnumpy(), boxes_this_image, imdb.classes, scale)
t3 = time.time() - t
t = time.time()
logger.info('testing %d/%d data %.4fs net %.4fs post %.4fs' % (i, imdb.num_images, t1, t2, t3))
i += 1
det_file = os.path.join(imdb.cache_path, imdb.name + '_detections.pkl')
with open(det_file, 'wb') as f:
pickle.dump(all_boxes, f, protocol=pickle.HIGHEST_PROTOCOL)
imdb.evaluate_detections(all_boxes)
def pred_eval_mask(predictor, test_data, imdb, roidb, result_path, vis=False, thresh=1e-1):
"""
wrapper for calculating offline validation for faster data analysis
in this example, all threshold are set by hand
:param predictor: Predictor
:param test_data: data iterator, must be non-shuffle
:param imdb: image database
:param vis: controls visualization
:param thresh: valid detection threshold
:return:
"""
assert vis or not test_data.shuffle
data_names = [k[0] for k in test_data.provide_data]
nms = py_nms_wrapper(config.TEST.NMS)
num_images = imdb.num_images
i = 0
t = time.time()
results_list = []
all_boxes = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
all_masks = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
for im_info, data_batch in test_data:
roi_rec = roidb[i]
t1 = time.time() - t
t = time.time()
scores, boxes, data_dict, mask_output = im_detect_mask(predictor, data_batch, data_names)
t2 = time.time() - t
t = time.time()
CLASSES = imdb.classes
label = np.argmax(scores, axis=1)
label = label[:, np.newaxis]
for cls in CLASSES:
cls_ind = CLASSES.index(cls)
cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)]
cls_masks = mask_output[:, cls_ind, :, :]
cls_scores = scores[:, cls_ind, np.newaxis]
keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0]
cls_masks = cls_masks[keep, :, :]
dets = np.hstack((cls_boxes, cls_scores)).astype(np.float32)[keep, :]
keep = nms(dets)
all_boxes[cls_ind][i] = dets[keep, :]
all_masks[cls_ind][i] = cls_masks[keep, :]
boxes_this_image = [[]] + [all_boxes[cls_ind][i] for cls_ind in range(1, imdb.num_classes)]
masks_this_image = [[]] + [all_masks[cls_ind][i] for cls_ind in range(1, imdb.num_classes)]
results_list.append({'image': roi_rec['image'],
'im_info': im_info,
'boxes': boxes_this_image,
'masks': masks_this_image})
t3 = time.time() - t
t = time.time()
print 'testing {}/{} data {:.4f}s net {:.4f}s post {:.4f}s'.format(i, imdb.num_images, t1, t2, t3)
i += 1
results_pack = {'all_boxes': all_boxes,
'all_masks': all_masks,
'results_list': results_list}
imdb.evaluate_mask(results_pack)
def demo_maskrcnn(network,
ctx,
prefix,
epoch,
img_path,
vis=True,
has_rpn=True,
thresh=0.001):
assert has_rpn, "Only has_rpn==True has been supported."
#sym = eval('get_' + network + '_mask_test')(num_classes=config.NUM_CLASSES, num_anchors=config.NUM_ANCHORS)
sym = eval('get_' + network + '_mask_test')(num_classes=config.NUM_CLASSES)
arg_params, aux_params = load_param(prefix,
epoch,
convert=True,
ctx=ctx,
process=True)
for k, v in arg_params.iteritems():
print(k, v.shape)
max_image_shape = (1, 3, 1024, 1024)
max_data_shapes = [("data", max_image_shape), ("im_info", (1, 3))]
mod = MutableModule(symbol=sym,
data_names=["data", "im_info"],
label_names=None,
max_data_shapes=max_data_shapes,
context=ctx)
mod.bind(data_shapes=max_data_shapes,
label_shapes=None,
for_training=False)
mod.init_params(arg_params=arg_params, aux_params=aux_params)
class OneDataBatch():
def __init__(self, img):
im_info = mx.nd.array([[img.shape[0], img.shape[1], 1.0]])
img = np.transpose(img, (2, 0, 1))
img = img[np.newaxis, (2, 1, 0)]
self.data = [mx.nd.array(img), im_info]
self.label = None
self.provide_label = None
self.provide_data = [("data", (1, 3, img.shape[2], img.shape[3])),
("im_info", (1, 3))]
img_ori = cv2.imread(img_path)
batch = OneDataBatch(img_ori)
mod.forward(batch, False)
results = mod.get_outputs()
output = dict(zip(mod.output_names, results))
rois = output['rois_output'].asnumpy()[:, 1:]
scores = output['cls_prob_reshape_output'].asnumpy()[0]
bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0]
mask_output = output['mask_prob_output'].asnumpy()
pred_boxes = bbox_pred(rois, bbox_deltas)
pred_boxes = clip_boxes(pred_boxes, [img_ori.shape[0], img_ori.shape[1]])
nms = py_nms_wrapper(config.TEST.NMS)
#nms = processing_nms_wrapper(config.TEST.NMS, 0.7)
boxes = pred_boxes
CLASSES = ('__background__', 'text')
all_boxes = [[[] for _ in xrange(1)] for _ in xrange(len(CLASSES))]
all_masks = [[[] for _ in xrange(1)] for _ in xrange(len(CLASSES))]
label = np.argmax(scores, axis=1)
label = label[:, np.newaxis]
for cls in CLASSES:
cls_ind = CLASSES.index(cls)
cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)]
cls_masks = mask_output[:, cls_ind, :, :]
cls_scores = scores[:, cls_ind, np.newaxis]
#print cls_scores.shape, label.shape
keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0]
cls_masks = cls_masks[keep, :, :]
dets = np.hstack((cls_boxes, cls_scores)).astype(np.float32)[keep, :]
keep = nms(dets)
#print dets.shape, cls_masks.shape
all_boxes[cls_ind] = dets[keep, :]
all_masks[cls_ind] = cls_masks[keep, :, :]
boxes_this_image = [[]] + [all_boxes[j] for j in range(1, len(CLASSES))]
masks_this_image = [[]] + [all_masks[j] for j in range(1, len(CLASSES))]
import copy
import random
class_names = CLASSES
color_white = (255, 255, 255)
scale = 1.0
im = copy.copy(img_ori)
for j, name in enumerate(class_names):
if name == '__background__':
continue
color = (random.randint(0, 256), random.randint(0, 256),
random.randint(0, 256)) # generate a random color
dets = boxes_this_image[j]
masks = masks_this_image[j]
for i in range(len(dets)):
bbox = dets[i, :4] * scale
if bbox[2] == bbox[0] or bbox[3] == bbox[1] or bbox[0] == bbox[
1] or bbox[2] == bbox[3]:
continue
score = dets[i, -1]
bbox = map(int, bbox)
cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]),
color=color,
thickness=2)
cv2.putText(im,
'%s %.3f' % (class_names[j], score),
(bbox[0], bbox[1] + 10),
color=color_white,
fontFace=cv2.FONT_HERSHEY_COMPLEX,
fontScale=0.5)
mask = masks[i, :, :]
mask = cv2.resize(mask, (bbox[2] - bbox[0], (bbox[3] - bbox[1])),
interpolation=cv2.INTER_LINEAR)
mask[mask > 0.5] = 1
mask[mask <= 0.5] = 0
mask_color = random.randint(0, 255)
c = random.randint(0, 2)
target = im[bbox[1]:bbox[3], bbox[0]:bbox[2],
c] + mask_color * mask
target[target >= 255] = 255
im[bbox[1]:bbox[3], bbox[0]:bbox[2], c] = target
##im = im[:,:,(2,1,0)]
##plt.imshow(im)
cv2.imwrite("figures/test_result.jpg", im)
def cpu_mask_voting(masks,
boxes,
scores,
num_classes,
max_per_image,
im_width,
im_height,
nms_thresh,
merge_thresh,
binary_thresh=0.4):
"""
Wrapper function for mask voting, note we already know the class of boxes and masks
"""
masks = masks.astype(np.float32)
mask_size = masks.shape[-1]
nms = py_nms_wrapper(nms_thresh)
# apply nms and sort to get first images according to their scores
# Intermediate results
t_boxes = [[] for _ in xrange(num_classes)]
t_scores = [[] for _ in xrange(num_classes)]
t_all_scores = []
for i in xrange(1, num_classes):
dets = np.hstack((boxes.astype(np.float32), scores[:, i:i + 1]))
inds = nms(dets)
num_keep = min(len(inds), max_per_image)
inds = inds[:num_keep]
t_boxes[i] = boxes[inds]
t_scores[i] = scores[inds, i]
t_all_scores.extend(scores[inds, i])
sorted_scores = np.sort(t_all_scores)[::-1]
num_keep = min(len(sorted_scores), max_per_image)
thresh = max(sorted_scores[num_keep - 1], 1e-3)
for i in xrange(1, num_classes):
keep = np.where(t_scores[i] >= thresh)
t_boxes[i] = t_boxes[i][keep]
t_scores[i] = t_scores[i][keep]
num_detect = boxes.shape[0]
res_mask = [[] for _ in xrange(num_detect)]
for i in xrange(num_detect):
box = np.round(boxes[i]).astype(int)
mask = cv2.resize(masks[i, 0].astype(np.float32),
(box[2] - box[0] + 1, box[3] - box[1] + 1))
res_mask[i] = mask
list_result_box = [[] for _ in xrange(num_classes)]
list_result_mask = [[] for _ in xrange(num_classes)]
for c in xrange(1, num_classes):
num_boxes = len(t_boxes[c])
masks_ar = np.zeros((num_boxes, 1, mask_size, mask_size))
boxes_ar = np.zeros((num_boxes, 4))
for i in xrange(num_boxes):
# Get weights according to their segmentation scores
cur_ov = bbox_overlaps(boxes.astype(np.float),
t_boxes[c][i, np.newaxis].astype(np.float))
cur_inds = np.where(cur_ov >= merge_thresh)[0]
cur_weights = scores[cur_inds, c]
cur_weights = cur_weights / sum(cur_weights)
# Re-format mask when passing it to mask_aggregation
p_mask = [res_mask[j] for j in list(cur_inds)]
# do mask aggregation
orig_mask, boxes_ar[i] = mask_aggregation(boxes[cur_inds], p_mask,
cur_weights, im_width,
im_height, binary_thresh)
masks_ar[i, 0] = cv2.resize(orig_mask.astype(np.float32),
(mask_size, mask_size))
boxes_scored_ar = np.hstack((boxes_ar, t_scores[c][:, np.newaxis]))
list_result_box[c] = boxes_scored_ar
list_result_mask[c] = masks_ar
return list_result_mask, list_result_box
def post_processing(pklfileIn,
pklfileOut,
dataset,
image_set,
root_path,
dataset_path,
_use_nms=False,
_use_box_voting=False):
imdb = eval(dataset)(image_set, root_path, dataset_path)
if _use_nms:
nms = py_nms_wrapper(config.TEST.NMS)
if _use_box_voting:
box_voting = py_box_voting_wrapper(config.TEST.BOX_VOTING_IOU_THRESH,
config.TEST.BOX_VOTING_SCORE_THRESH,
with_nms=_use_nms)
pklfileIn = pklfileIn.split(',')
recs = []
for detfile in pklfileIn:
with open(detfile.strip(), 'r') as f:
rec = cPickle.load(f)
recs.append(rec)
num_classes = imdb.num_classes
num_images = imdb.num_images
all_boxes = recs[0]
# stack det result from different models
for rec_idx in xrange(1, len(recs)):
rec_per_model = recs[rec_idx]
for cls_idx in xrange(num_classes):
for img_idx in xrange(num_images):
print("processing {}-th file:{} {}".format(
rec_idx, cls_idx, img_idx))
#print(all_boxes[cls_idx][img_idx], rec_per_model[cls_idx][img_idx])
if len(rec_per_model[cls_idx][img_idx]) == 0:
continue
elif len(all_boxes[cls_idx][img_idx]) == 0:
all_boxes[cls_idx][img_idx] = rec_per_model[cls_idx][
img_idx]
else:
all_boxes[cls_idx][img_idx]\
= np.concatenate((all_boxes[cls_idx][img_idx], rec_per_model[cls_idx][img_idx]), axis=0)
# do nms/box voting
for i in xrange(num_images):
for j in xrange(1, num_classes):
cls_dets = all_boxes[j][i]
if cls_dets.size == 0:
continue
if _use_nms:
keep = nms(cls_dets)
if _use_box_voting:
nms_cls_dets = cls_dets[keep, :]
#print(nms_cls_dets)
all_boxes[j][i] = box_voting(nms_cls_dets, cls_dets)
else:
all_boxes[j][i] = cls_dets[keep, :]
else:
if _use_box_voting:
all_boxes[j][i] = box_voting(cls_dets)
# else: do nothing
det_file = os.path.join(imdb.cache_path, pklfileOut)
with open(det_file, 'wb') as f:
cPickle.dump(all_boxes, f, protocol=cPickle.HIGHEST_PROTOCOL)
imdb.evaluate_detections(all_boxes)
return all_boxes
def pred_eval(predictor,
test_data,
imdb,
vis=False,
thresh=1e-3,
logger=None,
ignore_cache=False):
det_file = os.path.join(imdb.result_path, imdb.name + '_detections.pkl')
seg_file = os.path.join(imdb.result_path, imdb.name + '_masks.pkl')
if os.path.exists(det_file) and os.path.exists(
seg_file) and not ignore_cache:
with open(det_file, 'rb') as f:
all_boxes = cPickle.load(f)
with open(seg_file, 'rb') as f:
all_masks = cPickle.load(f)
else:
assert vis or not test_data.shuffle
data_names = [k[0] for k in test_data.provide_data[0]]
print data_names
if not isinstance(test_data, PrefetchingIter):
test_data = PrefetchingIter(test_data)
# function pointers
nms = py_nms_wrapper(config.TEST.NMS)
mask_voting = gpu_mask_voting if config.TEST.USE_GPU_MASK_MERGE else cpu_mask_voting
max_per_image = 100 if config.TEST.USE_MASK_MERGE else -1
num_images = imdb.num_images
all_boxes = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
all_masks = [[[] for _ in xrange(num_images)]
for _ in xrange(imdb.num_classes)]
idx = 0
t = time.time()
print "start testing loop"
for data_batch in test_data:
data_batch = data_batch[1]
t1 = time.time() - t
t = time.time()
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in xrange(len(data_batch.data))
]
scores_all, _, boxes_all, masks_all, data_dict_all = im_detect(
predictor, data_batch, data_names, scales)
im_shapes = [
data_batch.data[i][0].shape[2:4]
for i in xrange(len(data_batch.data))
]
t2 = time.time() - t
t = time.time()
# post processing
for delta, (scores, boxes, masks, data_dict) in enumerate(
zip(scores_all, boxes_all, masks_all, data_dict_all)):
# mask output
if not config.TEST.USE_MASK_MERGE:
for j in range(1, imdb.num_classes):
indexes = np.where(scores[:, j] > thresh)[0]
cls_scores = scores[indexes, j, np.newaxis]
cls_masks = masks[indexes, 1, :, :]
try:
if config.CLASS_AGNOSTIC:
cls_boxes = boxes[indexes, :]
else:
raise Exception()
except:
cls_boxes = boxes[indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j][idx + delta] = cls_dets[keep, :]
all_masks[j][idx + delta] = cls_masks[keep, :]
else:
masks = masks[:, 1:, :, :]
im_height = np.round(im_shapes[delta][0] /
scales[delta]).astype('int')
im_width = np.round(im_shapes[delta][1] /
scales[delta]).astype('int')
boxes_ = clip_boxes(boxes, (im_height, im_width))
result_mask, result_box = mask_voting(
masks, boxes_, scores, imdb.num_classes, max_per_image,
im_width, im_height, config.TEST.NMS,
config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH)
for j in xrange(1, imdb.num_classes):
all_boxes[j][idx + delta] = result_box[j]
all_masks[j][idx + delta] = result_mask[j][:, 0, :, :]
# box output
all_boxes_ = [[] for _ in imdb.classes[1:]]
nms = py_nms_wrapper(NMS_THRESH)
for cls in imdb.classes[1:]:
cls_ind = imdb.classes.index(cls)
cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)]
cls_scores = scores[:, cls_ind, np.newaxis]
keep = np.where(cls_scores >= CONF_THRESH)[0]
#print cls, keep
dets = np.hstack(
(cls_boxes, cls_scores)).astype(np.float32)[keep, :]
keep = nms(dets)
all_boxes_[cls_ind - 1] = dets[keep, :]
boxes_this_image_ = [
all_boxes_[j] for j in range(len(imdb.classes) - 1)
]
if vis:
boxes_this_image = [[]] + [
all_boxes[j][idx + delta]
for j in range(1, imdb.num_classes)
]
masks_this_image = [[]] + [
all_masks[j][idx + delta]
for j in range(1, imdb.num_classes)
]
vis_all_mask(data_dict['data'].asnumpy(), boxes_this_image,
masks_this_image, imdb.classes, scales[delta])
vis_all_detection(data_dict['data'].asnumpy(),
boxes_this_image_, imdb.classes[1:],
scales[delta])
idx += test_data.batch_size
t3 = time.time() - t
t = time.time()
print 'testing {}/{} data {:.4f}s net {:.4f}s post {:.4f}s'.format(
idx, imdb.num_images, t1, t2, t3)
if logger:
logger.info(
'testing {}/{} data {:.4f}s net {:.4f}s post {:.4f}s'.
format(idx, imdb.num_images, t1, t2, t3))
with open(det_file, 'wb') as f:
cPickle.dump(all_boxes, f, protocol=cPickle.HIGHEST_PROTOCOL)
with open(seg_file, 'wb') as f:
cPickle.dump(all_masks, f, protocol=cPickle.HIGHEST_PROTOCOL)
info_str = imdb.evaluate_sds(all_boxes, all_masks)
if logger:
logger.info('evaluate detections: \n{}'.format(info_str))
