def _load_selective_search_roidb(self, gt_roidb):
filename = os.path.abspath(os.path.join(cfg.DATA_DIR,
'selective_search_data',
self.name + '.mat'))
assert os.path.exists(filename), \
'Selective search data not found at: {}'.format(filename)
raw_data = sio.loadmat(filename)['boxes'].ravel()
box_list = []
for i in xrange(raw_data.shape[0]):
boxes = raw_data[i][:, (1, 0, 3, 2)] - 1
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
keep = ds_utils.filter_small_boxes(boxes, self.config['min_size'])
boxes = boxes[keep, :]
box_list.append(boxes)
if self._image_set != 'test':
for i in xrange(raw_data.shape[0]):
boxes = raw_data[i][:, (1, 0, 3, 2)] - 1
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
keep = ds_utils.filter_small_boxes(boxes, self.config['min_size'])
boxes = boxes[keep, :]
box_list.append(boxes)
return self.create_roidb_from_box_list(box_list, gt_roidb)
def _load_selective_search_roidb(self, gt_roidb):
filename = os.path.abspath(
os.path.join(cfg.DATA_DIR, 'selective_search_data',
self.name + '.mat'))
assert os.path.exists(filename), \
'Selective search data not found at: {}'.format(filename)
raw_data = sio.loadmat(filename)['boxes'].ravel()
try:
raw_scores = sio.loadmat(filename)['boxScores'].ravel()
except:
raw_scores = [
np.ones(raw_data[i].shape[0]) * 0.6
for i in range(len(raw_data))
]
box_list = []
score_list = []
for i in range(raw_data.shape[0]):
boxes = raw_data[i][:, (1, 0, 3, 2)] - 1
scores = raw_scores[i].flatten()
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
scores = scores[keep]
keep = ds_utils.filter_small_boxes(boxes, self.config['min_size'])
boxes = boxes[keep, :]
scores = scores[keep]
box_list.append(boxes)
score_list.append(scores)
return self.create_roidb_from_box_list(box_list, score_list, gt_roidb)
def _load_proposals(self, method, gt_roidb):
"""
Load pre-computed proposals in the format provided by Jan Hosang:
http://www.mpi-inf.mpg.de/departments/computer-vision-and-multimodal-
computing/research/object-recognition-and-scene-understanding/how-
good-are-detection-proposals-really/
"""
box_list = []
top_k = self.config['top_k']
valid_methods = ['slide']
assert method in valid_methods
print 'Loading {} boxes'.format(method)
for i, index in enumerate(self._image_index):
if i % 1000 == 0:
print '{:d} / {:d}'.format(i + 1, len(self._image_index))
box_file = osp.join(cfg.DATA_DIR, 'ilsvrc_proposals', method,
'mat', self._get_box_file(index))
raw_data = sio.loadmat(box_file)['boxes']
boxes = np.maximum(raw_data[:, 0:4] - 1, 0).astype(np.uint16)
# Remove duplicate boxes and very small boxes and then take top k
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
keep = ds_utils.filter_small_boxes(boxes, self.config['min_size'])
boxes = boxes[keep, :]
boxes = boxes[:top_k, :]
box_list.append(boxes)
return self.create_roidb_from_box_list(box_list, gt_roidb)
def _load_mat_roidb(self, gt_roidb):
filename = self.config['mat_file']
assert os.path.exists(filename), \
'Proposal data not found at: {}'.format(filename)
if not self._coco:
raw_data = sio.loadmat(filename)['boxes'].ravel()
else:
raw_data = h5py.File(filename, 'r')
box_list = []
for i in xrange(self.num_images):
if i % 500 == 0:
print 'Loading *.mat {:06d}/{:06d}'.format(i, self.num_images)
if not self._coco:
boxes = raw_data[i] - 1
else:
boxes = raw_data[raw_data['boxes'][0][i]].value.T - 1
boxes = boxes.astype(np.uint16)
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
keep = ds_utils.filter_small_boxes(boxes, self.config['min_size'])
boxes = boxes[keep, :]
box_list.append(boxes)
return self.create_roidb_from_box_list(box_list, gt_roidb)
def _load_proposal(self, index):
filename = os.path.join(self._data_path, 'proposal', index + '.json')
#currently only one bbox is considered.
assert os.path.exists(
cache_file), 'Annotation {} has to be here'.format(filename)
raw_data = sio.loadmat(filename)['boxes_1'].ravel()
boxes_1 = raw_data
keep = ds_utils.unique_boxes(boxes_1)
boxes_1 = boxes_1[keep, :]
keep = ds_utils.filter_small_boxes(boxes_2, self.config['min_size'])
boxes_1 = boxes_1[keep, :]
raw_data = sio.loadmat(filename)['boxes_2'].ravel()
boxes_2 = raw_data
keep = ds_utils.unique_boxes(boxes_2)
boxes_2 = boxes_2[keep, :]
keep = ds_utils.filter_small_boxes(boxes_2, self.config['min_size'])
boxes_2 = boxes_2[keep, :]
return boxes1, boxes_2
def _load_proposals(self, method, gt_roidb):
"""
Load pre-computed proposals in the format provided by Jan Hosang:
http://www.mpi-inf.mpg.de/departments/computer-vision-and-multimodal-
computing/research/object-recognition-and-scene-understanding/how-
good-are-detection-proposals-really/
For MCG, use boxes from http://www.eecs.berkeley.edu/Research/Projects/
CS/vision/grouping/mcg/ and convert the file layout using
lib/datasets/tools/mcg_munge.py.
"""
box_list = []
imsize_list = []
top_k = self.config['top_k']
valid_methods = [
'MCG',
'selective_search',
'edge_boxes_AR',
'edge_boxes_70']
assert method in valid_methods
print 'Loading {} boxes'.format(method)
for i, index in enumerate(self._image_index):
if i % 1000 == 0:
print '{:d} / {:d}'.format(i + 1, len(self._image_index))
box_file = osp.join(
cfg.DATA_DIR, 'coco_proposals', method, 'mat',
self._get_box_file(index))
raw_data = sio.loadmat(box_file)['boxes']
boxes = np.maximum(raw_data - 1, 0).astype(np.uint16)
if method == 'MCG':
# Boxes from the MCG website are in (y1, x1, y2, x2) order
boxes = boxes[:, (1, 0, 3, 2)]
im_ann = self._COCO.loadImgs(index)[0]
width = im_ann['width']
height = im_ann['height']
imsize_list.append([height,width])
# Remove duplicate boxes and very small boxes and then take top k
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
keep = ds_utils.filter_small_boxes(boxes, self.config['min_size'])
boxes = boxes[keep, :]
boxes = boxes[:top_k, :]
boxes[:,0]=np.maximum(boxes[:,0]-cfg.MARGIN,0)
boxes[:,2]=np.minimum(boxes[:,2]+cfg.MARGIN,width-1)
boxes[:,1]=np.maximum(boxes[:,1]-cfg.MARGIN,0)
boxes[:,3]=np.minimum(boxes[:,3]+cfg.MARGIN,height-1)
box_list.append(boxes)
# Sanity check
ds_utils.validate_boxes(boxes, width=width, height=height)
return self.create_roidb_from_box_list(box_list, gt_roidb,imsize_list)
def _load_selective_search_roidb(self, gt_roidb):
filename = os.path.abspath(os.path.join(cfg.DATA_DIR, "selective_search_data", self.name + ".mat"))
assert os.path.exists(filename), "Selective search data not found at: {}".format(filename)
raw_data = sio.loadmat(filename)["boxes"].ravel()
box_list = []
for i in xrange(raw_data.shape[0]):
boxes = raw_data[i][:, (1, 0, 3, 2)] - 1
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
keep = ds_utils.filter_small_boxes(boxes, self.config["min_size"])
boxes = boxes[keep, :]
box_list.append(boxes)
return self.create_roidb_from_box_list(box_list, gt_roidb)
def _load_selective_search_roidb(self, gt_roidb):
filename = os.path.abspath(
os.path.join(cfg.DATA_DIR, 'selective_search_data',
self.name + '.mat'))
assert os.path.exists(filename), \
'Selective search data not found at: {}'.format(filename)
raw_data = sio.loadmat(filename)['boxes'].ravel()
box_list = []
for i in xrange(raw_data.shape[0]):
boxes = raw_data[i][:, (1, 0, 3, 2)] - 1
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
keep = ds_utils.filter_small_boxes(boxes, self.config['min_size'])
boxes = boxes[keep, :]
box_list.append(boxes)
return self.create_roidb_from_box_list(box_list, gt_roidb)
def load_selective_search_roidb(dataset_name):
name = dataset_name
filename = os.path.abspath(
os.path.join(cfg.DATA_DIR, 'selective_search_data', name + '.mat'))
assert os.path.exists(filename), \
'Selective search data not found at: {}'.format(filename)
raw_data = sio.loadmat(filename)['boxes'].ravel()
box_list = []
for i in xrange(raw_data.shape[0]):
boxes = raw_data[i][:, (1, 0, 3, 2)] - 1
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
keep = ds_utils.filter_small_boxes(boxes, 2)
boxes = boxes[keep, :]
box_list.append(boxes)
return box_list
def _load_selective_search_roidb(self, gt_roidb):
box_list = []
for index in self.image_index:
filename = os.path.join(self._data_path, 'Proposals',
index + '.xml')
try:
ann = self._load_pascal_annotation_xml(filename)
boxes = ann['boxes']
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
keep = ds_utils.filter_small_boxes(boxes,
self.config['min_size'])
boxes = boxes[keep, :]
except IOError:
# If file does not exist, skip
print 'Failed to load proposals from %s' % filename
boxes = []
box_list.append(boxes)
return self.create_roidb_from_box_list(np.array(box_list), gt_roidb)
def _load_custom_roidb(self, gt_roidb):
box_list = []
top_k = self.config['top_k']
print 'Loading custom boxes'
for i, index in enumerate(self._image_index):
if i % 1000 == 0:
print '{:d} / {:d}'.format(i + 1, len(self._image_index))
box_file = osp.join(
cfg.DATA_DIR, 'pascal_voc_proposals', 'VOC' + self._year, 'custom', 'mat',
self._get_box_file(index))
raw_data = sio.loadmat(box_file)['boxes']
boxes = np.maximum(raw_data[:, 0:4] - 1, 0).astype(np.uint16)
# Remove duplicate boxes and very small boxes and then take top k
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
keep = ds_utils.filter_small_boxes(boxes, self.config['min_size'])
boxes = boxes[keep, :]
boxes = boxes[:top_k, :]
box_list.append(boxes)
return self.create_roidb_from_box_list(box_list, gt_roidb)
def _load_mcg_roidb(self, gt_roidb):
box_list = []
score_list = []
total_roi = 0
up_1024 = 0
up_2048 = 0
up_3072 = 0
up_4096 = 0
if cfg.USE_FEEDBACK:
box_fb_list, score_fb_list = self._feedback_roidb(gt_roidb)
for i, index in enumerate(self._image_index):
if i % 1000 == 0:
print '{:d} / {:d}'.format(i + 1, len(self._image_index))
img_size = PIL.Image.open(self.image_path_at(i)).size
box_file = osp.join(cfg.DATA_DIR, 'coco_proposals', 'MCG', 'mat',
self._get_box_file(index))
raw_data = sio.loadmat(box_file)['boxes']
score_data = sio.loadmat(box_file)['scores']
# boxes = np.maximum(raw_data - 1, 0).astype(np.uint16)
boxes = raw_data.astype(np.uint16) - 1
scores = score_data.astype(np.float)
# Boxes from the MCG website are in (y1, x1, y2, x2) order
boxes = boxes[:, (1, 0, 3, 2)]
assert (boxes[:, 0] >= 0).all()
assert (boxes[:, 1] >= 0).all()
assert (boxes[:, 2] >= boxes[:, 0]).all()
assert (boxes[:, 3] >= boxes[:, 1]).all()
assert (boxes[:, 2] < img_size[0]).all()
assert (boxes[:, 3] < img_size[1]).all()
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
scores = scores[keep]
keep = ds_utils.filter_small_boxes(boxes, self.config['min_size'])
boxes = boxes[keep, :]
scores = scores[keep]
# sort by confidence
sorted_ind = np.argsort(-scores.flatten())
scores = scores[sorted_ind, :]
boxes = boxes[sorted_ind, :]
assert boxes.shape[0] == scores.shape[
0], 'box num({}) should equal score num({})'.format(
boxes.shape, scores.shape)
if cfg.USE_FEEDBACK:
insert_p = min(cfg.TRAIN.ROIS_PER_IM, boxes.shape[0])
boxes_h = boxes[:insert_p]
boxes_t = boxes[insert_p:]
scores_h = scores[:insert_p]
scores_t = scores[insert_p:]
boxes = np.vstack((boxes_h, box_fb_list[i], boxes_t))
scores = np.vstack((scores_h, score_fb_list[i], scores_t))
total_roi += boxes.shape[0]
if boxes.shape[0] > 1024:
up_1024 += 1
if boxes.shape[0] > 2048:
up_2048 += 1
if boxes.shape[0] > 3072:
up_3072 += 1
if boxes.shape[0] > 4096:
up_4096 += 1
box_list.append(boxes)
score_list.append(scores)
if cfg.USE_FEEDBACK:
cfg.TRAIN.ROIS_PER_IM += cfg.FEEDBACK_NUM
print 'total_roi: ', total_roi, ' ave roi: ', total_roi / len(box_list)
print 'up_1024: ', up_1024
print 'up_2048: ', up_2048
print 'up_3072: ', up_3072
print 'up_4096: ', up_4096
return self.create_roidb_from_box_list(box_list, gt_roidb, score_list)
def inference_by_path(self, image_path):
# TODO
# - Inference using image path
# Load the demo image
im = cv2.imread(image_path)
timer = Timer()
timer.tic()
raw_data=selective_search.ss(im)
boxes = np.maximum(raw_data[:, 0:4], 0).astype(np.uint16)
# Remove duplicate boxes and very small boxes and then take top k
keep = ds_utils.unique_boxes(boxes)
boxes = boxes[keep, :]
keep = ds_utils.filter_small_boxes(boxes, 50)
boxes = boxes[keep, :]
boxes = boxes[:300,:]
num_boxes = boxes.shape[0]
num_batch = 2
#if num_boxes > 2000:
# boxes = boxes[:2000, :]
# num_boxes = boxes.shape[0]
#num_batch = (num_boxes+150-1)/150
scores_batch = np.zeros((num_batch*150, 201), dtype=np.float32)
boxes_batch = np.zeros((num_batch*150, 4*201), dtype=np.float32)
rois = np.tile(boxes[0, :], (num_batch*150, 1))
rois[:num_boxes, :] = boxes
for j in xrange(int(num_batch)):
roi = rois[j*150:(j+1)*150, :]
score, box = im_detect(self.net, im, roi,0,0)
scores_batch[j*150:(j+1)*150, :] = score# [:,:,0,0]
boxes_batch[j*150:(j+1)*150, :] = box
scores = scores_batch[:num_boxes, :]
boxes = boxes_batch[:num_boxes, :]
timer.toc()
print ('Detection took {:.3f}s for '
'{:d} object proposals').format(timer.total_time, num_boxes)
# Visualize detections for each class
CONF_THRESH = 0.3
NMS_THRESH = 0.4
result = []
for cls_ind, cls in enumerate(self.CLASSES[1:]):
cls_ind +=1
cls_scores = scores[:,cls_ind]
cls_boxes = boxes[:, cls_ind*4:(cls_ind+1)*4]
cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32, copy=False)
keep = nms(cls_dets, NMS_THRESH)
cls_dets = cls_dets[keep, :]
idx = np.where(cls_dets[:, -1] >= CONF_THRESH)[0]
if len(idx) == 0:
continue
for i in idx:
bbox = cls_dets[i, :4]
score = cls_dets[i, -1]
result.append([(round(bbox[0],3),round(bbox[1],3),round(bbox[2]-bbox[0],3),round(bbox[3]-bbox[1],3)),{cls.encode('ascii','ignore') : round(score,3)}])
self.result = result
return self.result