def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb, self.cfg)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for data, data_pad in zip(data_list, data_tensor):
data['data'] = data_pad[np.newaxis, :]
new_label_list = []
for data, label in zip(data_list, label_list):
# infer label shape
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
# assign anchor for label
label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'], self.cfg,
self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border,
self.normalize_target, self.bbox_mean, self.bbox_std)
new_label_list.append(label)
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = -1 if key == 'label' else 0
all_label[key] = tensor_vstack([batch[key] for batch in new_label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb, self.cfg)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for data, data_pad in zip(data_list, data_tensor):
data['data'] = data_pad[np.newaxis, :]
new_label_list = []
for data, label in zip(data_list, label_list):
# infer label shape
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
# assign anchor for label
label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'], self.cfg,
self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border)
new_label_list.append(label)
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = -1 if key == 'label' else 0
all_label[key] = tensor_vstack([batch[key] for batch in new_label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def infer_shape(self):
_, feat_shape, _ = self.feat_sym.infer_shape(
data=self.max_data_shapes['data'])
im_info = [self.max_h, self.max_w, 1.0]
label = assign_anchor(feat_shape[0], np.zeros((0, 5)), im_info,
self.cfg, self.allowed_border)
label = [label[k] for k in self.label_names]
label_shapes = [
tuple([self.batch_size] + list(v.shape[1:])) for v in label
]
self._provide_label = zip(self.label_names, label_shapes)
self._provide_data = [(k, self.max_data_shapes[k])
for k in self.data_names]
def infer_shape(self, max_data_shape=None, max_label_shape=None):
""" Return maximum data and label shape for single gpu """
if max_data_shape is None:
max_data_shape = []
if max_label_shape is None:
max_label_shape = []
max_shapes = dict(max_data_shape + max_label_shape)
input_batch_size = max_shapes['data'][0]
im_info = [[max_shapes['data'][2], max_shapes['data'][3], 1.0]]
_, feat_shape, _ = self.feat_sym.infer_shape(**max_shapes)
label = assign_anchor(feat_shape[0], np.zeros((0, 5)), im_info, self.cfg,
self.feat_stride, self.anchor_scales, self.anchor_ratios, self.allowed_border)
label = [label[k] for k in self.label_name]
label_shape = [(k, tuple([input_batch_size] + list(v.shape[1:]))) for k, v in zip(self.label_name, label)]
return max_data_shape, label_shape
def infer_shape(self, max_data_shape=None, max_label_shape=None):
""" Return maximum data and label shape for single gpu """
if max_data_shape is None:
max_data_shape = []
if max_label_shape is None:
max_label_shape = []
max_shapes = dict(max_data_shape + max_label_shape)
input_batch_size = max_shapes['data'][0]
im_info = [[max_shapes['data'][2], max_shapes['data'][3], 1.0]]
_, feat_shape, _ = self.feat_sym.infer_shape(**max_shapes)
label = assign_anchor(feat_shape[0], np.zeros((0, 5)), im_info, self.cfg,
self.feat_stride, self.anchor_scales, self.anchor_ratios, self.allowed_border)
label = [label[k] for k in self.label_name]
label_shape = [(k, tuple([input_batch_size] + list(v.shape[1:]))) for k, v in zip(self.label_name, label)]
return max_data_shape, label_shape
def parfetch(self, iroidb):
# get testing data for multigpu
data, label = get_rpn_batch(iroidb, self.cfg)
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
# assign anchor for label
label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'],
self.cfg, self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border)
return {'data': data, 'label': label}
def parfetch(self, iroidb):
# get testing data for multigpu
data, label = get_rpn_batch(iroidb, self.cfg)
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
# assign anchor for label
label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'], self.cfg,
self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border)
return {'data': data, 'label': label}
def parfetch(self, iroidb):
# get testing data for multigpu
data, label = get_rpn_pair_mv_batch(iroidb, self.cfg)
#for k, v in data.items():
# print(k, v)
data_shape = {k: v.shape for k, v in data.items()}
#print(data_shape)
del data_shape['im_info']
del data_shape['data']
data_shape1 = copy.deepcopy(data_shape)
del data_shape1['eq_flag']
del data_shape1['motion_vector']
_, feat_shape, _ = self.feat_conv_3x3_relu.infer_shape(**data_shape1)
#print('feat_shape: ', feat_shape)
#print('shape: ', data['motion_vector'].shape)
#print("size: ", int(feat_shape[0][2]), int(feat_shape[0][3]))
data['motion_vector'] = data['motion_vector'].astype('float64')
data['motion_vector'] = cv2.resize(
data['motion_vector'],
(int(feat_shape[0][3]), int(feat_shape[0][2])),
interpolation=cv2.INTER_AREA)
#print('data[\'motion_vector\'].shape: ', data['motion_vector'].shape)
data['motion_vector'] = transform(data['motion_vector'], [0, 0])
#print('data[\'motion_vector\'].shape: ', data['motion_vector'].shape)
#print("data['motion_vector']: ", data['motion_vector'])
#data['motion_vector'] = cv2.resize(data['motion_vector'], (36, 63))
data_shape = {k: v.shape for k, v in data.items()}
#print(data_shape)
del data_shape['im_info']
del data_shape['data']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
# assign anchor for label
label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'],
self.cfg, self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border,
self.normalize_target, self.bbox_mean,
self.bbox_std)
return {'data': data, 'label': label}
def parfetch_offline_memory_fromrec(self, iroidb):
# get testing data for multigpu
data, label = get_rpn_triple_batch_fromrec_offline(
iroidb, self.cfg, self.video_index_dict, self.rec)
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
# assign anchor for label
label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'],
self.cfg, self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border,
self.normalize_target, self.bbox_mean,
self.bbox_std)
return {'data': data, 'label': label}
def parfetch(self, iroidb):
# get testing data for multigpu
data, label = get_rpn_pair_batch(iroidb, self.cfg)
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
# assign anchor for label
label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'], self.cfg,
self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border,
self.normalize_target, self.bbox_mean, self.bbox_std)
# feat_height, feat_width = feat_shape[-2:]
# gt_boxes = np.copy(data['gt_boxes']) / self.feat_stride
# gt_boxes = np.around(gt_boxes).astype(np.int)
# gt_boxes[..., 0] = np.maximum(gt_boxes[..., 0], 0)
# gt_boxes[..., 1] = np.maximum(gt_boxes[..., 1], 0)
# gt_boxes[..., 2] = np.minimum(gt_boxes[..., 2], feat_width)
# gt_boxes[..., 3] = np.minimum(gt_boxes[..., 3], feat_height)
# feat_weight = np.zeros([feat_shape[0], feat_shape[-2], feat_shape[-1]])
# i = 0
#
# def setWeights(gt_box):
# x1, y1, x2, y2, _ = gt_box
# feat_weight[i, y1:y2, x1:x2] = 1
#
# # only support the one image
# for index in range(feat_shape[0]):
# i = index
# list(map(setWeights, gt_boxes[index]))
# # feat_weight = np.tile(np.expand_dims(feat_weight, axis=1), (1, 1024, 1, 1))
# label['wrap_feat_weight'] = feat_weight
return {'data': data, 'label': label}
def parfetch(self, iroidb):
# get testing data for multigpu
data, label = get_rpn_triple_batch(iroidb, self.cfg)
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
data['occluded'] = label['occluded']
data['delta_bef_gt'] = label['delta_bef_gt']
data['delta_aft_gt'] = label['delta_aft_gt']
# assign anchor for label
label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'],
self.cfg, self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border,
self.normalize_target, self.bbox_mean,
self.bbox_std)
#print '###################################begin parfetch##########################'
#print 'data[gt_boxes]', data['gt_boxes']
#print 'data[delta_bef]', data['delta_bef']
#print 'data[delta_aft]', data['delta_aft']
return {'data': data, 'label': label}
def get_rpn_batch(roidb,
target_scale,
sym,
cfg,
data_buf,
allowed_border=0,
max_gts=100,
kps_dim=0,
stride=32):
"""
allowed_border:
max_gts:
max number of groundtruths
kps_dim:
when trainning with keypoints, set kps_dim >0
"""
num_images = len(roidb)
assert num_images > 0, 'empty list !'
# get images
t0 = time.time() ###
target_size, max_size = target_scale
imgs, roidb = get_image(roidb, target_size, max_size)
max_h = max([img.shape[0] for img in imgs])
max_w = max([img.shape[1] for img in imgs])
stride = float(stride)
max_h = int(np.ceil(max_h / stride) * stride)
max_w = int(np.ceil(max_w / stride) * stride)
t1 = time.time() ###
# assign anchor labels
anchor_labels = []
_, feat_shape, _ = sym.infer_shape(data=(num_images, 3, max_h, max_w))
for i in range(num_images):
if roidb[i]['gt_classes'].size > 0:
assert np.sum(roidb[i]['gt_classes'] ==
0) == 0, 'should not have background boxes!'
gt_boxes = roidb[i]['boxes']
im_info = [max_h, max_w, roidb[i]['im_info'][2]]
# assign anchors
anchor_labels.append(
assign_anchor(feat_shape[0], gt_boxes, im_info, cfg,
allowed_border))
t2 = time.time() ###
# shapes
shapes = {
'data': (num_images, 3, max_h, max_w),
'im_info': (num_images, 3),
'gt_boxes': (num_images, max_gts, 5),
'label':
tuple([num_images] + list(anchor_labels[0]['label'].shape[1:])),
'bbox_target':
tuple([num_images] + list(anchor_labels[0]['bbox_target'].shape[1:])),
'bbox_weight':
tuple([num_images] + list(anchor_labels[0]['bbox_weight'].shape[1:])),
}
if kps_dim > 0:
shapes['gt_kps'] = ((num_images, max_gts, kps_dim))
# reshape buffers
batch = dict()
for k in data_buf:
s = shapes[k]
c = np.prod(s)
batch[k] = np.frombuffer(data_buf[k], dtype=np.float32,
count=c).reshape(s)
batch[k].fill(0)
# transform image data and gt labels
bgr_means = cfg.network.PIXEL_MEANS
for i, img in enumerate(imgs):
h, w = img.shape[:2]
for j in range(3):
batch['data'][i, j, :h, :w] = img[:, :, 2 - j] - bgr_means[2 - j]
batch['im_info'][i, :] = [max_h, max_w, roidb[i]['im_info'][2]]
num_gt = roidb[i]['boxes'].shape[0]
batch['gt_boxes'][i, :num_gt, :4] = roidb[i]['boxes']
batch['gt_boxes'][i, :num_gt, 4] = roidb[i]['gt_classes']
if kps_dim > 0:
batch['gt_kps'][i, :num_gt] = roidb[i]['keypoints']
batch['label'][i] = anchor_labels[i]['label']
batch['bbox_target'][i] = anchor_labels[i]['bbox_target']
batch['bbox_weight'][i] = anchor_labels[i]['bbox_weight']
t3 = time.time() ###
#print 't_image=%.3f\tt_assign=%.3f\tt_trans=%.3f\tt_all=%.3f' % (t1-t0, t2-t1, t3-t2, t3-t0) ###
return shapes