def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slices
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get each device
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rcnn_batch(iroidb)
data_list.append(data)
label_list.append(label)
all_data = dict()
for key in data_list[0].keys():
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in label_list[0].keys():
all_label[key] = tensor_vstack([batch[key] for batch in label_list])
self.data = [mx.nd.array(all_data[name]) for name in self.data_name]
self.label = [mx.nd.array(all_label[name]) for name in self.label_name]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slices
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get each device
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rcnn_batch(iroidb)
data_list.append(data)
label_list.append(label)
all_data = dict()
for key in data_list[0].keys():
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in label_list[0].keys():
all_label[key] = tensor_vstack(
[batch[key] for batch in label_list])
self.data = [mx.nd.array(all_data[name]) for name in self.data_name]
self.label = [mx.nd.array(all_label[name]) for name in self.label_name]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for data, data_pad in zip(data_list, data_tensor):
data['data'] = data_pad[np.newaxis, :]
new_label_list = []
for data, label in zip(data_list, label_list):
# infer label shape
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
data['gt_keypoints'] = label['gt_keypoints'][np.newaxis, :, :]
# assign anchor for label
label = assign_anchor(feat_shape, label['gt_boxes'],
data['im_info'], self.feat_stride,
self.anchor_scales, self.anchor_ratios,
self.allowed_border)
new_label_list.append(label)
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = -1 if key == 'label' else 0
all_label[key] = tensor_vstack(
[batch[key] for batch in new_label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for data, data_pad in zip(data_list, data_tensor):
data['data'] = data_pad[np.newaxis, :]
new_label_list = []
for data, label in zip(data_list, label_list):
# infer label shape
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
# assign anchor for label
label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'],
self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border)
new_label_list.append(label)
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = -1 if key == 'label' else 0
all_label[key] = tensor_vstack([batch[key] for batch in new_label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self):
deq = self.q_out.get()
print('q_out got')
data_list, label_list = deq
for data, label in zip(data_list, label_list):
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
feat_shape_list = []
for s in range(len(self.feat_stride)):
_, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
feat_shape_list.append(feat_shape)
#for k in self.label_name:
# label[k] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
im_info = data['im_info']
gt_boxes = label['gt_boxes']
gt_label = {'gt_boxes': gt_boxes}
label_dict = {}
head_label_dict = assign_anchor_fpn(feat_shape_list,
gt_label,
im_info,
False,
prefix='head')
label_dict.update(head_label_dict)
if config.FACE_LANDMARK:
gt_landmarks = label['gt_landmarks']
gt_label['gt_landmarks'] = gt_landmarks
face_label_dict = assign_anchor_fpn(feat_shape_list,
gt_label,
im_info,
config.FACE_LANDMARK,
prefix='face')
label_dict.update(face_label_dict)
#print('im_info', im_info.shape)
#print(gt_boxes.shape)
for k in self.label_name:
label[k] = label_dict[k]
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = 0 if key.startswith('bbox_') else -1
#print('label vstack', key, pad, len(label_list), file=sys.stderr)
all_label[key] = tensor_vstack(
[batch[key] for batch in label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for i_card in range(len(data_list)):
data_list[i_card]['data'] = data_tensor[
i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES]
for data, label in zip(data_list, label_list):
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
feat_shape_list = []
for s in range(len(self.feat_stride)):
_, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
feat_shape_list.append(feat_shape)
label['label'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
label['bbox_target'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
label['bbox_weight'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
for im_i in range(config.TRAIN.BATCH_IMAGES):
im_info = data['im_info'][im_i]
gt_boxes = label['gt_boxes'][im_i][0]
label_dict = \
assign_anchor_fpn(feat_shape_list, gt_boxes, im_info,
self.feat_stride,
self.anchor_scales,
self.anchor_ratios,
self.allowed_border)
label['label'][im_i] = label_dict['label']
label['bbox_target'][im_i] = label_dict['bbox_target']
label['bbox_weight'][im_i] = label_dict['bbox_weight']
label['label'] = np.vstack(label['label'])
label['bbox_target'] = np.vstack(label['bbox_target'])
label['bbox_weight'] = np.vstack(label['bbox_weight'])
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = 0 if key == 'weight' else -1
all_label[key] = tensor_vstack([batch[key] for batch in label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def _make_data_and_labels_afp(self, im_array_list, input_data_list):
#do not distribute rois on levels
data_list = []
label_list = []
mask_size = (config.ROI_SIZE[0]*2, config.ROI_SIZE[1]*2)
rois_num = 0
max_rois_num = 0
for _data in input_data_list:
for im_i in _data:
rois_num = _data[im_i]['rois'].shape[0]
max_rois_num = max(rois_num, max_rois_num)
rois_num = max_rois_num
# align to num_imgs
num_imgs = len(input_data_list[0])
if rois_num == 0:
rois_num = num_imgs
elif rois_num % num_imgs != 0:
ex = num_imgs - rois_num % num_imgs
rois_num += ex
#print('num', rois_num, num_imgs)
for im_array, data_on_imgs in zip(im_array_list, input_data_list):
num_imgs = len(data_on_imgs)
bucket_size = rois_num
for im_i in range(num_imgs):
_rois = data_on_imgs['img_%s' % im_i]['rois']
_labels = data_on_imgs['img_%s' % im_i]['labels']
_bbox_targets = data_on_imgs['img_%s' % im_i]['bbox_targets']
_bbox_weights = data_on_imgs['img_%s' % im_i]['bbox_weights']
_mask_targets = data_on_imgs['img_%s' % im_i]['mask_targets']
_mask_weights = data_on_imgs['img_%s' % im_i]['mask_weights']
rois_num = _rois.shape[0]
if rois_num < bucket_size:
num_pad = bucket_size - rois_num
rois_pad = np.array([[12, 34, 56, 78]] * num_pad)
labels_pad = np.array([-1] * num_pad)
bbox_targets_pad = np.array([[1, 2, 3, 4] * config.NUM_CLASSES] * num_pad)
bbox_weights_pad = np.array([[0, 0, 0, 0] * config.NUM_CLASSES] * num_pad)
mask_targets_pad = np.zeros((num_pad, config.NUM_CLASSES)+mask_size,
dtype=np.int8)
mask_weights_pad = np.zeros((num_pad, config.NUM_CLASSES, 1, 1), dtype=np.int8)
data_on_imgs['img_%s' % im_i]['rois'] = np.concatenate(
[_rois, rois_pad])
data_on_imgs['img_%s' % im_i]['labels'] = np.concatenate(
[_labels, labels_pad])
data_on_imgs['img_%s' % im_i]['bbox_targets'] = np.concatenate(
[_bbox_targets, bbox_targets_pad])
data_on_imgs['img_%s' % im_i]['bbox_weights'] = np.concatenate(
[_bbox_weights, bbox_weights_pad])
data_on_imgs['img_%s' % im_i]['mask_targets'] = np.concatenate(
[_mask_targets, mask_targets_pad])
data_on_imgs['img_%s' % im_i]['mask_weights'] = np.concatenate(
[_mask_weights, mask_weights_pad])
rois_on_imgs = []
labels_on_imgs = []
bbox_targets_on_imgs = []
bbox_weights_on_imgs = []
mask_targets_on_imgs = []
mask_weights_on_imgs = []
for im_i in range(num_imgs):
im_rois = data_on_imgs['img_%s' % im_i]['rois']
labels = data_on_imgs['img_%s' % im_i]['labels']
bbox_targets = data_on_imgs['img_%s' % im_i]['bbox_targets']
bbox_weights = data_on_imgs['img_%s' % im_i]['bbox_weights']
mask_targets = data_on_imgs['img_%s' % im_i]['mask_targets']
mask_weights = data_on_imgs['img_%s' % im_i]['mask_weights']
_rois = im_rois
batch_index = im_i * np.ones((_rois.shape[0], 1))
rois_on_imgs.append(np.hstack((batch_index, _rois)))
labels_on_imgs.append(labels)
bbox_targets_on_imgs.append(bbox_targets)
bbox_weights_on_imgs.append(bbox_weights)
mask_targets_on_imgs.append(mask_targets)
mask_weights_on_imgs.append(mask_weights)
label = dict()
label.update({'label': np.reshape(np.concatenate(labels_on_imgs, axis=0),
[num_imgs, -1])})
label.update({'bbox_target': np.reshape(
np.concatenate(bbox_targets_on_imgs, axis=0), [num_imgs, -1])})
label.update({'bbox_weight': np.reshape(
np.concatenate(bbox_weights_on_imgs, axis=0), [num_imgs, -1])})
label.update({'mask_target': np.reshape(
np.concatenate(mask_targets_on_imgs, axis=0),
[num_imgs, -1, config.NUM_CLASSES, mask_size[0], mask_size[1]])})
label.update({'mask_weight': np.reshape(
np.concatenate(mask_weights_on_imgs, axis=0),
[num_imgs, -1, config.NUM_CLASSES, 1, 1])})
# Stack batch data, and update dict
data = dict()
data.update({'data': im_array})
rois_array = np.array(rois_on_imgs)
data.update({'rois': rois_array})
data_list.append(data)
label_list.append(label)
all_data = dict()
for key in data_list[0].keys():
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in label_list[0].keys():
all_label[key] = tensor_vstack([batch[key] for batch in label_list])
return all_data, all_label
def _make_data_and_labels(self, im_array_list, levels_data_list):
data_list = []
label_list = []
mask_size = (config.ROI_SIZE[0]*2, config.ROI_SIZE[1]*2)
rois_num_on_levels = {'stride%s' % s: 0 for s in config.RCNN_FEAT_STRIDE}
for s in config.RCNN_FEAT_STRIDE:
max_rois_num = 0
for levels_data in levels_data_list:
for im_i in levels_data:
rois_num = levels_data[im_i]['rois_on_levels']['stride%s' % s].shape[0]
max_rois_num = max(rois_num, max_rois_num)
rois_num_on_levels['stride%s' % s] = max_rois_num
# align to num_imgs
num_imgs = len(levels_data_list[0])
for s in config.RCNN_FEAT_STRIDE:
if rois_num_on_levels['stride%s' % s] == 0:
rois_num_on_levels['stride%s' % s] = num_imgs
continue
if rois_num_on_levels['stride%s' % s] % num_imgs != 0:
ex = num_imgs - rois_num_on_levels['stride%s' % s] % num_imgs
rois_num_on_levels['stride%s' % s] += ex
print('num', rois_num_on_levels, num_imgs)
for im_array, data_on_imgs in zip(im_array_list, levels_data_list):
num_imgs = len(data_on_imgs)
for s in config.RCNN_FEAT_STRIDE:
bucket_size = rois_num_on_levels['stride%s' % s]
for im_i in range(num_imgs):
_rois = data_on_imgs['img_%s' % im_i]['rois_on_levels']['stride%s' % s]
_labels = data_on_imgs['img_%s' % im_i]['labels_on_levels']['stride%s' % s]
_bbox_targets = data_on_imgs['img_%s' % im_i]['bbox_targets_on_levels']['stride%s' % s]
_bbox_weights = data_on_imgs['img_%s' % im_i]['bbox_weights_on_levels']['stride%s' % s]
_mask_targets = data_on_imgs['img_%s' % im_i]['mask_targets_on_levels']['stride%s' % s]
_mask_weights = data_on_imgs['img_%s' % im_i]['mask_weights_on_levels']['stride%s' % s]
rois_num = _rois.shape[0]
if rois_num < bucket_size:
num_pad = bucket_size - rois_num
rois_pad = np.array([[12, 34, 56, 78]] * num_pad)
labels_pad = np.array([-1] * num_pad)
bbox_targets_pad = np.array([[1, 2, 3, 4] * config.NUM_CLASSES] * num_pad)
bbox_weights_pad = np.array([[0, 0, 0, 0] * config.NUM_CLASSES] * num_pad)
mask_targets_pad = np.zeros((num_pad, config.NUM_CLASSES)+mask_size,
dtype=np.int8)
mask_weights_pad = np.zeros((num_pad, config.NUM_CLASSES, 1, 1), dtype=np.int8)
data_on_imgs['img_%s' % im_i]['rois_on_levels']['stride%s' % s] = np.concatenate(
[_rois, rois_pad])
data_on_imgs['img_%s' % im_i]['labels_on_levels']['stride%s' % s] = np.concatenate(
[_labels, labels_pad])
data_on_imgs['img_%s' % im_i]['bbox_targets_on_levels']['stride%s' % s] = np.concatenate(
[_bbox_targets, bbox_targets_pad])
data_on_imgs['img_%s' % im_i]['bbox_weights_on_levels']['stride%s' % s] = np.concatenate(
[_bbox_weights, bbox_weights_pad])
data_on_imgs['img_%s' % im_i]['mask_targets_on_levels']['stride%s' % s] = np.concatenate(
[_mask_targets, mask_targets_pad])
data_on_imgs['img_%s' % im_i]['mask_weights_on_levels']['stride%s' % s] = np.concatenate(
[_mask_weights, mask_weights_pad])
rois_on_imgs = dict()
labels_on_imgs = dict()
bbox_targets_on_imgs = dict()
bbox_weights_on_imgs = dict()
mask_targets_on_imgs = dict()
mask_weights_on_imgs = dict()
for s in config.RCNN_FEAT_STRIDE:
rois_on_imgs.update({'stride%s' % s: list()})
labels_on_imgs.update({'stride%s' % s: list()})
bbox_targets_on_imgs.update({'stride%s' % s: list()})
bbox_weights_on_imgs.update({'stride%s' % s: list()})
mask_targets_on_imgs.update({'stride%s' % s: list()})
mask_weights_on_imgs.update({'stride%s' % s: list()})
for im_i in range(num_imgs):
for s in config.RCNN_FEAT_STRIDE:
im_rois_on_levels = data_on_imgs['img_%s' % im_i]['rois_on_levels']
labels_on_levels = data_on_imgs['img_%s' % im_i]['labels_on_levels']
bbox_targets_on_levels = data_on_imgs['img_%s' % im_i]['bbox_targets_on_levels']
bbox_weights_on_levels = data_on_imgs['img_%s' % im_i]['bbox_weights_on_levels']
mask_targets_on_levels = data_on_imgs['img_%s' % im_i]['mask_targets_on_levels']
mask_weights_on_levels = data_on_imgs['img_%s' % im_i]['mask_weights_on_levels']
_rois = im_rois_on_levels['stride%s' % s]
batch_index = im_i * np.ones((_rois.shape[0], 1))
rois_on_imgs['stride%s' % s].append(np.hstack((batch_index, _rois)))
labels_on_imgs['stride%s' % s].append(labels_on_levels['stride%s' % s])
bbox_targets_on_imgs['stride%s' % s].append(bbox_targets_on_levels['stride%s' % s])
bbox_weights_on_imgs['stride%s' % s].append(bbox_weights_on_levels['stride%s' % s])
mask_targets_on_imgs['stride%s' % s].append(mask_targets_on_levels['stride%s' % s])
mask_weights_on_imgs['stride%s' % s].append(mask_weights_on_levels['stride%s' % s])
label = dict()
for s in config.RCNN_FEAT_STRIDE:
label.update({'label_stride%s' % s: np.reshape(np.concatenate(labels_on_imgs['stride%s' % s], axis=0),
[num_imgs, -1])})
label.update({'bbox_target_stride%s' % s: np.reshape(
np.concatenate(bbox_targets_on_imgs['stride%s' % s], axis=0), [num_imgs, -1])})
label.update({'bbox_weight_stride%s' % s: np.reshape(
np.concatenate(bbox_weights_on_imgs['stride%s' % s], axis=0), [num_imgs, -1])})
label.update({'mask_target_stride%s' % s: np.reshape(
np.concatenate(mask_targets_on_imgs['stride%s' % s], axis=0),
[num_imgs, -1, config.NUM_CLASSES, mask_size[0], mask_size[1]])})
label.update({'mask_weight_stride%s' % s: np.reshape(
np.concatenate(mask_weights_on_imgs['stride%s' % s], axis=0),
[num_imgs, -1, config.NUM_CLASSES, 1, 1])})
# Stack batch data, and update dict
data = dict()
data.update({'data': im_array})
for s in config.RCNN_FEAT_STRIDE:
rois_array = np.array(rois_on_imgs['stride%s' % s])
data.update({'rois_stride%s' % s: rois_array})
data_list.append(data)
label_list.append(label)
all_data = dict()
for key in data_list[0].keys():
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in label_list[0].keys():
all_label[key] = tensor_vstack([batch[key] for batch in label_list])
return all_data, all_label
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for i_card in range(len(data_list)):
data_list[i_card]['data'] = data_tensor[
i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES]
for data, label in zip(data_list, label_list):
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
feat_shape_list = []
for s in range(len(self.feat_stride)):
_, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
feat_shape_list.append(feat_shape)
label['label'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
label['bbox_target'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
label['bbox_weight'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
for im_i in range(config.TRAIN.BATCH_IMAGES):
im_info = data['im_info'][im_i]
gt_boxes = label['gt_boxes'][im_i][0]
label_dict = \
assign_anchor_fpn(feat_shape_list, gt_boxes, im_info,
self.feat_stride,
self.anchor_scales,
self.anchor_ratios,
self.allowed_border)
label['label'][im_i] = label_dict['label']
label['bbox_target'][im_i] = label_dict['bbox_target']
label['bbox_weight'][im_i] = label_dict['bbox_weight']
label['label'] = np.vstack(label['label'])
label['bbox_target'] = np.vstack(label['bbox_target'])
label['bbox_weight'] = np.vstack(label['bbox_weight'])
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = 0 if key == 'weight' else -1
all_label[key] = tensor_vstack([batch[key] for batch in label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def _make_data_and_labels(self, im_array_list, levels_data_list):
data_list = []
label_list = []
rois_num_on_levels = {'stride%s' % s: 0 for s in config.RCNN_FEAT_STRIDE}
for s in config.RCNN_FEAT_STRIDE:
max_rois_num = 0
for levels_data in levels_data_list:
for im_i in levels_data:
rois_num = levels_data[im_i]['rois_on_levels']['stride%s' % s].shape[0]
max_rois_num = max(rois_num, max_rois_num)
rois_num_on_levels['stride%s' % s] = max_rois_num
# align to num_imgs
num_imgs = len(levels_data_list[0])
for s in config.RCNN_FEAT_STRIDE:
if rois_num_on_levels['stride%s' % s] == 0:
rois_num_on_levels['stride%s' % s] = num_imgs
continue
if rois_num_on_levels['stride%s' % s] % num_imgs != 0:
ex = num_imgs - rois_num_on_levels['stride%s' % s] % num_imgs
rois_num_on_levels['stride%s' % s] += ex
for im_array, data_on_imgs in zip(im_array_list, levels_data_list):
num_imgs = len(data_on_imgs)
for s in config.RCNN_FEAT_STRIDE:
bucket_size = rois_num_on_levels['stride%s' % s]
for im_i in range(num_imgs):
_rois = data_on_imgs['img_%s' % im_i]['rois_on_levels']['stride%s' % s]
_labels = data_on_imgs['img_%s' % im_i]['labels_on_levels']['stride%s' % s]
_bbox_targets = data_on_imgs['img_%s' % im_i]['bbox_targets_on_levels']['stride%s' % s]
_bbox_weights = data_on_imgs['img_%s' % im_i]['bbox_weights_on_levels']['stride%s' % s]
_mask_targets = data_on_imgs['img_%s' % im_i]['mask_targets_on_levels']['stride%s' % s]
_mask_weights = data_on_imgs['img_%s' % im_i]['mask_weights_on_levels']['stride%s' % s]
rois_num = _rois.shape[0]
if rois_num < bucket_size:
num_pad = bucket_size - rois_num
rois_pad = np.array([[12, 34, 56, 78]] * num_pad)
labels_pad = np.array([-1] * num_pad)
bbox_targets_pad = np.array([[1, 2, 3, 4] * config.NUM_CLASSES] * num_pad)
bbox_weights_pad = np.array([[0, 0, 0, 0] * config.NUM_CLASSES] * num_pad)
mask_targets_pad = np.zeros((num_pad, config.NUM_CLASSES, 28, 28),
dtype=np.int8)
mask_weights_pad = np.zeros((num_pad, config.NUM_CLASSES, 1, 1), dtype=np.int8)
data_on_imgs['img_%s' % im_i]['rois_on_levels']['stride%s' % s] = np.concatenate(
[_rois, rois_pad])
data_on_imgs['img_%s' % im_i]['labels_on_levels']['stride%s' % s] = np.concatenate(
[_labels, labels_pad])
data_on_imgs['img_%s' % im_i]['bbox_targets_on_levels']['stride%s' % s] = np.concatenate(
[_bbox_targets, bbox_targets_pad])
data_on_imgs['img_%s' % im_i]['bbox_weights_on_levels']['stride%s' % s] = np.concatenate(
[_bbox_weights, bbox_weights_pad])
data_on_imgs['img_%s' % im_i]['mask_targets_on_levels']['stride%s' % s] = np.concatenate(
[_mask_targets, mask_targets_pad])
data_on_imgs['img_%s' % im_i]['mask_weights_on_levels']['stride%s' % s] = np.concatenate(
[_mask_weights, mask_weights_pad])
rois_on_imgs = dict()
labels_on_imgs = dict()
bbox_targets_on_imgs = dict()
bbox_weights_on_imgs = dict()
mask_targets_on_imgs = dict()
mask_weights_on_imgs = dict()
for s in config.RCNN_FEAT_STRIDE:
rois_on_imgs.update({'stride%s' % s: list()})
labels_on_imgs.update({'stride%s' % s: list()})
bbox_targets_on_imgs.update({'stride%s' % s: list()})
bbox_weights_on_imgs.update({'stride%s' % s: list()})
mask_targets_on_imgs.update({'stride%s' % s: list()})
mask_weights_on_imgs.update({'stride%s' % s: list()})
for im_i in range(num_imgs):
for s in config.RCNN_FEAT_STRIDE:
im_rois_on_levels = data_on_imgs['img_%s' % im_i]['rois_on_levels']
labels_on_levels = data_on_imgs['img_%s' % im_i]['labels_on_levels']
bbox_targets_on_levels = data_on_imgs['img_%s' % im_i]['bbox_targets_on_levels']
bbox_weights_on_levels = data_on_imgs['img_%s' % im_i]['bbox_weights_on_levels']
mask_targets_on_levels = data_on_imgs['img_%s' % im_i]['mask_targets_on_levels']
mask_weights_on_levels = data_on_imgs['img_%s' % im_i]['mask_weights_on_levels']
_rois = im_rois_on_levels['stride%s' % s]
batch_index = im_i * np.ones((_rois.shape[0], 1))
rois_on_imgs['stride%s' % s].append(np.hstack((batch_index, _rois)))
labels_on_imgs['stride%s' % s].append(labels_on_levels['stride%s' % s])
bbox_targets_on_imgs['stride%s' % s].append(bbox_targets_on_levels['stride%s' % s])
bbox_weights_on_imgs['stride%s' % s].append(bbox_weights_on_levels['stride%s' % s])
mask_targets_on_imgs['stride%s' % s].append(mask_targets_on_levels['stride%s' % s])
mask_weights_on_imgs['stride%s' % s].append(mask_weights_on_levels['stride%s' % s])
label = dict()
for s in config.RCNN_FEAT_STRIDE:
label.update({'label_stride%s' % s: np.reshape(np.concatenate(labels_on_imgs['stride%s' % s], axis=0),
[num_imgs, -1])})
label.update({'bbox_target_stride%s' % s: np.reshape(
np.concatenate(bbox_targets_on_imgs['stride%s' % s], axis=0), [num_imgs, -1])})
label.update({'bbox_weight_stride%s' % s: np.reshape(
np.concatenate(bbox_weights_on_imgs['stride%s' % s], axis=0), [num_imgs, -1])})
label.update({'mask_target_stride%s' % s: np.reshape(
np.concatenate(mask_targets_on_imgs['stride%s' % s], axis=0),
[num_imgs, -1, config.NUM_CLASSES, 28, 28])})
label.update({'mask_weight_stride%s' % s: np.reshape(
np.concatenate(mask_weights_on_imgs['stride%s' % s], axis=0),
[num_imgs, -1, config.NUM_CLASSES, 1, 1])})
# Stack batch data, and update dict
data = dict()
data.update({'data': im_array})
for s in config.RCNN_FEAT_STRIDE:
rois_array = np.array(rois_on_imgs['stride%s' % s])
data.update({'rois_stride%s' % s: rois_array})
data_list.append(data)
label_list.append(label)
all_data = dict()
for key in data_list[0].keys():
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in label_list[0].keys():
all_label[key] = tensor_vstack([batch[key] for batch in label_list])
return all_data, all_label
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
assert cur_to == cur_from + self.batch_size
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_crop_batch(iroidb)
data_list += data
label_list += label
#data_list.append(data)
#label_list.append(label)
# pad data first and then assign anchor (read label)
#data_tensor = tensor_vstack([batch['data'] for batch in data_list])
#for i_card in range(len(data_list)):
# data_list[i_card]['data'] = data_tensor[
# i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES]
for data, label in zip(data_list, label_list):
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
feat_shape_list = []
for s in range(len(self.feat_stride)):
_, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
feat_shape_list.append(feat_shape)
#for k in self.label_name:
# label[k] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
im_info = data['im_info']
gt_boxes = label['gt_boxes']
gt_landmarks = label['gt_landmarks']
#print('im_info', im_info.shape)
#print(gt_boxes.shape)
label_dict = assign_anchor_fpn(feat_shape_list, gt_boxes,
gt_landmarks, im_info)
for k in self.label_name:
label[k] = label_dict[k]
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = 0 if key.startswith('bbox_') else -1
#print('label vstack', key, pad, len(label_list), file=sys.stderr)
all_label[key] = tensor_vstack(
[batch[key] for batch in label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for i_card in range(len(data_list)):
data_list[i_card]['data'] = data_tensor[i_card *
config.TRAIN.BATCH_IMAGES:
(1 + i_card) *
config.TRAIN.BATCH_IMAGES]
for data, label in zip(data_list, label_list):
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
feat_shape_list = []
for s in range(len(self.feat_stride)):
_, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
feat_shape_list.append(feat_shape)
for k in self.label_name:
label[k] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
for im_i in range(config.TRAIN.BATCH_IMAGES):
#im_info = data['im_info'][im_i]
#gt_boxes = label['gt_boxes'][im_i][0]
im_info = data['im_info']
gt_boxes = label['gt_boxes']
gt_landmarks = label['gt_landmarks']
#print('im_info', im_info.shape)
#print(gt_boxes.shape)
vis = True if self.visid >= 0 else False
label_dict = assign_anchor_fpn(feat_shape_list, gt_boxes,
gt_landmarks, im_info, vis)
#do visualize for debug here
if self.visid >= 0 and self.visid < 10:
for _roi in roidb:
print('image', _roi['image'], file=sys.stderr)
self.visid += 1
anchors = label_dict['anchors'].copy()
_im = data['data'].copy()
#print(label_dict['olabel'].shape)
_label = label_dict['olabel'].copy()
_gt_boxes = gt_boxes.copy().astype(np.int)
filename = './vis/A%d.png' % self.visid
_im = _im[0].transpose((1, 2, 0))
_im = _im[..., ::-1] #bgr
for c in range(3):
_im[:, :, c] += config.PIXEL_MEANS[c]
_im = _im.astype(np.uint8).copy()
fg_inds = np.where(_label == 1)[0]
print(_im.shape,
_label.shape,
anchors.shape,
len(fg_inds),
_gt_boxes.shape,
file=sys.stderr)
#draw FG anchors
_bc = 0
for a in range(anchors.shape[0]):
anchor = anchors[a].astype(np.int)
l = _label[a]
if l != 1:
continue
#print('drawing', _im.shape, anchor)
cv2.rectangle(_im, (anchor[0], anchor[1]),
(anchor[2], anchor[3]), (255, 0, 0), 1)
_bc += 1
for a in range(_gt_boxes.shape[0]):
_box = _gt_boxes[a]
cv2.rectangle(_im, (_box[0], _box[1]),
(_box[2], _box[3]), (0, 0, 255), 1)
print('draw to', filename, _bc, file=sys.stderr)
cv2.imwrite(filename, _im)
for k in self.label_name:
#print('0in_loader', k, label_dict[k].shape, file=sys.stderr)
label[k][im_i] = label_dict[k]
for k in self.label_name:
label[k] = np.vstack(label[k])
#print('in_loader', k, label[k].shape, file=sys.stderr)
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = 0 if key.startswith('bbox_') else -1
#print('label vstack', key, pad, len(label_list), file=sys.stderr)
all_label[key] = tensor_vstack(
[batch[key] for batch in label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_fpn_maskrcnn_batch(roidb):
"""
return a dictionary that contains raw data.
"""
num_images = len(roidb)
imgs, roidb = get_image(roidb, scale=config.TRAIN.SCALE)
im_array = tensor_vstack(imgs)
assert config.TRAIN.BATCH_ROIS % config.TRAIN.BATCH_IMAGES == 0, \
'BATCHIMAGES {} must divide BATCH_ROIS {}'.format(config.TRAIN.BATCH_IMAGES, config.TRAIN.BATCH_ROIS)
rois_per_image = config.TRAIN.BATCH_ROIS / config.TRAIN.BATCH_IMAGES
fg_rois_per_image = np.round(config.TRAIN.FG_FRACTION *
rois_per_image).astype(int)
rois_on_imgs = dict()
labels_on_imgs = dict()
bbox_targets_on_imgs = dict()
bbox_weights_on_imgs = dict()
mask_targets_on_imgs = dict()
mask_weights_on_imgs = dict()
for s in config.RCNN_FEAT_STRIDE:
rois_on_imgs.update({'stride%s' % s: list()})
labels_on_imgs.update({'stride%s' % s: list()})
bbox_targets_on_imgs.update({'stride%s' % s: list()})
bbox_weights_on_imgs.update({'stride%s' % s: list()})
mask_targets_on_imgs.update({'stride%s' % s: list()})
mask_weights_on_imgs.update({'stride%s' % s: list()})
# Sample rois
level_related_data_on_imgs = {}
for im_i in range(num_images):
roi_rec = roidb[im_i]
# infer num_classes from gt_overlaps
num_classes = roi_rec['gt_overlaps'].shape[1]
# label = class RoI has max overlap with
rois = roi_rec['boxes']
labels = roi_rec['max_classes']
overlaps = roi_rec['max_overlaps']
bbox_targets = roi_rec['bbox_targets']
im_info = roi_rec['im_info']
mask_targets = roi_rec['mask_targets']
mask_labels = roi_rec['mask_labels']
mask_inds = roi_rec['mask_inds']
assign_levels = roi_rec['assign_levels']
im_rois_on_levels, labels_on_levels, bbox_targets_on_levels, bbox_weights_on_levels, mask_targets_on_levels, mask_weights_on_levels = \
sample_rois_fpn(rois, assign_levels, fg_rois_per_image, rois_per_image, num_classes,
labels, overlaps, bbox_targets, mask_targets=mask_targets, mask_labels=mask_labels, mask_inds=mask_inds, im_info=im_info)
level_related_data_on_imgs.update({
'img_%s' % im_i: {
'rois_on_levels': im_rois_on_levels,
'labels_on_levels': labels_on_levels,
'bbox_targets_on_levels': bbox_targets_on_levels,
'bbox_weights_on_levels': bbox_weights_on_levels,
'mask_targets_on_levels': mask_targets_on_levels,
'mask_weights_on_levels': mask_weights_on_levels,
}
})
return im_array, level_related_data_on_imgs
def get_batch(self, roidb):
label_name = config.label_name
data_name = config.data_name
# get testing data for multigpu
data_list = []
label_list = []
# TODO：获取单个GPU的rpn_batch，data = {'data': im_array, 'im_info': im_info}，label = {'gt_landmarks'，'gt_boxes'}
data, label = get_crop_batch(roidb) # 返回真实label
# print('core/loader: label.shape = ', np.array(label).shape,'\n')
data_list += data
label_list += label # 每个元素为一张图片
# print('core/loader: label_list.shape = ',np.array(label_list).shape)
# print('core/loader: before!! label_list[0] = ',label_list[0])
select_stride = 0
for data, label in zip(data_list, label_list):
# 这里的label是dict
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
im_info = data['im_info']
gt_boxes = label['gt_boxes']
# print('loader/get_batch: in the gt_label! gt_bboxes.shape=',gt_boxes.shape)
gt_label = {'gt_boxes': gt_boxes}
label_dict = {}
if config.FACE_LANDMARK:
gt_landmarks = label['gt_landmarks']
# print('loader/get_batch: in the gt_label! gt_landmarks.shape=', gt_landmarks.shape)
gt_label['gt_landmarks'] = gt_landmarks
# TODO 上面把label赋值给gt_label的操作好像是没有意义的，gt_label与label没有区别. but, 后面label好像变了，但是否影响这里呢？
# ta = datetime.datetime.now()
# TODO：产生训练label
face_label_dict = self.aa.assign_anchor_fpn(
gt_label,
im_info,
config.FACE_LANDMARK,
prefix='face',
select_stride=select_stride)
# print('face_label_dict.keys = ',face_label_dict.keys())
# tb = datetime.datetime.now()
# self._times[0] += (tb-ta).total_seconds()
label_dict.update(face_label_dict)
# print('im_info', im_info.shape)
# print(gt_boxes.shape)
for k in label_name:
label[k] = label_dict[
k] # TODO 这里实际上是在更新label_list, 由zip函数返回的label应该是引用关系
# print('core/loader: after!! label_list[0] = ', label_list[0])
all_data = dict()
for key in data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in label_name:
pad = 0 if key.startswith('bbox_') else -1
# print('label vstack', key, pad, len(label_list), file=sys.stderr)
all_label[key] = tensor_vstack(
[batch[key] for batch in label_list],
pad=pad) # 这里的batch其实就是dict，这个函数的作用是将所有的图片的相同key的list叠起来
# print('batch_key_list len = ',len([batch['gt_boxes'] for batch in label_list]))
labels = {}
data = [np.array(all_data[key]) for key in data_name]
# print('label_list len = ', len([np.array(all_label[key])for key in label_name]))
label_d = {}
for key in label_name:
label_d[key] = all_label[key]
# label = [np.array()for key in label_name] #该list是按照顺序存的，要记住label和本list中的array的对应顺序
# for key in label_d.keys():
# print('{}: {}'.format(key,label_d[key].shape))
return data, label_d
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
assert cur_to == cur_from + self.batch_size
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_crop_batch(iroidb)
data_list += data
label_list += label
#data_list.append(data)
#label_list.append(label)
# pad data first and then assign anchor (read label)
#data_tensor = tensor_vstack([batch['data'] for batch in data_list])
#for i_card in range(len(data_list)):
# data_list[i_card]['data'] = data_tensor[
# i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES]
#iiddxx = 0
select_stride = 0
if config.RANDOM_FEAT_STRIDE:
select_stride = random.choice(config.RPN_FEAT_STRIDE)
for data, label in zip(data_list, label_list):
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
feat_shape_list = []
for s in range(len(self.feat_stride)):
_, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
feat_shape_list.append(feat_shape)
im_info = data['im_info']
gt_boxes = label['gt_boxes']
gt_label = {'gt_boxes': gt_boxes}
if config.USE_BLUR:
gt_blur = label['gt_blur']
gt_label['gt_blur'] = gt_blur
if self._debug:
img = data['data'].copy()[0].transpose(
(1, 2, 0))[:, :, ::-1].copy()
print('DEBUG SHAPE', data['data'].shape,
label['gt_boxes'].shape)
box = label['gt_boxes'].copy()[0][0:4].astype(np.int)
cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]),
(0, 255, 0), 2)
filename = './debugout/%d.png' % (self._debug_id)
print('debug write', filename)
cv2.imwrite(filename, img)
self._debug_id += 1
#print('DEBUG', img.shape, bbox.shape)
label_dict = {}
if config.HEAD_BOX:
head_label_dict = self.aa.assign_anchor_fpn(
gt_label,
im_info,
False,
prefix='head',
select_stride=select_stride)
label_dict.update(head_label_dict)
if config.FACE_LANDMARK:
gt_landmarks = label['gt_landmarks']
gt_label['gt_landmarks'] = gt_landmarks
#ta = datetime.datetime.now()
#face_label_dict = assign_anchor_fpn(feat_shape_list, gt_label, im_info, config.FACE_LANDMARK, prefix='face', select_stride = select_stride)
face_label_dict = self.aa.assign_anchor_fpn(
gt_label,
im_info,
config.FACE_LANDMARK,
prefix='face',
select_stride=select_stride)
#tb = datetime.datetime.now()
#self._times[0] += (tb-ta).total_seconds()
label_dict.update(face_label_dict)
#for k in label_dict:
# print(k, label_dict[k].shape)
if config.CASCADE > 0:
pad_gt_boxes = np.empty(
(1, config.TRAIN.MAX_BBOX_PER_IMAGE, 5), dtype=np.float32)
pad_gt_boxes.fill(-1)
pad_gt_boxes[0, 0:gt_boxes.shape[0], :] = gt_boxes
label_dict['gt_boxes'] = pad_gt_boxes
#print('im_info', im_info.shape)
#print(gt_boxes.shape)
for k in self.label_name:
label[k] = label_dict[k]
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = 0 if key.startswith('bbox_') else -1
#print('label vstack', key, pad, len(label_list), file=sys.stderr)
all_label[key] = tensor_vstack(
[batch[key] for batch in label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def forward(self, is_train, req, in_data, out_data, aux):
forward_start = time.time()
class_id = [0, 24, 25, 26, 27, 28, 31, 32, 33]
feat, deconv1_weight, deconv2_weight, conv1_weight, conv2_weight, gt, rois, label = in_data
if rois.shape[1] == 5:
rois = rois[:, 1:]
for blob in [
feat, deconv1_weight, deconv2_weight, conv1_weight,
conv2_weight
]:
blob.attach_grad()
self.in_data_list.append(blob)
with log_time("asnumpy"):
gt = gt.asnumpy()
label = label.astype(int).asnumpy()
rois = rois.asnumpy()
rois_stride4 = np.ceil(rois / 4).astype(int)
feat_np = feat.asnumpy()
fg_indexes = np.where(label != 0)[0]
fg_rois = rois[fg_indexes]
fg_roi_areas = (fg_rois[:, 2] - fg_rois[:, 0]) * (fg_rois[:, 3] -
fg_rois[:, 1])
fg_small_indexes = fg_indexes[np.where(fg_roi_areas < 5000)[0]]
fg_middle_indexes = fg_indexes[np.where(
np.logical_and(fg_roi_areas >= 5000, fg_roi_areas < 30000))]
fg_large_indexes = fg_indexes[np.where(fg_roi_areas >= 30000)[0]]
self.fg_small_rois = rois_stride4[fg_small_indexes]
self.fg_middle_rois = rois_stride4[fg_middle_indexes]
loss = mx.nd.array([0.0])
if len(self.fg_small_rois) > 0:
with log_time("prepare small"):
feat_tensor_list = []
for roi in rois_stride4[fg_small_indexes]:
x0, y0, x1, y1 = roi
feat_tensor_list.append(feat_np[:, :, y0:y1, x0:x1])
pad_small_feat = tensor_vstack(feat_tensor_list)
pad_small_feat = mx.nd.array(pad_small_feat, ctx=feat.context)
pad_small_feat.attach_grad()
self.in_data_list.append(pad_small_feat)
gt_tensor_list = []
for i in fg_small_indexes:
x0, y0, x1, y1 = rois[i]
x0, y0, x1, y1 = int(round(x0)), int(round(y0)), int(
round(x1)), int(round(y1))
crop_gt = gt[0, y0:y1, x0:x1].copy()
h, w = crop_gt.shape
if h > w:
rand_points = [
np.arange(h),
np.random.randint(0, w, (h, ))
]
else:
rand_points = [
np.random.randint(0, h, (w, )),
np.arange(w)
]
points = crop_gt[rand_points]
ids, counts = np.unique(points, return_counts=True)
indexes_sort_counts = np.argsort(counts)
mask = None
for idx in indexes_sort_counts:
if np.floor(ids[idx] / self.seg_code) == class_id[int(
label[i])]:
crop_gt[crop_gt != ids[idx]] = 0.
crop_gt[crop_gt == ids[idx]] = 1.
mask = crop_gt
break
if mask is None:
mask = np.zeros_like(crop_gt)
tmp = np.zeros(shape=(1, self.num_class) + mask.shape[-2:])
tmp[0, label[i]] = mask
gt_tensor_list.append(tmp)
pad_small_gt = tensor_vstack(gt_tensor_list)
pad_small_gt = mx.nd.array(pad_small_gt, ctx=feat.context)
with log_time("deconv small"):
with mx.autograd.record():
deconv1 = mx.nd.Deconvolution(pad_small_feat,
weight=deconv1_weight,
num_filter=32,
kernel=(2, 2),
stride=(2, 2),
no_bias=True,
cudnn_tune="off")
conv1 = mx.nd.Convolution(deconv1,
weight=conv1_weight,
num_filter=32,
kernel=(3, 3),
pad=(1, 1),
no_bias=True,
cudnn_tune="off")
relu1 = mx.nd.Activation(conv1, act_type="relu")
deconv2 = mx.nd.Deconvolution(relu1,
weight=deconv2_weight,
num_filter=32,
kernel=(2, 2),
stride=(2, 2),
pad=(2, 2),
no_bias=True,
cudnn_tune="off")
fullconv = mx.nd.Convolution(deconv2,
weight=conv2_weight,
num_filter=self.num_class,
kernel=(1, 1),
no_bias=True,
cudnn_tune="off")
prob = mx.nd.Activation(fullconv, act_type="sigmoid")
crop_mask = mx.nd.Crop(pad_small_gt, prob)
loss = loss + mx.nd.mean(mx.nd.square(crop_mask - prob))
if len(self.fg_middle_rois) > 0:
with log_time("prepare middle"):
feat_tensor_list = []
for roi in rois_stride4[fg_middle_indexes]:
x0, y0, x1, y1 = roi
feat_tensor_list.append(feat_np[:, :, y0:y1, x0:x1])
pad_middle_feat = tensor_vstack(feat_tensor_list)
pad_middle_feat = mx.nd.array(pad_middle_feat,
ctx=feat.context)
pad_middle_feat.attach_grad()
self.in_data_list.append(pad_middle_feat)
gt_tensor_list = []
for i in fg_middle_indexes:
x0, y0, x1, y1 = rois[i]
x0, y0, x1, y1 = int(round(x0)), int(round(y0)), int(
round(x1)), int(round(y1))
crop_gt = gt[0, y0:y1, x0:x1].copy()
h, w = crop_gt.shape
if h > w:
rand_points = [
np.arange(h),
np.random.randint(0, w, (h, ))
]
else:
rand_points = [
np.random.randint(0, h, (w, )),
np.arange(w)
]
points = crop_gt[rand_points]
ids, counts = np.unique(points, return_counts=True)
indexes_sort_counts = np.argsort(counts)
mask = None
for idx in indexes_sort_counts:
if np.floor(ids[idx] / self.seg_code) == class_id[int(
label[i])]:
crop_gt[crop_gt != ids[idx]] = 0.
crop_gt[crop_gt == ids[idx]] = 1.
mask = crop_gt
break
if mask is None:
mask = np.zeros_like(crop_gt)
tmp = np.zeros(shape=(1, self.num_class) + mask.shape[-2:])
tmp[0, label[i]] = mask
gt_tensor_list.append(tmp)
pad_middle_gt = tensor_vstack(gt_tensor_list)
pad_middle_gt = mx.nd.array(pad_middle_gt, ctx=feat.context)
with log_time("deconv middle"):
with mx.autograd.record():
deconv1 = mx.nd.Deconvolution(pad_middle_feat,
weight=deconv1_weight,
num_filter=32,
kernel=(2, 2),
stride=(2, 2),
no_bias=True,
cudnn_tune="off")
conv1 = mx.nd.Convolution(deconv1,
weight=conv1_weight,
num_filter=32,
kernel=(3, 3),
pad=(1, 1),
no_bias=True,
cudnn_tune="off")
relu1 = mx.nd.Activation(conv1, act_type="relu")
deconv2 = mx.nd.Deconvolution(relu1,
weight=deconv2_weight,
num_filter=32,
kernel=(2, 2),
stride=(2, 2),
pad=(2, 2),
no_bias=True,
cudnn_tune="off")
fullconv = mx.nd.Convolution(deconv2,
weight=conv2_weight,
num_filter=self.num_class,
kernel=(1, 1),
no_bias=True,
cudnn_tune="off")
prob = mx.nd.Activation(fullconv, act_type="sigmoid")
crop_mask = mx.nd.Crop(pad_middle_gt, prob)
loss = loss + mx.nd.mean(mx.nd.square(crop_mask - prob))
with mx.autograd.record():
with log_time("large rois"):
for i in fg_large_indexes.tolist():
# crop gt
roi = np.ceil(rois[i]).astype(
int) # slice is left inclusive and right exclusive
x0, y0, x1, y1 = roi
if x0 >= x1 or y0 >= y1:
continue
crop_gt = gt[0, y0:y1, x0:x1].copy()
h, w = crop_gt.shape
if h > w:
rand_points = [
np.arange(h),
np.random.randint(0, w, (h, ))
]
else:
rand_points = [
np.random.randint(0, h, (w, )),
np.arange(w)
]
points = crop_gt[rand_points]
ids, counts = np.unique(points, return_counts=True)
indexes_sort_counts = np.argsort(counts)
# gt_copy = gt[0].copy()
# cv2.rectangle(gt_copy, (x0, y0), (x1, y1), color=255*255, thickness=2)
# cv2.imshow("", gt_copy.astype(np.uint16))
# cv2.waitKey()
mask = None
for idx in indexes_sort_counts[::-1]:
if np.floor(ids[idx] /
self.seg_code) == class_id[label[i]]:
crop_gt[crop_gt != ids[idx]] = 0
crop_gt[crop_gt == ids[idx]] = 1
mask = crop_gt
# cv2.imshow("", mask)
# cv2.waitKey()
break
if mask is None:
mask = np.zeros_like(crop_gt)
tmp = np.zeros(shape=(1, self.num_class) + mask.shape[-2:])
tmp[0, label[i]] = mask
mask = mx.nd.array(tmp, feat.context)
roi_stride4 = np.ceil(rois[i] / 4).astype(int)
x0, y0, x1, y1 = roi_stride4
crop_feat = mx.nd.slice(feat,
begin=(0, 0, y0, x0),
end=(1, 128, y1, x1))
deconv1 = mx.nd.Deconvolution(crop_feat,
weight=deconv1_weight,
num_filter=32,
kernel=(2, 2),
stride=(2, 2),
no_bias=True,
cudnn_tune="off")
conv1 = mx.nd.Convolution(deconv1,
weight=conv1_weight,
num_filter=32,
kernel=(3, 3),
pad=(1, 1),
no_bias=True,
cudnn_tune="off")
relu1 = mx.nd.Activation(conv1, act_type="relu")
deconv2 = mx.nd.Deconvolution(relu1,
weight=deconv2_weight,
num_filter=32,
kernel=(2, 2),
stride=(2, 2),
pad=(2, 2),
no_bias=True,
cudnn_tune="off")
fullconv = mx.nd.Convolution(deconv2,
weight=conv2_weight,
num_filter=self.num_class,
kernel=(1, 1),
no_bias=True,
cudnn_tune="off")
prob = mx.nd.Activation(fullconv, act_type="sigmoid")
crop_mask = mx.nd.Crop(mask, prob)
loss = loss + mx.nd.mean(mx.nd.square(crop_mask - prob))
loss = loss / len(fg_indexes)
loss = loss * self.grad_scale
logging.debug("loss: %.1f" % loss.asscalar())
self.loss = loss
self.assign(out_data[0], req[0], loss)
forward_stop = time.time()
logging.debug("forward_consume: %.1fms\n" %
((forward_stop - forward_start) * 1000))
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
#print('get_rpn_batch')
data, label = get_rpn_batch(iroidb, self.use_data_augmentation)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for data, data_pad in zip(data_list, data_tensor):
data['data'] = data_pad[np.newaxis, :]
new_label_list = []
#print(label_list)
#print("label_list")
for data, label in zip(data_list, label_list):
# infer label shape
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
# assign anchor for label
#print(data['im_info'])
#print('data_shape')
#print(data_shape)
new_label_list.append(
self._assign_anchor(data_shape, label['gt_boxes'],
data['im_info']))
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
# print(len(new_label_list))
all_label = dict()
all_label['labelp2'] = new_label_list[0][0]['label']
all_label['labelp3'] = new_label_list[0][1]['label']
all_label['labelp4'] = new_label_list[0][2]['label']
all_label['labelp5'] = new_label_list[0][3]['label']
#for key in self.label_name:
all_label['bbox_targetp2'] = new_label_list[0][0]['bbox_target']
all_label['bbox_targetp3'] = new_label_list[0][1]['bbox_target']
all_label['bbox_targetp4'] = new_label_list[0][2]['bbox_target']
all_label['bbox_targetp5'] = new_label_list[0][3]['bbox_target']
#for key in self.label_name:
# pad = -1 if key == 'label' else 0
all_label['bbox_weightp2'] = new_label_list[0][0]['bbox_weight']
all_label['bbox_weightp3'] = new_label_list[0][1]['bbox_weight']
all_label['bbox_weightp4'] = new_label_list[0][2]['bbox_weight']
all_label['bbox_weightp5'] = new_label_list[0][3]['bbox_weight']
#for key in self.label_name:
# all_label[key] = [batch[key] for batch in new_label_list]
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
