def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slices
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get each device
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rcnn_batch(iroidb)
data_list.append(data)
label_list.append(label)
all_data = dict()
for key in data_list[0].keys():
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in label_list[0].keys():
all_label[key] = tensor_vstack([batch[key] for batch in label_list])
self.data = [mx.nd.array(all_data[name]) for name in self.data_name]
self.label = [mx.nd.array(all_label[name]) for name in self.label_name]
def decide_slices(self, data_shapes):
"""Decide the slices for each context according to the workload.
Parameters
----------
data_shapes : list
list of (name, shape) specifying the shapes for the input data or label.
"""
assert len(data_shapes) > 0
major_axis = [DataDesc.get_batch_axis(x.layout) for x in data_shapes]
for (name, shape), axis in zip(data_shapes, major_axis):
if axis == -1:
continue
batch_size = shape[axis]
if self.batch_size is not None:
assert batch_size == self.batch_size, ("all data must have the same batch size: "
+ ("batch_size = %d, but " % self.batch_size)
+ ("%s has shape %s" % (name, shape)))
else:
self.batch_size = batch_size
self.slices = _split_input_slice(self.batch_size, self.workload)
return major_axis
def _get_train_batch(self):
"""
utilize minibatch sampling, e.g. 2 images and 64 rois per image
:return: training batch (e.g. 128 samples)
"""
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
cur_from = self.cur
cur_to = cur_from + self.batch_size
if cur_to <= self.size:
roidb = [self.roidb[i] for i in range(cur_from, cur_to)]
else:
pad = cur_to - self.size
roidb = self.roidb[cur_from:] + self.roidb[:pad]
batch_list = []
for islice in slices:
num_im = islice.stop - islice.start
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
batch = minibatch.get_minibatch(iroidb, self.num_classes, self.ctx)
batch_list.append(batch)
all_batch = dict()
for key in batch_list[0].keys():
all_batch[key] = tensor_vstack([batch[key] for batch in batch_list])
return all_batch
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slices
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
im_array_list = []
levels_data_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
im_array, levels_data = get_fpn_maskrcnn_batch(iroidb)
im_array_list.append(im_array)
levels_data_list.append(levels_data)
all_data, all_label = self._make_data_and_labels(im_array_list, levels_data_list)
self.data = [mx.nd.array(all_data[name]) for name in self.data_name]
self.label = [mx.nd.array(all_label[name]) for name in self.label_name]
def get_batch_parallel(self):
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
segdb = [self.segdb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
multiprocess_results = []
for idx, islice in enumerate(slices):
isegdb = [segdb[i] for i in range(islice.start, islice.stop)]
multiprocess_results.append(self.pool.apply_async(parfetch, (self.config, self.crop_width, self.crop_height, isegdb)))
rst = [multiprocess_result.get() for multiprocess_result in multiprocess_results]
all_data = [_['data'] for _ in rst]
all_label = [_['label'] for _ in rst]
self.data = [[mx.nd.array(data[key]) for key in self.data_name] for data in all_data]
self.label = [[mx.nd.array(label[key]) for key in self.label_name] for label in all_label]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb, self.cfg)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for data, data_pad in zip(data_list, data_tensor):
data['data'] = data_pad[np.newaxis, :]
new_label_list = []
for data, label in zip(data_list, label_list):
# infer label shape
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
# assign anchor for label
label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'], self.cfg,
self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border,
self.normalize_target, self.bbox_mean, self.bbox_std)
new_label_list.append(label)
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = -1 if key == 'label' else 0
all_label[key] = tensor_vstack([batch[key] for batch in new_label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self):
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
if self.mode == 'test':
self.data, self.label = minibatch.get_minibatch(roidb, self.num_classes, self.mode)
else:
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = minibatch.get_minibatch(iroidb, self.num_classes, self.mode)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for data, data_pad in zip(data_list, data_tensor):
data['data'] = data_pad[np.newaxis, :]
new_label_list = []
for data, label in zip(data_list, label_list):
# infer label shape
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# assign anchor for label
label = minibatch.assign_anchor(feat_shape, label['gt_boxes'], data['im_info'],
self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border)
del data['im_info']
new_label_list.append(label)
all_data = dict()
for key in ['data']:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
all_label['label'] = tensor_vstack([batch['label'] for batch in new_label_list], pad=-1)
for key in ['bbox_target', 'bbox_inside_weight', 'bbox_outside_weight']:
all_label[key] = tensor_vstack([batch[key] for batch in new_label_list])
self.data = [mx.nd.array(all_data['data'])]
self.label = [mx.nd.array(all_label['label']),
mx.nd.array(all_label['bbox_target']),
mx.nd.array(all_label['bbox_inside_weight']),
mx.nd.array(all_label['bbox_outside_weight'])]
def get_batch_individual(self):
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
rst = []
for idx, islice in enumerate(slices):
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
rst.append(self.parfetch(iroidb))
all_data = [_['data'] for _ in rst]
all_label = [_['label'] for _ in rst]
self.data = [[mx.nd.array(data[key]) for key in self.data_name] for data in all_data]
self.label = [[mx.nd.array(label[key]) for key in self.label_name] for label in all_label]
def get_batch(self):
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
if self.mode == 'test':
self.data, self.label = minibatch.get_minibatch(roidb, self.num_classes, self.mode)
else:
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = minibatch.get_minibatch(iroidb, self.num_classes, self.mode)
data_list.append(data)
label_list.append(label)
all_data = dict()
for key in data_list[0].keys():
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in label_list[0].keys():
all_label[key] = tensor_vstack([batch[key] for batch in label_list])
self.data = [mx.nd.array(all_data['data']),
mx.nd.array(all_data['rois'])]
self.label = [mx.nd.array(all_label['label']),
mx.nd.array(all_label['bbox_target']),
mx.nd.array(all_label['bbox_inside_weight']),
mx.nd.array(all_label['bbox_outside_weight'])]
def get_batch_parallel(self):
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
pairdb = [self.pairdb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
multiprocess_results = []
for idx, islice in enumerate(slices):
"""
ipairdb = [pairdb[i] for i in range(islice.start, islice.stop)]
multiprocess_results.append(self.pool.apply_async(
get_data_pair_train_batch, (ipairdb, self.config)))
"""
for i in range(islice.start, islice.stop):
multiprocess_results.append(
self.pool.apply_async(get_data_pair_train_batch,
([pairdb[i]], self.config)))
if False:
temp = get_data_pair_train_batch([pairdb[islice.start]],
self.config) # for debug
print('**' * 20)
print(pairdb[0]['image_observed'])
print('data:')
for k in temp['data'].keys():
print("\t{}, {}".format(k, temp['data'][k].shape))
print('label:')
for k in temp['label'].keys():
print("\t{}, {}".format(k, temp['label'][k].shape))
print(temp['label']['rot'])
print(temp['label']['trans'])
from lib.pair_matching.RT_transform import calc_rt_dist_m
r_dist, t_dist = calc_rt_dist_m(temp['data']['src_pose'][0],
temp['data']['tgt_pose'][0])
print("{}: R_dist: {}, T_dist: {}".format(self.cur, r_dist,
t_dist))
print('**' * 20)
image_real = (
temp['data']['image_observed'][0].transpose([1, 2, 0]) +
128).astype(np.uint8)
print(np.max(image_real))
print(np.min(image_real))
image_rendered = (
temp['data']['image_rendered'][0].transpose([1, 2, 0]) +
128).astype(np.uint8)
mask_real_gt = np.squeeze(temp['label']['mask_gt_observed'])
mask_real_est = np.squeeze(temp['data']['mask_observed'])
mask_rendered = np.squeeze(temp['data']['mask_rendered'])
if 'flow' in temp['label']:
print('in loader, flow: ', temp['label']['flow'].shape,
np.unique(temp['label']['flow']))
print('in loader, flow weights: ',
temp['label']['flow_weights'].shape,
np.unique(temp['label']['flow_weights']))
import matplotlib.pyplot as plt
plt.subplot(2, 3, 1)
plt.imshow(mask_real_est)
plt.subplot(2, 3, 2)
plt.imshow(mask_real_gt)
plt.subplot(2, 3, 3)
plt.imshow(mask_rendered)
plt.subplot(2, 3, 4)
plt.imshow(image_real)
plt.subplot(2, 3, 5)
plt.imshow(image_rendered)
plt.show()
# plt.savefig('image_real_rendered_{}'.format(self.cur), aspect='normal')
rst = [
multiprocess_result.get()
for multiprocess_result in multiprocess_results
]
batch_per_gpu = int(self.batch_size / len(ctx))
data_list = []
label_list = []
for i in range(len(ctx)):
sample_data_list = [_['data'] for _ in rst]
sample_label_list = [_['label'] for _ in rst]
batch_data = {}
batch_label = {}
for key in sample_data_list[0]:
batch_data[key] = \
my_tensor_vstack([sample_data_list[j][key] for j in range(i*batch_per_gpu, (i+1)*batch_per_gpu)])
for key in sample_label_list[0]:
batch_label[key] = \
my_tensor_vstack([sample_label_list[j][key] for j in range(i*batch_per_gpu, (i+1)*batch_per_gpu)])
data_list.append(batch_data)
label_list.append(batch_label)
"""
data_list = [_['data'] for _ in rst]
label_list = [_['label'] for _ in rst]
"""
self.data = [[mx.nd.array(data_on_i[key]) for key in self.data_name]
for data_on_i in data_list]
self.label = [[
mx.nd.array(label_on_i[key]) for key in self.label_name
] for label_on_i in label_list]
def get_batch(self):
# slice roidb
#t = time.time()
debug = True
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(
iroidb,
self.config.SCALES,
self.config.input_mean,
stride=self.config.IMAGE_STRIDE
) #data:image, label:gt_boxes[n,5]:box+label
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for data, data_pad in zip(data_list, data_tensor):
data['data'] = data_pad[np.newaxis, :]
new_label_list = []
for data, label in zip(data_list, label_list): #image and gt_bbox
# infer label shape
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
feat_shapes = []
#print '###################data_shape:',data_shape
for feat_sym in self.feat_sym:
_, feat_shape, _ = feat_sym.infer_shape(
**data_shape
) #get size of feature map for rpn, there are 4 feat_sym in fpn
feat_shape = [int(i) for i in feat_shape[0]]
feat_shapes.append(feat_shape)
#print feat_shapes
gt_boxes = copy.deepcopy(label['gt_boxes'])
# assign anchor for label
#print 'image_info: ', data['im_info']
#t = time.time()
feat_shape_ = np.array(feat_shapes[0])
#print len(feat_shapes)
for i in range(1, len(feat_shapes)):
a = np.array(feat_shapes[i])
feat_shape_ = np.vstack((feat_shape_, a))
#print feat_shape_.shape
data['feat_shape'] = feat_shape_[np.newaxis, :, :]
#t_1 = time.time()
label = fpn_assign_anchor(feat_shapes, label['gt_boxes'],
data['im_info'], self.config,
self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border)
if gt_boxes.shape[0] == 0:
gt_boxes = -1 * np.ones((1, 5), dtype=np.float32)
data['gt_boxes'] = gt_boxes[np.newaxis, :, :] #[1,n,5]
new_label_list.append(label)
all_data = dict()
for key in self.data_name: #['data', 'im_info', 'gt_boxes']
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
#all_data: data:[4,H,W,C], im_info:[4,1,3], gt_boxes:[4,n,5]
all_label = dict()
for key in self.label_name: #['label', 'bbox_target', 'bbox_weight']
pad = -1 if (key == 'label1' or key == 'label2' or key == 'label3'
or key == 'label4' or key == 'label5') else 0
all_label[key] = tensor_vstack(
[batch[key] for batch in new_label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
assert cur_to==cur_from+self.batch_size
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_crop_batch(iroidb)
data_list += data
label_list += label
#data_list.append(data)
#label_list.append(label)
# pad data first and then assign anchor (read label)
#data_tensor = tensor_vstack([batch['data'] for batch in data_list])
#for i_card in range(len(data_list)):
# data_list[i_card]['data'] = data_tensor[
# i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES]
#iiddxx = 0
select_stride = 0
if config.RANDOM_FEAT_STRIDE:
select_stride = random.choice(config.RPN_FEAT_STRIDE)
for data, label in zip(data_list, label_list):
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
feat_shape_list = []
for s in range(len(self.feat_stride)):
_, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
feat_shape_list.append(feat_shape)
im_info = data['im_info']
gt_boxes = label['gt_boxes']
gt_label = {'gt_boxes':gt_boxes}
if config.USE_BLUR:
gt_blur = label['gt_blur']
gt_label['gt_blur'] = gt_blur
if self._debug:
img = data['data'].copy()[0].transpose( (1,2,0) )[:,:,::-1].copy()
print('DEBUG SHAPE', data['data'].shape, label['gt_boxes'].shape)
box = label['gt_boxes'].copy()[0][0:4].astype(np.int)
cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), (0, 255, 0), 2)
filename = './debugout/%d.png' % (self._debug_id)
print('debug write', filename)
cv2.imwrite(filename, img)
self._debug_id+=1
#print('DEBUG', img.shape, bbox.shape)
label_dict = {}
if config.HEAD_BOX:
head_label_dict = self.aa.assign_anchor_fpn(gt_label, im_info, False, prefix='head', select_stride = select_stride)
label_dict.update(head_label_dict)
if config.FACE_LANDMARK:
gt_landmarks = label['gt_landmarks']
gt_label['gt_landmarks'] = gt_landmarks
#ta = datetime.datetime.now()
#face_label_dict = assign_anchor_fpn(feat_shape_list, gt_label, im_info, config.FACE_LANDMARK, prefix='face', select_stride = select_stride)
face_label_dict = self.aa.assign_anchor_fpn(gt_label, im_info, config.FACE_LANDMARK, prefix='face', select_stride = select_stride)
#tb = datetime.datetime.now()
#self._times[0] += (tb-ta).total_seconds()
label_dict.update(face_label_dict)
#for k in label_dict:
# print(k, label_dict[k].shape)
if config.CASCADE>0:
pad_gt_boxes = np.empty( (1, config.TRAIN.MAX_BBOX_PER_IMAGE, 5), dtype=np.float32)
pad_gt_boxes.fill(-1)
pad_gt_boxes[0, 0:gt_boxes.shape[0],:] = gt_boxes
label_dict['gt_boxes'] = pad_gt_boxes
#print('im_info', im_info.shape)
#print(gt_boxes.shape)
for k in self.label_name:
label[k] = label_dict[k]
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = 0 if key.startswith('bbox_') else -1
#print('label vstack', key, pad, len(label_list), file=sys.stderr)
all_label[key] = tensor_vstack([batch[key] for batch in label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for i_card in range(len(data_list)):
data_list[i_card]['data'] = data_tensor[
i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES]
for data, label in zip(data_list, label_list):
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
feat_shape_list = []
for s in range(len(self.feat_stride)):
_, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
feat_shape_list.append(feat_shape)
label['label'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
label['bbox_target'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
label['bbox_weight'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
for im_i in range(config.TRAIN.BATCH_IMAGES):
im_info = data['im_info'][im_i]
gt_boxes = label['gt_boxes'][im_i][0]
label_dict = \
assign_anchor_fpn(feat_shape_list, gt_boxes, im_info,
self.feat_stride,
self.anchor_scales,
self.anchor_ratios,
self.allowed_border)
label['label'][im_i] = label_dict['label']
label['bbox_target'][im_i] = label_dict['bbox_target']
label['bbox_weight'][im_i] = label_dict['bbox_weight']
label['label'] = np.vstack(label['label'])
label['bbox_target'] = np.vstack(label['bbox_target'])
label['bbox_weight'] = np.vstack(label['bbox_weight'])
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = 0 if key == 'weight' else -1
all_label[key] = tensor_vstack([batch[key] for batch in label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for i_card in range(len(data_list)):
data_list[i_card]['data'] = data_tensor[i_card *
config.TRAIN.BATCH_IMAGES:
(1 + i_card) *
config.TRAIN.BATCH_IMAGES]
for data, label in zip(data_list, label_list):
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
feat_shape_list = []
for s in range(len(self.feat_stride)):
_, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
feat_shape_list.append(feat_shape)
for k in self.label_name:
label[k] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
for im_i in range(config.TRAIN.BATCH_IMAGES):
#im_info = data['im_info'][im_i]
#gt_boxes = label['gt_boxes'][im_i][0]
im_info = data['im_info']
gt_boxes = label['gt_boxes']
gt_landmarks = label['gt_landmarks']
#print('im_info', im_info.shape)
#print(gt_boxes.shape)
vis = True if self.visid >= 0 else False
label_dict = assign_anchor_fpn(feat_shape_list, gt_boxes,
gt_landmarks, im_info, vis)
#do visualize for debug here
if self.visid >= 0 and self.visid < 10:
for _roi in roidb:
print('image', _roi['image'], file=sys.stderr)
self.visid += 1
anchors = label_dict['anchors'].copy()
_im = data['data'].copy()
#print(label_dict['olabel'].shape)
_label = label_dict['olabel'].copy()
_gt_boxes = gt_boxes.copy().astype(np.int)
filename = './vis/A%d.png' % self.visid
_im = _im[0].transpose((1, 2, 0))
_im = _im[..., ::-1] #bgr
for c in range(3):
_im[:, :, c] += config.PIXEL_MEANS[c]
_im = _im.astype(np.uint8).copy()
fg_inds = np.where(_label == 1)[0]
print(_im.shape,
_label.shape,
anchors.shape,
len(fg_inds),
_gt_boxes.shape,
file=sys.stderr)
#draw FG anchors
_bc = 0
for a in range(anchors.shape[0]):
anchor = anchors[a].astype(np.int)
l = _label[a]
if l != 1:
continue
#print('drawing', _im.shape, anchor)
cv2.rectangle(_im, (anchor[0], anchor[1]),
(anchor[2], anchor[3]), (255, 0, 0), 1)
_bc += 1
for a in range(_gt_boxes.shape[0]):
_box = _gt_boxes[a]
cv2.rectangle(_im, (_box[0], _box[1]),
(_box[2], _box[3]), (0, 0, 255), 1)
print('draw to', filename, _bc, file=sys.stderr)
cv2.imwrite(filename, _im)
for k in self.label_name:
#print('0in_loader', k, label_dict[k].shape, file=sys.stderr)
label[k][im_i] = label_dict[k]
for k in self.label_name:
label[k] = np.vstack(label[k])
#print('in_loader', k, label[k].shape, file=sys.stderr)
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = 0 if key.startswith('bbox_') else -1
#print('label vstack', key, pad, len(label_list), file=sys.stderr)
all_label[key] = tensor_vstack(
[batch[key] for batch in label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def fit(self, X, y=None, eval_data=None, eval_metric='acc',
epoch_end_callback=None, batch_end_callback=None, kvstore='local', logger=None,
work_load_list=None, monitor=None, eval_batch_end_callback=None):
train_data = self._init_iter(X, y, is_train=True)
eval_data = self._init_eval_iter(eval_data)
# prepare sym
if self.sym_gen:
self.symbol = self.sym_gen(train_data.default_bucket_key) # pylint: disable=no-member
self._check_arguments()
self.kwargs["sym"] = self.symbol
# prepare args name and shape
arg_names, param_names, aux_names = \
self._init_params(dict(train_data.provide_data+train_data.provide_label))
# setup metric
if not isinstance(eval_metric, metric.EvalMetric):
eval_metric = metric.create(eval_metric)
# create kvstore
(kvstore, update_on_kvstore) = _create_kvstore(
kvstore, len(self.ctx), self.arg_params)
# param_idx2name = {}
# if update_on_kvstore:
# param_idx2name.update(enumerate(param_names))
# else:
# for i, n in enumerate(param_names):
# for k in range(len(self.ctx)):
# param_idx2name[i*len(self.ctx)+k] = n
# self.kwargs["param_idx2name"] = param_idx2name
# init optmizer
if isinstance(self.optimizer, str):
batch_size = train_data.batch_size
if kvstore and kvstore.type == 'dist_sync':
batch_size *= kvstore.num_workers
optimizer = opt.create(self.optimizer,
rescale_grad=(1.0/batch_size),
**(self.kwargs))
elif isinstance(self.optimizer, opt.Optimizer):
optimizer = self.optimizer
num_device = len(self.ctx)
logger.info('Start training with %s', str(self.ctx))
if work_load_list is None:
work_load_list = [1] * num_device
assert isinstance(work_load_list, list) and len(work_load_list) == num_device, \
"Invalid settings for work load. "
from mxnet.executor_manager import _split_input_slice
self.slices = _split_input_slice(train_data.batch_size, work_load_list)
execgrp = ExecuterGroup(self.symbol, arg_names, param_names, self.ctx,
self.slices, train_data)
if self.sym_gen is not None:
self.execgrp_bucket = {train_data.default_bucket_key: self.execgrp}
# executor_manager.set_params(arg_params, aux_params)
train_data.reset()
for epoch in range(self.begin_epoch, self.num_epoch):
# Training phase
tic = time.time()
eval_metric.reset()
nbatch = 0
# Iterate over training data.
#Into Epoch
#########################
#record acc
acc_hist = []
while True:
do_reset = True
for data_batch in train_data:
executor_manager.load_data_batch(data_batch)
if monitor is not None:
monitor.tic()
executor_manager.forward(is_train=True)
executor_manager.backward()
if update_on_kvstore:
_update_params_on_kvstore(executor_manager.param_arrays,
executor_manager.grad_arrays,
kvstore)
else:
_update_params(executor_manager.param_arrays,
executor_manager.grad_arrays,
updater=updater,
num_device=len(ctx),
kvstore=kvstore)
if monitor is not None:
monitor.toc_print()
eval_metric.reset()
executor_manager.update_metric(eval_metric, data_batch.label)
name_value = eval_metric.get_name_value()
for name, value in name_value:
acc_hist.append(value)
# logger.info('Epoch[%d] Training-%s=%f', epoch, name, value)
nbatch += 1
# batch callback (for print purpose)
if batch_end_callback != None:
batch_end_params = BatchEndParam(epoch=epoch,
nbatch=nbatch,
eval_metric=eval_metric,
locals=locals())
if isinstance(batch_end_callback, list):
for call in batch_end_callback:
call(batch_end_params)
else:
batch_end_callback(batch_end_params)
# this epoch is done possibly earlier
if epoch_size is not None and nbatch >= epoch_size:
do_reset = False
break
if do_reset is True:
logger.debug('Epoch[%d] Resetting Data Iterator', epoch)
train_data.reset()
logger.debug('Epoch[%d] Resetting Eval Metric', epoch)
eval_metric.reset()
# this epoch is done
if epoch_size is None or nbatch >= epoch_size:
break
toc = time.time()
logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc - tic))
# print epoch ,'<<<<<<<<<<<<'
if epoch_end_callback or epoch + 1 == end_epoch:
executor_manager.copy_to(arg_params, aux_params)
if epoch_end_callback != None:
if isinstance(epoch_end_callback, list):
for call in epoch_end_callback:
call(epoch, symbol, arg_params, aux_params,
acc_hist)
else:
epoch_end_callback(epoch, symbol, arg_params, aux_params,
acc_hist)
# evaluation
# print 'enter evaluation'
if eval_data:
eval_metric.reset()
eval_data.reset()
for i, eval_batch in enumerate(eval_data):
executor_manager.load_data_batch(eval_batch)
executor_manager.forward(is_train=False)
executor_manager.update_metric(eval_metric, eval_batch.label)
if eval_batch_end_callback != None:
batch_end_params = BatchEndParam(epoch=epoch,
nbatch=i,
eval_metric=eval_metric,
locals=locals())
if isinstance(eval_batch_end_callback, list):
for call in eval_batch_end_callback:
call(batch_end_params)
else:
eval_batch_end_callback(batch_end_params)
name_value = eval_metric.get_name_value()
for name, value in name_value:
logger.info('E[%d] V %s:%f', epoch, name, value)
# print 'Epoch[%d] Validation=%f' % (epoch, value)
# end of all epochs
return
def get_batch(self):
data_list = []
label_list = []
if self.nThreads == 0:
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [
self.roidb[self.index[i]] for i in range(cur_from, cur_to)
]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb)
data_list.append(data)
label_list.append(label)
else:
for p in range(len(self.ctx)):
data, label = self.multithread_get_rpn_batch()
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list
]) # to unify the shape of data
k = 0
for data in data_list:
bs = data['data'].shape[0]
data['data'] = data_tensor[k:k + bs, :]
k += bs
# re-organize images and labels for 3DCE
new_label_list = []
num_smp = config.TRAIN.SAMPLES_PER_BATCH
num_image = config.NUM_IMAGES_3DCE
num_slices = config.NUM_SLICES
for data, label1 in zip(data_list, label_list):
data0 = data['data']
im_info0 = data['im_info']
gt0 = label1['gt_boxes']
data_new = np.empty((num_smp * num_image, num_slices,
data0.shape[2], data0.shape[3]))
iminfo_new = np.empty((num_smp * num_image, 3))
gt_new = np.zeros((num_smp * num_image, gt0.shape[1], 5))
for p in range(num_smp):
for q in range(num_image):
data_new[p * num_image +
q, :, :, :] = data0[p, q * num_slices:(q + 1) *
num_slices, :, :]
iminfo_new[p * num_image + q, :] = im_info0[p, :]
if q == (num_image - 1) / 2:
gt_new[p * num_image + q, :, :] = gt0[p, :, :]
data['data'] = data_new
data['im_info'] = iminfo_new
label1['gt_boxes'] = gt_new
# infer label shape
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label1['gt_boxes'] #[np.newaxis, :, :]
# assign anchor for label
for i in range(len(label1['gt_boxes'])):
label = assign_anchor(feat_shape, label1['gt_boxes'][i],
data['im_info'][i:i + 1, :],
self.feat_stride, self.anchor_scales,
self.anchor_ratios, self.allowed_border)
if i % num_image != (num_image - 1) / 2:
label['label'][:] = -1
label['bbox_weight'][:] = 0
new_label_list.append(label)
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = -1 if key == 'label' else 0
all_label[key] = tensor_vstack(
[batch[key] for batch in new_label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for i_card in range(len(data_list)):
data_list[i_card]['data'] = data_tensor[
i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES]
for data, label in zip(data_list, label_list):
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
feat_shape_list = []
for s in range(len(self.feat_stride)):
_, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
feat_shape_list.append(feat_shape)
label['label'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
label['bbox_target'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
label['bbox_weight'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
for im_i in range(config.TRAIN.BATCH_IMAGES):
im_info = data['im_info'][im_i]
gt_boxes = label['gt_boxes'][im_i][0]
label_dict = \
assign_anchor_fpn(feat_shape_list, gt_boxes, im_info,
self.feat_stride,
self.anchor_scales,
self.anchor_ratios,
self.allowed_border)
label['label'][im_i] = label_dict['label']
label['bbox_target'][im_i] = label_dict['bbox_target']
label['bbox_weight'][im_i] = label_dict['bbox_weight']
label['label'] = np.vstack(label['label'])
label['bbox_target'] = np.vstack(label['bbox_target'])
label['bbox_weight'] = np.vstack(label['bbox_weight'])
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = 0 if key == 'weight' else -1
all_label[key] = tensor_vstack([batch[key] for batch in label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self):
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
if self.mode == 'test':
self.data, self.label = minibatch.get_minibatch(
roidb, self.num_classes, self.mode)
else:
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = minibatch.get_minibatch(iroidb, self.num_classes,
self.mode)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for data, data_pad in zip(data_list, data_tensor):
data['data'] = data_pad[np.newaxis, :]
new_label_list = []
for data, label in zip(data_list, label_list):
# infer label shape
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
# assign anchor for label
label = minibatch.assign_anchor(feat_shape, label['gt_boxes'],
data['im_info'],
self.feat_stride,
self.anchor_scales,
self.anchor_ratios,
self.allowed_border)
del data['im_info']
new_label_list.append(label)
all_data = dict()
for key in ['data']:
all_data[key] = tensor_vstack(
[batch[key] for batch in data_list])
all_label = dict()
all_label['label'] = tensor_vstack(
[batch['label'] for batch in new_label_list], pad=-1)
for key in [
'bbox_target', 'bbox_inside_weight', 'bbox_outside_weight'
]:
all_label[key] = tensor_vstack(
[batch[key] for batch in new_label_list])
self.data = [mx.nd.array(all_data['data'])]
self.label = [
mx.nd.array(all_label['label']),
mx.nd.array(all_label['bbox_target']),
mx.nd.array(all_label['bbox_inside_weight']),
mx.nd.array(all_label['bbox_outside_weight'])
]
def get_batch_parallel(self):
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
segdb = [self.segdb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# debug = []
# for idx, islice in enumerate(slices):
# isegdb = [segdb[i] for i in range(islice.start, islice.stop) if i < len(segdb)]
# resuelts = parfetch(self.config, self.crop_width, self.crop_width, isegdb)
# debug.append(resuelts)
# import pdb; pdb.set_trace()
multiprocess_results = []
# import pdb; pdb.set_trace()
for idx, islice in enumerate(slices):
isegdb = [
segdb[i] for i in range(islice.start, islice.stop)
if i < len(segdb)
]
# parfetch(config, crop_width, crop_height, isegdb)
# import pdb; pdb.set_trace()
# try:
# import pdb; pdb.set_trace()
# print("Here:",isegdb[0]['image'])
# print("Here:",isegdb[0]['seg_cls_path'])
try:
multiprocess_results.append(
self.pool.apply_async(parfetch,
(self.config, self.crop_width,
self.crop_height, isegdb)))
except:
print("add to pool error")
import pdb
pdb.set_trace()
# print(isegdb)
# multiprocess_results.append(
# parfetch(self.config, self.crop_width, self.crop_height, isegdb))
# import pdb; pdb.set_trace()
# rst = []
# for idx, item in enumerate(multiprocess_results):
# # import pdb; pdb.set_trace()
# try:
# rst.append(item.get())
# except:
# print("can't get data")
# import pdb; pdb.set_trace()
rst = [
multiprocess_result.get()
for multiprocess_result in multiprocess_results
]
all_data = [_['data'] for _ in rst]
all_label = [_['label'] for _ in rst]
self.data = [[mx.nd.array(data[key]) for key in self.data_name]
for data in all_data]
self.label = [[mx.nd.array(label[key]) for key in self.label_name]
for label in all_label]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
assert cur_to == cur_from + self.batch_size
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_crop_batch(iroidb)
data_list += data
label_list += label
#data_list.append(data)
#label_list.append(label)
# pad data first and then assign anchor (read label)
#data_tensor = tensor_vstack([batch['data'] for batch in data_list])
#for i_card in range(len(data_list)):
# data_list[i_card]['data'] = data_tensor[
# i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES]
for data, label in zip(data_list, label_list):
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
feat_shape_list = []
for s in range(len(self.feat_stride)):
_, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
feat_shape = [int(i) for i in feat_shape[0]]
feat_shape_list.append(feat_shape)
#for k in self.label_name:
# label[k] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
im_info = data['im_info']
gt_boxes = label['gt_boxes']
gt_landmarks = label['gt_landmarks']
#print('im_info', im_info.shape)
#print(gt_boxes.shape)
label_dict = assign_anchor_fpn(feat_shape_list, gt_boxes,
gt_landmarks, im_info)
for k in self.label_name:
label[k] = label_dict[k]
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = 0 if key.startswith('bbox_') else -1
#print('label vstack', key, pad, len(label_list), file=sys.stderr)
all_label[key] = tensor_vstack(
[batch[key] for batch in label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
data, label = get_rpn_batch(iroidb, self.cfg)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for data, data_pad in zip(data_list, data_tensor):
data['data'] = data_pad[np.newaxis, :]
new_label_list = []
for data, label in zip(data_list, label_list):
# infer label shape
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
_, feat_shape_p3, _ = self.feat_sym_p3.infer_shape(**data_shape)
feat_shape_p3 = [int(i) for i in feat_shape_p3[0]]
_, feat_shape_p4, _ = self.feat_sym_p4.infer_shape(**data_shape)
feat_shape_p4 = [int(i) for i in feat_shape_p4[0]]
_, feat_shape_p5, _ = self.feat_sym_p5.infer_shape(**data_shape)
feat_shape_p5 = [int(i) for i in feat_shape_p5[0]]
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
# assign anchor for label
label = assign_anchor(feat_shape_p3, feat_shape_p4, feat_shape_p5,
label['gt_boxes'], data['im_info'], self.cfg,
self.feat_stride_p3, self.anchor_scales_p3,
self.anchor_ratios_p3, self.feat_stride_p4,
self.anchor_scales_p4, self.anchor_ratios_p4,
self.feat_stride_p5, self.anchor_scales_p5,
self.anchor_ratios_p5, self.allowed_border)
new_label_list.append(label)
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
all_label = dict()
for key in self.label_name:
pad = -1 if key == 'label' else 0
all_label[key] = tensor_vstack(
[batch[key] for batch in new_label_list], pad=pad)
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_minibatch(roidb, num_classes, ctx, work_load_list=None):
"""
return minibatch of images in roidb
:param roidb: subset of main database
:param num_classes: number of classes is used in bbox regression targets
:return: minibatch: {'data', 'rois', 'labels', 'bbox_targets', 'bbox_inside_weights', 'bbox_outside_weights'}
"""
num_images = len(roidb)
random_scale_indexes = npr.randint(0, high=len(config.TRAIN.SCALES), size=num_images)
assert config.TRAIN.BATCH_SIZE % num_images == 0, \
'num_images {} must devide BATCHSIZE {}'.format(num_images, config.TRAIN.BATCH_SIZE)
rois_per_image = config.TRAIN.BATCH_SIZE / num_images
fg_rois_per_image = np.round(config.TRAIN.FG_FRACTION * rois_per_image).astype(int)
# im_array: [num_images, c, h, w]
im_array, im_scales = get_image_array(roidb, config.TRAIN.SCALES, random_scale_indexes)
rois_array = list()
labels_array = list()
bbox_targets_array = list()
bbox_inside_array = list()
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(num_images, work_load_list)
idx_in_slice = []
for islice in slices:
num_im = islice.stop - islice.start
for i in range(num_im):
idx_in_slice.append(i)
for im_i, idx in enumerate(idx_in_slice):
im_rois, labels, bbox_targets, bbox_inside_weights, overlaps = \
sample_rois(roidb[im_i], fg_rois_per_image, rois_per_image, num_classes)
# project im_rois
# do not round roi
rois = im_rois * im_scales[im_i]
batch_index = idx * np.ones((rois.shape[0], 1))
rois_array_this_image = np.hstack((batch_index, rois))
rois_array.append(rois_array_this_image)
# add labels
labels_array.append(labels)
bbox_targets_array.append(bbox_targets)
bbox_inside_array.append(bbox_inside_weights)
rois_array = np.vstack(rois_array)
labels_array = np.hstack(labels_array)
bbox_targets_array = np.vstack(bbox_targets_array)
bbox_inside_array = np.vstack(bbox_inside_array)
bbox_outside_array = np.array(bbox_inside_array > 0).astype(np.float32)
minibatch = {'data': im_array,
'rois': rois_array,
'labels': labels_array,
'bbox_targets': bbox_targets_array,
'bbox_inside_weights': bbox_inside_array,
'bbox_outside_weights': bbox_outside_array}
return minibatch
def get_batch(self):
# slice roidb
cur_from = self.cur
cur_to = min(cur_from + self.batch_size, self.size)
roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
# decide multi device slice
work_load_list = self.work_load_list
ctx = self.ctx
if work_load_list is None:
work_load_list = [1] * len(ctx)
assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
"Invalid settings for work load. "
slices = _split_input_slice(self.batch_size, work_load_list)
# get testing data for multigpu
data_list = []
label_list = []
for islice in slices:
iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
#print('get_rpn_batch')
data, label = get_rpn_batch(iroidb, self.use_data_augmentation)
data_list.append(data)
label_list.append(label)
# pad data first and then assign anchor (read label)
data_tensor = tensor_vstack([batch['data'] for batch in data_list])
for data, data_pad in zip(data_list, data_tensor):
data['data'] = data_pad[np.newaxis, :]
new_label_list = []
#print(label_list)
#print("label_list")
for data, label in zip(data_list, label_list):
# infer label shape
data_shape = {k: v.shape for k, v in data.items()}
del data_shape['im_info']
# assign anchor for label
#print(data['im_info'])
#print('data_shape')
#print(data_shape)
new_label_list.append(
self._assign_anchor(data_shape, label['gt_boxes'],
data['im_info']))
# add gt_boxes to data for e2e
data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]
all_data = dict()
for key in self.data_name:
all_data[key] = tensor_vstack([batch[key] for batch in data_list])
# print(len(new_label_list))
all_label = dict()
all_label['labelp2'] = new_label_list[0][0]['label']
all_label['labelp3'] = new_label_list[0][1]['label']
all_label['labelp4'] = new_label_list[0][2]['label']
all_label['labelp5'] = new_label_list[0][3]['label']
#for key in self.label_name:
all_label['bbox_targetp2'] = new_label_list[0][0]['bbox_target']
all_label['bbox_targetp3'] = new_label_list[0][1]['bbox_target']
all_label['bbox_targetp4'] = new_label_list[0][2]['bbox_target']
all_label['bbox_targetp5'] = new_label_list[0][3]['bbox_target']
#for key in self.label_name:
# pad = -1 if key == 'label' else 0
all_label['bbox_weightp2'] = new_label_list[0][0]['bbox_weight']
all_label['bbox_weightp3'] = new_label_list[0][1]['bbox_weight']
all_label['bbox_weightp4'] = new_label_list[0][2]['bbox_weight']
all_label['bbox_weightp5'] = new_label_list[0][3]['bbox_weight']
#for key in self.label_name:
# all_label[key] = [batch[key] for batch in new_label_list]
self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def fit(self,
X,
y=None,
eval_data=None,
eval_metric='acc',
epoch_end_callback=None,
batch_end_callback=None,
kvstore='local',
logger=None,
work_load_list=None,
monitor=None,
eval_batch_end_callback=None):
train_data = self._init_iter(X, y, is_train=True)
eval_data = self._init_eval_iter(eval_data)
# prepare sym
if self.sym_gen:
self.symbol = self.sym_gen(train_data.default_bucket_key) # pylint: disable=no-member
self._check_arguments()
self.kwargs["sym"] = self.symbol
# prepare args name and shape
arg_names, param_names, aux_names = \
self._init_params(dict(train_data.provide_data+train_data.provide_label))
# setup metric
if not isinstance(eval_metric, metric.EvalMetric):
eval_metric = metric.create(eval_metric)
# create kvstore
(kvstore, update_on_kvstore) = _create_kvstore(kvstore, len(self.ctx),
self.arg_params)
# param_idx2name = {}
# if update_on_kvstore:
# param_idx2name.update(enumerate(param_names))
# else:
# for i, n in enumerate(param_names):
# for k in range(len(self.ctx)):
# param_idx2name[i*len(self.ctx)+k] = n
# self.kwargs["param_idx2name"] = param_idx2name
# init optmizer
if isinstance(self.optimizer, str):
batch_size = train_data.batch_size
if kvstore and kvstore.type == 'dist_sync':
batch_size *= kvstore.num_workers
optimizer = opt.create(self.optimizer,
rescale_grad=(1.0 / batch_size),
**(self.kwargs))
elif isinstance(self.optimizer, opt.Optimizer):
optimizer = self.optimizer
num_device = len(self.ctx)
logger.info('Start training with %s', str(self.ctx))
if work_load_list is None:
work_load_list = [1] * num_device
assert isinstance(work_load_list, list) and len(work_load_list) == num_device, \
"Invalid settings for work load. "
from mxnet.executor_manager import _split_input_slice
self.slices = _split_input_slice(train_data.batch_size, work_load_list)
execgrp = ExecuterGroup(self.symbol, arg_names, param_names, self.ctx,
self.slices, train_data)
if self.sym_gen is not None:
self.execgrp_bucket = {train_data.default_bucket_key: self.execgrp}
# executor_manager.set_params(arg_params, aux_params)
train_data.reset()
for epoch in range(self.begin_epoch, self.num_epoch):
# Training phase
tic = time.time()
eval_metric.reset()
nbatch = 0
# Iterate over training data.
#Into Epoch
#########################
#record acc
acc_hist = []
while True:
do_reset = True
for data_batch in train_data:
executor_manager.load_data_batch(data_batch)
if monitor is not None:
monitor.tic()
executor_manager.forward(is_train=True)
executor_manager.backward()
if update_on_kvstore:
_update_params_on_kvstore(
executor_manager.param_arrays,
executor_manager.grad_arrays, kvstore)
else:
_update_params(executor_manager.param_arrays,
executor_manager.grad_arrays,
updater=updater,
num_device=len(ctx),
kvstore=kvstore)
if monitor is not None:
monitor.toc_print()
eval_metric.reset()
executor_manager.update_metric(eval_metric,
data_batch.label)
name_value = eval_metric.get_name_value()
for name, value in name_value:
acc_hist.append(value)
# logger.info('Epoch[%d] Training-%s=%f', epoch, name, value)
nbatch += 1
# batch callback (for print purpose)
if batch_end_callback != None:
batch_end_params = BatchEndParam(
epoch=epoch,
nbatch=nbatch,
eval_metric=eval_metric,
locals=locals())
if isinstance(batch_end_callback, list):
for call in batch_end_callback:
call(batch_end_params)
else:
batch_end_callback(batch_end_params)
# this epoch is done possibly earlier
if epoch_size is not None and nbatch >= epoch_size:
do_reset = False
break
if do_reset is True:
logger.debug('Epoch[%d] Resetting Data Iterator', epoch)
train_data.reset()
logger.debug('Epoch[%d] Resetting Eval Metric', epoch)
eval_metric.reset()
# this epoch is done
if epoch_size is None or nbatch >= epoch_size:
break
toc = time.time()
logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc - tic))
# print epoch ,'<<<<<<<<<<<<'
if epoch_end_callback or epoch + 1 == end_epoch:
executor_manager.copy_to(arg_params, aux_params)
if epoch_end_callback != None:
if isinstance(epoch_end_callback, list):
for call in epoch_end_callback:
call(epoch, symbol, arg_params, aux_params, acc_hist)
else:
epoch_end_callback(epoch, symbol, arg_params, aux_params,
acc_hist)
# evaluation
# print 'enter evaluation'
if eval_data:
eval_metric.reset()
eval_data.reset()
for i, eval_batch in enumerate(eval_data):
executor_manager.load_data_batch(eval_batch)
executor_manager.forward(is_train=False)
executor_manager.update_metric(eval_metric,
eval_batch.label)
if eval_batch_end_callback != None:
batch_end_params = BatchEndParam(
epoch=epoch,
nbatch=i,
eval_metric=eval_metric,
locals=locals())
if isinstance(eval_batch_end_callback, list):
for call in eval_batch_end_callback:
call(batch_end_params)
else:
eval_batch_end_callback(batch_end_params)
name_value = eval_metric.get_name_value()
for name, value in name_value:
logger.info('E[%d] V %s:%f', epoch, name, value)
# print 'Epoch[%d] Validation=%f' % (epoch, value)
# end of all epochs
return