def get_batch(self): # slice roidb cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slices work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) # get each device data_list = [] label_list = [] for islice in slices: iroidb = [roidb[i] for i in range(islice.start, islice.stop)] data, label = get_rcnn_batch(iroidb) data_list.append(data) label_list.append(label) all_data = dict() for key in data_list[0].keys(): all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in label_list[0].keys(): all_label[key] = tensor_vstack([batch[key] for batch in label_list]) self.data = [mx.nd.array(all_data[name]) for name in self.data_name] self.label = [mx.nd.array(all_label[name]) for name in self.label_name]
def get_batch(self): # slice roidb cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slices work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) # get each device data_list = [] label_list = [] for islice in slices: iroidb = [roidb[i] for i in range(islice.start, islice.stop)] data, label = get_rcnn_batch(iroidb) data_list.append(data) label_list.append(label) all_data = dict() for key in data_list[0].keys(): all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in label_list[0].keys(): all_label[key] = tensor_vstack( [batch[key] for batch in label_list]) self.data = [mx.nd.array(all_data[name]) for name in self.data_name] self.label = [mx.nd.array(all_label[name]) for name in self.label_name]
def get_batch(self): # slice roidb cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slice work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) # get testing data for multigpu data_list = [] label_list = [] for islice in slices: iroidb = [roidb[i] for i in range(islice.start, islice.stop)] data, label = get_rpn_batch(iroidb) data_list.append(data) label_list.append(label) # pad data first and then assign anchor (read label) data_tensor = tensor_vstack([batch['data'] for batch in data_list]) for data, data_pad in zip(data_list, data_tensor): data['data'] = data_pad[np.newaxis, :] new_label_list = [] for data, label in zip(data_list, label_list): # infer label shape data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] _, feat_shape, _ = self.feat_sym.infer_shape(**data_shape) feat_shape = [int(i) for i in feat_shape[0]] # add gt_boxes to data for e2e data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :] data['gt_keypoints'] = label['gt_keypoints'][np.newaxis, :, :] # assign anchor for label label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'], self.feat_stride, self.anchor_scales, self.anchor_ratios, self.allowed_border) new_label_list.append(label) all_data = dict() for key in self.data_name: all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in self.label_name: pad = -1 if key == 'label' else 0 all_label[key] = tensor_vstack( [batch[key] for batch in new_label_list], pad=pad) self.data = [mx.nd.array(all_data[key]) for key in self.data_name] self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self): # slice roidb cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slice work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) # get testing data for multigpu data_list = [] label_list = [] for islice in slices: iroidb = [roidb[i] for i in range(islice.start, islice.stop)] data, label = get_rpn_batch(iroidb) data_list.append(data) label_list.append(label) # pad data first and then assign anchor (read label) data_tensor = tensor_vstack([batch['data'] for batch in data_list]) for data, data_pad in zip(data_list, data_tensor): data['data'] = data_pad[np.newaxis, :] new_label_list = [] for data, label in zip(data_list, label_list): # infer label shape data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] _, feat_shape, _ = self.feat_sym.infer_shape(**data_shape) feat_shape = [int(i) for i in feat_shape[0]] # add gt_boxes to data for e2e data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :] # assign anchor for label label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'], self.feat_stride, self.anchor_scales, self.anchor_ratios, self.allowed_border) new_label_list.append(label) all_data = dict() for key in self.data_name: all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in self.label_name: pad = -1 if key == 'label' else 0 all_label[key] = tensor_vstack([batch[key] for batch in new_label_list], pad=pad) self.data = [mx.nd.array(all_data[key]) for key in self.data_name] self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self): deq = self.q_out.get() print('q_out got') data_list, label_list = deq for data, label in zip(data_list, label_list): data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] feat_shape_list = [] for s in range(len(self.feat_stride)): _, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape) feat_shape = [int(i) for i in feat_shape[0]] feat_shape_list.append(feat_shape) #for k in self.label_name: # label[k] = [0 for i in range(config.TRAIN.BATCH_IMAGES)] im_info = data['im_info'] gt_boxes = label['gt_boxes'] gt_label = {'gt_boxes': gt_boxes} label_dict = {} head_label_dict = assign_anchor_fpn(feat_shape_list, gt_label, im_info, False, prefix='head') label_dict.update(head_label_dict) if config.FACE_LANDMARK: gt_landmarks = label['gt_landmarks'] gt_label['gt_landmarks'] = gt_landmarks face_label_dict = assign_anchor_fpn(feat_shape_list, gt_label, im_info, config.FACE_LANDMARK, prefix='face') label_dict.update(face_label_dict) #print('im_info', im_info.shape) #print(gt_boxes.shape) for k in self.label_name: label[k] = label_dict[k] all_data = dict() for key in self.data_name: all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in self.label_name: pad = 0 if key.startswith('bbox_') else -1 #print('label vstack', key, pad, len(label_list), file=sys.stderr) all_label[key] = tensor_vstack( [batch[key] for batch in label_list], pad=pad) self.data = [mx.nd.array(all_data[key]) for key in self.data_name] self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self): # slice roidb cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slice work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) # get testing data for multigpu data_list = [] label_list = [] for islice in slices: iroidb = [roidb[i] for i in range(islice.start, islice.stop)] data, label = get_rpn_batch(iroidb) data_list.append(data) label_list.append(label) # pad data first and then assign anchor (read label) data_tensor = tensor_vstack([batch['data'] for batch in data_list]) for i_card in range(len(data_list)): data_list[i_card]['data'] = data_tensor[ i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES] for data, label in zip(data_list, label_list): data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] feat_shape_list = [] for s in range(len(self.feat_stride)): _, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape) feat_shape = [int(i) for i in feat_shape[0]] feat_shape_list.append(feat_shape) label['label'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)] label['bbox_target'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)] label['bbox_weight'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)] for im_i in range(config.TRAIN.BATCH_IMAGES): im_info = data['im_info'][im_i] gt_boxes = label['gt_boxes'][im_i][0] label_dict = \ assign_anchor_fpn(feat_shape_list, gt_boxes, im_info, self.feat_stride, self.anchor_scales, self.anchor_ratios, self.allowed_border) label['label'][im_i] = label_dict['label'] label['bbox_target'][im_i] = label_dict['bbox_target'] label['bbox_weight'][im_i] = label_dict['bbox_weight'] label['label'] = np.vstack(label['label']) label['bbox_target'] = np.vstack(label['bbox_target']) label['bbox_weight'] = np.vstack(label['bbox_weight']) all_data = dict() for key in self.data_name: all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in self.label_name: pad = 0 if key == 'weight' else -1 all_label[key] = tensor_vstack([batch[key] for batch in label_list], pad=pad) self.data = [mx.nd.array(all_data[key]) for key in self.data_name] self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def _make_data_and_labels_afp(self, im_array_list, input_data_list): #do not distribute rois on levels data_list = [] label_list = [] mask_size = (config.ROI_SIZE[0]*2, config.ROI_SIZE[1]*2) rois_num = 0 max_rois_num = 0 for _data in input_data_list: for im_i in _data: rois_num = _data[im_i]['rois'].shape[0] max_rois_num = max(rois_num, max_rois_num) rois_num = max_rois_num # align to num_imgs num_imgs = len(input_data_list[0]) if rois_num == 0: rois_num = num_imgs elif rois_num % num_imgs != 0: ex = num_imgs - rois_num % num_imgs rois_num += ex #print('num', rois_num, num_imgs) for im_array, data_on_imgs in zip(im_array_list, input_data_list): num_imgs = len(data_on_imgs) bucket_size = rois_num for im_i in range(num_imgs): _rois = data_on_imgs['img_%s' % im_i]['rois'] _labels = data_on_imgs['img_%s' % im_i]['labels'] _bbox_targets = data_on_imgs['img_%s' % im_i]['bbox_targets'] _bbox_weights = data_on_imgs['img_%s' % im_i]['bbox_weights'] _mask_targets = data_on_imgs['img_%s' % im_i]['mask_targets'] _mask_weights = data_on_imgs['img_%s' % im_i]['mask_weights'] rois_num = _rois.shape[0] if rois_num < bucket_size: num_pad = bucket_size - rois_num rois_pad = np.array([[12, 34, 56, 78]] * num_pad) labels_pad = np.array([-1] * num_pad) bbox_targets_pad = np.array([[1, 2, 3, 4] * config.NUM_CLASSES] * num_pad) bbox_weights_pad = np.array([[0, 0, 0, 0] * config.NUM_CLASSES] * num_pad) mask_targets_pad = np.zeros((num_pad, config.NUM_CLASSES)+mask_size, dtype=np.int8) mask_weights_pad = np.zeros((num_pad, config.NUM_CLASSES, 1, 1), dtype=np.int8) data_on_imgs['img_%s' % im_i]['rois'] = np.concatenate( [_rois, rois_pad]) data_on_imgs['img_%s' % im_i]['labels'] = np.concatenate( [_labels, labels_pad]) data_on_imgs['img_%s' % im_i]['bbox_targets'] = np.concatenate( [_bbox_targets, bbox_targets_pad]) data_on_imgs['img_%s' % im_i]['bbox_weights'] = np.concatenate( [_bbox_weights, bbox_weights_pad]) data_on_imgs['img_%s' % im_i]['mask_targets'] = np.concatenate( [_mask_targets, mask_targets_pad]) data_on_imgs['img_%s' % im_i]['mask_weights'] = np.concatenate( [_mask_weights, mask_weights_pad]) rois_on_imgs = [] labels_on_imgs = [] bbox_targets_on_imgs = [] bbox_weights_on_imgs = [] mask_targets_on_imgs = [] mask_weights_on_imgs = [] for im_i in range(num_imgs): im_rois = data_on_imgs['img_%s' % im_i]['rois'] labels = data_on_imgs['img_%s' % im_i]['labels'] bbox_targets = data_on_imgs['img_%s' % im_i]['bbox_targets'] bbox_weights = data_on_imgs['img_%s' % im_i]['bbox_weights'] mask_targets = data_on_imgs['img_%s' % im_i]['mask_targets'] mask_weights = data_on_imgs['img_%s' % im_i]['mask_weights'] _rois = im_rois batch_index = im_i * np.ones((_rois.shape[0], 1)) rois_on_imgs.append(np.hstack((batch_index, _rois))) labels_on_imgs.append(labels) bbox_targets_on_imgs.append(bbox_targets) bbox_weights_on_imgs.append(bbox_weights) mask_targets_on_imgs.append(mask_targets) mask_weights_on_imgs.append(mask_weights) label = dict() label.update({'label': np.reshape(np.concatenate(labels_on_imgs, axis=0), [num_imgs, -1])}) label.update({'bbox_target': np.reshape( np.concatenate(bbox_targets_on_imgs, axis=0), [num_imgs, -1])}) label.update({'bbox_weight': np.reshape( np.concatenate(bbox_weights_on_imgs, axis=0), [num_imgs, -1])}) label.update({'mask_target': np.reshape( np.concatenate(mask_targets_on_imgs, axis=0), [num_imgs, -1, config.NUM_CLASSES, mask_size[0], mask_size[1]])}) label.update({'mask_weight': np.reshape( np.concatenate(mask_weights_on_imgs, axis=0), [num_imgs, -1, config.NUM_CLASSES, 1, 1])}) # Stack batch data, and update dict data = dict() data.update({'data': im_array}) rois_array = np.array(rois_on_imgs) data.update({'rois': rois_array}) data_list.append(data) label_list.append(label) all_data = dict() for key in data_list[0].keys(): all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in label_list[0].keys(): all_label[key] = tensor_vstack([batch[key] for batch in label_list]) return all_data, all_label
def _make_data_and_labels(self, im_array_list, levels_data_list): data_list = [] label_list = [] mask_size = (config.ROI_SIZE[0]*2, config.ROI_SIZE[1]*2) rois_num_on_levels = {'stride%s' % s: 0 for s in config.RCNN_FEAT_STRIDE} for s in config.RCNN_FEAT_STRIDE: max_rois_num = 0 for levels_data in levels_data_list: for im_i in levels_data: rois_num = levels_data[im_i]['rois_on_levels']['stride%s' % s].shape[0] max_rois_num = max(rois_num, max_rois_num) rois_num_on_levels['stride%s' % s] = max_rois_num # align to num_imgs num_imgs = len(levels_data_list[0]) for s in config.RCNN_FEAT_STRIDE: if rois_num_on_levels['stride%s' % s] == 0: rois_num_on_levels['stride%s' % s] = num_imgs continue if rois_num_on_levels['stride%s' % s] % num_imgs != 0: ex = num_imgs - rois_num_on_levels['stride%s' % s] % num_imgs rois_num_on_levels['stride%s' % s] += ex print('num', rois_num_on_levels, num_imgs) for im_array, data_on_imgs in zip(im_array_list, levels_data_list): num_imgs = len(data_on_imgs) for s in config.RCNN_FEAT_STRIDE: bucket_size = rois_num_on_levels['stride%s' % s] for im_i in range(num_imgs): _rois = data_on_imgs['img_%s' % im_i]['rois_on_levels']['stride%s' % s] _labels = data_on_imgs['img_%s' % im_i]['labels_on_levels']['stride%s' % s] _bbox_targets = data_on_imgs['img_%s' % im_i]['bbox_targets_on_levels']['stride%s' % s] _bbox_weights = data_on_imgs['img_%s' % im_i]['bbox_weights_on_levels']['stride%s' % s] _mask_targets = data_on_imgs['img_%s' % im_i]['mask_targets_on_levels']['stride%s' % s] _mask_weights = data_on_imgs['img_%s' % im_i]['mask_weights_on_levels']['stride%s' % s] rois_num = _rois.shape[0] if rois_num < bucket_size: num_pad = bucket_size - rois_num rois_pad = np.array([[12, 34, 56, 78]] * num_pad) labels_pad = np.array([-1] * num_pad) bbox_targets_pad = np.array([[1, 2, 3, 4] * config.NUM_CLASSES] * num_pad) bbox_weights_pad = np.array([[0, 0, 0, 0] * config.NUM_CLASSES] * num_pad) mask_targets_pad = np.zeros((num_pad, config.NUM_CLASSES)+mask_size, dtype=np.int8) mask_weights_pad = np.zeros((num_pad, config.NUM_CLASSES, 1, 1), dtype=np.int8) data_on_imgs['img_%s' % im_i]['rois_on_levels']['stride%s' % s] = np.concatenate( [_rois, rois_pad]) data_on_imgs['img_%s' % im_i]['labels_on_levels']['stride%s' % s] = np.concatenate( [_labels, labels_pad]) data_on_imgs['img_%s' % im_i]['bbox_targets_on_levels']['stride%s' % s] = np.concatenate( [_bbox_targets, bbox_targets_pad]) data_on_imgs['img_%s' % im_i]['bbox_weights_on_levels']['stride%s' % s] = np.concatenate( [_bbox_weights, bbox_weights_pad]) data_on_imgs['img_%s' % im_i]['mask_targets_on_levels']['stride%s' % s] = np.concatenate( [_mask_targets, mask_targets_pad]) data_on_imgs['img_%s' % im_i]['mask_weights_on_levels']['stride%s' % s] = np.concatenate( [_mask_weights, mask_weights_pad]) rois_on_imgs = dict() labels_on_imgs = dict() bbox_targets_on_imgs = dict() bbox_weights_on_imgs = dict() mask_targets_on_imgs = dict() mask_weights_on_imgs = dict() for s in config.RCNN_FEAT_STRIDE: rois_on_imgs.update({'stride%s' % s: list()}) labels_on_imgs.update({'stride%s' % s: list()}) bbox_targets_on_imgs.update({'stride%s' % s: list()}) bbox_weights_on_imgs.update({'stride%s' % s: list()}) mask_targets_on_imgs.update({'stride%s' % s: list()}) mask_weights_on_imgs.update({'stride%s' % s: list()}) for im_i in range(num_imgs): for s in config.RCNN_FEAT_STRIDE: im_rois_on_levels = data_on_imgs['img_%s' % im_i]['rois_on_levels'] labels_on_levels = data_on_imgs['img_%s' % im_i]['labels_on_levels'] bbox_targets_on_levels = data_on_imgs['img_%s' % im_i]['bbox_targets_on_levels'] bbox_weights_on_levels = data_on_imgs['img_%s' % im_i]['bbox_weights_on_levels'] mask_targets_on_levels = data_on_imgs['img_%s' % im_i]['mask_targets_on_levels'] mask_weights_on_levels = data_on_imgs['img_%s' % im_i]['mask_weights_on_levels'] _rois = im_rois_on_levels['stride%s' % s] batch_index = im_i * np.ones((_rois.shape[0], 1)) rois_on_imgs['stride%s' % s].append(np.hstack((batch_index, _rois))) labels_on_imgs['stride%s' % s].append(labels_on_levels['stride%s' % s]) bbox_targets_on_imgs['stride%s' % s].append(bbox_targets_on_levels['stride%s' % s]) bbox_weights_on_imgs['stride%s' % s].append(bbox_weights_on_levels['stride%s' % s]) mask_targets_on_imgs['stride%s' % s].append(mask_targets_on_levels['stride%s' % s]) mask_weights_on_imgs['stride%s' % s].append(mask_weights_on_levels['stride%s' % s]) label = dict() for s in config.RCNN_FEAT_STRIDE: label.update({'label_stride%s' % s: np.reshape(np.concatenate(labels_on_imgs['stride%s' % s], axis=0), [num_imgs, -1])}) label.update({'bbox_target_stride%s' % s: np.reshape( np.concatenate(bbox_targets_on_imgs['stride%s' % s], axis=0), [num_imgs, -1])}) label.update({'bbox_weight_stride%s' % s: np.reshape( np.concatenate(bbox_weights_on_imgs['stride%s' % s], axis=0), [num_imgs, -1])}) label.update({'mask_target_stride%s' % s: np.reshape( np.concatenate(mask_targets_on_imgs['stride%s' % s], axis=0), [num_imgs, -1, config.NUM_CLASSES, mask_size[0], mask_size[1]])}) label.update({'mask_weight_stride%s' % s: np.reshape( np.concatenate(mask_weights_on_imgs['stride%s' % s], axis=0), [num_imgs, -1, config.NUM_CLASSES, 1, 1])}) # Stack batch data, and update dict data = dict() data.update({'data': im_array}) for s in config.RCNN_FEAT_STRIDE: rois_array = np.array(rois_on_imgs['stride%s' % s]) data.update({'rois_stride%s' % s: rois_array}) data_list.append(data) label_list.append(label) all_data = dict() for key in data_list[0].keys(): all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in label_list[0].keys(): all_label[key] = tensor_vstack([batch[key] for batch in label_list]) return all_data, all_label
def _make_data_and_labels(self, im_array_list, levels_data_list): data_list = [] label_list = [] rois_num_on_levels = {'stride%s' % s: 0 for s in config.RCNN_FEAT_STRIDE} for s in config.RCNN_FEAT_STRIDE: max_rois_num = 0 for levels_data in levels_data_list: for im_i in levels_data: rois_num = levels_data[im_i]['rois_on_levels']['stride%s' % s].shape[0] max_rois_num = max(rois_num, max_rois_num) rois_num_on_levels['stride%s' % s] = max_rois_num # align to num_imgs num_imgs = len(levels_data_list[0]) for s in config.RCNN_FEAT_STRIDE: if rois_num_on_levels['stride%s' % s] == 0: rois_num_on_levels['stride%s' % s] = num_imgs continue if rois_num_on_levels['stride%s' % s] % num_imgs != 0: ex = num_imgs - rois_num_on_levels['stride%s' % s] % num_imgs rois_num_on_levels['stride%s' % s] += ex for im_array, data_on_imgs in zip(im_array_list, levels_data_list): num_imgs = len(data_on_imgs) for s in config.RCNN_FEAT_STRIDE: bucket_size = rois_num_on_levels['stride%s' % s] for im_i in range(num_imgs): _rois = data_on_imgs['img_%s' % im_i]['rois_on_levels']['stride%s' % s] _labels = data_on_imgs['img_%s' % im_i]['labels_on_levels']['stride%s' % s] _bbox_targets = data_on_imgs['img_%s' % im_i]['bbox_targets_on_levels']['stride%s' % s] _bbox_weights = data_on_imgs['img_%s' % im_i]['bbox_weights_on_levels']['stride%s' % s] _mask_targets = data_on_imgs['img_%s' % im_i]['mask_targets_on_levels']['stride%s' % s] _mask_weights = data_on_imgs['img_%s' % im_i]['mask_weights_on_levels']['stride%s' % s] rois_num = _rois.shape[0] if rois_num < bucket_size: num_pad = bucket_size - rois_num rois_pad = np.array([[12, 34, 56, 78]] * num_pad) labels_pad = np.array([-1] * num_pad) bbox_targets_pad = np.array([[1, 2, 3, 4] * config.NUM_CLASSES] * num_pad) bbox_weights_pad = np.array([[0, 0, 0, 0] * config.NUM_CLASSES] * num_pad) mask_targets_pad = np.zeros((num_pad, config.NUM_CLASSES, 28, 28), dtype=np.int8) mask_weights_pad = np.zeros((num_pad, config.NUM_CLASSES, 1, 1), dtype=np.int8) data_on_imgs['img_%s' % im_i]['rois_on_levels']['stride%s' % s] = np.concatenate( [_rois, rois_pad]) data_on_imgs['img_%s' % im_i]['labels_on_levels']['stride%s' % s] = np.concatenate( [_labels, labels_pad]) data_on_imgs['img_%s' % im_i]['bbox_targets_on_levels']['stride%s' % s] = np.concatenate( [_bbox_targets, bbox_targets_pad]) data_on_imgs['img_%s' % im_i]['bbox_weights_on_levels']['stride%s' % s] = np.concatenate( [_bbox_weights, bbox_weights_pad]) data_on_imgs['img_%s' % im_i]['mask_targets_on_levels']['stride%s' % s] = np.concatenate( [_mask_targets, mask_targets_pad]) data_on_imgs['img_%s' % im_i]['mask_weights_on_levels']['stride%s' % s] = np.concatenate( [_mask_weights, mask_weights_pad]) rois_on_imgs = dict() labels_on_imgs = dict() bbox_targets_on_imgs = dict() bbox_weights_on_imgs = dict() mask_targets_on_imgs = dict() mask_weights_on_imgs = dict() for s in config.RCNN_FEAT_STRIDE: rois_on_imgs.update({'stride%s' % s: list()}) labels_on_imgs.update({'stride%s' % s: list()}) bbox_targets_on_imgs.update({'stride%s' % s: list()}) bbox_weights_on_imgs.update({'stride%s' % s: list()}) mask_targets_on_imgs.update({'stride%s' % s: list()}) mask_weights_on_imgs.update({'stride%s' % s: list()}) for im_i in range(num_imgs): for s in config.RCNN_FEAT_STRIDE: im_rois_on_levels = data_on_imgs['img_%s' % im_i]['rois_on_levels'] labels_on_levels = data_on_imgs['img_%s' % im_i]['labels_on_levels'] bbox_targets_on_levels = data_on_imgs['img_%s' % im_i]['bbox_targets_on_levels'] bbox_weights_on_levels = data_on_imgs['img_%s' % im_i]['bbox_weights_on_levels'] mask_targets_on_levels = data_on_imgs['img_%s' % im_i]['mask_targets_on_levels'] mask_weights_on_levels = data_on_imgs['img_%s' % im_i]['mask_weights_on_levels'] _rois = im_rois_on_levels['stride%s' % s] batch_index = im_i * np.ones((_rois.shape[0], 1)) rois_on_imgs['stride%s' % s].append(np.hstack((batch_index, _rois))) labels_on_imgs['stride%s' % s].append(labels_on_levels['stride%s' % s]) bbox_targets_on_imgs['stride%s' % s].append(bbox_targets_on_levels['stride%s' % s]) bbox_weights_on_imgs['stride%s' % s].append(bbox_weights_on_levels['stride%s' % s]) mask_targets_on_imgs['stride%s' % s].append(mask_targets_on_levels['stride%s' % s]) mask_weights_on_imgs['stride%s' % s].append(mask_weights_on_levels['stride%s' % s]) label = dict() for s in config.RCNN_FEAT_STRIDE: label.update({'label_stride%s' % s: np.reshape(np.concatenate(labels_on_imgs['stride%s' % s], axis=0), [num_imgs, -1])}) label.update({'bbox_target_stride%s' % s: np.reshape( np.concatenate(bbox_targets_on_imgs['stride%s' % s], axis=0), [num_imgs, -1])}) label.update({'bbox_weight_stride%s' % s: np.reshape( np.concatenate(bbox_weights_on_imgs['stride%s' % s], axis=0), [num_imgs, -1])}) label.update({'mask_target_stride%s' % s: np.reshape( np.concatenate(mask_targets_on_imgs['stride%s' % s], axis=0), [num_imgs, -1, config.NUM_CLASSES, 28, 28])}) label.update({'mask_weight_stride%s' % s: np.reshape( np.concatenate(mask_weights_on_imgs['stride%s' % s], axis=0), [num_imgs, -1, config.NUM_CLASSES, 1, 1])}) # Stack batch data, and update dict data = dict() data.update({'data': im_array}) for s in config.RCNN_FEAT_STRIDE: rois_array = np.array(rois_on_imgs['stride%s' % s]) data.update({'rois_stride%s' % s: rois_array}) data_list.append(data) label_list.append(label) all_data = dict() for key in data_list[0].keys(): all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in label_list[0].keys(): all_label[key] = tensor_vstack([batch[key] for batch in label_list]) return all_data, all_label
def get_batch(self): # slice roidb cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) assert cur_to == cur_from + self.batch_size roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slice work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) # get testing data for multigpu data_list = [] label_list = [] for islice in slices: iroidb = [roidb[i] for i in range(islice.start, islice.stop)] data, label = get_crop_batch(iroidb) data_list += data label_list += label #data_list.append(data) #label_list.append(label) # pad data first and then assign anchor (read label) #data_tensor = tensor_vstack([batch['data'] for batch in data_list]) #for i_card in range(len(data_list)): # data_list[i_card]['data'] = data_tensor[ # i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES] for data, label in zip(data_list, label_list): data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] feat_shape_list = [] for s in range(len(self.feat_stride)): _, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape) feat_shape = [int(i) for i in feat_shape[0]] feat_shape_list.append(feat_shape) #for k in self.label_name: # label[k] = [0 for i in range(config.TRAIN.BATCH_IMAGES)] im_info = data['im_info'] gt_boxes = label['gt_boxes'] gt_landmarks = label['gt_landmarks'] #print('im_info', im_info.shape) #print(gt_boxes.shape) label_dict = assign_anchor_fpn(feat_shape_list, gt_boxes, gt_landmarks, im_info) for k in self.label_name: label[k] = label_dict[k] all_data = dict() for key in self.data_name: all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in self.label_name: pad = 0 if key.startswith('bbox_') else -1 #print('label vstack', key, pad, len(label_list), file=sys.stderr) all_label[key] = tensor_vstack( [batch[key] for batch in label_list], pad=pad) self.data = [mx.nd.array(all_data[key]) for key in self.data_name] self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_batch(self): # slice roidb cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slice work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) # get testing data for multigpu data_list = [] label_list = [] for islice in slices: iroidb = [roidb[i] for i in range(islice.start, islice.stop)] data, label = get_rpn_batch(iroidb) data_list.append(data) label_list.append(label) # pad data first and then assign anchor (read label) data_tensor = tensor_vstack([batch['data'] for batch in data_list]) for i_card in range(len(data_list)): data_list[i_card]['data'] = data_tensor[i_card * config.TRAIN.BATCH_IMAGES: (1 + i_card) * config.TRAIN.BATCH_IMAGES] for data, label in zip(data_list, label_list): data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] feat_shape_list = [] for s in range(len(self.feat_stride)): _, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape) feat_shape = [int(i) for i in feat_shape[0]] feat_shape_list.append(feat_shape) for k in self.label_name: label[k] = [0 for i in range(config.TRAIN.BATCH_IMAGES)] for im_i in range(config.TRAIN.BATCH_IMAGES): #im_info = data['im_info'][im_i] #gt_boxes = label['gt_boxes'][im_i][0] im_info = data['im_info'] gt_boxes = label['gt_boxes'] gt_landmarks = label['gt_landmarks'] #print('im_info', im_info.shape) #print(gt_boxes.shape) vis = True if self.visid >= 0 else False label_dict = assign_anchor_fpn(feat_shape_list, gt_boxes, gt_landmarks, im_info, vis) #do visualize for debug here if self.visid >= 0 and self.visid < 10: for _roi in roidb: print('image', _roi['image'], file=sys.stderr) self.visid += 1 anchors = label_dict['anchors'].copy() _im = data['data'].copy() #print(label_dict['olabel'].shape) _label = label_dict['olabel'].copy() _gt_boxes = gt_boxes.copy().astype(np.int) filename = './vis/A%d.png' % self.visid _im = _im[0].transpose((1, 2, 0)) _im = _im[..., ::-1] #bgr for c in range(3): _im[:, :, c] += config.PIXEL_MEANS[c] _im = _im.astype(np.uint8).copy() fg_inds = np.where(_label == 1)[0] print(_im.shape, _label.shape, anchors.shape, len(fg_inds), _gt_boxes.shape, file=sys.stderr) #draw FG anchors _bc = 0 for a in range(anchors.shape[0]): anchor = anchors[a].astype(np.int) l = _label[a] if l != 1: continue #print('drawing', _im.shape, anchor) cv2.rectangle(_im, (anchor[0], anchor[1]), (anchor[2], anchor[3]), (255, 0, 0), 1) _bc += 1 for a in range(_gt_boxes.shape[0]): _box = _gt_boxes[a] cv2.rectangle(_im, (_box[0], _box[1]), (_box[2], _box[3]), (0, 0, 255), 1) print('draw to', filename, _bc, file=sys.stderr) cv2.imwrite(filename, _im) for k in self.label_name: #print('0in_loader', k, label_dict[k].shape, file=sys.stderr) label[k][im_i] = label_dict[k] for k in self.label_name: label[k] = np.vstack(label[k]) #print('in_loader', k, label[k].shape, file=sys.stderr) all_data = dict() for key in self.data_name: all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in self.label_name: pad = 0 if key.startswith('bbox_') else -1 #print('label vstack', key, pad, len(label_list), file=sys.stderr) all_label[key] = tensor_vstack( [batch[key] for batch in label_list], pad=pad) self.data = [mx.nd.array(all_data[key]) for key in self.data_name] self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def get_fpn_maskrcnn_batch(roidb): """ return a dictionary that contains raw data. """ num_images = len(roidb) imgs, roidb = get_image(roidb, scale=config.TRAIN.SCALE) im_array = tensor_vstack(imgs) assert config.TRAIN.BATCH_ROIS % config.TRAIN.BATCH_IMAGES == 0, \ 'BATCHIMAGES {} must divide BATCH_ROIS {}'.format(config.TRAIN.BATCH_IMAGES, config.TRAIN.BATCH_ROIS) rois_per_image = config.TRAIN.BATCH_ROIS / config.TRAIN.BATCH_IMAGES fg_rois_per_image = np.round(config.TRAIN.FG_FRACTION * rois_per_image).astype(int) rois_on_imgs = dict() labels_on_imgs = dict() bbox_targets_on_imgs = dict() bbox_weights_on_imgs = dict() mask_targets_on_imgs = dict() mask_weights_on_imgs = dict() for s in config.RCNN_FEAT_STRIDE: rois_on_imgs.update({'stride%s' % s: list()}) labels_on_imgs.update({'stride%s' % s: list()}) bbox_targets_on_imgs.update({'stride%s' % s: list()}) bbox_weights_on_imgs.update({'stride%s' % s: list()}) mask_targets_on_imgs.update({'stride%s' % s: list()}) mask_weights_on_imgs.update({'stride%s' % s: list()}) # Sample rois level_related_data_on_imgs = {} for im_i in range(num_images): roi_rec = roidb[im_i] # infer num_classes from gt_overlaps num_classes = roi_rec['gt_overlaps'].shape[1] # label = class RoI has max overlap with rois = roi_rec['boxes'] labels = roi_rec['max_classes'] overlaps = roi_rec['max_overlaps'] bbox_targets = roi_rec['bbox_targets'] im_info = roi_rec['im_info'] mask_targets = roi_rec['mask_targets'] mask_labels = roi_rec['mask_labels'] mask_inds = roi_rec['mask_inds'] assign_levels = roi_rec['assign_levels'] im_rois_on_levels, labels_on_levels, bbox_targets_on_levels, bbox_weights_on_levels, mask_targets_on_levels, mask_weights_on_levels = \ sample_rois_fpn(rois, assign_levels, fg_rois_per_image, rois_per_image, num_classes, labels, overlaps, bbox_targets, mask_targets=mask_targets, mask_labels=mask_labels, mask_inds=mask_inds, im_info=im_info) level_related_data_on_imgs.update({ 'img_%s' % im_i: { 'rois_on_levels': im_rois_on_levels, 'labels_on_levels': labels_on_levels, 'bbox_targets_on_levels': bbox_targets_on_levels, 'bbox_weights_on_levels': bbox_weights_on_levels, 'mask_targets_on_levels': mask_targets_on_levels, 'mask_weights_on_levels': mask_weights_on_levels, } }) return im_array, level_related_data_on_imgs
def get_batch(self, roidb): label_name = config.label_name data_name = config.data_name # get testing data for multigpu data_list = [] label_list = [] # TODO:获取单个GPU的rpn_batch,data = {'data': im_array, 'im_info': im_info},label = {'gt_landmarks','gt_boxes'} data, label = get_crop_batch(roidb) # 返回真实label # print('core/loader: label.shape = ', np.array(label).shape,'\n') data_list += data label_list += label # 每个元素为一张图片 # print('core/loader: label_list.shape = ',np.array(label_list).shape) # print('core/loader: before!! label_list[0] = ',label_list[0]) select_stride = 0 for data, label in zip(data_list, label_list): # 这里的label是dict data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] im_info = data['im_info'] gt_boxes = label['gt_boxes'] # print('loader/get_batch: in the gt_label! gt_bboxes.shape=',gt_boxes.shape) gt_label = {'gt_boxes': gt_boxes} label_dict = {} if config.FACE_LANDMARK: gt_landmarks = label['gt_landmarks'] # print('loader/get_batch: in the gt_label! gt_landmarks.shape=', gt_landmarks.shape) gt_label['gt_landmarks'] = gt_landmarks # TODO 上面把label赋值给gt_label的操作好像是没有意义的,gt_label与label没有区别. but, 后面label好像变了,但是否影响这里呢? # ta = datetime.datetime.now() # TODO:产生训练label face_label_dict = self.aa.assign_anchor_fpn( gt_label, im_info, config.FACE_LANDMARK, prefix='face', select_stride=select_stride) # print('face_label_dict.keys = ',face_label_dict.keys()) # tb = datetime.datetime.now() # self._times[0] += (tb-ta).total_seconds() label_dict.update(face_label_dict) # print('im_info', im_info.shape) # print(gt_boxes.shape) for k in label_name: label[k] = label_dict[ k] # TODO 这里实际上是在更新label_list, 由zip函数返回的label应该是引用关系 # print('core/loader: after!! label_list[0] = ', label_list[0]) all_data = dict() for key in data_name: all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in label_name: pad = 0 if key.startswith('bbox_') else -1 # print('label vstack', key, pad, len(label_list), file=sys.stderr) all_label[key] = tensor_vstack( [batch[key] for batch in label_list], pad=pad) # 这里的batch其实就是dict,这个函数的作用是将所有的图片的相同key的list叠起来 # print('batch_key_list len = ',len([batch['gt_boxes'] for batch in label_list])) labels = {} data = [np.array(all_data[key]) for key in data_name] # print('label_list len = ', len([np.array(all_label[key])for key in label_name])) label_d = {} for key in label_name: label_d[key] = all_label[key] # label = [np.array()for key in label_name] #该list是按照顺序存的,要记住label和本list中的array的对应顺序 # for key in label_d.keys(): # print('{}: {}'.format(key,label_d[key].shape)) return data, label_d
def get_batch(self): # slice roidb cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) assert cur_to == cur_from + self.batch_size roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slice work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) # get testing data for multigpu data_list = [] label_list = [] for islice in slices: iroidb = [roidb[i] for i in range(islice.start, islice.stop)] data, label = get_crop_batch(iroidb) data_list += data label_list += label #data_list.append(data) #label_list.append(label) # pad data first and then assign anchor (read label) #data_tensor = tensor_vstack([batch['data'] for batch in data_list]) #for i_card in range(len(data_list)): # data_list[i_card]['data'] = data_tensor[ # i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES] #iiddxx = 0 select_stride = 0 if config.RANDOM_FEAT_STRIDE: select_stride = random.choice(config.RPN_FEAT_STRIDE) for data, label in zip(data_list, label_list): data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] feat_shape_list = [] for s in range(len(self.feat_stride)): _, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape) feat_shape = [int(i) for i in feat_shape[0]] feat_shape_list.append(feat_shape) im_info = data['im_info'] gt_boxes = label['gt_boxes'] gt_label = {'gt_boxes': gt_boxes} if config.USE_BLUR: gt_blur = label['gt_blur'] gt_label['gt_blur'] = gt_blur if self._debug: img = data['data'].copy()[0].transpose( (1, 2, 0))[:, :, ::-1].copy() print('DEBUG SHAPE', data['data'].shape, label['gt_boxes'].shape) box = label['gt_boxes'].copy()[0][0:4].astype(np.int) cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), (0, 255, 0), 2) filename = './debugout/%d.png' % (self._debug_id) print('debug write', filename) cv2.imwrite(filename, img) self._debug_id += 1 #print('DEBUG', img.shape, bbox.shape) label_dict = {} if config.HEAD_BOX: head_label_dict = self.aa.assign_anchor_fpn( gt_label, im_info, False, prefix='head', select_stride=select_stride) label_dict.update(head_label_dict) if config.FACE_LANDMARK: gt_landmarks = label['gt_landmarks'] gt_label['gt_landmarks'] = gt_landmarks #ta = datetime.datetime.now() #face_label_dict = assign_anchor_fpn(feat_shape_list, gt_label, im_info, config.FACE_LANDMARK, prefix='face', select_stride = select_stride) face_label_dict = self.aa.assign_anchor_fpn( gt_label, im_info, config.FACE_LANDMARK, prefix='face', select_stride=select_stride) #tb = datetime.datetime.now() #self._times[0] += (tb-ta).total_seconds() label_dict.update(face_label_dict) #for k in label_dict: # print(k, label_dict[k].shape) if config.CASCADE > 0: pad_gt_boxes = np.empty( (1, config.TRAIN.MAX_BBOX_PER_IMAGE, 5), dtype=np.float32) pad_gt_boxes.fill(-1) pad_gt_boxes[0, 0:gt_boxes.shape[0], :] = gt_boxes label_dict['gt_boxes'] = pad_gt_boxes #print('im_info', im_info.shape) #print(gt_boxes.shape) for k in self.label_name: label[k] = label_dict[k] all_data = dict() for key in self.data_name: all_data[key] = tensor_vstack([batch[key] for batch in data_list]) all_label = dict() for key in self.label_name: pad = 0 if key.startswith('bbox_') else -1 #print('label vstack', key, pad, len(label_list), file=sys.stderr) all_label[key] = tensor_vstack( [batch[key] for batch in label_list], pad=pad) self.data = [mx.nd.array(all_data[key]) for key in self.data_name] self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
def forward(self, is_train, req, in_data, out_data, aux): forward_start = time.time() class_id = [0, 24, 25, 26, 27, 28, 31, 32, 33] feat, deconv1_weight, deconv2_weight, conv1_weight, conv2_weight, gt, rois, label = in_data if rois.shape[1] == 5: rois = rois[:, 1:] for blob in [ feat, deconv1_weight, deconv2_weight, conv1_weight, conv2_weight ]: blob.attach_grad() self.in_data_list.append(blob) with log_time("asnumpy"): gt = gt.asnumpy() label = label.astype(int).asnumpy() rois = rois.asnumpy() rois_stride4 = np.ceil(rois / 4).astype(int) feat_np = feat.asnumpy() fg_indexes = np.where(label != 0)[0] fg_rois = rois[fg_indexes] fg_roi_areas = (fg_rois[:, 2] - fg_rois[:, 0]) * (fg_rois[:, 3] - fg_rois[:, 1]) fg_small_indexes = fg_indexes[np.where(fg_roi_areas < 5000)[0]] fg_middle_indexes = fg_indexes[np.where( np.logical_and(fg_roi_areas >= 5000, fg_roi_areas < 30000))] fg_large_indexes = fg_indexes[np.where(fg_roi_areas >= 30000)[0]] self.fg_small_rois = rois_stride4[fg_small_indexes] self.fg_middle_rois = rois_stride4[fg_middle_indexes] loss = mx.nd.array([0.0]) if len(self.fg_small_rois) > 0: with log_time("prepare small"): feat_tensor_list = [] for roi in rois_stride4[fg_small_indexes]: x0, y0, x1, y1 = roi feat_tensor_list.append(feat_np[:, :, y0:y1, x0:x1]) pad_small_feat = tensor_vstack(feat_tensor_list) pad_small_feat = mx.nd.array(pad_small_feat, ctx=feat.context) pad_small_feat.attach_grad() self.in_data_list.append(pad_small_feat) gt_tensor_list = [] for i in fg_small_indexes: x0, y0, x1, y1 = rois[i] x0, y0, x1, y1 = int(round(x0)), int(round(y0)), int( round(x1)), int(round(y1)) crop_gt = gt[0, y0:y1, x0:x1].copy() h, w = crop_gt.shape if h > w: rand_points = [ np.arange(h), np.random.randint(0, w, (h, )) ] else: rand_points = [ np.random.randint(0, h, (w, )), np.arange(w) ] points = crop_gt[rand_points] ids, counts = np.unique(points, return_counts=True) indexes_sort_counts = np.argsort(counts) mask = None for idx in indexes_sort_counts: if np.floor(ids[idx] / self.seg_code) == class_id[int( label[i])]: crop_gt[crop_gt != ids[idx]] = 0. crop_gt[crop_gt == ids[idx]] = 1. mask = crop_gt break if mask is None: mask = np.zeros_like(crop_gt) tmp = np.zeros(shape=(1, self.num_class) + mask.shape[-2:]) tmp[0, label[i]] = mask gt_tensor_list.append(tmp) pad_small_gt = tensor_vstack(gt_tensor_list) pad_small_gt = mx.nd.array(pad_small_gt, ctx=feat.context) with log_time("deconv small"): with mx.autograd.record(): deconv1 = mx.nd.Deconvolution(pad_small_feat, weight=deconv1_weight, num_filter=32, kernel=(2, 2), stride=(2, 2), no_bias=True, cudnn_tune="off") conv1 = mx.nd.Convolution(deconv1, weight=conv1_weight, num_filter=32, kernel=(3, 3), pad=(1, 1), no_bias=True, cudnn_tune="off") relu1 = mx.nd.Activation(conv1, act_type="relu") deconv2 = mx.nd.Deconvolution(relu1, weight=deconv2_weight, num_filter=32, kernel=(2, 2), stride=(2, 2), pad=(2, 2), no_bias=True, cudnn_tune="off") fullconv = mx.nd.Convolution(deconv2, weight=conv2_weight, num_filter=self.num_class, kernel=(1, 1), no_bias=True, cudnn_tune="off") prob = mx.nd.Activation(fullconv, act_type="sigmoid") crop_mask = mx.nd.Crop(pad_small_gt, prob) loss = loss + mx.nd.mean(mx.nd.square(crop_mask - prob)) if len(self.fg_middle_rois) > 0: with log_time("prepare middle"): feat_tensor_list = [] for roi in rois_stride4[fg_middle_indexes]: x0, y0, x1, y1 = roi feat_tensor_list.append(feat_np[:, :, y0:y1, x0:x1]) pad_middle_feat = tensor_vstack(feat_tensor_list) pad_middle_feat = mx.nd.array(pad_middle_feat, ctx=feat.context) pad_middle_feat.attach_grad() self.in_data_list.append(pad_middle_feat) gt_tensor_list = [] for i in fg_middle_indexes: x0, y0, x1, y1 = rois[i] x0, y0, x1, y1 = int(round(x0)), int(round(y0)), int( round(x1)), int(round(y1)) crop_gt = gt[0, y0:y1, x0:x1].copy() h, w = crop_gt.shape if h > w: rand_points = [ np.arange(h), np.random.randint(0, w, (h, )) ] else: rand_points = [ np.random.randint(0, h, (w, )), np.arange(w) ] points = crop_gt[rand_points] ids, counts = np.unique(points, return_counts=True) indexes_sort_counts = np.argsort(counts) mask = None for idx in indexes_sort_counts: if np.floor(ids[idx] / self.seg_code) == class_id[int( label[i])]: crop_gt[crop_gt != ids[idx]] = 0. crop_gt[crop_gt == ids[idx]] = 1. mask = crop_gt break if mask is None: mask = np.zeros_like(crop_gt) tmp = np.zeros(shape=(1, self.num_class) + mask.shape[-2:]) tmp[0, label[i]] = mask gt_tensor_list.append(tmp) pad_middle_gt = tensor_vstack(gt_tensor_list) pad_middle_gt = mx.nd.array(pad_middle_gt, ctx=feat.context) with log_time("deconv middle"): with mx.autograd.record(): deconv1 = mx.nd.Deconvolution(pad_middle_feat, weight=deconv1_weight, num_filter=32, kernel=(2, 2), stride=(2, 2), no_bias=True, cudnn_tune="off") conv1 = mx.nd.Convolution(deconv1, weight=conv1_weight, num_filter=32, kernel=(3, 3), pad=(1, 1), no_bias=True, cudnn_tune="off") relu1 = mx.nd.Activation(conv1, act_type="relu") deconv2 = mx.nd.Deconvolution(relu1, weight=deconv2_weight, num_filter=32, kernel=(2, 2), stride=(2, 2), pad=(2, 2), no_bias=True, cudnn_tune="off") fullconv = mx.nd.Convolution(deconv2, weight=conv2_weight, num_filter=self.num_class, kernel=(1, 1), no_bias=True, cudnn_tune="off") prob = mx.nd.Activation(fullconv, act_type="sigmoid") crop_mask = mx.nd.Crop(pad_middle_gt, prob) loss = loss + mx.nd.mean(mx.nd.square(crop_mask - prob)) with mx.autograd.record(): with log_time("large rois"): for i in fg_large_indexes.tolist(): # crop gt roi = np.ceil(rois[i]).astype( int) # slice is left inclusive and right exclusive x0, y0, x1, y1 = roi if x0 >= x1 or y0 >= y1: continue crop_gt = gt[0, y0:y1, x0:x1].copy() h, w = crop_gt.shape if h > w: rand_points = [ np.arange(h), np.random.randint(0, w, (h, )) ] else: rand_points = [ np.random.randint(0, h, (w, )), np.arange(w) ] points = crop_gt[rand_points] ids, counts = np.unique(points, return_counts=True) indexes_sort_counts = np.argsort(counts) # gt_copy = gt[0].copy() # cv2.rectangle(gt_copy, (x0, y0), (x1, y1), color=255*255, thickness=2) # cv2.imshow("", gt_copy.astype(np.uint16)) # cv2.waitKey() mask = None for idx in indexes_sort_counts[::-1]: if np.floor(ids[idx] / self.seg_code) == class_id[label[i]]: crop_gt[crop_gt != ids[idx]] = 0 crop_gt[crop_gt == ids[idx]] = 1 mask = crop_gt # cv2.imshow("", mask) # cv2.waitKey() break if mask is None: mask = np.zeros_like(crop_gt) tmp = np.zeros(shape=(1, self.num_class) + mask.shape[-2:]) tmp[0, label[i]] = mask mask = mx.nd.array(tmp, feat.context) roi_stride4 = np.ceil(rois[i] / 4).astype(int) x0, y0, x1, y1 = roi_stride4 crop_feat = mx.nd.slice(feat, begin=(0, 0, y0, x0), end=(1, 128, y1, x1)) deconv1 = mx.nd.Deconvolution(crop_feat, weight=deconv1_weight, num_filter=32, kernel=(2, 2), stride=(2, 2), no_bias=True, cudnn_tune="off") conv1 = mx.nd.Convolution(deconv1, weight=conv1_weight, num_filter=32, kernel=(3, 3), pad=(1, 1), no_bias=True, cudnn_tune="off") relu1 = mx.nd.Activation(conv1, act_type="relu") deconv2 = mx.nd.Deconvolution(relu1, weight=deconv2_weight, num_filter=32, kernel=(2, 2), stride=(2, 2), pad=(2, 2), no_bias=True, cudnn_tune="off") fullconv = mx.nd.Convolution(deconv2, weight=conv2_weight, num_filter=self.num_class, kernel=(1, 1), no_bias=True, cudnn_tune="off") prob = mx.nd.Activation(fullconv, act_type="sigmoid") crop_mask = mx.nd.Crop(mask, prob) loss = loss + mx.nd.mean(mx.nd.square(crop_mask - prob)) loss = loss / len(fg_indexes) loss = loss * self.grad_scale logging.debug("loss: %.1f" % loss.asscalar()) self.loss = loss self.assign(out_data[0], req[0], loss) forward_stop = time.time() logging.debug("forward_consume: %.1fms\n" % ((forward_stop - forward_start) * 1000))
def get_batch(self): # slice roidb cur_from = self.cur cur_to = min(cur_from + self.batch_size, self.size) roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)] # decide multi device slice work_load_list = self.work_load_list ctx = self.ctx if work_load_list is None: work_load_list = [1] * len(ctx) assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \ "Invalid settings for work load. " slices = _split_input_slice(self.batch_size, work_load_list) # get testing data for multigpu data_list = [] label_list = [] for islice in slices: iroidb = [roidb[i] for i in range(islice.start, islice.stop)] #print('get_rpn_batch') data, label = get_rpn_batch(iroidb, self.use_data_augmentation) data_list.append(data) label_list.append(label) # pad data first and then assign anchor (read label) data_tensor = tensor_vstack([batch['data'] for batch in data_list]) for data, data_pad in zip(data_list, data_tensor): data['data'] = data_pad[np.newaxis, :] new_label_list = [] #print(label_list) #print("label_list") for data, label in zip(data_list, label_list): # infer label shape data_shape = {k: v.shape for k, v in data.items()} del data_shape['im_info'] # assign anchor for label #print(data['im_info']) #print('data_shape') #print(data_shape) new_label_list.append( self._assign_anchor(data_shape, label['gt_boxes'], data['im_info'])) # add gt_boxes to data for e2e data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :] all_data = dict() for key in self.data_name: all_data[key] = tensor_vstack([batch[key] for batch in data_list]) # print(len(new_label_list)) all_label = dict() all_label['labelp2'] = new_label_list[0][0]['label'] all_label['labelp3'] = new_label_list[0][1]['label'] all_label['labelp4'] = new_label_list[0][2]['label'] all_label['labelp5'] = new_label_list[0][3]['label'] #for key in self.label_name: all_label['bbox_targetp2'] = new_label_list[0][0]['bbox_target'] all_label['bbox_targetp3'] = new_label_list[0][1]['bbox_target'] all_label['bbox_targetp4'] = new_label_list[0][2]['bbox_target'] all_label['bbox_targetp5'] = new_label_list[0][3]['bbox_target'] #for key in self.label_name: # pad = -1 if key == 'label' else 0 all_label['bbox_weightp2'] = new_label_list[0][0]['bbox_weight'] all_label['bbox_weightp3'] = new_label_list[0][1]['bbox_weight'] all_label['bbox_weightp4'] = new_label_list[0][2]['bbox_weight'] all_label['bbox_weightp5'] = new_label_list[0][3]['bbox_weight'] #for key in self.label_name: # all_label[key] = [batch[key] for batch in new_label_list] self.data = [mx.nd.array(all_data[key]) for key in self.data_name] self.label = [mx.nd.array(all_label[key]) for key in self.label_name]