Exemple #1
0
    def get_batch(self):
        # slice roidb
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slices
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        # get each device
        data_list = []
        label_list = []
        for islice in slices:
            iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
            data, label = get_rcnn_batch(iroidb)
            data_list.append(data)
            label_list.append(label)

        all_data = dict()
        for key in data_list[0].keys():
            all_data[key] = tensor_vstack([batch[key] for batch in data_list])

        all_label = dict()
        for key in label_list[0].keys():
            all_label[key] = tensor_vstack([batch[key] for batch in label_list])

        self.data = [mx.nd.array(all_data[name]) for name in self.data_name]
        self.label = [mx.nd.array(all_label[name]) for name in self.label_name]
    def decide_slices(self, data_shapes):
        """Decide the slices for each context according to the workload.

        Parameters
        ----------
        data_shapes : list
            list of (name, shape) specifying the shapes for the input data or label.
        """
        assert len(data_shapes) > 0
        major_axis = [DataDesc.get_batch_axis(x.layout) for x in data_shapes]

        for (name, shape), axis in zip(data_shapes, major_axis):
            if axis == -1:
                continue

            batch_size = shape[axis]
            if self.batch_size is not None:
                assert batch_size == self.batch_size, ("all data must have the same batch size: "
                                                       + ("batch_size = %d, but " % self.batch_size)
                                                       + ("%s has shape %s" % (name, shape)))
            else:
                self.batch_size = batch_size
                self.slices = _split_input_slice(self.batch_size, self.workload)

        return major_axis
Exemple #3
0
    def _get_train_batch(self):
        """
        utilize minibatch sampling, e.g. 2 images and 64 rois per image
        :return: training batch (e.g. 128 samples)
        """
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        cur_from = self.cur
        cur_to = cur_from + self.batch_size
        if cur_to <= self.size:
            roidb = [self.roidb[i] for i in range(cur_from, cur_to)]
        else:
            pad = cur_to - self.size
            roidb = self.roidb[cur_from:] + self.roidb[:pad]

        batch_list = []
        for islice in slices:
            num_im = islice.stop - islice.start
            iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
            batch = minibatch.get_minibatch(iroidb, self.num_classes, self.ctx)
            batch_list.append(batch)

        all_batch = dict()
        for key in batch_list[0].keys():
            all_batch[key] = tensor_vstack([batch[key] for batch in batch_list])

        return all_batch
Exemple #4
0
    def get_batch(self):
        # slice roidb
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slices
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        im_array_list = []
        levels_data_list = []
        for islice in slices:
            iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
            im_array, levels_data = get_fpn_maskrcnn_batch(iroidb)
            im_array_list.append(im_array)
            levels_data_list.append(levels_data)

        all_data, all_label = self._make_data_and_labels(im_array_list, levels_data_list)
        self.data = [mx.nd.array(all_data[name]) for name in self.data_name]
        self.label = [mx.nd.array(all_label[name]) for name in self.label_name]
    def get_batch_parallel(self):
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        segdb = [self.segdb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slice
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        multiprocess_results = []

        for idx, islice in enumerate(slices):
            isegdb = [segdb[i] for i in range(islice.start, islice.stop)]
            multiprocess_results.append(self.pool.apply_async(parfetch, (self.config, self.crop_width, self.crop_height, isegdb)))

        rst = [multiprocess_result.get() for multiprocess_result in multiprocess_results]

        all_data = [_['data'] for _ in rst]
        all_label = [_['label'] for _ in rst]
        self.data = [[mx.nd.array(data[key]) for key in self.data_name] for data in all_data]
        self.label = [[mx.nd.array(label[key]) for key in self.label_name] for label in all_label]
Exemple #6
0
    def get_batch(self):
        # slice roidb
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slice
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        # get testing data for multigpu
        data_list = []
        label_list = []
        for islice in slices:
            iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
            data, label = get_rpn_batch(iroidb, self.cfg)
            data_list.append(data)
            label_list.append(label)

        # pad data first and then assign anchor (read label)
        data_tensor = tensor_vstack([batch['data'] for batch in data_list])
        for data, data_pad in zip(data_list, data_tensor):
            data['data'] = data_pad[np.newaxis, :]

        new_label_list = []
        for data, label in zip(data_list, label_list):
            # infer label shape
            data_shape = {k: v.shape for k, v in data.items()}
            del data_shape['im_info']
            _, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
            feat_shape = [int(i) for i in feat_shape[0]]

            # add gt_boxes to data for e2e
            data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]

            # assign anchor for label
            label = assign_anchor(feat_shape, label['gt_boxes'], data['im_info'], self.cfg,
                                  self.feat_stride, self.anchor_scales,
                                  self.anchor_ratios, self.allowed_border,
                                  self.normalize_target, self.bbox_mean, self.bbox_std)
            new_label_list.append(label)

        all_data = dict()
        for key in self.data_name:
            all_data[key] = tensor_vstack([batch[key] for batch in data_list])

        all_label = dict()
        for key in self.label_name:
            pad = -1 if key == 'label' else 0
            all_label[key] = tensor_vstack([batch[key] for batch in new_label_list], pad=pad)

        self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
        self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
Exemple #7
0
    def get_batch(self):
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
        if self.mode == 'test':
            self.data, self.label = minibatch.get_minibatch(roidb, self.num_classes, self.mode)
        else:
            work_load_list = self.work_load_list
            ctx = self.ctx
            if work_load_list is None:
                work_load_list = [1] * len(ctx)
            assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
                "Invalid settings for work load. "
            slices = _split_input_slice(self.batch_size, work_load_list)

            data_list = []
            label_list = []
            for islice in slices:
                iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
                data, label = minibatch.get_minibatch(iroidb, self.num_classes, self.mode)
                data_list.append(data)
                label_list.append(label)

            # pad data first and then assign anchor (read label)
            data_tensor = tensor_vstack([batch['data'] for batch in data_list])
            for data, data_pad in zip(data_list, data_tensor):
                data['data'] = data_pad[np.newaxis, :]

            new_label_list = []
            for data, label in zip(data_list, label_list):
                # infer label shape
                data_shape = {k: v.shape for k, v in data.items()}
                del data_shape['im_info']
                _, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
                feat_shape = [int(i) for i in feat_shape[0]]

                # assign anchor for label
                label = minibatch.assign_anchor(feat_shape, label['gt_boxes'], data['im_info'],
                                                self.feat_stride, self.anchor_scales,
                                                self.anchor_ratios, self.allowed_border)
                del data['im_info']
                new_label_list.append(label)

            all_data = dict()
            for key in ['data']:
                all_data[key] = tensor_vstack([batch[key] for batch in data_list])

            all_label = dict()
            all_label['label'] = tensor_vstack([batch['label'] for batch in new_label_list], pad=-1)
            for key in ['bbox_target', 'bbox_inside_weight', 'bbox_outside_weight']:
                all_label[key] = tensor_vstack([batch[key] for batch in new_label_list])

            self.data = [mx.nd.array(all_data['data'])]

            self.label = [mx.nd.array(all_label['label']),
                          mx.nd.array(all_label['bbox_target']),
                          mx.nd.array(all_label['bbox_inside_weight']),
                          mx.nd.array(all_label['bbox_outside_weight'])]
Exemple #8
0
 def get_batch_individual(self):
     cur_from = self.cur
     cur_to = min(cur_from + self.batch_size, self.size)
     roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
     # decide multi device slice
     work_load_list = self.work_load_list
     ctx = self.ctx
     if work_load_list is None:
         work_load_list = [1] * len(ctx)
     assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
         "Invalid settings for work load. "
     slices = _split_input_slice(self.batch_size, work_load_list)
     rst = []
     for idx, islice in enumerate(slices):
         iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
         rst.append(self.parfetch(iroidb))
     all_data = [_['data'] for _ in rst]
     all_label = [_['label'] for _ in rst]
     self.data = [[mx.nd.array(data[key]) for key in self.data_name] for data in all_data]
     self.label = [[mx.nd.array(label[key]) for key in self.label_name] for label in all_label]
Exemple #9
0
    def get_batch(self):
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
        if self.mode == 'test':
            self.data, self.label = minibatch.get_minibatch(roidb, self.num_classes, self.mode)
        else:
            work_load_list = self.work_load_list
            ctx = self.ctx
            if work_load_list is None:
                work_load_list = [1] * len(ctx)
            assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
                "Invalid settings for work load. "
            slices = _split_input_slice(self.batch_size, work_load_list)

            data_list = []
            label_list = []
            for islice in slices:
                iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
                data, label = minibatch.get_minibatch(iroidb, self.num_classes, self.mode)
                data_list.append(data)
                label_list.append(label)

            all_data = dict()
            for key in data_list[0].keys():
                all_data[key] = tensor_vstack([batch[key] for batch in data_list])

            all_label = dict()
            for key in label_list[0].keys():
                all_label[key] = tensor_vstack([batch[key] for batch in label_list])

            self.data = [mx.nd.array(all_data['data']),
                         mx.nd.array(all_data['rois'])]
            self.label = [mx.nd.array(all_label['label']),
                          mx.nd.array(all_label['bbox_target']),
                          mx.nd.array(all_label['bbox_inside_weight']),
                          mx.nd.array(all_label['bbox_outside_weight'])]
Exemple #10
0
    def get_batch_parallel(self):
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        pairdb = [self.pairdb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slice
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        multiprocess_results = []
        for idx, islice in enumerate(slices):
            """
            ipairdb = [pairdb[i] for i in range(islice.start, islice.stop)]
            multiprocess_results.append(self.pool.apply_async(
                get_data_pair_train_batch, (ipairdb, self.config)))
            """
            for i in range(islice.start, islice.stop):
                multiprocess_results.append(
                    self.pool.apply_async(get_data_pair_train_batch,
                                          ([pairdb[i]], self.config)))

        if False:
            temp = get_data_pair_train_batch([pairdb[islice.start]],
                                             self.config)  # for debug
            print('**' * 20)
            print(pairdb[0]['image_observed'])
            print('data:')
            for k in temp['data'].keys():
                print("\t{}, {}".format(k, temp['data'][k].shape))
            print('label:')
            for k in temp['label'].keys():
                print("\t{}, {}".format(k, temp['label'][k].shape))
            print(temp['label']['rot'])
            print(temp['label']['trans'])
            from lib.pair_matching.RT_transform import calc_rt_dist_m
            r_dist, t_dist = calc_rt_dist_m(temp['data']['src_pose'][0],
                                            temp['data']['tgt_pose'][0])
            print("{}: R_dist: {}, T_dist: {}".format(self.cur, r_dist,
                                                      t_dist))
            print('**' * 20)
            image_real = (
                temp['data']['image_observed'][0].transpose([1, 2, 0]) +
                128).astype(np.uint8)
            print(np.max(image_real))
            print(np.min(image_real))
            image_rendered = (
                temp['data']['image_rendered'][0].transpose([1, 2, 0]) +
                128).astype(np.uint8)
            mask_real_gt = np.squeeze(temp['label']['mask_gt_observed'])
            mask_real_est = np.squeeze(temp['data']['mask_observed'])
            mask_rendered = np.squeeze(temp['data']['mask_rendered'])
            if 'flow' in temp['label']:
                print('in loader, flow: ', temp['label']['flow'].shape,
                      np.unique(temp['label']['flow']))
                print('in loader, flow weights: ',
                      temp['label']['flow_weights'].shape,
                      np.unique(temp['label']['flow_weights']))
            import matplotlib.pyplot as plt
            plt.subplot(2, 3, 1)
            plt.imshow(mask_real_est)
            plt.subplot(2, 3, 2)
            plt.imshow(mask_real_gt)
            plt.subplot(2, 3, 3)
            plt.imshow(mask_rendered)
            plt.subplot(2, 3, 4)
            plt.imshow(image_real)
            plt.subplot(2, 3, 5)
            plt.imshow(image_rendered)
            plt.show()
            # plt.savefig('image_real_rendered_{}'.format(self.cur), aspect='normal')

        rst = [
            multiprocess_result.get()
            for multiprocess_result in multiprocess_results
        ]

        batch_per_gpu = int(self.batch_size / len(ctx))
        data_list = []
        label_list = []
        for i in range(len(ctx)):
            sample_data_list = [_['data'] for _ in rst]
            sample_label_list = [_['label'] for _ in rst]
            batch_data = {}
            batch_label = {}
            for key in sample_data_list[0]:
                batch_data[key] = \
                    my_tensor_vstack([sample_data_list[j][key] for j in range(i*batch_per_gpu, (i+1)*batch_per_gpu)])
            for key in sample_label_list[0]:
                batch_label[key] = \
                    my_tensor_vstack([sample_label_list[j][key] for j in range(i*batch_per_gpu, (i+1)*batch_per_gpu)])
            data_list.append(batch_data)
            label_list.append(batch_label)
        """
        data_list = [_['data'] for _ in rst]
        label_list = [_['label'] for _ in rst]
        """
        self.data = [[mx.nd.array(data_on_i[key]) for key in self.data_name]
                     for data_on_i in data_list]
        self.label = [[
            mx.nd.array(label_on_i[key]) for key in self.label_name
        ] for label_on_i in label_list]
    def get_batch(self):
        # slice roidb
        #t = time.time()
        debug = True

        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slice
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        # get testing data for multigpu
        data_list = []
        label_list = []
        for islice in slices:
            iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
            data, label = get_rpn_batch(
                iroidb,
                self.config.SCALES,
                self.config.input_mean,
                stride=self.config.IMAGE_STRIDE
            )  #data:image, label:gt_boxes[n,5]:box+label
            data_list.append(data)
            label_list.append(label)

        # pad data first and then assign anchor (read label)
        data_tensor = tensor_vstack([batch['data'] for batch in data_list])
        for data, data_pad in zip(data_list, data_tensor):
            data['data'] = data_pad[np.newaxis, :]
        new_label_list = []
        for data, label in zip(data_list, label_list):  #image and gt_bbox
            # infer label shape
            data_shape = {k: v.shape for k, v in data.items()}
            del data_shape['im_info']
            feat_shapes = []
            #print '###################data_shape:',data_shape
            for feat_sym in self.feat_sym:
                _, feat_shape, _ = feat_sym.infer_shape(
                    **data_shape
                )  #get size of feature map for rpn, there are 4 feat_sym in fpn
                feat_shape = [int(i) for i in feat_shape[0]]
                feat_shapes.append(feat_shape)
            #print feat_shapes
            gt_boxes = copy.deepcopy(label['gt_boxes'])
            # assign anchor for label
            #print 'image_info: ', data['im_info']
            #t = time.time()
            feat_shape_ = np.array(feat_shapes[0])
            #print len(feat_shapes)
            for i in range(1, len(feat_shapes)):
                a = np.array(feat_shapes[i])
                feat_shape_ = np.vstack((feat_shape_, a))
            #print feat_shape_.shape
            data['feat_shape'] = feat_shape_[np.newaxis, :, :]
            #t_1 = time.time()
            label = fpn_assign_anchor(feat_shapes, label['gt_boxes'],
                                      data['im_info'], self.config,
                                      self.feat_stride, self.anchor_scales,
                                      self.anchor_ratios, self.allowed_border)

            if gt_boxes.shape[0] == 0:
                gt_boxes = -1 * np.ones((1, 5), dtype=np.float32)
            data['gt_boxes'] = gt_boxes[np.newaxis, :, :]  #[1,n,5]
            new_label_list.append(label)

        all_data = dict()
        for key in self.data_name:  #['data', 'im_info', 'gt_boxes']
            all_data[key] = tensor_vstack([batch[key] for batch in data_list])
        #all_data: data:[4,H,W,C], im_info:[4,1,3], gt_boxes:[4,n,5]
        all_label = dict()
        for key in self.label_name:  #['label', 'bbox_target', 'bbox_weight']
            pad = -1 if (key == 'label1' or key == 'label2' or key == 'label3'
                         or key == 'label4' or key == 'label5') else 0
            all_label[key] = tensor_vstack(
                [batch[key] for batch in new_label_list], pad=pad)

        self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
        self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
Exemple #12
0
    def get_batch(self):
        # slice roidb
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        assert cur_to==cur_from+self.batch_size
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slice
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        # get testing data for multigpu
        data_list = []
        label_list = []
        for islice in slices:
            iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
            data, label = get_crop_batch(iroidb)
            data_list += data
            label_list += label
            #data_list.append(data)
            #label_list.append(label)

        # pad data first and then assign anchor (read label)
        #data_tensor = tensor_vstack([batch['data'] for batch in data_list])
        #for i_card in range(len(data_list)):
        #    data_list[i_card]['data'] = data_tensor[
        #                                i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES]

        #iiddxx = 0
        select_stride = 0
        if config.RANDOM_FEAT_STRIDE:
          select_stride = random.choice(config.RPN_FEAT_STRIDE)

        for data, label in zip(data_list, label_list):
            data_shape = {k: v.shape for k, v in data.items()}
            del data_shape['im_info']
            feat_shape_list = []
            for s in range(len(self.feat_stride)):
                _, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
                feat_shape = [int(i) for i in feat_shape[0]]
                feat_shape_list.append(feat_shape)
            im_info = data['im_info']
            gt_boxes = label['gt_boxes']
            gt_label = {'gt_boxes':gt_boxes}
            if config.USE_BLUR:
              gt_blur = label['gt_blur']
              gt_label['gt_blur'] = gt_blur
            if self._debug:
              img = data['data'].copy()[0].transpose( (1,2,0) )[:,:,::-1].copy()
              print('DEBUG SHAPE', data['data'].shape, label['gt_boxes'].shape)

              box = label['gt_boxes'].copy()[0][0:4].astype(np.int)
              cv2.rectangle(img, (box[0], box[1]), (box[2], box[3]), (0, 255, 0), 2)
              filename = './debugout/%d.png' % (self._debug_id)
              print('debug write', filename)
              cv2.imwrite(filename, img)
              self._debug_id+=1
              #print('DEBUG', img.shape, bbox.shape)
            label_dict = {}
            if config.HEAD_BOX:
              head_label_dict = self.aa.assign_anchor_fpn(gt_label, im_info, False, prefix='head', select_stride = select_stride)
              label_dict.update(head_label_dict)
            if config.FACE_LANDMARK:
              gt_landmarks = label['gt_landmarks']
              gt_label['gt_landmarks'] = gt_landmarks
            #ta = datetime.datetime.now()
            #face_label_dict = assign_anchor_fpn(feat_shape_list, gt_label, im_info, config.FACE_LANDMARK, prefix='face', select_stride = select_stride)
            face_label_dict = self.aa.assign_anchor_fpn(gt_label, im_info, config.FACE_LANDMARK, prefix='face', select_stride = select_stride)
            #tb = datetime.datetime.now()
            #self._times[0] += (tb-ta).total_seconds()
            label_dict.update(face_label_dict)
            #for k in label_dict:
            #  print(k, label_dict[k].shape)

            if config.CASCADE>0:
              pad_gt_boxes = np.empty( (1, config.TRAIN.MAX_BBOX_PER_IMAGE, 5), dtype=np.float32)
              pad_gt_boxes.fill(-1)
              pad_gt_boxes[0, 0:gt_boxes.shape[0],:] = gt_boxes
              label_dict['gt_boxes'] = pad_gt_boxes
            #print('im_info', im_info.shape)
            #print(gt_boxes.shape)
            for k in self.label_name:
              label[k] = label_dict[k]

        all_data = dict()
        for key in self.data_name:
            all_data[key] = tensor_vstack([batch[key] for batch in data_list])

        all_label = dict()
        for key in self.label_name:
            pad = 0 if key.startswith('bbox_') else -1
            #print('label vstack', key, pad, len(label_list), file=sys.stderr)
            all_label[key] = tensor_vstack([batch[key] for batch in label_list], pad=pad)

        self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
        self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
Exemple #13
0
    def get_batch(self):
        # slice roidb
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slice
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        # get testing data for multigpu
        data_list = []
        label_list = []
        for islice in slices:
            iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
            data, label = get_rpn_batch(iroidb)
            data_list.append(data)
            label_list.append(label)

        # pad data first and then assign anchor (read label)
        data_tensor = tensor_vstack([batch['data'] for batch in data_list])
        for i_card in range(len(data_list)):
            data_list[i_card]['data'] = data_tensor[
                                        i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES]

        for data, label in zip(data_list, label_list):
            data_shape = {k: v.shape for k, v in data.items()}
            del data_shape['im_info']
            feat_shape_list = []
            for s in range(len(self.feat_stride)):
                _, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
                feat_shape = [int(i) for i in feat_shape[0]]
                feat_shape_list.append(feat_shape)
            label['label'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
            label['bbox_target'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
            label['bbox_weight'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]

            for im_i in range(config.TRAIN.BATCH_IMAGES):
                im_info = data['im_info'][im_i]
                gt_boxes = label['gt_boxes'][im_i][0]
                label_dict = \
                    assign_anchor_fpn(feat_shape_list, gt_boxes, im_info,
                                  self.feat_stride,
                                  self.anchor_scales,
                                  self.anchor_ratios,
                                  self.allowed_border)
                label['label'][im_i] = label_dict['label']
                label['bbox_target'][im_i] = label_dict['bbox_target']
                label['bbox_weight'][im_i] = label_dict['bbox_weight']
            label['label'] = np.vstack(label['label'])
            label['bbox_target'] = np.vstack(label['bbox_target'])
            label['bbox_weight'] = np.vstack(label['bbox_weight'])

        all_data = dict()
        for key in self.data_name:
            all_data[key] = tensor_vstack([batch[key] for batch in data_list])

        all_label = dict()
        for key in self.label_name:
            pad = 0 if key == 'weight' else -1
            all_label[key] = tensor_vstack([batch[key] for batch in label_list], pad=pad)
        self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
        self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
    def get_batch(self):
        # slice roidb
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slice
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        # get testing data for multigpu
        data_list = []
        label_list = []
        for islice in slices:
            iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
            data, label = get_rpn_batch(iroidb)
            data_list.append(data)
            label_list.append(label)

        # pad data first and then assign anchor (read label)
        data_tensor = tensor_vstack([batch['data'] for batch in data_list])
        for i_card in range(len(data_list)):
            data_list[i_card]['data'] = data_tensor[i_card *
                                                    config.TRAIN.BATCH_IMAGES:
                                                    (1 + i_card) *
                                                    config.TRAIN.BATCH_IMAGES]

        for data, label in zip(data_list, label_list):
            data_shape = {k: v.shape for k, v in data.items()}
            del data_shape['im_info']
            feat_shape_list = []
            for s in range(len(self.feat_stride)):
                _, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
                feat_shape = [int(i) for i in feat_shape[0]]
                feat_shape_list.append(feat_shape)
            for k in self.label_name:
                label[k] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]

            for im_i in range(config.TRAIN.BATCH_IMAGES):
                #im_info = data['im_info'][im_i]
                #gt_boxes = label['gt_boxes'][im_i][0]
                im_info = data['im_info']
                gt_boxes = label['gt_boxes']
                gt_landmarks = label['gt_landmarks']
                #print('im_info', im_info.shape)
                #print(gt_boxes.shape)
                vis = True if self.visid >= 0 else False
                label_dict = assign_anchor_fpn(feat_shape_list, gt_boxes,
                                               gt_landmarks, im_info, vis)
                #do visualize for debug here
                if self.visid >= 0 and self.visid < 10:
                    for _roi in roidb:
                        print('image', _roi['image'], file=sys.stderr)
                    self.visid += 1
                    anchors = label_dict['anchors'].copy()
                    _im = data['data'].copy()
                    #print(label_dict['olabel'].shape)
                    _label = label_dict['olabel'].copy()
                    _gt_boxes = gt_boxes.copy().astype(np.int)
                    filename = './vis/A%d.png' % self.visid
                    _im = _im[0].transpose((1, 2, 0))
                    _im = _im[..., ::-1]  #bgr
                    for c in range(3):
                        _im[:, :, c] += config.PIXEL_MEANS[c]
                    _im = _im.astype(np.uint8).copy()
                    fg_inds = np.where(_label == 1)[0]
                    print(_im.shape,
                          _label.shape,
                          anchors.shape,
                          len(fg_inds),
                          _gt_boxes.shape,
                          file=sys.stderr)
                    #draw FG anchors
                    _bc = 0
                    for a in range(anchors.shape[0]):
                        anchor = anchors[a].astype(np.int)
                        l = _label[a]
                        if l != 1:
                            continue
                        #print('drawing', _im.shape, anchor)
                        cv2.rectangle(_im, (anchor[0], anchor[1]),
                                      (anchor[2], anchor[3]), (255, 0, 0), 1)
                        _bc += 1
                    for a in range(_gt_boxes.shape[0]):
                        _box = _gt_boxes[a]
                        cv2.rectangle(_im, (_box[0], _box[1]),
                                      (_box[2], _box[3]), (0, 0, 255), 1)
                    print('draw to', filename, _bc, file=sys.stderr)
                    cv2.imwrite(filename, _im)

                for k in self.label_name:
                    #print('0in_loader', k, label_dict[k].shape, file=sys.stderr)
                    label[k][im_i] = label_dict[k]
            for k in self.label_name:
                label[k] = np.vstack(label[k])
                #print('in_loader', k, label[k].shape, file=sys.stderr)

        all_data = dict()
        for key in self.data_name:
            all_data[key] = tensor_vstack([batch[key] for batch in data_list])

        all_label = dict()
        for key in self.label_name:
            pad = 0 if key.startswith('bbox_') else -1
            #print('label vstack', key, pad, len(label_list), file=sys.stderr)
            all_label[key] = tensor_vstack(
                [batch[key] for batch in label_list], pad=pad)
        self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
        self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
    def fit(self, X, y=None, eval_data=None, eval_metric='acc',
            epoch_end_callback=None, batch_end_callback=None, kvstore='local', logger=None,
            work_load_list=None, monitor=None, eval_batch_end_callback=None):

        train_data = self._init_iter(X, y, is_train=True)
        eval_data = self._init_eval_iter(eval_data)

        # prepare sym
        if self.sym_gen:
            self.symbol = self.sym_gen(train_data.default_bucket_key) # pylint: disable=no-member
            self._check_arguments()
        self.kwargs["sym"] = self.symbol

        # prepare args name and shape
        arg_names, param_names, aux_names = \
                self._init_params(dict(train_data.provide_data+train_data.provide_label))

        # setup metric
        if not isinstance(eval_metric, metric.EvalMetric):
            eval_metric = metric.create(eval_metric)

        # create kvstore
        (kvstore, update_on_kvstore) = _create_kvstore(
            kvstore, len(self.ctx), self.arg_params)

        # param_idx2name = {}
        # if update_on_kvstore:
        #     param_idx2name.update(enumerate(param_names))
        # else:
        #     for i, n in enumerate(param_names):
        #         for k in range(len(self.ctx)):
        #             param_idx2name[i*len(self.ctx)+k] = n
        # self.kwargs["param_idx2name"] = param_idx2name

        # init optmizer
        if isinstance(self.optimizer, str):
            batch_size = train_data.batch_size
            if kvstore and kvstore.type == 'dist_sync':
                batch_size *= kvstore.num_workers
            optimizer = opt.create(self.optimizer,
                                   rescale_grad=(1.0/batch_size),
                                   **(self.kwargs))
        elif isinstance(self.optimizer, opt.Optimizer):
            optimizer = self.optimizer


        num_device = len(self.ctx)
        logger.info('Start training with %s', str(self.ctx))

        if work_load_list is None:
            work_load_list = [1] * num_device
        assert isinstance(work_load_list, list) and len(work_load_list) == num_device, \
            "Invalid settings for work load. "

        from mxnet.executor_manager import _split_input_slice
        self.slices = _split_input_slice(train_data.batch_size, work_load_list)

        execgrp = ExecuterGroup(self.symbol, arg_names, param_names, self.ctx,
                                                 self.slices, train_data)

        if self.sym_gen is not None:
            self.execgrp_bucket = {train_data.default_bucket_key: self.execgrp}

        # executor_manager.set_params(arg_params, aux_params)

        train_data.reset()
        for epoch in range(self.begin_epoch, self.num_epoch):
            # Training phase

            tic = time.time()
            eval_metric.reset()
            nbatch = 0
            # Iterate over training data.

            #Into Epoch
            #########################
            #record acc
            acc_hist = []

            while True:                        
                do_reset = True
                for data_batch in train_data:
                    executor_manager.load_data_batch(data_batch)

                    if monitor is not None:
                        monitor.tic()

                    executor_manager.forward(is_train=True)
                    executor_manager.backward()
                    
                    if update_on_kvstore:
                        _update_params_on_kvstore(executor_manager.param_arrays,
                                                  executor_manager.grad_arrays,
                                                  kvstore)
                    else:
                        _update_params(executor_manager.param_arrays,
                                       executor_manager.grad_arrays,
                                       updater=updater,
                                       num_device=len(ctx),
                                       kvstore=kvstore)

                    if monitor is not None:
                        monitor.toc_print()

                    eval_metric.reset()
                    executor_manager.update_metric(eval_metric, data_batch.label)
                    name_value = eval_metric.get_name_value()

                    for name, value in name_value:
                        acc_hist.append(value)
                        # logger.info('Epoch[%d] Training-%s=%f', epoch, name, value)


                    nbatch += 1
                    # batch callback (for print purpose)
                    if batch_end_callback != None:
                        batch_end_params = BatchEndParam(epoch=epoch,
                                                         nbatch=nbatch,
                                                         eval_metric=eval_metric,
                                                         locals=locals())
                        if isinstance(batch_end_callback, list):
                            for call in batch_end_callback:
                                call(batch_end_params)
                        else:
                            batch_end_callback(batch_end_params)

                    # this epoch is done possibly earlier
                    if epoch_size is not None and nbatch >= epoch_size:
                        do_reset = False
                        break

                if do_reset is True:
                    logger.debug('Epoch[%d] Resetting Data Iterator', epoch)
                    train_data.reset()
                    logger.debug('Epoch[%d] Resetting Eval Metric', epoch)
                    eval_metric.reset()

                # this epoch is done
                if epoch_size is None or nbatch >= epoch_size:
                    break

            toc = time.time()
            logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc - tic))

            # print epoch ,'<<<<<<<<<<<<'
            if epoch_end_callback or epoch + 1 == end_epoch:
                executor_manager.copy_to(arg_params, aux_params)

            if epoch_end_callback != None:
                if isinstance(epoch_end_callback, list):
                    for call in epoch_end_callback:
                        call(epoch, symbol, arg_params, aux_params, 
                            acc_hist)
                else:
                    epoch_end_callback(epoch, symbol, arg_params, aux_params, 
                            acc_hist)

            # evaluation
            # print 'enter evaluation'
            if eval_data:
                eval_metric.reset()
                eval_data.reset()
                for i, eval_batch in enumerate(eval_data):
                    executor_manager.load_data_batch(eval_batch)
                    executor_manager.forward(is_train=False)
                    executor_manager.update_metric(eval_metric, eval_batch.label)
                    if eval_batch_end_callback != None:
                        batch_end_params = BatchEndParam(epoch=epoch,
                                                         nbatch=i,
                                                         eval_metric=eval_metric,
                                                         locals=locals())
                        if isinstance(eval_batch_end_callback, list):
                            for call in eval_batch_end_callback:
                                call(batch_end_params)
                        else:
                            eval_batch_end_callback(batch_end_params)
                name_value = eval_metric.get_name_value()
                for name, value in name_value:
                    logger.info('E[%d] V %s:%f', epoch, name, value)
                    # print 'Epoch[%d] Validation=%f' % (epoch, value)
        # end of all epochs
        return
Exemple #16
0
    def get_batch(self):
        data_list = []
        label_list = []
        if self.nThreads == 0:
            # slice roidb
            cur_from = self.cur
            cur_to = min(cur_from + self.batch_size, self.size)
            roidb = [
                self.roidb[self.index[i]] for i in range(cur_from, cur_to)
            ]

            # decide multi device slice
            work_load_list = self.work_load_list
            ctx = self.ctx
            if work_load_list is None:
                work_load_list = [1] * len(ctx)
            assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
                "Invalid settings for work load. "
            slices = _split_input_slice(self.batch_size, work_load_list)

            # get testing data for multigpu
            for islice in slices:
                iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
                data, label = get_rpn_batch(iroidb)
                data_list.append(data)
                label_list.append(label)

        else:
            for p in range(len(self.ctx)):
                data, label = self.multithread_get_rpn_batch()
                data_list.append(data)
                label_list.append(label)

        # pad data first and then assign anchor (read label)
        data_tensor = tensor_vstack([batch['data'] for batch in data_list
                                     ])  # to unify the shape of data
        k = 0
        for data in data_list:
            bs = data['data'].shape[0]
            data['data'] = data_tensor[k:k + bs, :]
            k += bs

        # re-organize images and labels for 3DCE
        new_label_list = []
        num_smp = config.TRAIN.SAMPLES_PER_BATCH
        num_image = config.NUM_IMAGES_3DCE
        num_slices = config.NUM_SLICES
        for data, label1 in zip(data_list, label_list):
            data0 = data['data']
            im_info0 = data['im_info']
            gt0 = label1['gt_boxes']
            data_new = np.empty((num_smp * num_image, num_slices,
                                 data0.shape[2], data0.shape[3]))
            iminfo_new = np.empty((num_smp * num_image, 3))
            gt_new = np.zeros((num_smp * num_image, gt0.shape[1], 5))
            for p in range(num_smp):
                for q in range(num_image):
                    data_new[p * num_image +
                             q, :, :, :] = data0[p, q * num_slices:(q + 1) *
                                                 num_slices, :, :]
                    iminfo_new[p * num_image + q, :] = im_info0[p, :]
                    if q == (num_image - 1) / 2:
                        gt_new[p * num_image + q, :, :] = gt0[p, :, :]

            data['data'] = data_new
            data['im_info'] = iminfo_new
            label1['gt_boxes'] = gt_new

            # infer label shape
            data_shape = {k: v.shape for k, v in data.items()}
            del data_shape['im_info']
            _, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
            feat_shape = [int(i) for i in feat_shape[0]]

            # add gt_boxes to data for e2e
            data['gt_boxes'] = label1['gt_boxes']  #[np.newaxis, :, :]

            # assign anchor for label
            for i in range(len(label1['gt_boxes'])):
                label = assign_anchor(feat_shape, label1['gt_boxes'][i],
                                      data['im_info'][i:i + 1, :],
                                      self.feat_stride, self.anchor_scales,
                                      self.anchor_ratios, self.allowed_border)
                if i % num_image != (num_image - 1) / 2:
                    label['label'][:] = -1
                    label['bbox_weight'][:] = 0

                new_label_list.append(label)

        all_data = dict()
        for key in self.data_name:
            all_data[key] = tensor_vstack([batch[key] for batch in data_list])

        all_label = dict()
        for key in self.label_name:
            pad = -1 if key == 'label' else 0
            all_label[key] = tensor_vstack(
                [batch[key] for batch in new_label_list], pad=pad)

        self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
        self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
Exemple #17
0
    def get_batch(self):
        # slice roidb
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slice
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        # get testing data for multigpu
        data_list = []
        label_list = []
        for islice in slices:
            iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
            data, label = get_rpn_batch(iroidb)
            data_list.append(data)
            label_list.append(label)

        # pad data first and then assign anchor (read label)
        data_tensor = tensor_vstack([batch['data'] for batch in data_list])
        for i_card in range(len(data_list)):
            data_list[i_card]['data'] = data_tensor[
                                        i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES]

        for data, label in zip(data_list, label_list):
            data_shape = {k: v.shape for k, v in data.items()}
            del data_shape['im_info']
            feat_shape_list = []
            for s in range(len(self.feat_stride)):
                _, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
                feat_shape = [int(i) for i in feat_shape[0]]
                feat_shape_list.append(feat_shape)
            label['label'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
            label['bbox_target'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
            label['bbox_weight'] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]

            for im_i in range(config.TRAIN.BATCH_IMAGES):
                im_info = data['im_info'][im_i]
                gt_boxes = label['gt_boxes'][im_i][0]
                label_dict = \
                    assign_anchor_fpn(feat_shape_list, gt_boxes, im_info,
                                  self.feat_stride,
                                  self.anchor_scales,
                                  self.anchor_ratios,
                                  self.allowed_border)
                label['label'][im_i] = label_dict['label']
                label['bbox_target'][im_i] = label_dict['bbox_target']
                label['bbox_weight'][im_i] = label_dict['bbox_weight']
            label['label'] = np.vstack(label['label'])
            label['bbox_target'] = np.vstack(label['bbox_target'])
            label['bbox_weight'] = np.vstack(label['bbox_weight'])

        all_data = dict()
        for key in self.data_name:
            all_data[key] = tensor_vstack([batch[key] for batch in data_list])

        all_label = dict()
        for key in self.label_name:
            pad = 0 if key == 'weight' else -1
            all_label[key] = tensor_vstack([batch[key] for batch in label_list], pad=pad)
        self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
        self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
Exemple #18
0
    def get_batch(self):
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]
        if self.mode == 'test':
            self.data, self.label = minibatch.get_minibatch(
                roidb, self.num_classes, self.mode)
        else:
            work_load_list = self.work_load_list
            ctx = self.ctx
            if work_load_list is None:
                work_load_list = [1] * len(ctx)
            assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
                "Invalid settings for work load. "
            slices = _split_input_slice(self.batch_size, work_load_list)

            data_list = []
            label_list = []
            for islice in slices:
                iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
                data, label = minibatch.get_minibatch(iroidb, self.num_classes,
                                                      self.mode)
                data_list.append(data)
                label_list.append(label)

            # pad data first and then assign anchor (read label)
            data_tensor = tensor_vstack([batch['data'] for batch in data_list])
            for data, data_pad in zip(data_list, data_tensor):
                data['data'] = data_pad[np.newaxis, :]

            new_label_list = []
            for data, label in zip(data_list, label_list):
                # infer label shape
                data_shape = {k: v.shape for k, v in data.items()}
                del data_shape['im_info']
                _, feat_shape, _ = self.feat_sym.infer_shape(**data_shape)
                feat_shape = [int(i) for i in feat_shape[0]]

                # assign anchor for label
                label = minibatch.assign_anchor(feat_shape, label['gt_boxes'],
                                                data['im_info'],
                                                self.feat_stride,
                                                self.anchor_scales,
                                                self.anchor_ratios,
                                                self.allowed_border)
                del data['im_info']
                new_label_list.append(label)

            all_data = dict()
            for key in ['data']:
                all_data[key] = tensor_vstack(
                    [batch[key] for batch in data_list])

            all_label = dict()
            all_label['label'] = tensor_vstack(
                [batch['label'] for batch in new_label_list], pad=-1)
            for key in [
                    'bbox_target', 'bbox_inside_weight', 'bbox_outside_weight'
            ]:
                all_label[key] = tensor_vstack(
                    [batch[key] for batch in new_label_list])

            self.data = [mx.nd.array(all_data['data'])]

            self.label = [
                mx.nd.array(all_label['label']),
                mx.nd.array(all_label['bbox_target']),
                mx.nd.array(all_label['bbox_inside_weight']),
                mx.nd.array(all_label['bbox_outside_weight'])
            ]
    def get_batch_parallel(self):

        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        segdb = [self.segdb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slice
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        # debug = []
        # for idx, islice in enumerate(slices):
        #     isegdb = [segdb[i] for i in range(islice.start, islice.stop) if i < len(segdb)]
        #     resuelts = parfetch(self.config, self.crop_width, self.crop_width, isegdb)
        #     debug.append(resuelts)
        # import pdb; pdb.set_trace()

        multiprocess_results = []
        # import pdb; pdb.set_trace()
        for idx, islice in enumerate(slices):
            isegdb = [
                segdb[i] for i in range(islice.start, islice.stop)
                if i < len(segdb)
            ]
            # parfetch(config, crop_width, crop_height, isegdb)
            # import pdb; pdb.set_trace()
            # try:
            # import pdb; pdb.set_trace()
            # print("Here:",isegdb[0]['image'])
            # print("Here:",isegdb[0]['seg_cls_path'])
            try:
                multiprocess_results.append(
                    self.pool.apply_async(parfetch,
                                          (self.config, self.crop_width,
                                           self.crop_height, isegdb)))
            except:
                print("add to pool error")
                import pdb
                pdb.set_trace()
                # print(isegdb)
            # multiprocess_results.append(
            # parfetch(self.config, self.crop_width, self.crop_height, isegdb))
        # import pdb; pdb.set_trace()
        # rst = []
        # for idx, item in enumerate(multiprocess_results):
        #     # import pdb; pdb.set_trace()
        #     try:
        #         rst.append(item.get())
        #     except:
        #         print("can't get data")
        #         import pdb; pdb.set_trace()
        rst = [
            multiprocess_result.get()
            for multiprocess_result in multiprocess_results
        ]

        all_data = [_['data'] for _ in rst]
        all_label = [_['label'] for _ in rst]

        self.data = [[mx.nd.array(data[key]) for key in self.data_name]
                     for data in all_data]
        self.label = [[mx.nd.array(label[key]) for key in self.label_name]
                      for label in all_label]
    def get_batch(self):
        # slice roidb
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        assert cur_to == cur_from + self.batch_size
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slice
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        # get testing data for multigpu
        data_list = []
        label_list = []
        for islice in slices:
            iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
            data, label = get_crop_batch(iroidb)
            data_list += data
            label_list += label
            #data_list.append(data)
            #label_list.append(label)

        # pad data first and then assign anchor (read label)
        #data_tensor = tensor_vstack([batch['data'] for batch in data_list])
        #for i_card in range(len(data_list)):
        #    data_list[i_card]['data'] = data_tensor[
        #                                i_card * config.TRAIN.BATCH_IMAGES:(1 + i_card) * config.TRAIN.BATCH_IMAGES]

        for data, label in zip(data_list, label_list):
            data_shape = {k: v.shape for k, v in data.items()}
            del data_shape['im_info']
            feat_shape_list = []
            for s in range(len(self.feat_stride)):
                _, feat_shape, _ = self.feat_sym[s].infer_shape(**data_shape)
                feat_shape = [int(i) for i in feat_shape[0]]
                feat_shape_list.append(feat_shape)
            #for k in self.label_name:
            #  label[k] = [0 for i in range(config.TRAIN.BATCH_IMAGES)]
            im_info = data['im_info']
            gt_boxes = label['gt_boxes']
            gt_landmarks = label['gt_landmarks']
            #print('im_info', im_info.shape)
            #print(gt_boxes.shape)
            label_dict = assign_anchor_fpn(feat_shape_list, gt_boxes,
                                           gt_landmarks, im_info)
            for k in self.label_name:
                label[k] = label_dict[k]

        all_data = dict()
        for key in self.data_name:
            all_data[key] = tensor_vstack([batch[key] for batch in data_list])

        all_label = dict()
        for key in self.label_name:
            pad = 0 if key.startswith('bbox_') else -1
            #print('label vstack', key, pad, len(label_list), file=sys.stderr)
            all_label[key] = tensor_vstack(
                [batch[key] for batch in label_list], pad=pad)
        self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
        self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
Exemple #21
0
    def get_batch(self):
        # slice roidb
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slice
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        # get testing data for multigpu
        data_list = []
        label_list = []
        for islice in slices:
            iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
            data, label = get_rpn_batch(iroidb, self.cfg)
            data_list.append(data)
            label_list.append(label)

        # pad data first and then assign anchor (read label)
        data_tensor = tensor_vstack([batch['data'] for batch in data_list])
        for data, data_pad in zip(data_list, data_tensor):
            data['data'] = data_pad[np.newaxis, :]

        new_label_list = []
        for data, label in zip(data_list, label_list):
            # infer label shape
            data_shape = {k: v.shape for k, v in data.items()}
            del data_shape['im_info']

            _, feat_shape_p3, _ = self.feat_sym_p3.infer_shape(**data_shape)
            feat_shape_p3 = [int(i) for i in feat_shape_p3[0]]
            _, feat_shape_p4, _ = self.feat_sym_p4.infer_shape(**data_shape)
            feat_shape_p4 = [int(i) for i in feat_shape_p4[0]]
            _, feat_shape_p5, _ = self.feat_sym_p5.infer_shape(**data_shape)
            feat_shape_p5 = [int(i) for i in feat_shape_p5[0]]

            # add gt_boxes to data for e2e
            data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]

            # assign anchor for label
            label = assign_anchor(feat_shape_p3, feat_shape_p4, feat_shape_p5,
                                  label['gt_boxes'], data['im_info'], self.cfg,
                                  self.feat_stride_p3, self.anchor_scales_p3,
                                  self.anchor_ratios_p3, self.feat_stride_p4,
                                  self.anchor_scales_p4, self.anchor_ratios_p4,
                                  self.feat_stride_p5, self.anchor_scales_p5,
                                  self.anchor_ratios_p5, self.allowed_border)
            new_label_list.append(label)

        all_data = dict()
        for key in self.data_name:
            all_data[key] = tensor_vstack([batch[key] for batch in data_list])

        all_label = dict()
        for key in self.label_name:
            pad = -1 if key == 'label' else 0
            all_label[key] = tensor_vstack(
                [batch[key] for batch in new_label_list], pad=pad)

        self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
        self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
Exemple #22
0
def get_minibatch(roidb, num_classes, ctx, work_load_list=None):
    """
    return minibatch of images in roidb
    :param roidb: subset of main database
    :param num_classes: number of classes is used in bbox regression targets
    :return: minibatch: {'data', 'rois', 'labels', 'bbox_targets', 'bbox_inside_weights', 'bbox_outside_weights'}
    """
    num_images = len(roidb)
    random_scale_indexes = npr.randint(0, high=len(config.TRAIN.SCALES), size=num_images)
    assert config.TRAIN.BATCH_SIZE % num_images == 0, \
        'num_images {} must devide BATCHSIZE {}'.format(num_images, config.TRAIN.BATCH_SIZE)
    rois_per_image = config.TRAIN.BATCH_SIZE / num_images
    fg_rois_per_image = np.round(config.TRAIN.FG_FRACTION * rois_per_image).astype(int)

    # im_array: [num_images, c, h, w]
    im_array, im_scales = get_image_array(roidb, config.TRAIN.SCALES, random_scale_indexes)
    rois_array = list()
    labels_array = list()
    bbox_targets_array = list()
    bbox_inside_array = list()

    if work_load_list is None:
        work_load_list = [1] * len(ctx)
    assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
        "Invalid settings for work load. "
    slices = _split_input_slice(num_images, work_load_list)

    idx_in_slice = []
    for islice in slices:
        num_im = islice.stop - islice.start
        for i in range(num_im):
            idx_in_slice.append(i)
    for im_i, idx in enumerate(idx_in_slice):
        im_rois, labels, bbox_targets, bbox_inside_weights, overlaps = \
            sample_rois(roidb[im_i], fg_rois_per_image, rois_per_image, num_classes)

        # project im_rois
        # do not round roi
        rois = im_rois * im_scales[im_i]
        batch_index = idx * np.ones((rois.shape[0], 1))
        rois_array_this_image = np.hstack((batch_index, rois))
        rois_array.append(rois_array_this_image)

        # add labels
        labels_array.append(labels)
        bbox_targets_array.append(bbox_targets)
        bbox_inside_array.append(bbox_inside_weights)

    rois_array = np.vstack(rois_array)
    labels_array = np.hstack(labels_array)
    bbox_targets_array = np.vstack(bbox_targets_array)
    bbox_inside_array = np.vstack(bbox_inside_array)
    bbox_outside_array = np.array(bbox_inside_array > 0).astype(np.float32)

    minibatch = {'data': im_array,
                 'rois': rois_array,
                 'labels': labels_array,
                 'bbox_targets': bbox_targets_array,
                 'bbox_inside_weights': bbox_inside_array,
                 'bbox_outside_weights': bbox_outside_array}
    return minibatch
Exemple #23
0
    def get_batch(self):
        # slice roidb
        cur_from = self.cur
        cur_to = min(cur_from + self.batch_size, self.size)
        roidb = [self.roidb[self.index[i]] for i in range(cur_from, cur_to)]

        # decide multi device slice
        work_load_list = self.work_load_list
        ctx = self.ctx
        if work_load_list is None:
            work_load_list = [1] * len(ctx)
        assert isinstance(work_load_list, list) and len(work_load_list) == len(ctx), \
            "Invalid settings for work load. "
        slices = _split_input_slice(self.batch_size, work_load_list)

        # get testing data for multigpu
        data_list = []
        label_list = []
        for islice in slices:
            iroidb = [roidb[i] for i in range(islice.start, islice.stop)]
            #print('get_rpn_batch')
            data, label = get_rpn_batch(iroidb, self.use_data_augmentation)
            data_list.append(data)
            label_list.append(label)

        # pad data first and then assign anchor (read label)
        data_tensor = tensor_vstack([batch['data'] for batch in data_list])
        for data, data_pad in zip(data_list, data_tensor):
            data['data'] = data_pad[np.newaxis, :]

        new_label_list = []
        #print(label_list)
        #print("label_list")
        for data, label in zip(data_list, label_list):
            # infer label shape
            data_shape = {k: v.shape for k, v in data.items()}
            del data_shape['im_info']
            # assign anchor for label
            #print(data['im_info'])
            #print('data_shape')
            #print(data_shape)
            new_label_list.append(
                self._assign_anchor(data_shape, label['gt_boxes'],
                                    data['im_info']))

            # add gt_boxes to data for e2e
            data['gt_boxes'] = label['gt_boxes'][np.newaxis, :, :]

        all_data = dict()
        for key in self.data_name:
            all_data[key] = tensor_vstack([batch[key] for batch in data_list])


#        print(len(new_label_list))
        all_label = dict()

        all_label['labelp2'] = new_label_list[0][0]['label']
        all_label['labelp3'] = new_label_list[0][1]['label']
        all_label['labelp4'] = new_label_list[0][2]['label']
        all_label['labelp5'] = new_label_list[0][3]['label']
        #for key in self.label_name:
        all_label['bbox_targetp2'] = new_label_list[0][0]['bbox_target']
        all_label['bbox_targetp3'] = new_label_list[0][1]['bbox_target']
        all_label['bbox_targetp4'] = new_label_list[0][2]['bbox_target']
        all_label['bbox_targetp5'] = new_label_list[0][3]['bbox_target']
        #for key in self.label_name:
        #    pad = -1 if key == 'label' else 0
        all_label['bbox_weightp2'] = new_label_list[0][0]['bbox_weight']
        all_label['bbox_weightp3'] = new_label_list[0][1]['bbox_weight']
        all_label['bbox_weightp4'] = new_label_list[0][2]['bbox_weight']
        all_label['bbox_weightp5'] = new_label_list[0][3]['bbox_weight']
        #for key in self.label_name:
        #    all_label[key] = [batch[key] for batch in new_label_list]

        self.data = [mx.nd.array(all_data[key]) for key in self.data_name]
        self.label = [mx.nd.array(all_label[key]) for key in self.label_name]
Exemple #24
0
    def fit(self,
            X,
            y=None,
            eval_data=None,
            eval_metric='acc',
            epoch_end_callback=None,
            batch_end_callback=None,
            kvstore='local',
            logger=None,
            work_load_list=None,
            monitor=None,
            eval_batch_end_callback=None):

        train_data = self._init_iter(X, y, is_train=True)
        eval_data = self._init_eval_iter(eval_data)

        # prepare sym
        if self.sym_gen:
            self.symbol = self.sym_gen(train_data.default_bucket_key)  # pylint: disable=no-member
            self._check_arguments()
        self.kwargs["sym"] = self.symbol

        # prepare args name and shape
        arg_names, param_names, aux_names = \
                self._init_params(dict(train_data.provide_data+train_data.provide_label))

        # setup metric
        if not isinstance(eval_metric, metric.EvalMetric):
            eval_metric = metric.create(eval_metric)

        # create kvstore
        (kvstore, update_on_kvstore) = _create_kvstore(kvstore, len(self.ctx),
                                                       self.arg_params)

        # param_idx2name = {}
        # if update_on_kvstore:
        #     param_idx2name.update(enumerate(param_names))
        # else:
        #     for i, n in enumerate(param_names):
        #         for k in range(len(self.ctx)):
        #             param_idx2name[i*len(self.ctx)+k] = n
        # self.kwargs["param_idx2name"] = param_idx2name

        # init optmizer
        if isinstance(self.optimizer, str):
            batch_size = train_data.batch_size
            if kvstore and kvstore.type == 'dist_sync':
                batch_size *= kvstore.num_workers
            optimizer = opt.create(self.optimizer,
                                   rescale_grad=(1.0 / batch_size),
                                   **(self.kwargs))
        elif isinstance(self.optimizer, opt.Optimizer):
            optimizer = self.optimizer

        num_device = len(self.ctx)
        logger.info('Start training with %s', str(self.ctx))

        if work_load_list is None:
            work_load_list = [1] * num_device
        assert isinstance(work_load_list, list) and len(work_load_list) == num_device, \
            "Invalid settings for work load. "

        from mxnet.executor_manager import _split_input_slice
        self.slices = _split_input_slice(train_data.batch_size, work_load_list)

        execgrp = ExecuterGroup(self.symbol, arg_names, param_names, self.ctx,
                                self.slices, train_data)

        if self.sym_gen is not None:
            self.execgrp_bucket = {train_data.default_bucket_key: self.execgrp}

        # executor_manager.set_params(arg_params, aux_params)

        train_data.reset()
        for epoch in range(self.begin_epoch, self.num_epoch):
            # Training phase

            tic = time.time()
            eval_metric.reset()
            nbatch = 0
            # Iterate over training data.

            #Into Epoch
            #########################
            #record acc
            acc_hist = []

            while True:
                do_reset = True
                for data_batch in train_data:
                    executor_manager.load_data_batch(data_batch)

                    if monitor is not None:
                        monitor.tic()

                    executor_manager.forward(is_train=True)
                    executor_manager.backward()

                    if update_on_kvstore:
                        _update_params_on_kvstore(
                            executor_manager.param_arrays,
                            executor_manager.grad_arrays, kvstore)
                    else:
                        _update_params(executor_manager.param_arrays,
                                       executor_manager.grad_arrays,
                                       updater=updater,
                                       num_device=len(ctx),
                                       kvstore=kvstore)

                    if monitor is not None:
                        monitor.toc_print()

                    eval_metric.reset()
                    executor_manager.update_metric(eval_metric,
                                                   data_batch.label)
                    name_value = eval_metric.get_name_value()

                    for name, value in name_value:
                        acc_hist.append(value)
                        # logger.info('Epoch[%d] Training-%s=%f', epoch, name, value)

                    nbatch += 1
                    # batch callback (for print purpose)
                    if batch_end_callback != None:
                        batch_end_params = BatchEndParam(
                            epoch=epoch,
                            nbatch=nbatch,
                            eval_metric=eval_metric,
                            locals=locals())
                        if isinstance(batch_end_callback, list):
                            for call in batch_end_callback:
                                call(batch_end_params)
                        else:
                            batch_end_callback(batch_end_params)

                    # this epoch is done possibly earlier
                    if epoch_size is not None and nbatch >= epoch_size:
                        do_reset = False
                        break

                if do_reset is True:
                    logger.debug('Epoch[%d] Resetting Data Iterator', epoch)
                    train_data.reset()
                    logger.debug('Epoch[%d] Resetting Eval Metric', epoch)
                    eval_metric.reset()

                # this epoch is done
                if epoch_size is None or nbatch >= epoch_size:
                    break

            toc = time.time()
            logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc - tic))

            # print epoch ,'<<<<<<<<<<<<'
            if epoch_end_callback or epoch + 1 == end_epoch:
                executor_manager.copy_to(arg_params, aux_params)

            if epoch_end_callback != None:
                if isinstance(epoch_end_callback, list):
                    for call in epoch_end_callback:
                        call(epoch, symbol, arg_params, aux_params, acc_hist)
                else:
                    epoch_end_callback(epoch, symbol, arg_params, aux_params,
                                       acc_hist)

            # evaluation
            # print 'enter evaluation'
            if eval_data:
                eval_metric.reset()
                eval_data.reset()
                for i, eval_batch in enumerate(eval_data):
                    executor_manager.load_data_batch(eval_batch)
                    executor_manager.forward(is_train=False)
                    executor_manager.update_metric(eval_metric,
                                                   eval_batch.label)
                    if eval_batch_end_callback != None:
                        batch_end_params = BatchEndParam(
                            epoch=epoch,
                            nbatch=i,
                            eval_metric=eval_metric,
                            locals=locals())
                        if isinstance(eval_batch_end_callback, list):
                            for call in eval_batch_end_callback:
                                call(batch_end_params)
                        else:
                            eval_batch_end_callback(batch_end_params)
                name_value = eval_metric.get_name_value()
                for name, value in name_value:
                    logger.info('E[%d] V %s:%f', epoch, name, value)
                    # print 'Epoch[%d] Validation=%f' % (epoch, value)
        # end of all epochs
        return