class TargetAssigner(object): def __init__(self, assigner_config): # some compositions self.similarity_calc = similarity_calc_builder.build( assigner_config['similarity_calc_config']) self.bbox_coder = bbox_coder_builder.build( assigner_config['coder_config']) self.bbox_coder_3d = bbox_coder_builder.build({'type': 'bbox_3d'}) self.keypoint_coder = bbox_coder_builder.build({ 'type': 'keypoint', 'size': [56, 56] }) self.matcher = matcher_builder.build(assigner_config['matcher_config']) self.analyzer = Analyzer() self.fg_thresh = assigner_config['fg_thresh'] self.bg_thresh = assigner_config['bg_thresh'] # self.clobber_positives = assigner_config['clobber_positives'] @property def stat(self): return self.analyzer.stat def assign(self, bboxes, gt_boxes, gt_boxes_3d, gt_labels=None, cls_prob=None, match=None): """ Assign each bboxes with label and bbox targets for training Args: bboxes: shape(N,K,4), encoded by xxyy gt_boxes: shape(N,M,4), encoded likes as bboxes """ # import ipdb # ipdb.set_trace() # usually IoU overlaps is used as metric bboxes = bboxes.detach() match_quality_matrix = self.similarity_calc.compare_batch( bboxes, gt_boxes) # match 0.7 for truly recall calculation # if self.fg_thresh < 0.7: fake_match = self.matcher.match_batch(match_quality_matrix, 0.7) self.analyzer.analyze(fake_match, gt_boxes.shape[1]) # match # shape(N,K) match = self.matcher.match_batch(match_quality_matrix, self.fg_thresh) assigned_overlaps_batch = self.matcher.assigned_overlaps_batch # self.analyzer.analyze(match, gt_boxes.shape[1]) # else: # # fake data # assigned_overlaps_batch = torch.zeros_like(match).float() # true_match = self.matcher.match_batch(match_quality_matrix, 0.7) # self.analyzer.analyze(true_match, gt_boxes.shape[1]) # get assigned infomation # shape (num_batch,num_boxes) # assign regression targets reg_targets, reg_targets_3d = self._assign_regression_targets( match, bboxes, gt_boxes, gt_boxes_3d) # assign classification targets cls_targets = self._assign_classification_targets(match, gt_labels) # create regression weights reg_weights, reg_weights_3d = self._create_regression_weights( assigned_overlaps_batch) # create classification weights cls_weights = self._create_classification_weights( assigned_overlaps_batch) #################################### # postprocess #################################### # match == -1 means unmatched reg_targets[match == -1] = 0 cls_targets[match == -1] = 0 reg_weights[match == -1] = 0 reg_weights_3d[match == -1] = 0 # as for cls weights, ignore according to bg_thresh if self.bg_thresh > 0: ignored_bg = (assigned_overlaps_batch > self.bg_thresh) & (match == -1) cls_weights[ignored_bg] = 0 return cls_targets, reg_targets, cls_weights, reg_weights, reg_targets_3d, reg_weights_3d def _create_regression_weights(self, assigned_overlaps_batch): """ Args: assigned_overlaps_batch: shape (num_batch,num_boxes) Returns: reg_weights: shape(num_batch,num_boxes,4) """ # gamma = 2 # return torch.pow(1 - assigned_overlaps_batch, gamma).detach() return torch.ones_like(assigned_overlaps_batch), torch.ones_like( assigned_overlaps_batch) def _create_classification_weights(self, assigned_overlaps_batch): """ All samples can be used for calculating loss,So reserve all. """ cls_weights = torch.ones_like(assigned_overlaps_batch) return cls_weights def _assign_regression_targets(self, match, bboxes, gt_boxes, gt_boxes_3d): """ Args: match: Tensor(num_batch,num_boxes) gt_boxes: Tensor(num_batch,num_gt_boxes,4) Returns: reg_targets: Tensor(num_batch,num_boxes,4) """ # shape(num_batch,num_boxes,4) batch_size = gt_boxes.shape[0] offset = torch.arange(0, batch_size) * gt_boxes.size(1) match += offset.view(batch_size, 1).type_as(match) assigned_gt_boxes = gt_boxes.view(-1, 4)[match.view(-1)].view( batch_size, -1, 4) num_cols = gt_boxes_3d.shape[-1] assigned_gt_boxes_3d = gt_boxes_3d.view(-1, num_cols)[match.view(-1)].view( batch_size, -1, num_cols) reg_targets_batch = self.bbox_coder.encode_batch( bboxes, assigned_gt_boxes) reg_targets_batch_3d = self.bbox_coder_3d.encode_batch_bbox( assigned_gt_boxes_3d[0], assigned_gt_boxes[0]) # import ipdb # ipdb.set_trace() # no batch format keypoints = assigned_gt_boxes_3d[0][:, 3:].view(-1, 4, 2) keypoint_heatmap = self.keypoint_coder.encode_keypoint_heatmap( bboxes[0], keypoints) num_rois = reg_targets_batch_3d.shape[0] reg_targets_batch_3d = torch.cat( [reg_targets_batch_3d[:, :3], keypoint_heatmap.view(num_rois, -1)], dim=-1) # no need grad_fn return reg_targets_batch, reg_targets_batch_3d.unsqueeze(0) def _assign_classification_targets(self, match, gt_labels): """ Just return the countpart labels Note that use zero to represent background labels For the first stage, generate binary labels, For the second stage generate countpart gt_labels """ # binary labels classifcation if gt_labels is None: # consider it as binary classification problem return self._generate_binary_labels(match) # multiple labels classification batch_size = match.shape[0] offset = torch.arange(0, batch_size) * gt_labels.size(1) match += offset.view(batch_size, 1).type_as(match) cls_targets_batch = gt_labels.view(-1)[match.view(-1)].view( batch_size, match.shape[1]) return cls_targets_batch def _generate_binary_labels(self, match): gt_labels_batch = torch.ones_like(match).long() return gt_labels_batch
class TargetAssigner(object): def __init__(self, assigner_config): # some compositions self.similarity_calc = similarity_calc_builder.build( assigner_config['similarity_calc_config']) self.bbox_coder = bbox_coder_builder.build( assigner_config['coder_config']) self.matcher = matcher_builder.build(assigner_config['matcher_config']) self.analyzer = Analyzer() self.fg_thresh = assigner_config['fg_thresh'] self.bg_thresh = assigner_config['bg_thresh'] # self.clobber_positives = assigner_config['clobber_positives'] @property def stat(self): return self.analyzer.stat def assign(self, bboxes, gt_boxes, voxel_centers, gt_boxes_3d, gt_labels=None): """ Assign each bboxes with label and bbox targets for training Args: bboxes: shape(N,K,4), encoded by xxyy gt_boxes: shape(N,M,4), encoded likes as bboxes """ # usually IoU overlaps is used as metric bboxes = bboxes.detach() match_quality_matrix = self.similarity_calc.compare_batch( bboxes, gt_boxes) # match 0.7 for truly recall calculation # if self.fg_thresh < 0.7: fake_match = self.matcher.match_batch(match_quality_matrix, 0.7) self.analyzer.analyze(fake_match, gt_boxes.shape[1]) match = self.matcher.match_batch(match_quality_matrix, self.fg_thresh) assigned_overlaps_batch = self.matcher.assigned_overlaps_batch # assign regression targets reg_targets = self._assign_regression_targets(match, voxel_centers, gt_boxes_3d) # assign classification targets cls_targets = self._assign_classification_targets( voxel_centers, gt_boxes_3d) # create regression weights reg_weights = self._create_regression_weights(assigned_overlaps_batch) # create classification weights cls_weights = self._create_classification_weights(cls_targets) #################################### # postprocess #################################### # match == -1 means unmatched reg_targets[match == -1] = 0 cls_targets[match == -1] = 0 reg_weights[match == -1] = 0 # as for cls weights, ignore according to bg_thresh # if self.bg_thresh > 0: ignored_bg = (assigned_overlaps_batch > self.bg_thresh) & (match == -1) cls_weights[ignored_bg] = 0 return cls_weights, reg_weights, cls_targets, reg_targets def _create_regression_weights(self, assigned_overlaps_batch): """ Args: assigned_overlaps_batch: shape (num_batch,num_boxes) Returns: reg_weights: shape(num_batch,num_boxes,4) """ # gamma = 2 # return torch.pow(1 - assigned_overlaps_batch, gamma).detach() return torch.ones_like(assigned_overlaps_batch) def _create_classification_weights(self, scores_map): """ All samples can be used for calculating loss,So reserve all. """ cls_weights = torch.ones_like(scores_map) cls_weights[scores_map < 5e-2] = 1e-2 return cls_weights def _assign_regression_targets(self, match, voxel_centers, gt_boxes_3d): """ Args: match: Tensor(num_batch,num_boxes) gt_boxes: Tensor(num_batch,num_gt_boxes,4) Returns: reg_targets: Tensor(num_batch,num_boxes,4) """ # shape(num_batch,num_boxes,4) batch_size = gt_boxes_3d.shape[0] offset = torch.arange(0, batch_size) * gt_boxes_3d.size(1) match += offset.view(batch_size, 1).type_as(match) num_cols = gt_boxes_3d.shape[-1] assigned_gt_boxes_3d = gt_boxes_3d.view(-1, num_cols)[match.view(-1)].view( batch_size, -1, num_cols) reg_targets_batch_3d = self.bbox_coder.encode_batch_bbox( voxel_centers, assigned_gt_boxes_3d) # import ipdb # ipdb.set_trace() angle_box_4c = self.bbox_coder.encode_batch_angle_box_4c( assigned_gt_boxes_3d) reg_targets_batch_3d = torch.cat( [reg_targets_batch_3d[:, :, :6], angle_box_4c], dim=-1) return reg_targets_batch_3d def _assign_classification_targets(self, voxel_centers, gt_boxes_3d): return self.bbox_coder.encode_batch_labels(voxel_centers, gt_boxes_3d) # def _assign_classification_targets(self, match, gt_labels): # """ # Just return the countpart labels # Note that use zero to represent background labels # For the first stage, generate binary labels, For the second stage # generate countpart gt_labels # """ # # binary labels classifcation # if gt_labels is None: # # consider it as binary classification problem # return self._generate_binary_labels(match) # # multiple labels classification # batch_size = match.shape[0] # offset = torch.arange(0, batch_size) * gt_labels.size(1) # match += offset.view(batch_size, 1).type_as(match) # cls_targets_batch = gt_labels.view(-1)[match.view(-1)].view( # batch_size, match.shape[1]) # return cls_targets_batch def _generate_binary_labels(self, match): gt_labels_batch = torch.ones_like(match).long() return gt_labels_batch
class TargetAssigner(object): def __init__(self, assigner_config): # some compositions self.similarity_calc = similarity_calc_builder.build( assigner_config['similarity_calc_config']) self.bbox_coder = bbox_coder_builder.build( assigner_config['coder_config']) self.matcher = matcher_builder.build(assigner_config['matcher_config']) self.analyzer = Analyzer() # cls thresh self.fg_thresh_cls = assigner_config['fg_thresh_cls'] self.bg_thresh_cls = assigner_config['bg_thresh_cls'] # bbox thresh self.fg_thresh_reg = assigner_config['fg_thresh_reg'] self.bg_thresh_reg = assigner_config['bg_thresh_reg'] if assigner_config.get('fake_match_thresh') is not None: self.fake_match_thresh = assigner_config['fake_match_thresh'] else: # default value self.fake_match_thresh = 0.7 @property def stat(self): return self.analyzer.stat def assign(self, bboxes, gt_boxes, new_anchor, gt_labels=None, cls_prob=None, ret_iou=False): """ Assign each bboxes with label and bbox targets for training Args: bboxes: shape(N,K,4), encoded by xxyy gt_boxes: shape(N,M,4), encoded likes as bboxes """ # usually IoU overlaps is used as metric bboxes = bboxes.detach() new_anchor = new_anchor.detach() match_quality_matrix = self.similarity_calc.compare_batch( bboxes, gt_boxes) # match 0.7 for truly recall calculation # if self.fg_thresh_cls < 0.7: fake_match = self.matcher.match_batch(match_quality_matrix, self.fake_match_thresh) stats = self.analyzer.analyze(fake_match, gt_boxes.shape[1]) ################################# # handle cls ################################# cls_match = self.matcher.match_batch(match_quality_matrix, self.fg_thresh_cls) cls_assigned_overlaps_batch = self.matcher.assigned_overlaps_batch stats['iou'] = cls_assigned_overlaps_batch # assign classification targets cls_targets = self._assign_classification_targets(cls_match, gt_labels) # create classification weights cls_weights = self._create_classification_weights( cls_assigned_overlaps_batch) cls_targets[cls_match == -1] = 0 # as for cls weights, ignore according to bg_thresh if self.bg_thresh_cls > 0: ignored_bg = (cls_assigned_overlaps_batch > self.bg_thresh_cls) & (cls_match == -1) cls_weights[ignored_bg] = 0 ################################## # handle reg ################################## reg_match = self.matcher.match_batch(match_quality_matrix, self.fg_thresh_reg) reg_assigned_overlaps_batch = self.matcher.assigned_overlaps_batch # assign regression targets reg_targets = self._assign_regression_targets(reg_match, new_anchor, gt_boxes) # create regression weights reg_weights = self._create_regression_weights( reg_assigned_overlaps_batch) reg_targets[reg_match == -1] = 0 reg_weights[reg_match == -1] = 0 ret = [cls_targets, reg_targets, cls_weights, reg_weights, stats] if ret_iou: ret.append(match_quality_matrix) return ret def _create_regression_weights(self, assigned_overlaps_batch): """ Args: assigned_overlaps_batch: shape (num_batch,num_boxes) Returns: reg_weights: shape(num_batch,num_boxes,4) """ # gamma = 2 # return torch.pow(1 - assigned_overlaps_batch, gamma).detach() return torch.ones_like(assigned_overlaps_batch) def _create_classification_weights(self, assigned_overlaps_batch): """ All samples can be used for calculating loss,So reserve all. """ cls_weights = torch.ones_like(assigned_overlaps_batch) return cls_weights def _assign_regression_targets(self, match, bboxes, gt_boxes): """ Args: match: Tensor(num_batch,num_boxes) gt_boxes: Tensor(num_batch,num_gt_boxes,4) Returns: reg_targets: Tensor(num_batch,num_boxes,4) """ # shape(num_batch,num_boxes,4) batch_size = gt_boxes.shape[0] offset = torch.arange(0, batch_size) * gt_boxes.size(1) match += offset.view(batch_size, 1).type_as(match) assigned_gt_boxes = gt_boxes.view(-1, 4)[match.view(-1)].view( batch_size, -1, 4) reg_targets_batch = self.bbox_coder.encode_batch( bboxes, assigned_gt_boxes) # no need grad_fn return reg_targets_batch def _assign_classification_targets(self, match, gt_labels): """ Just return the countpart labels Note that use zero to represent background labels For the first stage, generate binary labels, For the second stage generate countpart gt_labels """ # binary labels classifcation if gt_labels is None: # consider it as binary classification problem return self._generate_binary_labels(match) # multiple labels classification batch_size = match.shape[0] offset = torch.arange(0, batch_size) * gt_labels.size(1) match += offset.view(batch_size, 1).type_as(match) cls_targets_batch = gt_labels.view(-1)[match.view(-1)].view( batch_size, match.shape[1]) return cls_targets_batch def _generate_binary_labels(self, match): gt_labels_batch = torch.ones_like(match).long() return gt_labels_batch
class NewTargetAssigner(object): def __init__(self, assigner_config): # some compositions self.reg_similarity_calc = similarity_calc_builder.build( assigner_config['similarity_calc_config']) # make sure match boxes when they are enough close for classification self.cls_similarity_calc = CenterSimilarityCalc() # make sure more bbox can be optimized # self.reg_similarity_calc = ScaleSimilarityCalc() self.bbox_coder = bbox_coder_builder.build( assigner_config['coder_config']) self.matcher = matcher_builder.build(assigner_config['matcher_config']) self.analyzer = Analyzer() self.fg_thresh = assigner_config['fg_thresh'] self.bg_thresh = assigner_config['bg_thresh'] # self.clobber_positives = assigner_config['clobber_positives'] @property def stat(self): return self.analyzer.stat def assign(self, bboxes, gt_boxes, gt_labels=None, cls_prob=None, window=None): """ match policy depends on reg_match_matrix cls targets depends on cls_match_quality_matrix Args: bboxes: shape(N,K,4), encoded by xxyy gt_boxes: shape(N,M,4), encoded likes as bboxes """ # import ipdb # ipdb.set_trace() # usually IoU overlaps is used as metric import ipdb ipdb.set_trace() bboxes = bboxes.detach() # preprocess bboxes keep = box_ops.window_filter(bboxes.view(-1, 4), window) # just use IoU here # shape(N,K,M) cls_match_quality_matrix = self.cls_similarity_calc.compare_batch( bboxes, gt_boxes) reg_match_quality_matrix = self.reg_similarity_calc.compare_batch( bboxes, gt_boxes) # shape(N,K) match = self.matcher.match_batch(reg_match_quality_matrix, self.fg_thresh) match.view(-1)[~keep] = -1 # some statistics about result of match self.analyzer.analyze(match, gt_boxes.shape[1]) # get assigned infomation # shape (num_batch,num_boxes) # assigned_overlaps_batch = self.matcher.assigned_overlaps_batch num = match.numel() M = cls_match_quality_matrix.shape[2] row = torch.arange(0, num).type_as(match) assigned_overlaps_batch = cls_match_quality_matrix.view( -1, M)[row, match.view(-1)].view_as(match) ####################### # reg ####################### # assign regression targets reg_targets = self._assign_regression_targets(match, bboxes, gt_boxes) # create regression weights reg_weights = self._create_regression_weights(assigned_overlaps_batch) ####################### # cls ####################### # create classification weights # TODO(which criterion to use for calculate cls weights,cls_match or # reg_match) cls_weights = self._create_classification_weights( assigned_overlaps_batch) # assign classification targets cls_targets = self._assign_classification_targets( match, gt_labels, cls_match_quality_matrix) #################################### # postprocess #################################### # match == -1 means unmatched reg_targets[match == -1] = 0 cls_targets[match == -1] = 0 reg_weights[match == -1] = 0 # as for cls weights, ignore according to bg_thresh if self.bg_thresh > 0: ignored_bg = (assigned_overlaps_batch > self.bg_thresh) & (match == -1) cls_weights[ignored_bg] = 0 return cls_targets, reg_targets, cls_weights, reg_weights def _create_regression_weights(self, assigned_overlaps_batch): """ Args: assigned_overlaps_batch: shape (num_batch,num_boxes) Returns: reg_weights: shape(num_batch,num_boxes,4) """ # gamma = 2 # return torch.pow(1 - assigned_overlaps_batch, gamma).detach() # return torch.ones_like(assigned_overlaps_batch) # return 10 * (F.sigmoid(assigned_overlaps_batch) - 0.5) return torch.ones_like(assigned_overlaps_batch) def _create_classification_weights(self, assigned_overlaps_batch): """ All samples can be used for calculating loss,So reserve all. """ cls_weights = torch.ones_like(assigned_overlaps_batch) return cls_weights # return torch.exp(2 * assigned_overlaps_batch) def _assign_regression_targets(self, match, bboxes, gt_boxes): """ Args: match: Tensor(num_batch,num_boxes) gt_boxes: Tensor(num_batch,num_gt_boxes,4) Returns: reg_targets: Tensor(num_batch,num_boxes,4) """ # shape(num_batch,num_boxes,4) batch_size = gt_boxes.shape[0] offset = torch.arange(0, batch_size) * gt_boxes.size(1) match += offset.view(batch_size, 1).type_as(match) assigned_gt_boxes = gt_boxes.view(-1, 4)[match.view(-1)].view( batch_size, -1, 4) reg_targets_batch = self.bbox_coder.encode_batch( bboxes, assigned_gt_boxes) # no need grad_fn return reg_targets_batch def _assign_classification_targets(self, match, gt_labels, match_quality_matrix): """ Just return the countpart labels Note that use zero to represent background labels For the first stage, generate binary labels, For the second stage generate countpart gt_labels """ # binary labels classifcation # if gt_labels is None: # consider it as binary classification problem return self._generate_binary_labels(match, match_quality_matrix) # multiple labels classification # TODO(as for multiple cls ,match_quality_matrix should also be # considered) # batch_size = match.shape[0] # offset = torch.arange(0, batch_size) * gt_labels.size(1) # match += offset.view(batch_size, 1).type_as(match) # cls_targets_batch = gt_labels.view(-1)[match.view(-1)].view( # batch_size, match.shape[1]) # return cls_targets_batch def _generate_binary_labels(self, match, match_quality_matrix): """ Select cls_target from matrix according to match Args: match: shape(N,K) match_quality_matrix: shape(N,K,M) """ num = match.numel() row = torch.arange(0, num).type_as(match) M = match_quality_matrix.shape[2] cls_targets_batch = match_quality_matrix.view( -1, M)[row, match.view(-1)].view_as(match) return cls_targets_batch
class LEDTargetAssigner(object): def __init__(self, assigner_config): # some compositions self.similarity_calc = similarity_calc_builder.build( assigner_config['similarity_calc_config']) self.bbox_coder = bbox_coder_builder.build( assigner_config['coder_config']) self.matcher = matcher_builder.build(assigner_config['matcher_config']) self.analyzer = Analyzer() self.fg_thresh = assigner_config['fg_thresh'] self.bg_thresh = assigner_config['bg_thresh'] # self.clobber_positives = assigner_config['clobber_positives'] self.iog_similarity_calc = similarity_calc_builder.build( {'type': 'iog'}) self.iod_similarity_calc = similarity_calc_builder.build( {'type': 'iod'}) @property def stat(self): return self.analyzer.stat def assign(self, bboxes, gt_boxes, gt_labels=None, cls_prob=None, input_size=None): """ Assign each bboxes with label and bbox targets for training Args: bboxes: shape(N,K,4), encoded by xxyy gt_boxes: shape(N,M,4), encoded likes as bboxes """ # import ipdb # ipdb.set_trace() # usually IoU overlaps is used as metric bboxes = bboxes.detach() if input_size is not None: # clip bbox # import ipdb # ipdb.set_trace() bboxes = box_ops.clip_boxes(bboxes, input_size) match_quality_matrix = self.similarity_calc.compare_batch( bboxes, gt_boxes) iog_match_quality_matrix = self.iog_similarity_calc.compare_batch( bboxes, gt_boxes) iod_match_quality_matrix = self.iod_similarity_calc.compare_batch( bboxes, gt_boxes) self.matcher.iog_match_quality_matrix_batch = iog_match_quality_matrix self.matcher.iod_match_quality_matrix_batch = iod_match_quality_matrix # nonsuppress_match = self.matcher.match_batch(match_quality_matrix, 0) # self.assigned_iog_batch = self._assign_iox(iog_match_quality_matrix, # nonsuppress_match) # self.assigned_iod_batch = self._assign_iox(iod_match_quality_matrix, # nonsuppress_match) # match # shape(N,K) # used for being matched for each one match = self.matcher.match_batch(match_quality_matrix, self.fg_thresh) self.analyzer.analyze(match, gt_boxes.shape[1]) # get assigned infomation # shape (num_batch,num_boxes) assigned_overlaps_batch = self.matcher.assigned_overlaps_batch # assign regression targets reg_targets = self._assign_regression_targets(match, bboxes, gt_boxes) # assign classification targets cls_targets = self._assign_classification_targets(match, gt_labels) # cls_targets = assigned_overlaps_batch # create regression weights reg_weights = self._create_regression_weights(assigned_overlaps_batch) # create classification weights cls_weights = self._create_classification_weights( assigned_overlaps_batch) #################################### # postprocess #################################### # match == -1 means unmatched reg_targets[match == -1] = 0 # cls_targets[match == -1] = 0 reg_weights[match == -1] = 0 # as for cls weights, ignore according to bg_thresh # if self.bg_thresh > 0: # ignored_bg = (assigned_overlaps_batch > self.bg_thresh) & ( # match == -1) # cls_weights[ignored_bg] = 0 return cls_targets, reg_targets, cls_weights, reg_weights def _assign_iox(self): pass def _create_regression_weights(self, assigned_overlaps_batch): """ Args: assigned_overlaps_batch: shape (num_batch,num_boxes) Returns: reg_weights: shape(num_batch,num_boxes,4) """ # gamma = 2 # return torch.pow(1 - assigned_overlaps_batch, gamma).detach() return torch.ones_like(assigned_overlaps_batch) def _create_classification_weights(self, assigned_overlaps_batch): """ All samples can be used for calculating loss,So reserve all. """ cls_weights = torch.ones_like(assigned_overlaps_batch) return cls_weights def _assign_regression_targets(self, match, bboxes, gt_boxes): """ Args: match: Tensor(num_batch,num_boxes) gt_boxes: Tensor(num_batch,num_gt_boxes,4) Returns: reg_targets: Tensor(num_batch,num_boxes,4) """ # shape(num_batch,num_boxes,4) batch_size = gt_boxes.shape[0] offset = torch.arange(0, batch_size) * gt_boxes.size(1) match += offset.view(batch_size, 1).type_as(match) assigned_gt_boxes = gt_boxes.view(-1, 4)[match.view(-1)].view( batch_size, -1, 4) reg_targets_batch = self.bbox_coder.encode_batch( bboxes, assigned_gt_boxes) # no need grad_fn return reg_targets_batch def _assign_classification_targets(self, match, gt_labels): """ Just return the countpart labels Note that use zero to represent background labels For the first stage, generate binary labels, For the second stage generate countpart gt_labels """ # binary labels classifcation if gt_labels is None: # consider it as binary classification problem return self._generate_binary_labels(match) # multiple labels classification batch_size = match.shape[0] offset = torch.arange(0, batch_size) * gt_labels.size(1) match += offset.view(batch_size, 1).type_as(match) cls_targets_batch = gt_labels.view(-1)[match.view(-1)].view( batch_size, match.shape[1]) return cls_targets_batch def _generate_binary_labels(self, match): gt_labels_batch = torch.ones_like(match).long() return gt_labels_batch
class TargetAssigner(object): def __init__(self, assigner_config): # some compositions self.similarity_calc = similarity_calc_builder.build( assigner_config['similarity_calc_config']) self.bbox_coder = bbox_coder_builder.build( assigner_config['coder_config']) self.matcher = matcher_builder.build(assigner_config['matcher_config']) self.analyzer = Analyzer() # cls thresh self.fg_thresh_cls = assigner_config['fg_thresh_cls'] self.bg_thresh_cls = assigner_config['bg_thresh_cls'] # bbox thresh self.fg_thresh_reg = assigner_config['fg_thresh_reg'] # self.bg_thresh_reg = assigner_config['bg_thresh_reg'] @property def stat(self): return self.analyzer.stat def assign(self, bboxes, gt_boxes, gt_labels=None, cls_prob=None): """ Assign each bboxes with label and bbox targets for training Args: bboxes: shape(N,K,4), encoded by xxyy gt_boxes: shape(N,M,4), encoded likes as bboxes """ # usually IoU overlaps is used as metric bboxes = bboxes.detach() match_quality_matrix = self.similarity_calc.compare_batch( bboxes, gt_boxes) # match 0.7 for truly recall calculation # if self.fg_thresh_cls < 0.7: fake_match = self.matcher.match_batch(match_quality_matrix, 0.7) self.analyzer.analyze(fake_match, gt_boxes.shape[1]) ################################# # handle cls ################################# # cls_match = self.matcher.match_batch(match_quality_matrix, # self.fg_thresh_cls) # cls_assigned_overlaps_batch = self.matcher.assigned_overlaps_batch # assign classification targets # cls_targets = self._assign_classification_targets(cls_match, gt_labels) # create classification weights # cls_weights = self._create_classification_weights( # cls_assigned_overlaps_batch) # cls_targets[cls_match == -1] = 0 # as for cls weights, ignore according to bg_thresh # if self.bg_thresh_cls > 0: # ignored_bg = (cls_assigned_overlaps_batch > self.bg_thresh_cls) & ( # cls_match == -1) # cls_weights[ignored_bg] = 0 ################################## # handle reg ################################## reg_match = self.matcher.match_batch(match_quality_matrix, self.fg_thresh_reg) reg_assigned_overlaps_batch = self.matcher.assigned_overlaps_batch # assign regression targets reg_targets, inter_boxes = self._assign_regression_targets( reg_match, bboxes, gt_boxes) # create regression weights reg_weights = self._create_regression_weights( reg_assigned_overlaps_batch) reg_targets[reg_match == -1] = 0 reg_weights[reg_match == -1] = 0 cls_targets = self._assign_classification_targets( reg_match, gt_labels, inter_boxes, bboxes) # all is ones cls_weights = self._create_classification_weights( reg_assigned_overlaps_batch) return cls_targets, reg_targets, cls_weights, reg_weights def _create_regression_weights(self, assigned_overlaps_batch): """ Args: assigned_overlaps_batch: shape (num_batch,num_boxes) Returns: reg_weights: shape(num_batch,num_boxes,4) """ # gamma = 2 # return torch.pow(1 - assigned_overlaps_batch, gamma).detach() return torch.ones_like(assigned_overlaps_batch) def _create_classification_weights(self, assigned_overlaps_batch): """ All samples can be used for calculating loss,So reserve all. """ cls_weights = torch.ones_like(assigned_overlaps_batch) return cls_weights def _assign_regression_targets(self, match, bboxes, gt_boxes): """ Args: match: Tensor(num_batch,num_boxes) gt_boxes: Tensor(num_batch,num_gt_boxes,4) Returns: reg_targets: Tensor(num_batch,num_boxes,4) """ # shape(num_batch,num_boxes,4) batch_size = gt_boxes.shape[0] offset = torch.arange(0, batch_size) * gt_boxes.size(1) match += offset.view(batch_size, 1).type_as(match) assigned_gt_boxes = gt_boxes.view(-1, 4)[match.view(-1)].view( batch_size, -1, 4) reg_targets_batch = self.bbox_coder.encode_batch( bboxes, assigned_gt_boxes) inter_boxes = box_ops.intersection(bboxes, assigned_gt_boxes) # no need grad_fn return reg_targets_batch, inter_boxes def _assign_classification_targets(self, match, gt_labels, inter_boxes, proposals): """ Generate cls map for segmentation loss Args: pass Returns: cls_map_targets """ # hard code pooling_size = 8 # get label for each bbox batch_size = match.shape[0] offset = torch.arange(0, batch_size) * gt_labels.size(1) match += offset.view(batch_size, 1).type_as(match) cls_targets_batch = gt_labels.view(-1)[match.view(-1)].view( batch_size, match.shape[1]) # set bg cls_targets_batch[match == -1] = 0 # generate map according to label for segmentation loss h = proposals[:, :, 3] - proposals[:, :, 1] w = proposals[:, :, 2] - proposals[:, :, 0] sub_bin_w = w / pooling_size sub_bin_h = h / pooling_size # pass offset_xmin = inter_boxes[:, :, 0] - proposals[:, :, 0] offset_xmax = inter_boxes[:, :, 2] - proposals[:, :, 0] offset_ymin = inter_boxes[:, :, 1] - proposals[:, :, 1] offset_ymax = inter_boxes[:, :, 3] - proposals[:, :, 1] offset_xmin_ind = (offset_xmin / sub_bin_w).round().int() offset_ymin_ind = (offset_ymin / sub_bin_h).round().int() offset_xmax_ind = (offset_xmax / sub_bin_w).round().int() offset_ymax_ind = (offset_ymax / sub_bin_h).round().int() # select not empty bbox from inter boxes # cond = (inter_boxes[:, :, 2] - inter_boxes[:, :, 0] + 1 > 0) & ( # inter_boxes[:, :, 3] - inter_boxes[:, :, 1] + 1 > 0) # import ipdb # ipdb.set_trace() num = pooling_size * pooling_size offset_xmin_ind = offset_xmin_ind.unsqueeze(-1).expand(-1, -1, num) offset_ymin_ind = offset_ymin_ind.unsqueeze(-1).expand(-1, -1, num) offset_xmax_ind = offset_xmax_ind.unsqueeze(-1).expand(-1, -1, num) offset_ymax_ind = offset_ymax_ind.unsqueeze(-1).expand(-1, -1, num) x = torch.range(0, pooling_size - 1).type_as(offset_xmin_ind) y = torch.range(0, pooling_size - 1).type_as(offset_xmin_ind) xx, yy = ops.meshgrid(x, y) coord = torch.stack([xx, yy], dim=-1) coord = coord.expand(inter_boxes.shape[0], inter_boxes.shape[1], -1, -1) # shape(N,M,49) cond = (coord[:, :, :, 0] < offset_xmax_ind) & (coord[:, :, :, 0] >= offset_xmin_ind) & ( coord[:, :, :, 1] < offset_ymax_ind) & (coord[:, :, :, 1] >= offset_ymin_ind) # torch.nonzero(cond) # cls_gate_map shape(N,M,49) # cond = cond.view(cond.shape[0], cond.shape[1], pooling_size, # pooling_size) cls_gate_map = cond.int() # reverse when bg # cls_gate_map[cls_targets_batch == 0] = ( # 1 - cls_gate_map)[cls_targets_batch == 0] return cls_gate_map.long() def _generate_binary_labels(self, match): gt_labels_batch = torch.ones_like(match).long() return gt_labels_batch
class RefineTargetAssigner(object): def __init__(self, assigner_config): # some compositions self.similarity_calc = similarity_calc_builder.build( assigner_config['similarity_calc_config']) self.bbox_coder = bbox_coder_builder.build( assigner_config['coder_config']) self.matcher = matcher_builder.build(assigner_config['matcher_config']) self.analyzer = Analyzer() self.fg_thresh = assigner_config['fg_thresh'] self.bg_thresh = assigner_config['bg_thresh'] # self.clobber_positives = assigner_config['clobber_positives'] @property def stat(self): return self.analyzer.stat def assign(self, bboxes, gt_boxes, gt_labels=None, cls_prob=None): """ Assign each bboxes with label and bbox targets for training Args: bboxes: shape(N,K,4), encoded by xxyy gt_boxes: shape(N,M,4), encoded likes as bboxes """ # import ipdb # ipdb.set_trace() # usually IoU overlaps is used as metric bboxes = bboxes.detach() match_quality_matrix = self.similarity_calc.compare_batch( bboxes, gt_boxes) # match # shape(N,K) match = self.matcher.match_batch(match_quality_matrix, self.fg_thresh) self.analyzer.analyze(match, gt_boxes.shape[1]) # get assigned infomation # shape (num_batch,num_boxes) assigned_overlaps_batch = self.matcher.assigned_overlaps_batch # assign regression targets reg_targets = self._assign_regression_targets(match, bboxes, gt_boxes) # assign classification targets # cls_targets = self._assign_classification_targets(match, gt_labels, # assigned_overlaps_batch) cls_targets = assigned_overlaps_batch.clone() # create regression weights reg_weights = self._create_regression_weights(assigned_overlaps_batch) # create classification weights cls_weights = self._create_classification_weights( assigned_overlaps_batch) #################################### # postprocess #################################### # match == -1 means unmatched reg_targets[match == -1] = 0 cls_targets[match == -1] = 0 reg_weights[match == -1] = 0 # as for cls weights, ignore according to bg_thresh if self.bg_thresh > 0: ignored_bg = (assigned_overlaps_batch > self.bg_thresh) & (match == -1) cls_weights[ignored_bg] = 0 return cls_targets, reg_targets, cls_weights, reg_weights def _create_regression_weights(self, assigned_overlaps_batch): """ Args: assigned_overlaps_batch: shape (num_batch,num_boxes) Returns: reg_weights: shape(num_batch,num_boxes,4) """ # gamma = 2 # return torch.pow(1 - assigned_overlaps_batch, gamma).detach() # return torch.ones_like(assigned_overlaps_batch return assigned_overlaps_batch.clone() def _create_classification_weights(self, assigned_overlaps_batch): """ All samples can be used for calculating loss,So reserve all. """ # cls_weights = torch.ones_like(assigned_overlaps_batch) # return cls_weights return assigned_overlaps_batch.clone() def _assign_regression_targets(self, match, bboxes, gt_boxes): """ Args: match: Tensor(num_batch,num_boxes) gt_boxes: Tensor(num_batch,num_gt_boxes,4) Returns: reg_targets: Tensor(num_batch,num_boxes,4) """ # shape(num_batch,num_boxes,4) batch_size = gt_boxes.shape[0] offset = torch.arange(0, batch_size) * gt_boxes.size(1) match += offset.view(batch_size, 1).type_as(match) assigned_gt_boxes = gt_boxes.view(-1, 4)[match.view(-1)].view( batch_size, -1, 4) reg_targets_batch = self.bbox_coder.encode_batch( bboxes, assigned_gt_boxes) # no need grad_fn return reg_targets_batch def _assign_classification_targets(self, match, gt_labels, match_quality_matrix): """ Just return the countpart labels Note that use zero to represent background labels For the first stage, generate binary labels, For the second stage generate countpart gt_labels """ # binary labels classifcation # if gt_labels is None: # consider it as binary classification problem return self._generate_binary_labels(match, match_quality_matrix) # multiple labels classification # TODO(as for multiple cls ,match_quality_matrix should also be # considered) # batch_size = match.shape[0] # offset = torch.arange(0, batch_size) * gt_labels.size(1) # match += offset.view(batch_size, 1).type_as(match) # cls_targets_batch = gt_labels.view(-1)[match.view(-1)].view( # batch_size, match.shape[1]) # return cls_targets_batch def _generate_binary_labels(self, match, match_quality_matrix): """ Select cls_target from matrix according to match Args: match: shape(N,K) match_quality_matrix: shape(N,K,M) """ num = match.numel() row = torch.arange(0, num).type_as(match) M = match_quality_matrix.shape[2] cls_targets_batch = match_quality_matrix.view( -1, M)[row, match.view(-1)].view_as(match) return cls_targets_batch
class SemanticTargetAssigner(object): def __init__(self, assigner_config): # some compositions self.similarity_calc = similarity_calc_builder.build( assigner_config['similarity_calc_config']) self.bbox_coder = bbox_coder_builder.build( assigner_config['coder_config']) self.matcher = matcher_builder.build(assigner_config['matcher_config']) self.analyzer = Analyzer() self.fg_thresh = assigner_config['fg_thresh'] self.bg_thresh = assigner_config['bg_thresh'] # self.clobber_positives = assigner_config['clobber_positives'] @property def stat(self): return self.analyzer.stat def assign(self, bboxes, gt_boxes, gt_labels=None, cls_prob=None): """ Assign each bboxes with label and bbox targets for training Args: bboxes: shape(N,K,4), encoded by xxyy gt_boxes: shape(N,M,4), encoded likes as bboxes """ # import ipdb # ipdb.set_trace() # usually IoU overlaps is used as metric bboxes = bboxes.detach() match_quality_matrix = self.similarity_calc.compare_batch( bboxes, gt_boxes) # match # shape(N,K) match = self.matcher.match_batch(match_quality_matrix, self.fg_thresh) self.analyzer.analyze(match, gt_boxes.shape[1]) # get assigned infomation # shape (num_batch,num_boxes) assigned_overlaps_batch = self.matcher.assigned_overlaps_batch # assign regression targets reg_targets = self._assign_regression_targets(match, bboxes, gt_boxes) # assign classification targets cls_targets = self._assign_classification_targets(match, gt_labels) # create regression weights reg_weights = self._create_regression_weights(assigned_overlaps_batch) # create classification weights # cls_weights = self._create_classification_weights( # assigned_overlaps_batch) cls_weights = reg_weights.clone() #################################### # postprocess #################################### # match == -1 means unmatched reg_targets[match == -1] = 0 cls_targets[match == -1] = 0 reg_weights[match == -1] = 0 # as for cls weights, ignore according to bg_thresh if self.bg_thresh > 0: ignored_bg = (assigned_overlaps_batch > self.bg_thresh) & (match == -1) cls_weights[ignored_bg] = 0 return cls_targets, reg_targets, cls_weights, reg_weights def _create_regression_weights(self, assigned_overlaps_batch): """ Args: assigned_overlaps_batch: shape (num_batch,num_boxes) Returns: reg_weights: shape(num_batch,num_boxes,4) """ # gamma = 2 # return torch.pow(1 - assigned_overlaps_batch, gamma).detach() # import ipdb # ipdb.set_trace() # reg_weights = torch.empty_like(assigned_overlaps_batch) # sum_batch = assigned_overlaps_batch.sum() # reg_weights_0 = assigned_overlaps_batch[assigned_overlaps_batch < # 0.5].sum() / sum_batch # reg_weights_1 = assigned_overlaps_batch[( # assigned_overlaps_batch >= 0.5) & (assigned_overlaps_batch < 0.6 # )].sum() / sum_batch # reg_weights_2 = assigned_overlaps_batch[( # assigned_overlaps_batch >= 0.6) & (assigned_overlaps_batch < 0.7 # )].sum() / sum_batch # reg_weights_3 = assigned_overlaps_batch[assigned_overlaps_batch >= # 0.7].sum() / sum_batch # if reg_weights_0: # reg_weights_0 = 1 / reg_weights_0 # if reg_weights_1: # reg_weights_1 = 1 / reg_weights_1 # if reg_weights_2: # reg_weights_2 = 1 / reg_weights_2 # if reg_weights_3: # reg_weights_3 = 1 / reg_weights_3 # reg_weights.fill_(reg_weights_0) # reg_weights[assigned_overlaps_batch > 0.5] = reg_weights_1 # reg_weights[assigned_overlaps_batch > 0.6] = reg_weights_2 # reg_weights[assigned_overlaps_batch > 0.7] = reg_weights_3 # import ipdb # ipdb.set_trace() reg_weights = torch.ones_like(assigned_overlaps_batch) num_0 = torch.nonzero(assigned_overlaps_batch < 0.5).numel() num_1 = torch.nonzero((assigned_overlaps_batch >= 0.5) & (assigned_overlaps_batch < 0.6)).numel() num_2 = torch.nonzero((assigned_overlaps_batch >= 0.6) & (assigned_overlaps_batch < 0.7)).numel() num_3 = torch.nonzero(assigned_overlaps_batch >= 0.7).numel() reg_weights /= num_0 if num_1: reg_weights[assigned_overlaps_batch > 0.5] = 1 / num_1 if num_2: reg_weights[assigned_overlaps_batch > 0.6] = 1 / num_2 if num_3: reg_weights[assigned_overlaps_batch > 0.7] = 1 / num_3 return reg_weights def _create_classification_weights(self, assigned_overlaps_batch): """ All samples can be used for calculating loss,So reserve all. """ # cls_weights = torch.ones_like(assigned_overlaps_batch) # return cls_weights cls_weights = torch.ones_like(assigned_overlaps_batch) cls_weights[assigned_overlaps_batch > 0.5] = 2 cls_weights[assigned_overlaps_batch > 0.6] = 3 cls_weights[assigned_overlaps_batch > 0.7] = 4 return cls_weights def _assign_regression_targets(self, match, bboxes, gt_boxes): """ Args: match: Tensor(num_batch,num_boxes) gt_boxes: Tensor(num_batch,num_gt_boxes,4) Returns: reg_targets: Tensor(num_batch,num_boxes,4) """ # shape(num_batch,num_boxes,4) batch_size = gt_boxes.shape[0] offset = torch.arange(0, batch_size) * gt_boxes.size(1) match += offset.view(batch_size, 1).type_as(match) assigned_gt_boxes = gt_boxes.view(-1, 4)[match.view(-1)].view( batch_size, -1, 4) reg_targets_batch = self.bbox_coder.encode_batch( bboxes, assigned_gt_boxes) # no need grad_fn return reg_targets_batch def _assign_classification_targets(self, match, gt_labels): """ Just return the countpart labels Note that use zero to represent background labels For the first stage, generate binary labels, For the second stage generate countpart gt_labels """ # binary labels classifcation if gt_labels is None: # consider it as binary classification problem return self._generate_binary_labels(match) # multiple labels classification batch_size = match.shape[0] offset = torch.arange(0, batch_size) * gt_labels.size(1) match += offset.view(batch_size, 1).type_as(match) cls_targets_batch = gt_labels.view(-1)[match.view(-1)].view( batch_size, match.shape[1]) return cls_targets_batch def _generate_binary_labels(self, match): gt_labels_batch = torch.ones_like(match).long() return gt_labels_batch