def visualize_iou_pred(self, iou_features, center_box): center_box = center_box.view(1,1,4) sz_norm = center_box[...,2:].clone() center_box_rel = bbutils.rect_to_rel(center_box, sz_norm) pos_dist = 1.0 sz_dist = math.log(3.0) pos_step = 0.01 sz_step = 0.01 pos_scale = torch.arange(-pos_dist, pos_dist+pos_step, step=pos_step) sz_scale = torch.arange(-sz_dist, sz_dist+sz_step, step=sz_step) bbx = torch.zeros(1, pos_scale.numel(), 4) bbx[0,:,0] = pos_scale.clone() bby = torch.zeros(pos_scale.numel(), 1, 4) bby[:,0,1] = pos_scale.clone() bbw = torch.zeros(1, sz_scale.numel(), 4) bbw[0,:,2] = sz_scale.clone() bbh = torch.zeros(sz_scale.numel(), 1, 4) bbh[:,0,3] = sz_scale.clone() pos_boxes = bbutils.rel_to_rect((center_box_rel + bbx) + bby, sz_norm).view(1,-1,4).to(self.params.device) sz_boxes = bbutils.rel_to_rect((center_box_rel + bbw) + bbh, sz_norm).view(1,-1,4).to(self.params.device) pos_scores = self.net.bb_regressor.predict_iou(self.iou_modulation, iou_features, pos_boxes).exp() sz_scores = self.net.bb_regressor.predict_iou(self.iou_modulation, iou_features, sz_boxes).exp() show_tensor(pos_scores.view(pos_scale.numel(),-1), title='Position scores', fig_num=21) show_tensor(sz_scores.view(sz_scale.numel(),-1), title='Size scores', fig_num=22)
def optimize_boxes_relative(self, iou_features, init_boxes): """Optimize iounet boxes with the relative parametrization ised in PrDiMP""" output_boxes = init_boxes.view(1, -1, 4).to(self.params.device) step_length = self.params.box_refinement_step_length if isinstance(step_length, (tuple, list)): step_length = torch.Tensor([ step_length[0], step_length[0], step_length[1], step_length[1] ]).to(self.params.device).view(1, 1, 4) sz_norm = output_boxes[:, :1, 2:].clone() output_boxes_rel = bbutils.rect_to_rel(output_boxes, sz_norm) for i_ in range(self.params.box_refinement_iter): # forward pass bb_init_rel = output_boxes_rel.clone().detach() bb_init_rel.requires_grad = True bb_init = bbutils.rel_to_rect(bb_init_rel, sz_norm) outputs = self.net.bb_regressor.predict_iou( self.iou_modulation, iou_features, bb_init) if isinstance(outputs, (list, tuple)): outputs = outputs[0] outputs.backward(gradient=torch.ones_like(outputs)) # Update proposal output_boxes_rel = bb_init_rel + step_length * bb_init_rel.grad output_boxes_rel.detach_() step_length *= self.params.box_refinement_step_decay # for s in outputs.view(-1): # print('{:.2f} '.format(s.item()), end='') # print('') # print('') output_boxes = bbutils.rel_to_rect(output_boxes_rel, sz_norm) return output_boxes.view(-1, 4).cpu(), outputs.detach().view(-1).cpu()
def optimize_boxes(self, iou_features, init_boxes): # Optimize iounet boxes output_boxes = init_boxes.view(1, -1, 4).to(self.params.device) step_length = self.params.box_refinement_step_length init_step_length = self.params.box_refinement_step_length if isinstance(step_length, (tuple, list)): init_step_length = torch.Tensor([ step_length[0], step_length[0], step_length[1], step_length[1] ]).to(self.params.device).view(1, 1, 4) box_refinement_space = self.params.get('box_refinement_space', 'default') step_length = init_step_length * output_boxes.new_ones( 1, output_boxes.shape[1], 1) outputs_prev = -99999999 * output_boxes.new_ones( 1, output_boxes.shape[1]) step = torch.zeros_like(output_boxes) if box_refinement_space == 'default': # Optimization using bounding box space used in original IoUNet for i_ in range(self.params.box_refinement_iter): # forward pass bb_init = output_boxes.clone().detach() bb_init.requires_grad = True outputs = self.iou_predictor.predict_iou( self.target_feat, iou_features, bb_init) if isinstance(outputs, (list, tuple)): outputs = outputs[0] outputs.backward(gradient=torch.ones_like(outputs)) # Update mask and step length update_mask = (outputs.detach() > outputs_prev) | ( self.params.box_refinement_step_decay >= 1) update_mask_float = update_mask.view(1, -1, 1).float() step_length[ ~update_mask, :] *= self.params.box_refinement_step_decay outputs_prev = outputs.detach().clone() # Update proposal step = update_mask_float * step_length * bb_init.grad * bb_init[:, :, 2:].repeat( 1, 1, 2) - (1.0 - update_mask_float) * step output_boxes = bb_init + step output_boxes.detach_() elif box_refinement_space == 'relative': # Optimization using relative bounding box space sz_norm = output_boxes[:, :1, 2:].clone() output_boxes_rel = bbutils.rect_to_rel(output_boxes, sz_norm) for i_ in range(self.params.box_refinement_iter): # forward pass bb_init_rel = output_boxes_rel.clone().detach() bb_init_rel.requires_grad = True bb_init = bbutils.rel_to_rect(bb_init_rel, sz_norm) outputs = self.iou_predictor.predict_iou( self.target_feat, iou_features, bb_init) if isinstance(outputs, (list, tuple)): outputs = outputs[0] outputs.backward(gradient=torch.ones_like(outputs)) # Update mask and step length update_mask = (outputs.detach() > outputs_prev) | ( self.params.box_refinement_step_decay >= 1) update_mask_float = update_mask.view(1, -1, 1).float() step_length[ ~update_mask, :] *= self.params.box_refinement_step_decay outputs_prev = outputs.detach().clone() # Update proposal step = update_mask_float * step_length * bb_init_rel.grad - ( 1.0 - update_mask_float) * step output_boxes_rel = bb_init_rel + step output_boxes_rel.detach_() # for s in outputs.view(-1): # print('{:.2f} '.format(s.item()), end='') # print('') # print('') output_boxes = bbutils.rel_to_rect(output_boxes_rel, sz_norm) else: raise ValueError( 'Unknown box_refinement_space {}'.format(box_refinement_space)) return output_boxes.view(-1, 4).cpu(), outputs.detach().view(-1).cpu()