def visualize_iou_pred(self, iou_features, center_box):
center_box = center_box.view(1,1,4)
sz_norm = center_box[...,2:].clone()
center_box_rel = bbutils.rect_to_rel(center_box, sz_norm)
pos_dist = 1.0
sz_dist = math.log(3.0)
pos_step = 0.01
sz_step = 0.01
pos_scale = torch.arange(-pos_dist, pos_dist+pos_step, step=pos_step)
sz_scale = torch.arange(-sz_dist, sz_dist+sz_step, step=sz_step)
bbx = torch.zeros(1, pos_scale.numel(), 4)
bbx[0,:,0] = pos_scale.clone()
bby = torch.zeros(pos_scale.numel(), 1, 4)
bby[:,0,1] = pos_scale.clone()
bbw = torch.zeros(1, sz_scale.numel(), 4)
bbw[0,:,2] = sz_scale.clone()
bbh = torch.zeros(sz_scale.numel(), 1, 4)
bbh[:,0,3] = sz_scale.clone()
pos_boxes = bbutils.rel_to_rect((center_box_rel + bbx) + bby, sz_norm).view(1,-1,4).to(self.params.device)
sz_boxes = bbutils.rel_to_rect((center_box_rel + bbw) + bbh, sz_norm).view(1,-1,4).to(self.params.device)
pos_scores = self.net.bb_regressor.predict_iou(self.iou_modulation, iou_features, pos_boxes).exp()
sz_scores = self.net.bb_regressor.predict_iou(self.iou_modulation, iou_features, sz_boxes).exp()
show_tensor(pos_scores.view(pos_scale.numel(),-1), title='Position scores', fig_num=21)
show_tensor(sz_scores.view(sz_scale.numel(),-1), title='Size scores', fig_num=22)
def optimize_boxes_relative(self, iou_features, init_boxes):
"""Optimize iounet boxes with the relative parametrization ised in PrDiMP"""
output_boxes = init_boxes.view(1, -1, 4).to(self.params.device)
step_length = self.params.box_refinement_step_length
if isinstance(step_length, (tuple, list)):
step_length = torch.Tensor([
step_length[0], step_length[0], step_length[1], step_length[1]
]).to(self.params.device).view(1, 1, 4)
sz_norm = output_boxes[:, :1, 2:].clone()
output_boxes_rel = bbutils.rect_to_rel(output_boxes, sz_norm)
for i_ in range(self.params.box_refinement_iter):
# forward pass
bb_init_rel = output_boxes_rel.clone().detach()
bb_init_rel.requires_grad = True
bb_init = bbutils.rel_to_rect(bb_init_rel, sz_norm)
outputs = self.net.bb_regressor.predict_iou(
self.iou_modulation, iou_features, bb_init)
if isinstance(outputs, (list, tuple)):
outputs = outputs[0]
outputs.backward(gradient=torch.ones_like(outputs))
# Update proposal
output_boxes_rel = bb_init_rel + step_length * bb_init_rel.grad
output_boxes_rel.detach_()
step_length *= self.params.box_refinement_step_decay
# for s in outputs.view(-1):
# print('{:.2f} '.format(s.item()), end='')
# print('')
# print('')
output_boxes = bbutils.rel_to_rect(output_boxes_rel, sz_norm)
return output_boxes.view(-1, 4).cpu(), outputs.detach().view(-1).cpu()
def optimize_boxes(self, iou_features, init_boxes):
# Optimize iounet boxes
output_boxes = init_boxes.view(1, -1, 4).to(self.params.device)
step_length = self.params.box_refinement_step_length
init_step_length = self.params.box_refinement_step_length
if isinstance(step_length, (tuple, list)):
init_step_length = torch.Tensor([
step_length[0], step_length[0], step_length[1], step_length[1]
]).to(self.params.device).view(1, 1, 4)
box_refinement_space = self.params.get('box_refinement_space',
'default')
step_length = init_step_length * output_boxes.new_ones(
1, output_boxes.shape[1], 1)
outputs_prev = -99999999 * output_boxes.new_ones(
1, output_boxes.shape[1])
step = torch.zeros_like(output_boxes)
if box_refinement_space == 'default':
# Optimization using bounding box space used in original IoUNet
for i_ in range(self.params.box_refinement_iter):
# forward pass
bb_init = output_boxes.clone().detach()
bb_init.requires_grad = True
outputs = self.iou_predictor.predict_iou(
self.target_feat, iou_features, bb_init)
if isinstance(outputs, (list, tuple)):
outputs = outputs[0]
outputs.backward(gradient=torch.ones_like(outputs))
# Update mask and step length
update_mask = (outputs.detach() > outputs_prev) | (
self.params.box_refinement_step_decay >= 1)
update_mask_float = update_mask.view(1, -1, 1).float()
step_length[
~update_mask, :] *= self.params.box_refinement_step_decay
outputs_prev = outputs.detach().clone()
# Update proposal
step = update_mask_float * step_length * bb_init.grad * bb_init[:, :, 2:].repeat(
1, 1, 2) - (1.0 - update_mask_float) * step
output_boxes = bb_init + step
output_boxes.detach_()
elif box_refinement_space == 'relative':
# Optimization using relative bounding box space
sz_norm = output_boxes[:, :1, 2:].clone()
output_boxes_rel = bbutils.rect_to_rel(output_boxes, sz_norm)
for i_ in range(self.params.box_refinement_iter):
# forward pass
bb_init_rel = output_boxes_rel.clone().detach()
bb_init_rel.requires_grad = True
bb_init = bbutils.rel_to_rect(bb_init_rel, sz_norm)
outputs = self.iou_predictor.predict_iou(
self.target_feat, iou_features, bb_init)
if isinstance(outputs, (list, tuple)):
outputs = outputs[0]
outputs.backward(gradient=torch.ones_like(outputs))
# Update mask and step length
update_mask = (outputs.detach() > outputs_prev) | (
self.params.box_refinement_step_decay >= 1)
update_mask_float = update_mask.view(1, -1, 1).float()
step_length[
~update_mask, :] *= self.params.box_refinement_step_decay
outputs_prev = outputs.detach().clone()
# Update proposal
step = update_mask_float * step_length * bb_init_rel.grad - (
1.0 - update_mask_float) * step
output_boxes_rel = bb_init_rel + step
output_boxes_rel.detach_()
# for s in outputs.view(-1):
# print('{:.2f} '.format(s.item()), end='')
# print('')
# print('')
output_boxes = bbutils.rel_to_rect(output_boxes_rel, sz_norm)
else:
raise ValueError(
'Unknown box_refinement_space {}'.format(box_refinement_space))
return output_boxes.view(-1, 4).cpu(), outputs.detach().view(-1).cpu()
