def __call__(self, x: TensorList): """ Compute residuals :param x: [filters] :return: [data_terms, filter_regularizations] """ # Do convolution and compute residuals residuals = operation.conv2d(self.training_samples, x, mode='same').apply( self.response_activation) residuals = residuals - self.y for i in range(len(residuals)): tmp_data = torch.abs(residuals[i]) more_sample_weight = torch.max( torch.max(torch.max(tmp_data, 1)[0], 1)[0], 1)[0] + 0.01 more_sample_weight = more_sample_weight / torch.max( more_sample_weight[:torch.sum(self.sample_weights[i] > 0)]) max_loss_weight = torch.exp(-self.beta * more_sample_weight) residuals[i] = max_loss_weight.view(-1, 1, 1, 1) * residuals[i] residuals = self.sample_weights.sqrt().view(-1, 1, 1, 1) * residuals # Add regularization for projection matrix residuals.extend(self.filter_reg.apply(math.sqrt) * x) return residuals
def __call__(self, x: TensorList): if self.use_attetion: filter = x[:1] fc2 = x[1:2] fc1 = x[2:3] P = x[3:4] else: filter = x[:len(x) // 2] # w2 in paper P = x[len(x) // 2:] # w1 in paper # Compression module compressed_samples = operation.conv1x1(self.training_samples, P).apply( self.projection_activation) # Attention module if self.use_attetion: if cfg.TRACK.CHANNEL_ATTENTION: global_average = operation.adaptive_avg_pool2d( compressed_samples, 1) temp_variables = operation.conv1x1(global_average, fc1).apply( self.att_activation) channel_attention = operation.sigmoid( operation.conv1x1(temp_variables, fc2)) else: channel_attention = TensorList([ torch.zeros(compressed_samples[0].size(0), compressed_samples[0].size(1), 1, 1).cuda() ]) if cfg.TRACK.SPATIAL_ATTENTION == 'none': spatial_attention = TensorList([ torch.zeros(compressed_samples[0].size(0), 1, compressed_samples[0].size(2), compressed_samples[0].size(3)).cuda() ]) elif cfg.TRACK.SPATIAL_ATTENTION == 'pool': spatial_attention = operation.spatial_attention( compressed_samples, dim=1, keepdim=True) else: raise NotImplementedError('No spatial attention Implemented') compressed_samples = operation.matmul(compressed_samples, spatial_attention) + \ operation.matmul(compressed_samples, channel_attention) # Filter module residuals = operation.conv2d(compressed_samples, filter, mode='same').apply( self.response_activation) residuals = residuals - self.y residuals = self.sample_weights.sqrt().view(-1, 1, 1, 1) * residuals residuals.extend(self.filter_reg.apply(math.sqrt) * filter) if self.use_attetion: residuals.extend(self.projection_reg.apply(math.sqrt) * fc2) residuals.extend(self.projection_reg.apply(math.sqrt) * fc1) residuals.extend(self.projection_reg.apply(math.sqrt) * P) return residuals
def __call__(self, x: TensorList): """ Compute residuals :param x: [filters] :return: [data_terms, filter_regularizations] """ # Do convolution and compute residuals residuals = operation.conv2d(self.training_samples, x, mode='same').apply( self.response_activation) residuals = residuals - self.y residuals = self.sample_weights.sqrt().view(-1, 1, 1, 1) * residuals # Add regularization for projection matrix residuals.extend(self.filter_reg.apply(math.sqrt) * x) return residuals
def project_sample(self, x: TensorList, proj_matrix=None): # Apply projection matrix if proj_matrix is None: proj_matrix = self.projection_matrix compressed_samples = operation.conv2d(x, proj_matrix).apply( self.projection_activation) if self.use_attention_layer: if cfg.TRACK.CHANNEL_ATTENTION: global_average = operation.adaptive_avg_pool2d( compressed_samples, 1) temp_variables = operation.conv1x1(global_average, self.channel_att_fc1).apply( self.att_activation) channel_attention = operation.sigmoid( operation.conv1x1(temp_variables, self.channel_att_fc2)) else: channel_attention = TensorList([ torch.zeros(compressed_samples[0].size(0), compressed_samples[0].size(1), 1, 1).cuda() ]) if cfg.TRACK.SPATIAL_ATTENTION == 'none': spatial_attention = TensorList([ torch.zeros(compressed_samples[0].size(0), 1, compressed_samples[0].size(2), compressed_samples[0].size(3)).cuda() ]) elif cfg.TRACK.SPATIAL_ATTENTION == 'pool': spatial_attention = operation.spatial_attention( compressed_samples, dim=1, keepdim=True) compressed_samples = operation.matmul(compressed_samples, spatial_attention) + \ operation.matmul(compressed_samples, channel_attention) return compressed_samples
def ip_input(self, a: TensorList, b: TensorList): if self.use_attetion: a_filter = a[:1] a_f2 = a[1:2] a_f1 = a[2:3] a_P = a[3:] b_filter = b[:1] b_f2 = b[1:2] b_f1 = b[2:3] b_P = b[3:] ip_out = operation.conv2d(a_filter, b_filter).view(-1) ip_out += operation.conv2d(a_f2.view(1, -1, 1, 1), b_f2.view(1, -1, 1, 1)).view(-1) ip_out += operation.conv2d(a_f1.view(1, -1, 1, 1), b_f1.view(1, -1, 1, 1)).view(-1) ip_out += operation.conv2d(a_P.view(1, -1, 1, 1), b_P.view(1, -1, 1, 1)).view(-1) return ip_out.concat(ip_out.clone()).concat(ip_out.clone()).concat( ip_out.clone()) else: num = len(a) // 2 # Number of filters a_filter = a[:num] b_filter = b[:num] a_P = a[num:] b_P = b[num:] # Filter inner product # ip_out = a_filter.reshape(-1) @ b_filter.reshape(-1) ip_out = operation.conv2d(a_filter, b_filter).view(-1) # Add projection matrix part # ip_out += a_P.reshape(-1) @ b_P.reshape(-1) ip_out += operation.conv2d(a_P.view(1, -1, 1, 1), b_P.view(1, -1, 1, 1)).view(-1) # Have independent inner products for each filter return ip_out.concat(ip_out.clone())
def apply_filter(self, sample_x: TensorList): return operation.conv2d(sample_x, self.filter, mode='same')
def ip_input(self, a: TensorList, b: TensorList): # return a.reshape(-1) @ b.reshape(-1) # return (a * b).sum() return operation.conv2d(a, b).view(-1)