def __call__(self, x: TensorList):
"""
Compute residuals
:param x: [filters]
:return: [data_terms, filter_regularizations]
"""
# Do convolution and compute residuals
residuals = operation.conv2d(self.training_samples, x,
mode='same').apply(
self.response_activation)
residuals = residuals - self.y
for i in range(len(residuals)):
tmp_data = torch.abs(residuals[i])
more_sample_weight = torch.max(
torch.max(torch.max(tmp_data, 1)[0], 1)[0], 1)[0] + 0.01
more_sample_weight = more_sample_weight / torch.max(
more_sample_weight[:torch.sum(self.sample_weights[i] > 0)])
max_loss_weight = torch.exp(-self.beta * more_sample_weight)
residuals[i] = max_loss_weight.view(-1, 1, 1, 1) * residuals[i]
residuals = self.sample_weights.sqrt().view(-1, 1, 1, 1) * residuals
# Add regularization for projection matrix
residuals.extend(self.filter_reg.apply(math.sqrt) * x)
return residuals
def __call__(self, x: TensorList):
if self.use_attetion:
filter = x[:1]
fc2 = x[1:2]
fc1 = x[2:3]
P = x[3:4]
else:
filter = x[:len(x) // 2] # w2 in paper
P = x[len(x) // 2:] # w1 in paper
# Compression module
compressed_samples = operation.conv1x1(self.training_samples, P).apply(
self.projection_activation)
# Attention module
if self.use_attetion:
if cfg.TRACK.CHANNEL_ATTENTION:
global_average = operation.adaptive_avg_pool2d(
compressed_samples, 1)
temp_variables = operation.conv1x1(global_average, fc1).apply(
self.att_activation)
channel_attention = operation.sigmoid(
operation.conv1x1(temp_variables, fc2))
else:
channel_attention = TensorList([
torch.zeros(compressed_samples[0].size(0),
compressed_samples[0].size(1), 1, 1).cuda()
])
if cfg.TRACK.SPATIAL_ATTENTION == 'none':
spatial_attention = TensorList([
torch.zeros(compressed_samples[0].size(0), 1,
compressed_samples[0].size(2),
compressed_samples[0].size(3)).cuda()
])
elif cfg.TRACK.SPATIAL_ATTENTION == 'pool':
spatial_attention = operation.spatial_attention(
compressed_samples, dim=1, keepdim=True)
else:
raise NotImplementedError('No spatial attention Implemented')
compressed_samples = operation.matmul(compressed_samples, spatial_attention) + \
operation.matmul(compressed_samples, channel_attention)
# Filter module
residuals = operation.conv2d(compressed_samples, filter,
mode='same').apply(
self.response_activation)
residuals = residuals - self.y
residuals = self.sample_weights.sqrt().view(-1, 1, 1, 1) * residuals
residuals.extend(self.filter_reg.apply(math.sqrt) * filter)
if self.use_attetion:
residuals.extend(self.projection_reg.apply(math.sqrt) * fc2)
residuals.extend(self.projection_reg.apply(math.sqrt) * fc1)
residuals.extend(self.projection_reg.apply(math.sqrt) * P)
return residuals
def __call__(self, x: TensorList):
"""
Compute residuals
:param x: [filters]
:return: [data_terms, filter_regularizations]
"""
# Do convolution and compute residuals
residuals = operation.conv2d(self.training_samples, x,
mode='same').apply(
self.response_activation)
residuals = residuals - self.y
residuals = self.sample_weights.sqrt().view(-1, 1, 1, 1) * residuals
# Add regularization for projection matrix
residuals.extend(self.filter_reg.apply(math.sqrt) * x)
return residuals
def project_sample(self, x: TensorList, proj_matrix=None):
# Apply projection matrix
if proj_matrix is None:
proj_matrix = self.projection_matrix
compressed_samples = operation.conv2d(x, proj_matrix).apply(
self.projection_activation)
if self.use_attention_layer:
if cfg.TRACK.CHANNEL_ATTENTION:
global_average = operation.adaptive_avg_pool2d(
compressed_samples, 1)
temp_variables = operation.conv1x1(global_average,
self.channel_att_fc1).apply(
self.att_activation)
channel_attention = operation.sigmoid(
operation.conv1x1(temp_variables, self.channel_att_fc2))
else:
channel_attention = TensorList([
torch.zeros(compressed_samples[0].size(0),
compressed_samples[0].size(1), 1, 1).cuda()
])
if cfg.TRACK.SPATIAL_ATTENTION == 'none':
spatial_attention = TensorList([
torch.zeros(compressed_samples[0].size(0), 1,
compressed_samples[0].size(2),
compressed_samples[0].size(3)).cuda()
])
elif cfg.TRACK.SPATIAL_ATTENTION == 'pool':
spatial_attention = operation.spatial_attention(
compressed_samples, dim=1, keepdim=True)
compressed_samples = operation.matmul(compressed_samples, spatial_attention) + \
operation.matmul(compressed_samples, channel_attention)
return compressed_samples
def ip_input(self, a: TensorList, b: TensorList):
if self.use_attetion:
a_filter = a[:1]
a_f2 = a[1:2]
a_f1 = a[2:3]
a_P = a[3:]
b_filter = b[:1]
b_f2 = b[1:2]
b_f1 = b[2:3]
b_P = b[3:]
ip_out = operation.conv2d(a_filter, b_filter).view(-1)
ip_out += operation.conv2d(a_f2.view(1, -1, 1, 1),
b_f2.view(1, -1, 1, 1)).view(-1)
ip_out += operation.conv2d(a_f1.view(1, -1, 1, 1),
b_f1.view(1, -1, 1, 1)).view(-1)
ip_out += operation.conv2d(a_P.view(1, -1, 1, 1),
b_P.view(1, -1, 1, 1)).view(-1)
return ip_out.concat(ip_out.clone()).concat(ip_out.clone()).concat(
ip_out.clone())
else:
num = len(a) // 2 # Number of filters
a_filter = a[:num]
b_filter = b[:num]
a_P = a[num:]
b_P = b[num:]
# Filter inner product
# ip_out = a_filter.reshape(-1) @ b_filter.reshape(-1)
ip_out = operation.conv2d(a_filter, b_filter).view(-1)
# Add projection matrix part
# ip_out += a_P.reshape(-1) @ b_P.reshape(-1)
ip_out += operation.conv2d(a_P.view(1, -1, 1, 1),
b_P.view(1, -1, 1, 1)).view(-1)
# Have independent inner products for each filter
return ip_out.concat(ip_out.clone())
def apply_filter(self, sample_x: TensorList):
return operation.conv2d(sample_x, self.filter, mode='same')
def ip_input(self, a: TensorList, b: TensorList):
# return a.reshape(-1) @ b.reshape(-1)
# return (a * b).sum()
return operation.conv2d(a, b).view(-1)
