def rmac(x, L=3, eps=1e-6, nt=-1):
ovr = 0.4 # desired overlap of neighboring regions
steps = torch.Tensor([2, 3, 4, 5, 6, 7]) # possible regions for the long dimension
W = x.size(3)
H = x.size(2)
w = min(W, H)
w2 = math.floor(w / 2.0 - 1)
b = (max(H, W) - w) / (steps - 1)
(tmp, idx) = torch.min(torch.abs(((w ** 2 - w * b) / w ** 2) - ovr), 0) # steps(idx) regions for long dimension
# region overplus per dimension
Wd = 0;
Hd = 0;
if H < W:
Wd = idx.item() + 1
elif H > W:
Hd = idx.item() + 1
if nt == -1:
v = F.max_pool2d(x, (x.size(-2), x.size(-1)))
else:
v = F.lp_pool2d(x, nt, (x.size(-2), x.size(-1)))
v = v / (torch.norm(v, p=2, dim=1, keepdim=True) + eps).expand_as(v)
for l in range(1, L + 1):
wl = math.floor(2 * w / (l + 1))
wl2 = math.floor(wl / 2 - 1)
if l + Wd == 1:
b = 0
else:
b = (W - wl) / (l + Wd - 1)
cenW = torch.floor(wl2 + torch.Tensor(range(l - 1 + Wd + 1)) * b) - wl2 # center coordinates
if l + Hd == 1:
b = 0
else:
b = (H - wl) / (l + Hd - 1)
cenH = torch.floor(wl2 + torch.Tensor(range(l - 1 + Hd + 1)) * b) - wl2 # center coordinates
for i_ in cenH.tolist():
for j_ in cenW.tolist():
if wl == 0:
continue
R = x[:, :, (int(i_) + torch.Tensor(range(wl)).long()).tolist(), :]
R = R[:, :, :, (int(j_) + torch.Tensor(range(wl)).long()).tolist()]
if nt == -1:
vt = F.max_pool2d(R, (R.size(-2), R.size(-1)))
else:
vt = F.lp_pool2d(R, nt, (R.size(-2), R.size(-1)))
vt = vt / (torch.norm(vt, p=2, dim=1, keepdim=True) + eps).expand_as(vt)
v += vt
return v
def forward(self, x):
channel_att_sum = None
for pool_type in self.pool_types:
if pool_type == 'avg':
avg_pool = F.avg_pool2d(
x, (x.size(2), x.size(3)),
stride=(x.size(2),
x.size(3))) #kernel_size:(x.size(2), x.size(3)
channel_att_raw = self.mlp(avg_pool)
elif pool_type == 'max':
max_pool = F.max_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(max_pool)
elif pool_type == 'lp':
lp_pool = F.lp_pool2d(x,
2, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(lp_pool)
elif pool_type == 'lse':
# LSE pool only
lse_pool = logsumexp_2d(x)
channel_att_raw = self.mlp(lse_pool)
if channel_att_sum is None: #for循环第一次的时候，执行if，第二次for循环的时候执行else
channel_att_sum = channel_att_raw
else:
channel_att_sum = channel_att_sum + channel_att_raw #是mlp(avgpool(f))+mlp(maxpool(f))
scale = F.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(3).expand_as(
x)
#unsqueeze:在指定位置插入一个1维tensor
return x * scale
def squared_l2_distance(x, patches, stride):
"""
Arguments:
x: a float tensor with shape [C, H, W].
patches: a float tensor with shape [M, C, size, size], unnormalized.
stride: an integer.
Returns:
a float tensor with shape [N, M],
where N = n * m, n = 1 + floor((H - size)/stride),
and m = 1 + floor((W - size)/stride).
"""
# compute squared norms of patches
M = patches.size(0)
patch_norms = torch.pow(patches, 2).sum(dim=[1, 2, 3]) # shape [M]
# compute scalar products
x = x.unsqueeze(0)
products = F.conv2d(x, patches, stride=stride) # shape [1, M, n, m]
n, m = products.size()[2:]
N = n * m
products = products.squeeze(0).view(M, N)
# compute squared norms of patches from x
size = patches.size(2)
x_norms = F.lp_pool2d(x, norm_type=2, kernel_size=size, stride=stride) # shape [1, C, n, m]
x_norms = torch.pow(x_norms, 2).sum(dim=1).squeeze(0).view(N) # shape [N]
# |x - y|^2 = |x|^2 + |y|^2 - 2*(x, y)
distances = patch_norms.unsqueeze(1) + x_norms.unsqueeze(0) - 2.0 * products # shape [M, N]
return distances.t()
def forward(self, inputs: 'Tensor') -> 'Tensor':
""" Forward pass method for generalize LPPooling layer.
Parameters
----------
inputs : Tensor
input tensor.
Returns
-------
Tensor
result of pooling operation.
"""
x = pad(inputs, self.pad_sizes,
mode=self.padding_mode,
value=self._padding_value)
if self.ndims == 2:
pool_args = (x, self.norm_type, self.kernel_size[0], self.stride[0])
return F.lp_pool1d(*pool_args)
elif self.ndims == 3:
pool_args = (x, self.norm_type, self.kernel_size, self.stride)
return F.lp_pool2d(*pool_args)
elif self.ndims == 4:
return F.lp_pool3d(*pool_args)
def forward(self, x):
channel_att_sum = None
for pool_type in self.pool_types:
if pool_type == "avg":
avg_pool = F.avg_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.gate_c(avg_pool)
elif pool_type == "max":
max_pool = F.max_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.gate_c(max_pool)
elif pool_type == "lp":
lp_pool = F.lp_pool2d(x,
2, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.gate_c(lp_pool)
if channel_att_sum is None:
channel_att_sum = channel_att_raw
else:
channel_att_sum = channel_att_sum + channel_att_raw
if self.bam:
return channel_att_sum.unsqueeze(2).unsqueeze(3).expand_as(x)
else:
scale = F.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(
3).expand_as(x)
return x * scale
def forward(self, x):
channel_att_sum = None
for pool_type in self.pool_types:
if pool_type == 'avg':
avg_pool = F.avg_pool2d(
x, (x.size(2), x.size(3)),
stride=(x.size(2),
x.size(3))) #kernel_size:(x.size(2), x.size(3)
channel_att_raw = self.mlp(avg_pool)
elif pool_type == 'max':
max_pool = F.max_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(max_pool)
elif pool_type == 'lp':
lp_pool = F.lp_pool2d(x,
2, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(lp_pool)
elif pool_type == 'lse':
# LSE pool only
lse_pool = logsumexp_2d(x)
channel_att_raw = self.mlp(lse_pool)
if channel_att_sum is None: #forÑ»·µÚÒ»´ÎµÄʱºò£¬Ö´ÐÐif£¬µÚ¶þ´ÎforÑ»·µÄʱºòÖ´ÐÐelse
channel_att_sum = channel_att_raw
else:
channel_att_sum = channel_att_sum + channel_att_raw #ÊÇmlp(avgpool(f))+mlp(maxpool(f))
scale = F.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(3).expand_as(
x)
#unsqueeze:ÔÚÖ¸¶¨Î»ÖòåÈëÒ»¸ö1άtensor
return x * scale
def forward(self, x, landmark):
if isinstance(landmark, bool):
landmark = x
channel_att_sum = None
for pool_type in self.pool_types:
if pool_type=='avg':
avg_pool = F.avg_pool2d(landmark, (landmark.size(2), landmark.size(3)))
channel_att_raw = self.mlp(avg_pool)
elif pool_type=='max':
max_pool = F.max_pool2d(landmark, (landmark.size(2), landmark.size(3)))
channel_att_raw = self.mlp(max_pool)
elif pool_type=='lp':
lp_pool = F.lp_pool2d(landmark, 2, (landmark.size(2), landmark.size(3)))
channel_att_raw = self.mlp(lp_pool)
elif pool_type=='lse':
# LSE pool only
lse_pool = logsumexp_2d(x)
channel_att_raw = self.mlp( lse_pool )
if channel_att_sum is None:
channel_att_sum = channel_att_raw
else:
channel_att_sum = channel_att_sum + channel_att_raw
scale = torch.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(3).expand_as(x)
if self.dropout:
scale = self.dropout(scale)
return x * scale
def forward(self, x):
channel_att_sum = None
for pool_type in self.pool_types:
if pool_type == "avg":
avg_pool = F.avg_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(avg_pool)
elif pool_type == "max":
max_pool = F.max_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(max_pool)
elif pool_type == "lp":
lp_pool = F.lp_pool2d(x,
2, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(lp_pool)
elif pool_type == "lse":
lse_pool = logsumexp_2d(x)
channel_att_raw = self.mlp(lse_pool)
if channel_att_sum is None:
channel_att_sum = channel_att_raw
else:
channel_att_sum = channel_att_sum + channel_att_raw
scale = torch.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(
3).expand_as(x)
return x * scale
def forward(self, x):
channel_att_sum = None
for pool_type in self.pool_types:
if pool_type == 'avg':
avg_pool = F.avg_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(avg_pool)
elif pool_type == 'max':
max_pool = F.max_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(max_pool)
elif pool_type == 'lp':
lp_pool = F.lp_pool2d(x,
2, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(lp_pool)
elif pool_type == 'lse':
# LSE pool only
lse_pool = logsumexp_2d(x)
channel_att_raw = self.mlp(lse_pool)
if channel_att_sum is None:
channel_att_sum = channel_att_raw
else:
channel_att_sum = channel_att_sum + channel_att_raw
# scalecoe = F.sigmoid(channel_att_sum)
channel_att_sum = channel_att_sum.reshape(channel_att_sum.shape[0], 4,
4)
avg_weight = torch.mean(channel_att_sum, dim=2).unsqueeze(2)
avg_weight = avg_weight.expand(channel_att_sum.shape[0], 4,
4).reshape(channel_att_sum.shape[0], 16)
scale = F.sigmoid(avg_weight).unsqueeze(2).unsqueeze(3).expand_as(x)
return x * scale, scale
def forward(self, x):
channel_att_sum = None
for pool_type in self.pool_types:
if pool_type == 'avg':
avg_pool = F.avg_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(avg_pool)
elif pool_type == 'max':
max_pool = F.max_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(max_pool)
elif pool_type == 'lp':
lp_pool = F.lp_pool2d(x,
2, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(lp_pool)
elif pool_type == 'lse':
# LSE pool only
lse_pool = logsumexp_2d(x)
channel_att_raw = self.mlp(lse_pool)
if channel_att_sum is None:
channel_att_sum = channel_att_raw
else:
channel_att_sum = channel_att_sum + channel_att_raw
scale = F.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(3).expand_as(
x)
return x * scale
def forward(self, x):
b, c, h, w = x.size()
channel_att_sum = None
for pool_type in self.pool_types:
if pool_type == 'avg':
y = x.unsqueeze(0)
avg_pool = F.adaptive_avg_pool3d(y, (self.groups, 1, 1))
avg_pool = avg_pool.squeeze(0)
channel_att_raw = self.mlp(avg_pool)
elif pool_type == 'max':
y = x.unsqueeze(0)
max_pool = F.adaptive_max_pool3d(y, (self.groups, 1, 1))
max_pool = max_pool.squeeze(0)
channel_att_raw = self.mlp(max_pool)
elif pool_type == 'lp':
lp_pool = F.lp_pool2d(x,
2, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(lp_pool)
elif pool_type == 'lse':
# LSE pool only
lse_pool = logsumexp_2d(x)
channel_att_raw = self.mlp(lse_pool)
if channel_att_sum is None:
channel_att_sum = channel_att_raw
else:
channel_att_sum = channel_att_sum + channel_att_raw
# channel_att_sum=channel_att_sum.unsqueeze(1)
# channel_att_sum=F.upsample(channel_att_sum,c)
# channel_att_sum=channel_att_sum.squeeze(1)
scale = F.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(3).expand_as(
x)
return x * scale
def forward(self, x):
x = F.lp_pool2d(x, norm_type=2, kernel_size=3)
x = F.lp_pool2d(x, norm_type=2, kernel_size=4, stride=2)
x = F.lp_pool2d(x, norm_type=1, kernel_size=(1,3), stride=1, ceil_mode=False)
x = F.lp_pool2d(x, norm_type=1, kernel_size=(4,5), stride=(1,2), ceil_mode=True)
x = F.lp_pool2d(x, norm_type=1.2, kernel_size=(5,3), stride=(2,1), ceil_mode=False)
x = F.lp_pool2d(x, norm_type=0.5, kernel_size=2, stride=1, ceil_mode=True)
x = F.lp_pool2d(x, norm_type=0.1, kernel_size=(5,4), stride=1, ceil_mode=False)
return x
def forward(self, x, atten):
channel_att_sum = None
channel_ave = None
channel_max = None
for pool_type in self.pool_types:
if pool_type == 'avg':
avg_pool = F.avg_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
avg_pool = avg_pool.squeeze(3).squeeze(2)
if self.block_id == 0:
channel_att_raw = self.mlp(avg_pool)
elif self.block_id != 0:
avg_pool_combine = torch.cat((avg_pool, atten[:, :, 0]),
dim=1)
channel_att_raw = self.mlp(avg_pool_combine)
channel_ave = channel_att_raw.clone()
# print(channel_att_raw.size())
elif pool_type == 'max':
max_pool = F.max_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
max_pool = max_pool.squeeze(3).squeeze(2)
if self.block_id == 0:
channel_att_raw = self.mlp(max_pool)
elif self.block_id != 0:
max_pool_combine = torch.cat((max_pool, atten[:, :, 1]),
dim=1)
channel_att_raw = self.mlp(max_pool_combine)
channel_max = channel_att_raw.clone()
elif pool_type == 'lp':
lp_pool = F.lp_pool2d(x,
2, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(lp_pool)
elif pool_type == 'lse':
# LSE pool only
lse_pool = logsumexp_2d(x)
channel_att_raw = self.mlp(lse_pool)
if channel_att_sum is None:
channel_att_sum = channel_att_raw
else:
channel_att_sum = channel_att_sum + channel_att_raw
scale = F.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(3).expand_as(
x)
channel_ave = channel_ave.unsqueeze(2)
channel_max = channel_max.unsqueeze(2)
# print(torch.cat((channel_ave,channel_max),dim=2).size())
return x * scale, torch.cat((channel_ave, channel_max), dim=2)
def forward(self, x, warmup=False, **kwargs):
x = self.model.features(x)
prepool_y = y = self.pool_base(x, kernel_size=x.shape[-1])
if self.pool_aux is not None:
y += self.pool_aux(x, kernel_size=x.shape[-1])
if 'lp2' in self.pars.arch:
y += F.lp_pool2d(x, 2, kernel_size=x.shape[-1])
if 'lp3' in self.pars.arch:
y += F.lp_pool2d(x, 3, kernel_size=x.shape[-1])
y = y.view(len(x), -1)
if warmup:
x, y, prepool_y = x.detach(), y.detach(), prepool_y.detach()
z = self.model.last_linear(y)
if 'normalize' in self.name:
z = F.normalize(z, dim=-1)
return {
'embeds': z,
'avg_features': y,
'features': x,
'extra_embeds': prepool_y
}
def forward(self, xyz, Z, body23):
features_emb = None
features_23 = None
if self.Z:
features_emb = self.emb(Z)
if self.two_three:
features_23 = body23
if features_emb is None:
features = features_23.float()
else:
if features_23 is None:
features = features_emb.float()
else:
features = features_emb.float()
xyz = xyz.to(torch.float64)
features = features.to(torch.float64)
feature_list = []
for _, op in enumerate(self.clouds):
features = op(features, geometry=xyz)
if self.cloud_res:
feature_list.append(features)
features = features.to(torch.float64)
if self.cloud_res:
if len(feature_list) > 1:
features = torch.cat(feature_list, dim=2)
# CONCATENATE FEATURES: Z and 2&3-BODY --> FEED THEM INTO RESIDUAL LAYER
if self.final_res:
features = torch.cat(
[features_23.float(), features.float()], dim=2).double()
features = self.cloud_residual(features)
if 'sum' in self.feature_collation:
features = features.sum(1)
elif 'pool' in self.feature_collation:
# features = F.adaptive_avg_pool2d(features, (1, features.shape[2]))
# features = F.lp_pool2d(features, norm_type=1, kernel_size=(features.shape[1], 1), ceil_mode=False)
features = F.lp_pool2d(features,
norm_type=2,
kernel_size=(features.shape[1], 1),
ceil_mode=False)
features = features.squeeze()
for _, op in enumerate(self.collate):
# features = F.leaky_relu(op(features))
features = F.softplus(op(features))
return self.act(self.outputlayer(features))
def forward(self, inp):
x = self._relu_pool_drop(self.conv1(inp), kernel_size=4)
x = self._relu_pool_drop(self.conv2(x), kernel_size=2)
x = self._relu_pool_drop(self.conv3(x), kernel_size=2)
# Global temporal pooling
operations = [
F.avg_pool1d(x, kernel_size=x.size(2)),
F.max_pool1d(x, kernel_size=x.size(2)),
F.lp_pool2d(x, norm_type=2, kernel_size=(1, x.size(2)))
]
x = torch.cat(operations, 1)
x = x.view(1, 1, -1)
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
x = self.fc3(x)
return F.softmax(x)
def cosine_similarity(x, patches, stride):
"""
Arguments:
x: a float tensor with shape [C, H, W].
patches: a float tensor with shape [M, C, size, size], normalized.
stride: an integer.
Returns:
a float tensor with shape [N, M],
where N = n * m, n = 1 + floor((H - size)/stride),
and m = 1 + floor((W - size)/stride).
"""
M = patches.size(0)
x = x.unsqueeze(0)
products = F.conv2d(x, patches, stride=stride) # shape [1, M, n, m]
size = patches.size(2)
x_norms = F.lp_pool2d(x, norm_type=2, kernel_size=size, stride=stride) # shape [1, C, n, m]
x_norms = x_norms.norm(p=2, dim=1, keepdim=True) # shape [1, 1, n, m]
products /= (x_norms + EPSILON)
return products.squeeze(0).view(M, -1).t()
def forward(self, input):
x = input[0]
pre_channel_att = input[1]
channel_att_sum = None
for pool_type in self.pool_types:
if pool_type == 'avg':
avg_pool = F.avg_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(avg_pool)
elif pool_type == 'max':
max_pool = F.max_pool2d(x, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(max_pool)
elif pool_type == 'lp':
lp_pool = F.lp_pool2d(x,
2, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(lp_pool)
elif pool_type == 'lse':
# LSE pool only
lse_pool = logsumexp_2d(x)
channel_att_raw = self.mlp(lse_pool)
if channel_att_sum is None:
channel_att_sum = channel_att_raw
else:
channel_att_sum = channel_att_sum + channel_att_raw
if pre_channel_att is None:
channel_att_sum = channel_att_raw
else:
pre_channel_att = self.att_fc(pre_channel_att) if hasattr(
self, 'att_fc') else pre_channel_att
channel_att_sum = channel_att_sum + pre_channel_att
scale = torch.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(
3).expand_as(x)
return {0: x * scale, 1: channel_att_sum}
def forward(self, x):
channel_att_sum = None
for pool_type in self.pool_types:
if pool_type == 'avg':
# avg_pool2d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True)
avg_pool = F.avg_pool2d(x, (x.size(2), x.size(3)), stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(avg_pool)
elif pool_type == 'max':
max_pool = F.max_pool2d(x, (x.size(2), x.size(3)), stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(max_pool)
elif pool_type == 'lp':
lp_pool = F.lp_pool2d(x, 2, (x.size(2), x.size(3)), stride=(x.size(2), x.size*(3)))
channel_att_raw = self.mlp(lp_pool)
elif pool_type == 'lse':
# LSE pool
lse_pool = logsumexp_2d(x)
channel_att_raw = self.mlp(lse_pool)
if channel_att_sum is None:
channel_att_sum = channel_att_raw
else:
channel_att_sum = channel_att_sum + channel_att_raw
scale = F.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(3).expand_as(x)
return x * scale
def test_lp_pool2d(self):
#torch.nn.LPPool2d(norm_type, kernel_size, stride=None, ceil_mode=False)
inp = torch.randn(1, 32, 64, 64, device='cuda', dtype=self.dtype)
output = F.lp_pool2d(inp, 2, 3, stride=2, ceil_mode=True)
def gem(x, p: Parameter = 3, eps=1e-6):
# return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1. / p)
return F.lp_pool2d(F.threshold(x, eps, eps), p, (x.size(-2), x.size(-1)))
def forward(self, input: Tensor) -> Tensor:
input = self.quant_handle(input)
return F.lp_pool2d(input, float(self.norm_type), self.kernel_size,
self.stride, self.ceil_mode)
def pool(self, frame_feats):
if self.pool_fn == 'L2':
pool_feats = F.lp_pool2d(frame_feats, 2, self.spatial_dim)
elif self.pool_fn == 'avg':
pool_feats = F.avg_pool2d(frame_feats, self.spatial_dim)
return pool_feats
