def forward(self, x):
features = self.features(x)
out = F.relu(features, inplace=True)
last_duration = int(math.ceil(self.sample_duration / 16))
last_size = int(math.floor(self.sample_size / 32))
out = F.avg_pool3d(
out, kernel_size=(last_duration, last_size, last_size)).view(
features.size(0), -1)
out = self.classifier(out)
return out
def forward(self, x):
out = self.conv1(x)
out = self.maxpool(out)
out = self.layer1(out)
out = self.layer2(out)
out = self.layer3(out)
out = F.avg_pool3d(out, out.data.size()[-3:])
out = out.view(out.size(0), -1)
feature = self.fc1(out)
out = self.fc2(feature)
return feature, out
def basic(x, out):
y = F.avg_pool3d(x, kernel_size=1, stride=self.stride)
zero_pads = torch.Tensor(
y.size(0), out - y.size(1), y.size(2), y.size(3),
y.size(4)
).zero_().requires_grad_()
if torch.cuda.is_available() and isinstance(y.data, torch.cuda.FloatTensor):
zero_pads = zero_pads.cuda()
y = torch.cat([y, zero_pads], dim=1)
return y
def forward(self, x):
b, c, d, h, w = x.shape
xs = F.avg_pool3d(F.relu(self.conv1(x)), 2)
xs = F.avg_pool3d(F.relu(self.conv2(xs)), 2)
xs = F.avg_pool3d(F.relu(self.conv3(xs)), 2)
xs = xs.view(xs.size(0), -1)
self.regularisation_loss = 30.0 * torch.mean(torch.abs(xs))
# cap the displacement field by max_disp
xs = torch.tanh(self.fc(xs)) * self.max_disp
xs = xs.view(-1, *self.cp_grid_shape)
self.displacement_field = self.compute_displacement(
xs) + self.gen_3d_mesh_grid(d, h, w).unsqueeze(0)
# extract first channel for warping
img = x.narrow(dim=1, start=0, length=1)
# warp image
return self.warp_image(img).to(self.device)
def _downsample_basic_block(self, x, planes, stride):
out = F.avg_pool3d(x, kernel_size=1, stride=stride)
zero_pads = torch.zeros(out.size(0), planes - out.size(1), out.size(2),
out.size(3), out.size(4))
if isinstance(out.data, torch.cuda.FloatTensor):
zero_pads = zero_pads.cuda()
out = torch.cat([out.data, zero_pads], dim=1)
return out
def forward(self, x):
out = F.relu(self.bn1(self.conv1(x)))
out = self.layer1(out)
out = self.layer2(out)
out = self.layer3(out)
out = self.layer4(out)
out = F.avg_pool3d(out, 4)
out = out.view(out.size(0), -1)
out = self.drop(out)
out = self.linear(out)
return out
def forward(self, input):
# get X, namely eq(2)
pool_spikes = F.avg_pool3d(input, self.kernel_size)
output = LIF(pool_spikes, self.theta, self.leak, self.V_min)
if self.check_mode:
return (conv_spikes, *output)
else:
return output[1]
def forward(self, x):
y = self.base(x)
# print("base: ", y.size())
y = F.avg_pool3d(y, (2, y.size(3), y.size(4)), stride=1)
# print("avg_pool: ", y.size())
y = self.fc(y)
# print("fc: ", y.size())
y = y.view(y.size(0), y.size(1), y.size(2))
logits = torch.mean(y, 2)
return logits
def channel_avg_pool2d(self, x, face):
assert self.ndim(x) == 4
in_channels = self.shape(x)[1]
assert 0 <= face <= in_channels
assert in_channels % face == 0
x = self.expand_dims(x, 1)
pool_face = face, 1, 1
pool_stride = pool_face
pool_padding = 0
x = F.avg_pool3d(x, pool_face, pool_stride, pool_padding)
return self.squeeze(x, 1)
def forward(self, x):
features = self.features(x)
out = F.relu(features, inplace=True)
last_duration = math.ceil(self.sample_duration / 16)
last_size = math.floor(self.sample_size / 32)
out = F.avg_pool3d(out,
kernel_size=(last_duration, last_size,
last_size)).view(features.size(0), -1)
if self.last_fc:
out = self.classifier(out)
return out
def downsample_basic_block(x, planes, stride, no_cuda=False):
out = F.avg_pool3d(x, kernel_size=1, stride=stride)
zero_pads = torch.Tensor(out.size(0), planes - out.size(1), out.size(2),
out.size(3), out.size(4)).zero_()
if not no_cuda:
if isinstance(out.data, torch.cuda.FloatTensor):
zero_pads = zero_pads.cuda()
out = Variable(torch.cat([out.data, zero_pads], dim=1))
return out
def downsample_basic_block(x, planes, stride):
out = F.avg_pool3d(x, kernel_size=1, stride=stride)
zero_pads = torch.Tensor(
out.size(0), planes - out.size(1), out.size(2), out.size(3),
out.size(4)).zero_()
if isinstance(out.data, torch.cuda.FloatTensor):
zero_pads = zero_pads.cuda()
out = Variable(torch.cat([out.data, zero_pads], dim=1))
return out
def forward(self, x):
if torch.onnx.is_in_onnx_export() and not self.reduce_temporal:
glob_context = F.avg_pool3d(x,
(1, int(x.shape[3]), int(x.shape[4])),
stride=1,
padding=0)
else:
glob_context = self.avg_pool(x)
mask = self.fc(glob_context)
return mask * x
def forward(self, x, y):
x = x.permute(0, 2, 1, 3, 4).cuda()
features = self.features(x)
out = F.relu(features, inplace=True)
last_duration = int(math.ceil(self.sample_duration / 16))
last_size = int(math.floor(self.sample_size / 32))
out = F.avg_pool3d(out,
kernel_size=(1, 3, 1)).view(features.size(0), -1)
#out = self.classifier(out)
out = self.classifier1(out)
return out
def forward(self, x):
branch1x1 = self.branch1x1(x)
branch3x3 = self.branch3x3(x)
branch5x5 = self.branch5x5(x)
branch_pool = F.avg_pool3d(x, kernel_size=3, stride=1, padding=1)
branch_pool = self.branch_pool(branch_pool)
outputs = [branch1x1, branch3x3, branch5x5, branch_pool]
return torch.cat(outputs, 1)
def forward(self, x):
x = x.permute(0, 4, 1, 2, 3)
out = self.conv1(x)
out = self.layer1(out)
out = self.layer2(out)
out = self.layer3(out)
out = F.relu(self.bn1(out))
out = F.avg_pool3d(out, 8)
out = out.view(out.size(0), -1)
out = self.linear(out)
return out
def forward(self, x):
out = self.conv_1(x)
out = self.bn_1(out)
out = self.stage_1(out)
out = self.stage_2(out)
out = self.stage_3(out)
out = F.relu(out)
out = F.avg_pool3d(out, 8)
out = out.view(-1, self.in_chs[3])
out = self.fc_out(out)
out = nn.functional.softmax(out,dim=1)
return out
def forward(self, block):
# block: [B, N, C, SL, W, H]
### extract feature ###
(B, N, C, SL, H, W) = block.shape
block = block.view(B*N, C, SL, H, W)
feature = self.backbone(block)
del block
feature = F.avg_pool3d(feature, (self.last_duration, 1, 1), stride=(1, 1, 1))
feature_inf_all = feature.view(B, N, self.param['feature_size'], self.last_size, self.last_size) # before ReLU, (-inf, +inf)
feature = self.relu(feature) # [0, +inf)
feature = feature.view(B, N, self.param['feature_size'], self.last_size, self.last_size) # [B,N,D,6,6], [0, +inf)
feature_inf = feature_inf_all[:, N-self.pred_step::, :].contiguous()
del feature_inf_all
### aggregate, predict future ###
_, hidden = self.agg(feature[:, 0:N-self.pred_step, :].contiguous())
hidden = hidden[:,-1,:] # after tanh, (-1,1). get the hidden state of last layer, last time step
pred = []
for i in range(self.pred_step):
# sequentially pred future
p_tmp = self.network_pred(hidden)
pred.append(p_tmp)
_, hidden = self.agg(self.relu(p_tmp).unsqueeze(1), hidden.unsqueeze(0))
hidden = hidden[:,-1,:]
pred = torch.stack(pred, 1) # B, pred_step, xxx
del hidden
### Get similarity score ###
# pred: [B, pred_step, D, last_size, last_size]
# GT: [B, N, D, last_size, last_size]
N = self.pred_step
# dot product D dimension in pred-GT pair, get a 6d tensor. First 3 dims are from pred, last 3 dims are from GT.
pred = pred.permute(0,1,3,4,2).contiguous().view(B*self.pred_step*self.last_size**2, self.param['feature_size'])
feature_inf = feature_inf.permute(0,1,3,4,2).contiguous().view(B*N*self.last_size**2, self.param['feature_size']).transpose(0,1)
score = torch.matmul(pred, feature_inf).view(B, self.pred_step, self.last_size**2, B, N, self.last_size**2)
del feature_inf, pred
if self.mask is None: # only compute mask once
# mask meaning: -2: omit, -1: temporal neg (hard), 0: easy neg, 1: pos, -3: spatial neg
mask = torch.zeros((B, self.pred_step, self.last_size**2, B, N, self.last_size**2), dtype=torch.int8, requires_grad=False).detach().cuda()
mask[torch.arange(B), :, :, torch.arange(B), :, :] = -3 # spatial neg
for k in range(B):
mask[k, :, torch.arange(self.last_size**2), k, :, torch.arange(self.last_size**2)] = -1 # temporal neg
tmp = mask.permute(0, 2, 1, 3, 5, 4).contiguous().view(B*self.last_size**2, self.pred_step, B*self.last_size**2, N)
for j in range(B*self.last_size**2):
tmp[j, torch.arange(self.pred_step), j, torch.arange(N-self.pred_step, N)] = 1 # pos
mask = tmp.view(B, self.last_size**2, self.pred_step, B, self.last_size**2, N).permute(0,2,1,3,5,4)
self.mask = mask
return [score, self.mask]
def forward(self, inputs, output_per_pixel=False):
"""
:param inputs: BxSxHxW
:param output_per_pixel:
:return:
"""
res = []
batch_size = inputs.shape[0]
nb_input_slices = inputs.shape[1]
x = inputs.view(batch_size * nb_input_slices, 1, inputs.shape[2],
inputs.shape[3])
x = self.l1(x)
x = self.bn1(x)
x = torch.relu(x)
x = self.maxpool(x)
x = self.base_model.layer1(x)
x = self.base_model.layer2(x)
x = self.base_model.layer3(x)
x = self.base_model.layer4(x) # BSxCxHxW
base_model_features = x.shape[1]
x = x.view(batch_size, nb_input_slices, base_model_features,
x.shape[2], x.shape[3]) # BxSxCxHxW
x = x.permute((0, 2, 1, 3, 4)) # BxCxSxHxW
x = self.combine_conv(x)
if output_per_pixel:
res.append(
F.conv3d(torch.cat([x, x], dim=1),
self.fc.weight[:, :, None, None], self.fc.bias))
# x: BxCxSxHxW
avg_pool = F.avg_pool3d(x, (1, ) + x.shape[3:])
max_pool = F.max_pool3d(x, (1, ) + x.shape[3:])
avg_max_pool = torch.cat((avg_pool, max_pool), 1)
# x: Bx2CxSx1x1
x = avg_max_pool[:, :, :, 0, 0]
if self.dropout > 0:
x = F.dropout(x, self.dropout, self.training)
out = self.fc(x) # BxCxS
# x: Bx2CxS
# out = out[None, :, :]
out = out.permute(0, 2, 1) # BxSxC
if res:
res.append(out)
return res
else:
return out
def forward(self, inputs):
if_print = False
# Feature Extraction
e0 = self.ec0(inputs)
syn0 = self.ec1(e0)
e1 = self.pool0(syn0)
e2 = self.ec2(e1)
syn1 = self.ec3(e2)
if if_print:
print("e0 = %s" % str(e0.size()))
print("syn0 = %s" % str(syn0.size()))
print("e1 = %s" % str(e1.size()))
print("e2 = %s" % str(e2.size()))
print("syn1 = %s" % str(syn1.size()))
e3 = self.pool1(syn1)
e4 = self.ec4(e3)
syn2 = self.ec5(e4)
if if_print:
print("e3 = %s" % str(e3.size()))
print("e4 = %s" % str(e4.size()))
print("syn2 = %s" % str(syn2.size()))
e5 = self.pool2(syn2)
e6 = self.ec6(e5)
# e7 = self.ec7(e6)
if if_print:
print("e5 = %s" % str(e5.size()))
print("e6 = %s" % str(e6.size()))
# print("e7 = %s" % str(e7.size()))
batch_size = inputs.size(0) # inputs.shape[0]
pooled = F.avg_pool3d(e6, (8, 24, 16)).view(batch_size, -1)
# if if_print:
# print("pooled = %s" % str(pooled.size()))
#
#
# # # Attention Mechanism
# # g_conv1, att1 = self.compatibility_score1(syn2, e7)
# # g_conv2, att2 = self.compatibility_score2(syn1, e7)
# g_conv3, att3 = self.compatibility_score3(syn0, e6)
#
# g_conv3 = self.bn3D_3(g_conv3)
# # # flatten to get single feature vector
# fsizes = self.attention_filter_sizes
# # g1 = torch.sum(g_conv1.view(batch_size, fsizes[0], -1), dim=-1)
# # g2 = torch.sum(g_conv2.view(batch_size, fsizes[1], -1), dim=-1)
# # g3 = torch.sum(g_conv3.view(batch_size, fsizes[2], -1), dim=-1)
# # g3 = self.bn1D_3(g3)
# g3 = F.avg_pool3d(g_conv3, (64, 96, 128)).view(batch_size, -1)
y = self.classifier(pooled)
return y
def forward(self, in_tensor):
in_size = in_tensor.size()
avg_pool = F.avg_pool3d(in_tensor, (in_size[2], in_size[3], in_size[4]), stride=in_size[2])
# avg_pool: batch, channel, 1, 1, 1
print(avg_pool.size())
# somthing wrong here
output = self.gate_c(avg_pool)
print("oirginal", output.size())
output = output.unsqueeze(2).unsqueeze(3).unsqueeze(4).expand_as(in_tensor)
print("After unsqueeze", output.size())
return output
def MINDSSC(img, radius=2, dilation=2):
# see http://mpheinrich.de/pub/miccai2013_943_mheinrich.pdf for details on the MIND-SSC descriptor
# kernel size
kernel_size = radius * 2 + 1
# define start and end locations for self-similarity pattern
six_neighbourhood = torch.Tensor([[0, 1, 1], [1, 1, 0], [1, 0, 1],
[1, 1, 2], [2, 1, 1], [1, 2, 1]]).long()
# squared distances
dist = pdist_squared(six_neighbourhood.t().unsqueeze(0)).squeeze(0)
# define comparison mask
x, y = torch.meshgrid(torch.arange(6), torch.arange(6))
mask = ((x > y).view(-1) & (dist == 2).view(-1))
# build kernel
idx_shift1 = six_neighbourhood.unsqueeze(1).repeat(1, 6,
1).view(-1, 3)[mask, :]
idx_shift2 = six_neighbourhood.unsqueeze(0).repeat(6, 1,
1).view(-1, 3)[mask, :]
mshift1 = torch.zeros(12, 1, 3, 3, 3).cuda()
mshift1.view(-1)[torch.arange(12) * 27 + idx_shift1[:, 0] * 9 +
idx_shift1[:, 1] * 3 + idx_shift1[:, 2]] = 1
mshift2 = torch.zeros(12, 1, 3, 3, 3).cuda()
mshift2.view(-1)[torch.arange(12) * 27 + idx_shift2[:, 0] * 9 +
idx_shift2[:, 1] * 3 + idx_shift2[:, 2]] = 1
rpad1 = nn.ReplicationPad3d(dilation)
rpad2 = nn.ReplicationPad3d(radius)
# compute patch-ssd
ssd = F.avg_pool3d(rpad2(
(F.conv3d(rpad1(img), mshift1, dilation=dilation) -
F.conv3d(rpad1(img), mshift2, dilation=dilation))**2),
kernel_size,
stride=1)
# MIND equation
mind = ssd - torch.min(ssd, 1, keepdim=True)[0]
mind_var = torch.mean(mind, 1, keepdim=True)
mind_var_mean = mind_var.mean().cpu().data
mind_var = torch.clamp(mind_var, mind_var_mean * 0.001,
mind_var_mean * 1000)
mind /= mind_var
mind = torch.exp(-mind)
#permute to have same ordering as C++ code
mind = mind[:,
torch.Tensor([6, 8, 1, 11, 2, 10, 0, 7, 9, 4, 5, 3]).long(
), :, :, :]
return mind
def forward(self, x):
features = self.features(x)
out = F.relu(features, inplace=True)
last_duration = int(math.ceil(self.sample_duration / 16))
last_size0 = int(math.floor(self.sample_size0 / 32))
last_size1 = int(math.floor(self.sample_size1 / 32))
out = F.avg_pool3d(out,
kernel_size=(last_duration, last_size0,
last_size1)).view(
features.size(0), -1)
out = self.classifier(out)
return out
def logits(self, features):
if not self.training and self.test_time_pool:
x = F.avg_pool3d(features,
kernel_size=(self.frame_num // 8, 7, 7),
stride=1)
out = self.classifier(x)
# The extra test time pool should be pooling an img_size//32 - 6 size patch
out = adaptive_avgmax_pool2d(out, pool_type='avgmax')
else:
x = adaptive_avgmax_pool2d(features, pool_type='avg')
out = self.classifier(x)
return out.view(out.size(0), -1)
def forward(self, x):
x = F.avg_pool3d(x, kernel_size=3)
x = F.avg_pool3d(x, kernel_size=4, stride=2, padding=2)
x = F.avg_pool3d(x,
kernel_size=(1, 2, 3),
stride=1,
padding=(0, 1, 1),
ceil_mode=False,
count_include_pad=True)
x = F.avg_pool3d(x,
kernel_size=(3, 4, 5),
stride=(1, 2, 2),
padding=(1, 1, 2),
ceil_mode=True,
count_include_pad=False)
x = F.avg_pool3d(x,
kernel_size=(5, 4, 3),
stride=(2, 1, 1),
padding=1,
ceil_mode=False,
count_include_pad=True)
x = F.avg_pool3d(x,
kernel_size=2,
stride=1,
padding=0,
ceil_mode=True,
count_include_pad=True)
#x = F.avg_pool3d(x, kernel_size=(5,4,4), stride=1, padding=2, ceil_mode=False, count_include_pad=False, divisor_override=77)
return x
def adaptive_avgmax_pool2d(x,
pool_type='avg',
padding=0,
count_include_pad=False):
"""Selectable global pooling function with dynamic input kernel size
"""
if pool_type == 'avgmaxc':
x = torch.cat([
F.avg_pool3d(x,
kernel_size=(x.size(2), x.size(3), x.size(4)),
padding=padding,
count_include_pad=count_include_pad),
F.max_pool3d(x,
kernel_size=(x.size(2), x.size(3), x.size(4)),
padding=padding)
],
dim=1)
elif pool_type == 'avgmax':
x_avg = F.avg_pool3d(x,
kernel_size=(x.size(2), x.size(3), x.size(4)),
padding=padding,
count_include_pad=count_include_pad)
x_max = F.max_pool3d(x,
kernel_size=(x.size(2), x.size(3), x.size(4)),
padding=padding)
x = 0.5 * (x_avg + x_max)
elif pool_type == 'max':
x = F.max_pool3d(x,
kernel_size=(x.size(2), x.size(3), x.size(4)),
padding=padding)
else:
if pool_type != 'avg':
print(
'Invalid pool type %s specified. Defaulting to average pooling.'
% pool_type)
x = F.avg_pool3d(x,
kernel_size=(x.size(2), x.size(3), x.size(4)),
padding=padding,
count_include_pad=count_include_pad)
return x
def downsample_basic_block(x, planes, stride):
# decrease data resolution if stride not equals to 1
out = F.avg_pool3d(x, kernel_size=1, stride=stride)
# shape: (batch_size, channel, t, h, w)
# try to match the channel size
zero_pads = torch.Tensor(out.size(0), planes - out.size(1), out.size(2),
out.size(3), out.size(4)).zero_()
if isinstance(out.data, torch.cuda.FloatTensor):
zero_pads = zero_pads.cuda()
out = Variable(torch.cat([out.data, zero_pads], dim=1))
return out
def soft_pool3d(x, kernel_size=2, stride=None, force_inplace=False):
if x.is_cuda and not force_inplace:
x = CUDA_SOFTPOOL3d.apply(x, kernel_size, stride)
# Replace `NaN's if found
if torch.isnan(x).any():
return torch.nan_to_num(x)
return x
kernel_size = _triple(kernel_size)
if stride is None:
stride = kernel_size
else:
stride = _triple(stride)
# Get input sizes
_, c, d, h, w = x.size()
# Create per-element exponential value sum : Tensor [b x c x d x h x w]
e_x = torch.exp(x)
# Apply mask to input and pool and calculate the exponential sum
# Tensor: [b x c x d x h x w] -> [b x c x d' x h' x w']
return F.avg_pool3d(x.mul(e_x), kernel_size,
stride=stride).mul_(sum(kernel_size)).div_(
F.avg_pool3d(e_x, kernel_size,
stride=stride).mul_(sum(kernel_size)))
def forward(self, x):
output = F.relu(self.conv1_norm(self.conv1(x)))
output = F.relu(self.conv2_norm(self.conv2(output)))
if self.se_enable:
se = F.avg_pool3d(output, (output.size(2), output.size(3), output.size(4)))
se = F.relu(self.se_conv1(se))
se = F.sigmoid(self.se_conv2(se))
output = output * se
output += x
output = F.relu(output)
return output
def forward(self, x):
out = self.conv1(x)
out = self.maxpool(out)
out = self.layer1(out)
out = self.layer2(out)
out = self.layer3(out)
out = self.layer4(out)
out = F.avg_pool3d(out, out.data.size()[-3:])
out = out.view(out.size(0), -1)
normed_out = F.normalize(out, p=2, dim=1)
return out, normed_out
def forward(self, x):
if self.dim == 2:
w = F.avg_pool2d(x, x.size(2))
elif self.dim == 3:
w = F.avg_pool3d(x, x.size(2))
else:
raise ValueError(
'The spatial dimensionality of the kernel ({0:d}) is not supported.'
.format(self.dim))
w = F.relu(self.fc1(w))
w = self.fc2(w).sigmoid()
out = x * w
return out
def forward(self, x):
spatial_logits = self.spatial_logits(x)
temporal_logits = self.temporal_logits(
F.avg_pool3d(x, (1, int(x.shape[3]), int(x.shape[4])), stride=1, padding=0))
logits = self.scale * (spatial_logits + temporal_logits)
soft_mask = torch.sigmoid(logits)
if self.residual:
out = (1.0 + soft_mask) * x
else:
out = soft_mask * x
return out
def forward(self, input_tensor):
return functional.avg_pool3d(input_tensor, input_tensor.size()[2:]).view(input_tensor.size()[:2])
