def forward(self, input_tensor):
"""
:param input_tensor: X, shape = (batch_size, num_channels, D, H, W)
:return: output tensor
"""
batch_size, num_channels, D, H, W = input_tensor.size()
# Project:
# Average along channels and different axes
squeeze_tensor_w = F.adaptive_avg_pool3d(input_tensor, (1, 1, W))
squeeze_tensor_h = F.adaptive_avg_pool3d(input_tensor, (1, H, 1))
squeeze_tensor_d = F.adaptive_avg_pool3d(input_tensor, (D, 1, 1))
# tile tensors to original size and add:
final_squeeze_tensor = sum([
squeeze_tensor_w.view(batch_size, num_channels, 1, 1, W),
squeeze_tensor_h.view(batch_size, num_channels, 1, H, 1),
squeeze_tensor_d.view(batch_size, num_channels, D, 1, 1)
])
# Excitation:
final_squeeze_tensor = self.sigmoid(
self.conv_cT(self.relu(self.conv_c(final_squeeze_tensor))))
output_tensor = torch.mul(input_tensor, final_squeeze_tensor)
return output_tensor
def forward(self,x,y) :
dense_model1_output = self.f_1(x) ###Original Output size (1X528X94X94)
dense_model2_output = self.f_2(y)
output_size = dense_model2_output.size()
attention_input = dense_model2_output[0].unsqueeze(0)
attention_output = self.Attention_Network(attention_input)
attention_output = F.adaptive_avg_pool3d(attention_output, (1,1, 1))
patches = int((dense_model2_output.size())[0])
for i in range(1,patches):
attention_input = dense_model2_output[i].unsqueeze(0)
attention_output_patch = self.Attention_Network(attention_input) ###gate channels?? ---> output will be of dimension NXCXHXW (63 X 15 X 22 X 22) in our case
attention_output_patch = F.adaptive_avg_pool3d(attention_output_patch, (1,1, 1))
attention_output = torch.cat((attention_output,attention_output_patch),0)
##Now adding the patch attention network after we compute the channel and spatial attention for all the N patches and we get a outpt vector size of 160X3X94X94
patch_attention_input = torch.reshape((attention_output),(1,patches,1,1))
patch_attention_output = self.Patch_Attention(patch_attention_input)
patch_attention_output = torch.reshape((patch_attention_output),(patches,1,1,1))
final_attention_output = torch.mul(dense_model2_output,patch_attention_output) ###Multiplying the output of Patch attention with the input
###Combining both the original model and the patch based network### ---> Implies combining (1X528X94X94) with (63 X 15 X 22 X 22) we want ----> (64 X 543 X 2 X 2)
model1_avg_pool_output = F.adaptive_avg_pool2d(dense_model1_output, (1, 1)) ## 1 X 528 X 1 X 1
patch_avg_pool_output = F.adaptive_avg_pool2d(final_attention_output, (1, 1)) ## 63 X 15 X 1 X 1
patch_model_output = (torch.mean(patch_avg_pool_output,0)).unsqueeze(0) ### 1 X 15 X 1 X 1
patch_model_output = patch_model_output.view(patch_model_output.size(0), -1) ## 1 X 15
original_output = model1_avg_pool_output.view(model1_avg_pool_output.size(0), -1) ## 1 X 528
combined_out = torch.cat((original_output,patch_model_output),dim = 1)
out_flat = combined_out.view(combined_out.size(0), -1)
in_size = out_flat.size()[1]
num_classes = 2
out = self.lin_layer(out_flat)
return out
def forward(self, inputs):
_b, _c, _d, _h, _w = inputs.shape
min_dim = min(_d, _h, _w)
# reduce dimensionality to 2d
d = inputs
d = F.adaptive_avg_pool3d(d, (min_dim, 1, 1))
h = inputs.transpose(2, 4)
h = F.adaptive_avg_pool3d(h, (min_dim, 1, 1))
w = inputs.transpose(3, 4)
w = F.adaptive_avg_pool3d(w, (min_dim, 1, 1))
# stack three group of 2D feature maps
x = torch.cat((d, h, w), dim=1)
# depthwise conv
x = self._depthwise_conv(x)
x = self._bn1(x)
x = self._activate(x)
# reduce dimensionality to 1d
x = self._se_reduce(x)
x = self._activate(x)
# pointwise conv
x = self._project_conv(x).sigmoid()
d, h, w = torch.split(x, self._channels, dim=1)
h = h.transpose(2, 4)
w = w.transpose(3, 4)
mask = F.interpolate((d * h * w), size=(_d, _h, _w))
out = inputs * mask
return out
def forward(self, x):
self.input_imgs = x.detach().cpu().numpy()
features = self.features(x)
if self.out_block is None:
out = F.relu(features, inplace=True)
out = F.adaptive_avg_pool3d(out, 1)
out = torch.flatten(out, 1)
out = self.classifier(out)
elif self.out_block[:5] == "block":
out_size = max(int((10**4 / self.num_features)**(1 / 3)), 1)
out = F.adaptive_avg_pool3d(features,
out_size) # final dim ~ 10**4
out = torch.flatten(out, 1)
elif self.out_block == "simCLR":
out = F.relu(features, inplace=True)
out = F.adaptive_avg_pool3d(out, 1)
out = torch.flatten(out, 1)
out = self.hidden_representation(out)
out = F.relu(out, inplace=True)
out = self.head_projection(out)
elif self.out_block == "sup_simCLR":
out = F.relu(features, inplace=True)
out = F.adaptive_avg_pool3d(out, 1)
out = torch.flatten(out, 1)
out = self.hidden_representation(out)
out = F.relu(out, inplace=True)
out = self.head_projection(out)
out = torch.cat([out, self.classifier(out)], dim=1)
return out.squeeze(dim=1)
def forward(self, inputs):
_, _, d, h, w = inputs.shape
min_dim = min(d, h, w)
# reduce dimensionality to 2d
dh = inputs
dh = F.adaptive_avg_pool3d(dh, (min_dim, min_dim, 1)).squeeze(dim=-1)
hw = inputs.transpose(2, 4)
hw = F.adaptive_avg_pool3d(hw, (min_dim, min_dim, 1)).squeeze(dim=-1)
dw = inputs.transpose(3, 4)
dw = F.adaptive_avg_pool3d(dw, (min_dim, min_dim, 1)).squeeze(dim=-1)
# stack three group of 2D feature maps
x = torch.cat((dh, hw, dw), dim=1)
# depthwise conv
x = self._depthwise_conv(x)
x = self._bn1(x)
x = self._activate(x)
# reduce dimensionality to 1d
x = self._se_reduce(x)
x = self._activate(x)
# pointwise conv
x = self._project_conv(x).sigmoid()
dh, hw, dw = torch.split(x, self._channels, dim=1)
dh = dh.unsqueeze(dim=-1)
dw = dw.unsqueeze(dim=-2)
hw = hw.unsqueeze(dim=-3)
mask = F.interpolate((dh * dw * hw), size=(d, h, w))
out = inputs * mask
return out
def forward(self, input):
print('the size of input image is ', input.size())
# Context Path
y = self.conv1_xception39(input)
# print('the size of xception39 after conv1', y.size())
y = self.maxpool_xception39(y)
print(' level 1: 1 / 4 the size of xception39 after maxpool', y.size())
y = self.block1_xception39(y)
# print('the size of xception39 after block1', y.size())
y = self.block2_xception39(y)
print(' level 2: 1 / 8 the size of xception39 after block2', y.size())
y = self.block3_xception39(y)
# print(' level 3: 1 / 16 the size of xception39 after block3', y.size())
y = self.block4_xception39(y)
print(' level 3: 1 / 16 the size of xception39 after block4', y.size())
y = F.adaptive_avg_pool3d(y, (28, 28, 28))
y_arm = self.arm1_context_path(y)
print(' the size of image feature after first y_arm', y_arm.size())
y = self.block5_xception39(y)
# print('the size of xception39 after block5', y.size())
y_32 = self.block6_xception39(y)
print(' level 4: 1 / 32 the size of xception39 after block6', y.size())
y = F.adaptive_avg_pool3d(y_32, (28, 28, 28))
y_arm2 = self.arm2_context_path(y)
print(' the size of image feature after second y_arm', y_arm2.size())
y_32_up = F.adaptive_avg_pool3d(y_32, (28, 28, 28))
print(' the size of y_32_up is ', y_32_up.size())
# Concatenate the image feature of ARM1, ARM2 and y_32_up
y_cat = torch.cat([y_arm, y_arm2], dim=1)
y_cat = torch.cat([y_cat, y_32_up], dim=1)
print(' size of y_cat is ', y_cat.size())
# Spatial Path
sp = self.block1_spatial_path(input)
print(' level 1 of spatial path, the size of sp is ', sp.size())
sp = self.block2_spatial_path(sp)
print(' level 2 of spatial path, the size of sp is ', sp.size())
sp = self.block3_spatial_path(sp)
print(' level 3 of spatial path, the size of sp is ', sp.size())
# Concatenate the image feature after context path : y_cat and the image feature after spatial path : sp
y_cat = torch.cat([y_cat, sp], dim=1)
print(
' the size of image feature after the context path and spatial path is ',
y_cat.size())
y_cat = self.FFM(y_cat)
print(' the size of image feature after FFM', y_cat.size())
y_cat = F.adaptive_avg_pool3d(y_cat, (256, 256, 256))
print(' the size of image feature after FFM', y_cat.size())
# y = F.adaptive_avg_pool2d(y, (1, 1))
# y = y.view(y.size(0), -1)
# # print('the size of xception39 is ', y.size()[1])
# y = self.fc_xception39(y)
return y_cat
def forward(self, x):
if self.method == 'avg':
return F.adaptive_avg_pool3d(x, self.output_size)
elif self.method == 'avg':
return F.adaptive_max_pool3d(x, self.output_size)
else:
avg_pooled = F.adaptive_avg_pool3d(x, self.output_size)
max_pooled = F.adaptive_max_pool3d(x, self.output_size)
return avg_pooled + max_pooled
def sliding_window_extract(model, loader, save_dir, window_size, device):
# when you extarct features with sliding window,
# the features do not keep spatial and temporal dimension
# because they are too large
model.eval()
for sample in tqdm.tqdm(loader, total=len(loader)):
with torch.no_grad():
name = sample['name'][0]
# if features already exist, the below process will be passes.
if os.path.exists(os.path.join(save_dir, name + '.pth')):
continue
if os.path.exists(os.path.join(save_dir, 'motion', name + '.pth')):
continue
clip = sample['clip']
_, _, t, h, w = clip.shape
zeros = torch.zeros(1, 3, window_size - 1, h, w)
clip = torch.cat([clip, zeros], dim=2)
feats = []
for i in range(t):
x = clip[:, :, i:i + window_size].clone().detach().to(device)
feat = model.extract_features(x)
# if features are extracted by slowfast
if isinstance(feat, list):
semantic = F.adaptive_avg_pool3d(feat[0], output_size=1)
semantic = semantic.squeeze()
motion = F.adaptive_avg_pool3d(feat[1], output_size=1)
motion = motion.squeeze()
# after squeeze, dim 0 is channel dimension
concated_feat = torch.cat([semantic, motion], dim=0)
feats.append(concated_feat.to('cpu'))
else:
feat = F.adaptive_avg_pool3d(feat, output_size=1)
feat = feat.squeeze()
feats.append(feat.to('cpu'))
# separate concated slowfast features into semantic and motion features
if isinstance(feat, list):
feats = torch.stack(feats, dim=1)
c = len(semantic)
semantics = feats[:c]
motions = feats[c:]
torch.save(
semantics, os.path.join(save_dir, 'semantic', name + '.pth'))
torch.save(
motions, os.path.join(save_dir, 'motion', name + '.pth'))
else:
feats = torch.stack(feats, dim=1)
torch.save(
feats, os.path.join(save_dir, name + '.pth'))
def forward(self, x):
features = self.features(x)
out = F.relu(features, inplace=True)
out = F.adaptive_avg_pool3d(out, (1, 1, 1)).view(features.size(0), -1)
out = F.dropout(out, p=self.drop_rate, training=self.training)
out = self.classifier(out)
return out
def forward(self, x):
x = self.features(x)
x = F.relu(x, inplace=True)
x = F.adaptive_avg_pool3d(x, (1, 1, 1))
x = torch.flatten(x, 1)
x = self.classifier(x)
return x
def forward(self, x):
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.maxpool(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
num_frames = x.shape[2]
x = F.adaptive_avg_pool3d(x, output_size=(num_frames, 1, 1))
# N x 1024 x ((F/8)-1) x 1 x 1
x = x.squeeze(-1)
x = x.squeeze(-1)
x = x.transpose(1, 2)
n, c, nf = x.size()
x = x.contiguous().view(n * c, -1)
x = self.dropout(x)
x = self.fc(x)
x = x.view(n, c, -1)
# N x num_classes x ((F/8)-1)
logits = torch.mean(x, 1)
return logits
def temporal_pool(x, n_segment):
nt, c, h, w = x.size()
n_batch = nt // n_segment
x = x.view(n_batch, n_segment, c, h, w).transpose(1, 2) # n, c, t, h, w
x = F.adaptive_avg_pool3d(x, (n_segment // 2, h // 2, w // 2))
x = x.transpose(1, 2).contiguous().view(nt // 2, c, h // 2, w // 2)
return x
def forward(self, x):
out = self.features.forward(x)
out = F.adaptive_avg_pool3d(out, (1, 1, 1)).view(out.size(0), -1)
out = self.classifier(out)
return out
def extract(self, inp):
print("input: ", inp.size())
out = self.conv3d_1a_7x7(inp)
out = self.maxPool3d_2a_3x3(out)
out = self.conv3d_2b_1x1(out)
out = self.conv3d_2c_3x3(out)
out = self.maxPool3d_3a_3x3(out)
out = self.mixed_3b(out)
out = self.mixed_3c(out)
out = self.maxPool3d_4a_3x3(out)
out = self.mixed_4b(out)
out = self.mixed_4c(out)
out = self.mixed_4d(out)
out = self.mixed_4e(out)
out = self.mixed_4f(out)
out = self.maxPool3d_5a_2x2(out)
out = self.mixed_5b(out)
out = self.mixed_5c(out)
# print "mixed_5c: ", out.size()
out = F.adaptive_avg_pool3d(out, (None, 1, 1))
# print "avg_pool: ", out.size()
out = out.squeeze(3)
out = out.squeeze(3)
return out.transpose(1, 2)
def forward(self, x):
inp_x = torch.stack(
[torch.max(x, 1)[0],
torch.std(x, 1),
torch.mean(x, 1)], 1)
# inp_x = torch.cat([inp_x, x], 1)
# inp_x = torch.max(x, 1)[0].reshape(-1, 1, 52, 63, 53)
# inp_x = torch.mean(x, 1).reshape(-1, 1, 52, 63, 53)
x = self.conv1(inp_x)
x = self.bn1(x)
x = self.relu(x)
x = self.maxpool(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
x = F.adaptive_avg_pool3d(x, (1, 1, 1))
emb_3d = x.view((-1, self.fea_dim))
out = self.fc(emb_3d)
return out
def _compute_grad_weights(self, grads):
grads = self._normalize(grads)
if self.is_3d:
weights = F.adaptive_avg_pool3d(grads, 1)
else:
weights = F.adaptive_avg_pool2d(grads, 1)
return weights
def forward(self, x):
b, c, h, w = x.size()
channel_att_sum = None
for pool_type in self.pool_types:
if pool_type == 'avg':
y = x.unsqueeze(0)
avg_pool = F.adaptive_avg_pool3d(y, (self.groups, 1, 1))
avg_pool = avg_pool.squeeze(0)
channel_att_raw = self.mlp(avg_pool)
elif pool_type == 'max':
y = x.unsqueeze(0)
max_pool = F.adaptive_max_pool3d(y, (self.groups, 1, 1))
max_pool = max_pool.squeeze(0)
channel_att_raw = self.mlp(max_pool)
elif pool_type == 'lp':
lp_pool = F.lp_pool2d(x,
2, (x.size(2), x.size(3)),
stride=(x.size(2), x.size(3)))
channel_att_raw = self.mlp(lp_pool)
elif pool_type == 'lse':
# LSE pool only
lse_pool = logsumexp_2d(x)
channel_att_raw = self.mlp(lse_pool)
if channel_att_sum is None:
channel_att_sum = channel_att_raw
else:
channel_att_sum = channel_att_sum + channel_att_raw
# channel_att_sum=channel_att_sum.unsqueeze(1)
# channel_att_sum=F.upsample(channel_att_sum,c)
# channel_att_sum=channel_att_sum.squeeze(1)
scale = F.sigmoid(channel_att_sum).unsqueeze(2).unsqueeze(3).expand_as(
x)
return x * scale
def forward(self, x):
features = self.features(x)
out = F.relu(features, inplace=True)
out = F.adaptive_avg_pool3d(out, output_size=(1, 1, 1)).view(
features.size(0), -1)
out = self.classifier(out)
return out
def forward(self, inputs, drop_connect_rate=None):
"""
:param inputs: input tensor
:param drop_connect_rate: drop connect rate (float, between 0 and 1)
:return: output of block
"""
# Expansion and Depthwise Convolution
x = inputs
if self._block_args.expand_ratio != 1:
x = self._swish(self._bn0(self._expand_conv(inputs)))
x = self._swish(self._bn1(self._depthwise_conv(x)))
# Squeeze and Excitation
if self.has_se:
x_squeezed = F.adaptive_avg_pool3d(x, 1)
x_squeezed = self._se_expand(
self._swish(self._se_reduce(x_squeezed)))
x = torch.sigmoid(x_squeezed) * x
x = self._bn2(self._project_conv(x))
# Skip connection and drop connect
input_filters, output_filters = self._block_args.input_filters, self._block_args.output_filters
if self.id_skip and self._block_args.stride == 1 and input_filters == output_filters:
if drop_connect_rate:
x = drop_connect(x,
p=drop_connect_rate,
training=self.training)
x = x + inputs # skip connection
return x
def forward_multiframe(self, x, pool=True):
(B, T, C, H, W) = x.size()
x = x.contiguous()
x = x.view(B * T, C, H, W)
x = self.feature_extraction(x)
(_, C, H, W) = x.size()
x = x.view(B, T, C, H, W)
x = x.permute(0, 2, 1, 3, 4)
if not pool:
return x
if self.pool_type == 'avgpool':
x = F.adaptive_avg_pool3d(x, 1)
elif self.pool_type == 'maxpool':
x = F.adaptive_max_pool3d(x, 1)
if self.with_fc:
x = x.view(x.size(0), -1)
x = self.fc(x)
return x.view(x.size(0), -1, 1, 1)
else:
return x.view(x.size(0), -1, 1, 1)
return x
def forward(self, x):
features = self.features(x)
out = F.relu(features, inplace=True)
out = F.adaptive_avg_pool3d(out, (1, 1, 1))
out = out.view(-1, self.final_num_features)
out = self.classifier(out)
return out
def forward(self, x):
win_size = x.shape[1]
x = x.permute(0, 2, 1, 3, 4)
x = self.conv1(x)
# pdb.set_trace()
x = self.bn1(x)
x = F.avg_pool3d(x,(1,2,2))
x = self.relu(x)
x = self.conv2(x)
x = self.bn2(x)
x = F.avg_pool3d(x,(1,2,2))
x = self.relu(x)
x = self.conv3(x)
x = self.bn3(x)
x = F.avg_pool3d(x,(1,2,2))
x = self.relu(x)
x = self.conv4(x)
x = self.bn4(x)
x = F.avg_pool3d(x,(1,2,2))
x = self.relu(x)
x = self.conv5(x)
x = self.bn5(x)
x = F.avg_pool3d(x,(1,2,2))
x = self.relu(x)
x = F.adaptive_avg_pool3d(x, (win_size, 1, 1))
x = x.permute(0, 2, 1, 3, 4)
x = x.reshape(x.size(0), x.size(1), - 1)
return x
def global_average_pooling(inp: torch.Tensor) -> torch.Tensor:
if inp.ndim == 5:
return F.adaptive_avg_pool3d(inp, 1)
elif inp.ndim == 4:
return F.adaptive_avg_pool2d(inp, 1)
else:
raise NotImplementedError
def forward(self, x):
_, c, h, w = x.size()
# c2,h2,w2=int(c/4),int(h/4),int(w/4)
c2, h2, w2 = self.group, int(h / 4), int(w / 4)
# y=self.conv1(x)
y = x.clone()
y = y.unsqueeze(0)
y1 = F.adaptive_avg_pool3d(y, (c2, h2, w2))
y1 = y1.squeeze(0)
y1 = self.conv2_1(y1)
# y=y.unsqueeze(0)
y2 = F.adaptive_max_pool3d(y, (c2, h2, w2))
y2 = y2.squeeze(0)
y2 = self.conv2_2(y2)
# y1=y1.unsqueeze(1)
# y2=y2.unsqueeze(1)
# y3=torch.cat((y1,y2),dim=1)
# y3=self.conv(y3)
# y3=y3.squeeze(1)
y3 = y1 + y2
y3 = y3.unsqueeze(0)
y3 = F.upsample(y3, size=(c, h, w))
y3 = y3.squeeze(0)
# y=self.conv3(y)
y3 = F.sigmoid(y3)
# print(y3.size())
return y3 * x
def forward(self, inp):
# Preprocessing
inp = torch.cat([inp, inp, inp], 3)
inp = torch.cat([inp, inp, inp, inp, inp, inp], 2)
print("input: ", inp.size())
out = self.conv3d_1a_7x7(inp)
out = self.maxPool3d_2a_3x3(out)
out = self.conv3d_2b_1x1(out)
out = self.conv3d_2c_3x3(out)
out = self.maxPool3d_3a_3x3(out)
out = self.mixed_3b(out)
out = self.mixed_3c(out)
out = self.maxPool3d_4a_3x3(out)
out = self.mixed_4b(out)
out = self.mixed_4c(out)
out = self.mixed_4d(out)
out = self.mixed_4e(out)
out = self.mixed_4f(out)
out = self.maxPool3d_5a_2x2(out)
out = self.mixed_5b(out)
out = self.mixed_5c(out)
print("mixed_5c: ", out.size())
# out = self.avg_pool(out)
out = F.adaptive_avg_pool3d(out, (None, 1, 1))
print("avg_pool: ", out.size())
out = self.dropout(out)
out = self.conv3d_0c_1x1(out)
print("conv3d: ", out.size())
out = out.squeeze(3)
out = out.squeeze(3)
out = out.mean(2)
print("logits: ", out.size())
out_logits = out
out = self.softmax(out_logits)
return out, out_logits
def forward(self, x):
x = self.batch_norm1(x)
x = self.conv1(x)
x = self.relu(x)
x = self.pool1(x)
#print(x.size())
x = self.batch_norm2(x)
x = self.conv2(x)
x = self.relu(x)
x = self.pool2(x)
#print(x.size())
x = self.batch_norm3(x)
x = self.conv3(x)
x = self.relu(x)
x = self.pool3(x)
#print(x.size())
x = self.batch_norm4(x)
x = self.conv4(x)
x = self.relu(x)
x = self.pool4(x)
#print(x.size())
#m = nn.AvgPool3d((8, 8, 1), stride=(1, 1, 1))
#print(m(x).size())
# global pooling
x = F.adaptive_avg_pool3d(x, (1, 1, 1))
x = x.view(x.shape[0], -1)
if self.n_class != 0:
x = self.l1(x)
return x
def forward(self, x):
_, c, h, w = x.size()
c2, h2, w2 = int(c / 4), int(h / 4), int(w / 4)
y = x
y = y.unsqueeze(0)
y1 = F.adaptive_avg_pool3d(y, (c2, h2, w2))
y1 = y1.squeeze(0)
y2 = F.adaptive_max_pool3d(y, (c2, h2, w2))
y2 = y2.squeeze(0)
y3 = torch.cat((y1, y2), 1)
print(y3.size())
y3 = self.conv2(y3)
y3 = y3.unsqueeze(0)
y3 = F.upsample(y3, size=(c, h, w))
y3 = y3.squeeze(0)
# y=self.conv3(y)
y3 = F.sigmoid(y3)
# y2=F.adaptive_max_pool3d(y,(c2,h2,w2))
# y2=y2.squeeze(0)
# y2=self.conv2(y2)
# y2=y2.unsqueeze(0)
# y2=F.upsample(y2,size=(c,h,w))
# y2=y2.squeeze(0)
# # y=self.conv3(y)
# y2=F.sigmoid(y2)
return y3 * x
def forward(self, x, target=None, fcn_testing=False):
loss = dict()
if len(self.convs) > 0:
for i, conv in enumerate(self.convs):
x = conv(x)
if fcn_testing:
# fcn testing
if self.new_cls is None:
# create a conv head
self.new_cls = nn.Conv3d(self.indim,self.outdim,1,1,0).cuda()
self.new_cls.weight.copy_(self.fc.weight.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1))
self.new_cls.bias.copy_(self.fc.bias)
self.fc = None
# predict the calss map
x = self.new_cls(x)
return x
x = F.adaptive_avg_pool3d(x,1).squeeze(-1).squeeze(-1).squeeze(-1)
x = self.dropout(x)
x = self.fc(x)
if target is None:
return x
if self.multilabel:
loss['loss_aux'] = self.loss_weight * binary_weighted_multilabel_binary_cross_entropy(x, target, target.new_ones(target.shape))
else:
loss['loss_aux'] = self.loss_weight * F.cross_entropy(x, target)
return loss
def forward(self, x):
features = self.features(x)
out = F.relu(features, inplace=True)
out = F.adaptive_avg_pool3d(out, 1).squeeze()
# out = F.avg_pool3d(out, kernel_size=7, stride=1).view(features.size(0), -1)
out = self.classifier(out)
return out
def forward(self, x):
_, c, h, w = x.size()
c2, h2, w2 = self.group, int(h / 4), int(w / 4)
# print("x1:",x[0,0,0,:])
y = x
y = y.unsqueeze(0)
y = F.adaptive_avg_pool3d(y, (c2, h2, w2))
y = y.squeeze(0)
y = self.conv2(y)
y = y.unsqueeze(0)
y = F.interpolate(y, size=(c, h, w))
y = y.squeeze(0)
#second_time=time.time()
#print("time1",second_time-start_time)
# y=self.conv3(y)
# print(random.random())
if random.random() < 0.5:
# print("here")
y = self.prm_ours(y)
#print("time2",time.time()-second_time)
y = F.sigmoid(y)
#exit()
# print("y:",y[0,0,0,:])
# print("x2:",x[0,0,0,:])
# exit()
return y * x