def __init__(self):
super(CNNScore, self).__init__()
self.conv1 = nn.Conv3d(16, 96, kernel_size=1, stride=2)
self.fire2_squeeze = nn.Conv3d(96, 16, kernel_size=1)
self.fire2_expand1 = nn.Conv3d(16, 64, kernel_size=1)
# Padding = (k-1)/2 where k is the kernel size
self.fire2_expand2 = nn.Conv3d(16, 64, kernel_size=3, padding=1)
self.fire3_squeeze = nn.Conv3d(128, 16, kernel_size=1)
self.fire3_expand1 = nn.Conv3d(16, 64, kernel_size=1)
self.fire3_expand2 = nn.Conv3d(16, 64, kernel_size=3, padding=1)
self.fire4_squeeze = nn.Conv3d(128, 32, kernel_size=1)
self.fire4_expand1 = nn.Conv3d(32, 128, kernel_size=1)
self.fire4_expand2 = nn.Conv3d(32, 128, kernel_size=3, padding=1)
self.pool = nn.MaxPool3d(kernel_size=3, stride=2)
self.fire5_squeeze = nn.Conv3d(256, 32, kernel_size=1)
self.fire5_expand1 = nn.Conv3d(32, 128, kernel_size=1)
self.fire5_expand2 = nn.Conv3d(32, 128, kernel_size=3, padding=1)
self.fire6_squeeze = nn.Conv3d(256, 48, kernel_size=1)
self.fire6_expand1 = nn.Conv3d(48, 192, kernel_size=1)
self.fire6_expand2 = nn.Conv3d(48, 192, kernel_size=3, padding=1)
self.fire7_squeeze = nn.Conv3d(384, 48, kernel_size=1)
self.fire7_expand1 = nn.Conv3d(48, 192, kernel_size=1)
self.fire7_expand2 = nn.Conv3d(48, 192, kernel_size=3, padding=1)
self.fire8_squeeze = nn.Conv3d(384, 64, kernel_size=1)
self.fire8_expand1 = nn.Conv3d(64, 256, kernel_size=1)
self.fire8_expand2 = nn.Conv3d(64, 256, kernel_size=3, padding=1)
self.avg_pool = nn.AvgPool3d(kernel_size=3, padding=1)
self.dense1 = nn.Linear(512*2*2*2, 1)
def __init__(self, in_ch, out_ch, se=False, norm='bn'):
super(inconv, self).__init__()
self.conv1 = nn.Conv3d(in_ch, out_ch, kernel_size=(1,3,3), padding=(0,1,1), bias=False)
self.relu = nn.ReLU(inplace=True)
self.conv2 = SEBasicBlock(out_ch, out_ch, kernel_size=(1,3,3), norm=norm)
def __init__(self):
super(vgg16, self).__init__()
self.block1_output = nn.Sequential(
nn.Conv3d(3, 63, kernel_size=3, padding=1), nn.ReLU(inplace=True),
nn.Conv3d(64, 64, kernel_size=3, padding=1), nn.ReLU(inplace=True),
nn.MaxPool3d(kernel_size=2, stride=2))
self.block2_output = nn.Sequential(
nn.Conv3d(64, 128, kernel_size=3, padding=1),
nn.ReLU(inplace=True), nn.Conv3d(128,
128,
kernel_size=3,
padding=1), nn.ReLU(inplace=True),
nn.MaxPool3d(kernel_size=2, stride=2))
self.block3_output = nn.Sequential(
nn.Conv3d(128, 256, kernel_size=3, padding=1),
nn.ReLU(inplace=True), nn.Conv3d(256,
256,
kernel_size=3,
padding=1), nn.ReLU(inplace=True),
nn.Conv3d(256, 256, kernel_size=3,
padding=1), nn.ReLU(inplace=True),
nn.MaxPool3d(kernel_size=2, stride=2))
self.block4_output = nn.Sequential(
nn.Conv3d(256, 512, kernel_size=3, padding=1),
nn.ReLU(inplace=True), nn.Conv3d(512,
512,
kernel_size=3,
padding=1), nn.ReLU(inplace=True),
nn.Conv3d(512, 512, kernel_size=3,
padding=1), nn.ReLU(inplace=True))
self.block4_3output = nn.Sequential(
nn.MaxPool3d(kernel_size=2, stride=2))
self.block5_output = nn.Sequential(
nn.Conv3d(512, 512, kernel_size=3, padding=1),
nn.ReLU(inplace=True), nn.Conv3d(512,
512,
kernel_size=3,
padding=1), nn.ReLU(inplace=True),
nn.Conv3d(512, 512, kernel_size=3,
padding=1), nn.ReLU(inplace=True),
nn.MaxPool3d(kernel_size=2, stride=2))
self.block6_output = nn.Sequential(
nn.Conv3d(512, 1024, kernel_size=3, padding=1),
nn.ReLU(inplace=True), nn.MaxPool3d(kernel_size=2, stride=2))
self.block7_output = nn.Sequential(
nn.Conv3d(1024, 1024, kernel_size=1, padding=0),
nn.ReLU(inplace=True))
self.block7_1_output = nn.Sequential(
nn.MaxPool3d(kernel_size=2, striide=2))
self.block8_output = nn.Sequential(
nn.Conv3d(512, 128, kernel_size=1, padding=0),
nn.ReLU(inplace=True), nn.Conv3d(128,
256,
kernel_size=3,
padding=1), nn.ReLU(inplace=True))
self.block8_2_output = nn.Sequential(
nn.MaxPool3d(kernel_size=2, striide=2))
self.block9_output = nn.Sequential(
nn.Conv3d(256, 128, kernel_size=1, padding=0),
nn.ReLU(inplace=True), nn.Conv3d(128,
256,
kernel_size=3,
padding=1), nn.ReLU(inplace=True))
self.block9_2_output = nn.Sequential(
nn.MaxPool3d(kernel_size=2, striide=2))
self.block10_output = nn.Sequential(
nn.Conv3d(256, 128, kernel_size=1, padding=0),
nn.ReLU(inplace=True),
nn.Conv3d(128, 256, kernel_size=3, padding=1),
nn.ReLU(inplace=True),
)
self.block10_2_output = nn.Sequential(
nn.MaxPool3d(kernel_size=2, striide=2))
self.block11_output = nn.Sequential(
nn.Conv3d(256, 128, kernel_size=1, padding=0),
nn.ReLU(inplace=True), nn.Conv3d(128,
256,
kernel_size=3,
padding=1), nn.ReLU(inplace=True))
self.block11_2_output = nn.Sequential(
nn.MaxPool3d(kernel_size=2, striide=2))
def conv1x1x1(in_planes, out_planes, stride=1):
return nn.Conv3d(in_planes,
out_planes,
kernel_size=1,
stride=stride,
bias=False)
def __init__(self, img_sz, opt=None):
super(VoxelMorphMICCAI2019, self).__init__()
self.is_train = opt['tsk_set'][('train', False, 'if is in train mode')]
opt_voxelmorph = opt['tsk_set']['reg']['morph_miccai']
self.load_trained_affine_net = opt_voxelmorph[(
'load_trained_affine_net', False,
'if true load_trained_affine_net; if false, the affine network is not initialized'
)]
self.affine_refine_step = opt_voxelmorph[(
'affine_refine_step', 5,
"the multi-step num in affine refinement")]
self.using_affine_init = opt_voxelmorph[(
"using_affine_init", False,
"deploy affine network before the nonparametric network")]
self.affine_init_path = opt_voxelmorph[(
'affine_init_path', '', "the path of pretrained affine model")]
enc_filters = [16, 32, 32, 32, 32]
#dec_filters = [32, 32, 32, 8, 8]
dec_filters = [32, 32, 32, 32, 16]
self.enc_filter = enc_filters
self.dec_filter = dec_filters
input_channel = 2
output_channel = 3
self.input_channel = input_channel
self.output_channel = output_channel
self.img_sz = img_sz
self.low_res_img_sz = [int(x / 2) for x in img_sz]
self.spacing = 1. / (np.array(img_sz) - 1)
self.int_steps = 7
self.image_sigma = opt_voxelmorph[('image_sigma', 0.02, 'image_sigma')]
self.prior_lambda = opt_voxelmorph[('lambda_factor_in_vmr', 50,
'lambda_factor_in_vmr')]
self.prior_lambda_mean = opt_voxelmorph[('lambda_mean_factor_in_vmr',
50,
'lambda_mean_factor_in_vmr')]
self.flow_vol_shape = self.low_res_img_sz
self.D = self._degree_matrix(self.flow_vol_shape)
self.D = (self.D).cuda() # 1, 96, 40,40 3'
self.loss_fn = None
if self.using_affine_init:
self.init_affine_net(opt)
self.id_transform = None
else:
self.id_transform = gen_identity_map(self.img_sz, 1.0).cuda()
self.id_transform = self.id_transform.view(
[1] + list(self.id_transform.shape))
print("Attention, the affine net is not used")
"""to compatiable to the mesh setting in voxel morph"""
self.low_res_id_transform = gen_identity_map(self.img_sz,
0.5,
normalized=False).cuda()
self.encoders = nn.ModuleList()
self.decoders = nn.ModuleList()
#self.bilinear = Bilinear(zero_boundary=True)
self.bilinear = stn_nd.STN_ND_BCXYZ(np.array([1., 1., 1.]),
zero_boundary=True)
self.bilinear_img = Bilinear(zero_boundary=True)
for i in range(len(enc_filters)):
if i == 0:
self.encoders.append(
convBlock(input_channel,
enc_filters[i],
stride=1,
bias=True))
else:
self.encoders.append(
convBlock(enc_filters[i - 1],
enc_filters[i],
stride=2,
bias=True))
self.decoders.append(
convBlock(enc_filters[-1], dec_filters[0], stride=1, bias=True))
self.decoders.append(
convBlock(dec_filters[0] + enc_filters[3],
dec_filters[1],
stride=1,
bias=True))
self.decoders.append(
convBlock(dec_filters[1] + enc_filters[2],
dec_filters[2],
stride=1,
bias=True))
self.decoders.append(
convBlock(dec_filters[2] + enc_filters[1],
dec_filters[3],
stride=1,
bias=True))
self.decoders.append(
convBlock(dec_filters[3], dec_filters[4], stride=1, bias=True))
self.flow_mean = nn.Conv3d(dec_filters[-1],
output_channel,
kernel_size=3,
stride=1,
padding=1,
bias=True)
self.flow_sigma = nn.Conv3d(dec_filters[-1],
output_channel,
kernel_size=3,
stride=1,
padding=1,
bias=True)
self.flow_mean.weight.data.normal_(0., 1e-5)
self.flow_sigma.weight.data.normal_(0., 1e-10)
self.flow_sigma.bias.data = torch.Tensor([-10] * 3)
self.print_count = 0
def __init__(self, n_features=80, log_transform=True, colored=True):
super().__init__()
self.log_transform = log_transform
self.channels = 5 if colored else 2
# Encoder
self.enc1 = nn.Conv3d(self.channels,
n_features,
4,
stride=2,
padding=1)
self.bn1 = nn.BatchNorm3d(n_features)
self.enc2 = nn.Conv3d(n_features,
2 * n_features,
4,
stride=2,
padding=1)
self.bn2 = nn.BatchNorm3d(2 * n_features)
self.enc3 = nn.Conv3d(2 * n_features,
4 * n_features,
4,
stride=2,
padding=1)
self.bn3 = nn.BatchNorm3d(4 * n_features)
self.enc4 = nn.Conv3d(4 * n_features,
8 * n_features,
4,
stride=1,
padding=0)
self.bn4 = nn.BatchNorm3d(8 * n_features)
# Bottleneck
self.fc1 = nn.Linear(8 * n_features, 8 * n_features)
self.bn5 = nn.BatchNorm1d(8 * n_features)
self.fc2 = nn.Linear(8 * n_features, 8 * n_features)
self.bn6 = nn.BatchNorm1d(8 * n_features)
# Decoder
self.dec1 = nn.ConvTranspose3d(2 * 8 * n_features,
4 * n_features,
4,
stride=1,
padding=0)
self.dbn1 = nn.BatchNorm3d(4 * n_features)
self.dec2 = nn.ConvTranspose3d(2 * 4 * n_features,
2 * n_features,
4,
stride=2,
padding=1)
self.dbn2 = nn.BatchNorm3d(2 * n_features)
self.dec3 = nn.ConvTranspose3d(2 * 2 * n_features,
n_features,
4,
stride=2,
padding=1)
self.dbn3 = nn.BatchNorm3d(n_features)
self.dec4 = nn.ConvTranspose3d(2 * n_features,
self.channels - 1,
4,
stride=2,
padding=1)
def __init__(self):
super(PhysNet, self).__init__()
self.ConvBlock1 = nn.Sequential(
nn.Conv3d(3, 16, [1, 5, 5], stride=1, padding=[0, 2, 2]),
nn.BatchNorm3d(16),
nn.ReLU(inplace=True),
)
#######################
self.ConvBlock2 = nn.Sequential(
nn.Conv3d(16, 32, [1, 3, 3], stride=1, padding=[0, 1, 1]),
nn.BatchNorm3d(32),
nn.ReLU(inplace=True),
)
self.ConvBlock3 = nn.Sequential(
nn.Conv3d(32, 32, [3, 1, 1], stride=1, padding=[1, 0, 0]),
nn.BatchNorm3d(32),
nn.ReLU(inplace=True),
)
self.ConvBlock4 = nn.Sequential(
nn.Conv3d(32, 32, [1, 3, 3], stride=1, padding=[0, 1, 1]),
nn.BatchNorm3d(32),
nn.ReLU(inplace=True),
)
self.ConvBlock5 = nn.Sequential(
nn.Conv3d(32, 32, [3, 1, 1], stride=1, padding=[1, 0, 0]),
nn.BatchNorm3d(32),
nn.ReLU(inplace=True),
)
########################
self.ConvBlock6 = nn.Sequential(
nn.Conv3d(32, 64, [1, 3, 3], stride=1, padding=[0, 1, 1]),
nn.BatchNorm3d(64),
nn.ReLU(inplace=True),
)
self.ConvBlock7 = nn.Sequential(
nn.Conv3d(64, 64, [3, 1, 1], stride=1, padding=[1, 0, 0]),
nn.BatchNorm3d(64),
nn.ReLU(inplace=True),
)
self.ConvBlock8 = nn.Sequential(
nn.Conv3d(64, 64, [1, 3, 3], stride=1, padding=[0, 1, 1]),
nn.BatchNorm3d(64),
nn.ReLU(inplace=True),
)
self.ConvBlock9 = nn.Sequential(
nn.Conv3d(64, 64, [3, 1, 1], stride=1, padding=[1, 0, 0]),
nn.BatchNorm3d(64),
nn.ReLU(inplace=True),
)
########################
self.ConvBlock10 = nn.Sequential(
nn.Conv3d(64, 64, [1, 3, 3], stride=1, padding=[0, 1, 1]),
nn.BatchNorm3d(64),
nn.ReLU(inplace=True),
)
self.ConvBlock11 = nn.Sequential(
nn.Conv3d(64, 64, [3, 1, 1], stride=1, padding=[1, 0, 0]),
nn.BatchNorm3d(64),
nn.ReLU(inplace=True),
)
self.ConvBlock12 = nn.Sequential(
nn.Conv3d(64, 64, [1, 3, 3], stride=1, padding=[0, 1, 1]),
nn.BatchNorm3d(64),
nn.ReLU(inplace=True),
)
self.ConvBlock13 = nn.Sequential(
nn.Conv3d(64, 64, [3, 1, 1], stride=1, padding=[1, 0, 0]),
nn.BatchNorm3d(64),
nn.ReLU(inplace=True),
)
########################
self.ConvBlock14 = nn.Sequential( # padding?? padding=[0, 2, 2]
nn.Conv3d(64, 64, [1, 3, 3], stride=1, padding=[0, 1, 1]),
nn.BatchNorm3d(64),
nn.ReLU(inplace=True),
)
self.ConvBlock15 = nn.Sequential(
nn.Conv3d(64, 64, [3, 1, 1], stride=1, padding=[1, 0, 0]),
nn.BatchNorm3d(64),
nn.ReLU(inplace=True),
)
self.ConvBlock16 = nn.Conv3d(64, 1, [1, 1, 1], stride=1, padding=0)
self.AvgpoolSpa1 = nn.AvgPool3d((1, 2, 2), stride=(1, 2, 2))
self.AvgpoolSpa2 = nn.AvgPool3d((1, 7, 7), stride=(1, 2, 2))
self.MaxpoolSpa = nn.MaxPool3d((1, 2, 2), stride=(1, 2, 2))
def __init__(self, in_dims):
super(MatchHead, self).__init__()
self.feature = VoxRes(in_dims)
self.dims_align = nn.Conv3d(64, 1, 1, 1, 0)
def __init__(self, in_dims, emd_dims=32):
super(EmbHead, self).__init__()
self.feature = VoxRes(in_dims)
self.dims_align = nn.Conv3d(64, emd_dims, 1, 1, 0)
def __init__(self, opt):
super(VI_2D_Decoder_3, self).__init__()
self.opt = opt
dv = 2 if self.opt.double_size else 1
### decoder
self.dc0 = GatedConvolution(128,
128,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
padding=(0, 1, 1),
bias=False)
self.dc1 = GatedConvolution(128,
128,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
padding=(0, 1, 1),
bias=False)
#### UPCONV
self.dc1_1 = GatedUpConvolution(
(1, opt.crop_size // 4 // dv, opt.crop_size // 4 // dv),
128,
96,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
padding=(0, 1, 1),
bias=False)
self.dc2_1 = GatedConvolution(96 + 96,
96,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
padding=(0, 1, 1),
bias=False)
self.dc2_bt1 = GatedConvolution(96,
96,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
dilation=(1, 2, 2),
padding=(0, 2, 2),
bias=False)
self.dc2_bt2 = GatedConvolution(96,
96,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
dilation=(1, 4, 4),
padding=(0, 4, 4),
bias=False)
self.dc2_bt3 = GatedConvolution(96,
96,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
dilation=(1, 8, 8),
padding=(0, 8, 8),
bias=False)
#### UPCONV
self.dc2_2 = GatedUpConvolution(
(1, opt.crop_size // 2 // dv, opt.crop_size // 2 // dv),
96,
64,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
padding=(0, 1, 1),
bias=False)
self.dc3_1 = GatedConvolution(64 + 64,
64,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
padding=(0, 1, 1),
bias=False)
self.dc3_2 = GatedConvolution(64,
64,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
padding=(0, 1, 1),
bias=False)
#### UPCONV
self.dc4 = GatedUpConvolution(
(1, opt.crop_size // dv, opt.crop_size // dv),
64,
32,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
padding=(0, 1, 1),
bias=False)
if self.opt.double_size:
self.upsample = nn.Upsample(size=(1, opt.crop_size, opt.crop_size),
mode='trilinear')
self.dc5 = GatedConvolution(32,
16,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
padding=(0, 1, 1),
bias=False)
self.dc6 = nn.Conv3d(16,
3,
kernel_size=(1, 3, 3),
stride=(1, 1, 1),
padding=(0, 1, 1),
bias=False)
for m in self.modules():
if isinstance(m, nn.Conv3d) or isinstance(m, nn.Conv2d):
m.weight = nn.init.kaiming_normal_(m.weight, mode='fan_out')
elif isinstance(m, nn.BatchNorm3d) or isinstance(
m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def __init__(self, num_classes=12):
super(SSC_RGBD_GRFNet, self).__init__()
print('SSC_RGBD_GRFNet.')
# --- depth
c_in, c, c_out, dilation, residual = 1, 4, 8, 1, True
self.dep_feature2d = nn.Sequential(
nn.Conv2d(c_in, c_out, 1, 1, 0), # reduction
BottleneckDDR2d(c_out,
c,
c_out,
dilation=dilation,
residual=residual),
BottleneckDDR2d(c_out,
c,
c_out,
dilation=dilation,
residual=residual),
)
self.project_layer_dep = Project2Dto3D(240, 144,
240) # w=240, h=144, d=240
self.dep_feature3d = nn.Sequential(
DownsampleBlock3d(8, 16),
BottleneckDDR3d(c_in=16, c=8, c_out=16, dilation=1, residual=True),
DownsampleBlock3d(16, 64), # nn.MaxPool3d(kernel_size=2, stride=2)
BottleneckDDR3d(c_in=64, c=16, c_out=64, dilation=1,
residual=True),
)
# --- RGB
c_in, c, c_out, dilation, residual = 3, 4, 8, 1, True
self.rgb_feature2d = nn.Sequential(
nn.Conv2d(c_in, c_out, 1, 1, 0), # reduction
BottleneckDDR2d(c_out,
c,
c_out,
dilation=dilation,
residual=residual),
BottleneckDDR2d(c_out,
c,
c_out,
dilation=dilation,
residual=residual),
)
self.project_layer_rgb = Project2Dto3D(240, 144,
240) # w=240, h=144, d=240
self.rgb_feature3d = nn.Sequential(
DownsampleBlock3d(8, 16),
BottleneckDDR3d(c_in=16, c=8, c_out=16, dilation=1, residual=True),
DownsampleBlock3d(16, 64), # nn.MaxPool3d(kernel_size=2, stride=2)
BottleneckDDR3d(c_in=64, c=16, c_out=64, dilation=1,
residual=True),
)
# -------------1/4
ck = 64
c = ck // 4
# --- RGB
self.res3d_1r = BottleneckDDR3d(c_in=ck,
c=c,
c_out=ck,
dilation=2,
residual=True)
self.res3d_2r = BottleneckDDR3d(c_in=ck,
c=c,
c_out=ck,
dilation=3,
residual=True)
self.res3d_3r = BottleneckDDR3d(c_in=ck,
c=c,
c_out=ck,
dilation=5,
residual=True)
# --- Depth
self.res3d_1d = BottleneckDDR3d(c_in=ck,
c=c,
c_out=ck,
dilation=2,
residual=True)
self.res3d_2d = BottleneckDDR3d(c_in=ck,
c=c,
c_out=ck,
dilation=3,
residual=True)
self.res3d_3d = BottleneckDDR3d(c_in=ck,
c=c,
c_out=ck,
dilation=5,
residual=True)
# self.lstm = DDRConv3dLSTMCell(input_channels=128, hidden_channels=64, kernel_size=(3, 3, 3), bias=True)
self.gru = Conv3dGRUCell(input_channels=64,
hidden_channels=64,
kernel_size=3,
bias=True)
self.aspp = DDR_ASPP3d(c_in=ck, c=16, c_out=64)
self.conv_out = nn.Sequential(nn.Conv3d(320, 160, 1, 1, 0),
nn.ReLU(inplace=True),
nn.Conv3d(160, num_classes, 1, 1, 0))
# ---- weights init
for m in self.modules():
if isinstance(m, nn.Conv3d):
n = m.kernel_size[0] * m.kernel_size[1] * m.kernel_size[
2] * m.out_channels
# nn.init.xavier_normal(m.weight.data, gain=math.sqrt(2. / n))
# nn.init.xavier_uniform(m.weight.data, gain=math.sqrt(2. / n))
nn.init.xavier_uniform_(m.weight.data) # gain=1
# nn.init.constant(m.bias.data, 0)
if isinstance(m, nn.Conv2d):
nn.init.normal_(m.weight.data, mean=0, std=0.1)
def __init__(self, nInputChannels=3, n_classes=21, os=16, pretrained=False, _print=True, final_sigmoid=False,
hidden_layers=128, device=None):
if _print:
print("Constructing DeepLabv3+ model...")
print("Number of classes: {}".format(n_classes))
print("Output stride: {}".format(os))
print("Number of Input Channels: {}".format(nInputChannels))
super(DeepLabv3_plus_gcn_3d, self).__init__()
self.device = device
# Atrous Conv
self.xception_features = Xception(nInputChannels, os, pretrained)
# ASPP
if os == 16:
rates = [1, 6, 12, 18]
elif os == 8:
rates = [1, 12, 24, 36]
raise NotImplementedError
else:
raise NotImplementedError
self.aspp1 = ASPP_module_rate0(1024, 128, rate=rates[0])
self.aspp2 = ASPP_module(1024, 128, rate=rates[1])
self.aspp3 = ASPP_module(1024, 128, rate=rates[2])
self.aspp4 = ASPP_module(1024, 128, rate=rates[3])
self.relu = nn.ReLU()
self.global_avg_pool = nn.Sequential(nn.AdaptiveAvgPool3d((1, 1, 1)),
nn.Conv3d(1024, 128, 1, stride=1, bias=False),
nn.BatchNorm3d(128),
nn.ReLU()
)
self.concat_projection_conv1 = nn.Conv3d(640, 128, 1, bias=False)
self.concat_projection_bn1 = nn.BatchNorm3d(128)
# adopt [1x1, 48] for channel reduction.
self.feature_projection_conv1 = nn.Conv3d(128, 24, 1, bias=False)
self.feature_projection_bn1 = nn.BatchNorm3d(24)
# self.decoder = nn.Sequential(Decoder_module(152, 128),
# Decoder_module(128, 128)
# )
#
# self.featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=128, hidden_layers=hidden_layers,
# nodes=n_classes)
# self.graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers)
# self.graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers)
# self.graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers)
#
# self.graph_2_fea = gcn.Graph_to_Featuremaps_savemem(input_channels=128,
# output_channels=128,
# hidden_layers=hidden_layers,
# nodes=n_classes)
#
# self.skip_conv = nn.Sequential(*[nn.Conv3d(128, 128, kernel_size=1),
# nn.ReLU(True)])
#
# self.semantic = nn.Conv3d(128, n_classes, kernel_size=1, stride=1)
self.decoder = nn.Sequential(Decoder_module(152, 256),
Decoder_module(256, 256)
)
self.featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=256, hidden_layers=hidden_layers,
nodes=n_classes)
self.graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers)
self.graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers)
self.graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers)
self.graph_2_fea = gcn.Graph_to_Featuremaps_savemem(input_channels=256,
output_channels=256,
hidden_layers=hidden_layers,
nodes=n_classes)
self.skip_conv = nn.Sequential(*[nn.Conv3d(256, 256, kernel_size=1),
nn.ReLU(True)])
self.semantic = nn.Conv3d(256, n_classes, kernel_size=1, stride=1)
if final_sigmoid:
self.final_activation = nn.Sigmoid()
else:
self.final_activation = nn.Softmax(dim=1)
def __init__(self, inplanes=3, os=16, pretrained=False):
super(Xception, self).__init__()
if os == 16:
entry_block3_stride = (1, 2, 2)
middle_block_rate = 1
exit_block_rates = (1, 2)
elif os == 8:
entry_block3_stride = 1
middle_block_rate = 2
exit_block_rates = (2, 4)
else:
raise NotImplementedError
# Entry flow
self.conv1 = nn.Conv3d(inplanes, 16, 3, stride=(1,2,2), padding=1, bias=False)
self.bn1 = nn.BatchNorm3d(16)
self.relu = nn.ReLU(inplace=True)
self.conv2 = nn.Conv3d(16, 32, 3, stride=1, padding=1, bias=False)
self.bn2 = nn.BatchNorm3d(32)
self.block1 = Block(32, 64, reps=2, stride=(1,2,2), start_with_relu=False, grow_first=True)
self.block2 = Block2(64, 128, reps=2, stride=(1,2,2), start_with_relu=True, grow_first=True)
self.block3 = Block(128, 364, reps=2, stride=entry_block3_stride, start_with_relu=True, grow_first=True)
# Middle flow
# self.block4 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block5 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block6 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block7 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block8 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block9 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block10 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block11 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block12 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block13 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block14 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block15 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block16 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block17 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block18 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# self.block19 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True)
# Exit flow
self.block20 = Block(364, 512, reps=2, stride=1, dilation=exit_block_rates[0],
start_with_relu=True, grow_first=False, is_last=True)
self.conv3 = SeparableConv3d_aspp(512, 768, 3, stride=1, dilation=exit_block_rates[1],padding=exit_block_rates[1])
# self.bn3 = nn.BatchNorm3d(1536)
self.conv4 = SeparableConv3d_aspp(768, 768, 3, stride=1, dilation=exit_block_rates[1],padding=exit_block_rates[1])
# self.bn4 = nn.BatchNorm3d(1536)
self.conv5 = SeparableConv3d_aspp(768, 1024, 3, stride=1, dilation=exit_block_rates[1],padding=exit_block_rates[1])
# self.bn5 = nn.BatchNorm3d(2048)
# Init weights
# self.__init_weight()
# Load pretrained model
if pretrained:
self.__load_xception_pretrained()
def __init__(self, inplanes, planes, kernel_size=3, stride=1, padding=0, dilation=1, bias=False):
super(SeparableConv3d, self).__init__()
self.conv1 = nn.Conv3d(inplanes, inplanes, kernel_size, stride, padding, dilation,
groups=inplanes, bias=bias)
self.pointwise = nn.Conv3d(inplanes, planes, 1, 1, 0, 1, 1, bias=bias)
def __init__(self, growth_rate=12, shared_block_config=(8, 8), MC_block_config=(8, 8), CA_block_config=(8, 8), compression=0.5,
num_init_features=32, bn_size=4, drop_rate=0.2,
num_classes_MC=1, num_classes_CA=1, efficient=False):
super(DenseNet, self).__init__()
assert 0 < compression <= 1, 'compression of densenet should be between 0 and 1'
# First convolution
self.shared_blocks = nn.Sequential(OrderedDict([
('conv0', nn.Conv3d(1, num_init_features, kernel_size=5, stride=1, padding=2, bias=False)),
]))
# Shared blocks
num_features = num_init_features
for i, num_layers in enumerate(shared_block_config):
block = _DenseBlock(
num_layers=num_layers,
num_input_features=num_features,
bn_size=bn_size,
growth_rate=growth_rate,
drop_rate=drop_rate,
efficient=efficient,
)
self.shared_blocks.add_module('denseblock%d' % (i + 1), block)
num_features = num_features + num_layers * growth_rate
# if i != len(shared_block_config) - 1:
if True:
trans = _Transition_no_pool(num_input_features=num_features,
num_output_features=int(num_features * compression))
self.shared_blocks.add_module('transition%d' % (i + 1), trans)
num_features = int(num_features * compression)
num_shared_features = num_features
self.MC_blocks=nn.Sequential()
num_features=num_shared_features
for i, num_layers in enumerate(MC_block_config):
block = _DenseBlock(
num_layers=num_layers,
num_input_features=num_features,
bn_size=bn_size,
growth_rate=growth_rate,
drop_rate=drop_rate,
efficient=efficient,
)
self.MC_blocks.add_module('denseblock_%d' % (i + 1), block)
num_features = num_features + num_layers * growth_rate
if i != len(CA_block_config) - 1:
trans = _Transition(num_input_features=num_features,
num_output_features=int(num_features * compression))
self.MC_blocks.add_module('transition_%d' % (i + 1), trans)
num_features = int(num_features * compression)
# Final batch norm
self.MC_blocks.add_module('norm_final', nn.BatchNorm3d(num_features))
# Final ReLU
self.MC_blocks.add_module('relu_final', nn.ReLU(inplace=True))
# Final pool
self.MC_blocks.add_module('pool_final', nn.AdaptiveAvgPool3d(1))
num_features_MC = num_features
self.MC_classifier = nn.Linear(num_features_MC, num_classes_MC)
self.CA_blocks=nn.Sequential()
num_features=num_shared_features
for i, num_layers in enumerate(CA_block_config):
block = _DenseBlock(
num_layers=num_layers,
num_input_features=num_features,
bn_size=bn_size,
growth_rate=growth_rate,
drop_rate=drop_rate,
efficient=efficient,
)
self.CA_blocks.add_module('denseblock_%d' % (i + 1), block)
num_features = num_features + num_layers * growth_rate
if i != len(CA_block_config) - 1:
trans = _Transition(num_input_features=num_features,
num_output_features=int(num_features * compression))
self.CA_blocks.add_module('transition_%d' % (i + 1), trans)
num_features = int(num_features * compression)
# Final batch norm
self.CA_blocks.add_module('norm_final', nn.BatchNorm3d(num_features))
# Final ReLU
self.CA_blocks.add_module('relu_final', nn.ReLU(inplace=True))
# Final pool
self.CA_blocks.add_module('pool_final', nn.AdaptiveAvgPool3d(1))
num_features_CA = num_features
self.CA_classifier = nn.Linear(num_features_CA, num_classes_CA)
def conv3x3x3(in_planes, out_planes, stride=1):
# 3x3x3 convolution with padding
return nn.Conv3d(in_planes,out_planes, kernel_size=3, stride=stride, padding=1, bias=False)
def __init__(self, num_input_features, num_output_features):
super(_Transition_no_pool, self).__init__()
self.add_module('norm', nn.BatchNorm3d(num_input_features))
self.add_module('relu', nn.ReLU(inplace=True))
self.add_module('conv', nn.Conv3d(num_input_features, num_output_features,
kernel_size=1, stride=1, bias=False))
def __init__(self,
kernel_size = 4,
stride = 2,
padding = 1,
full_conv_limit = 4,
just_conv_limit = 1,
cube_edge = 128,
ch_mult = 2,
conv_bias = False,
leakyrelu_const = 0.01):
super(Encoder, self).__init__()
print("Encoder = encoder_v02.py")
"""
input: batch_size * channels * cube_edge * cube_edge * cube_edge
output: batch_size * (channel_multiplier * channels)
BatchNorm is also added.
Conv3D arguments=in_channels,out_channels,kernel_size,stride,padding
"""
"""
Convolutional Layers
"""
self.embed_cube_edge = cube_edge
conv_net = nn.Sequential()
channels = 1
layer = 1
"""
Convolutions with BatchNorm
"""
while layer <= full_conv_limit:
conv_net.add_module("Conv_{0}".format(layer),
nn.Conv3d(channels,
channels * ch_mult,
kernel_size = kernel_size,
stride = stride,
padding = padding,
bias = conv_bias))
conv_net.add_module("BatchNorm_{0}".format(layer),
nn.BatchNorm3d(channels * ch_mult))
conv_net.add_module("leakyrelu_{0}".format(layer),
nn.LeakyReLU(leakyrelu_const, inplace = True))
self.embed_cube_edge = calculate_conv_output_dim(D = self.embed_cube_edge,
K = kernel_size,
P = padding,
S = stride)
channels = channels * ch_mult
layer = layer + 1
"""
Convolutions without BatchNorm
"""
while layer <= just_conv_limit:
conv_net.add_module("Conv_{0}".format(layer),
nn.Conv3d(channels,
channels * ch_mult,
kernel_size = kernel_size,
stride = stride,
padding = padding,
bias = conv_bias))
channels = channels * ch_mult
layer = layer + 1
# conv_net.add_module("Conv_{0}".format(layer),
# nn.Conv3d(channels,
# channels * ch_mult,
# kernel_size = 4,
# stride = 2,
# padding = 1,
# bias = conv_bias))
# channels = channels * ch_mult
self.conv_net = conv_net
self.channels = channels
self.embed_cube_edge = calculate_conv_output_dim(D = self.embed_cube_edge,
K = kernel_size,
P = padding,
S = stride)
def __init__(self,
in_chs,
out_chs,
block,
feats,
layers,
stride=2,
groups=1,
dilation=1,
dropout=None,
**kwargs):
assert isinstance(out_chs, (int, tuple))
super(UNet, self).__init__()
up_mode = kwargs.pop('up_mode') if 'up_mode' in kwargs else 'tconv'
self.kwargs = kwargs
num_layers = len(feats) - 1
layers = _repeat(layers, num_layers)
stride = _repeat(stride, num_layers)
groups = _repeat(groups, num_layers)
dilation = _repeat(dilation, num_layers)
dropout = _repeat(dropout, num_layers * 2)
if type(block) != tuple:
block = (block, block)
self.stem = nn.Sequential(
conv3x3(in_chs, feats[0], stride=1),
_norm_layer(feats[0], **self.kwargs),
_act_layer(**self.kwargs),
conv3x3(feats[0], feats[0], stride=1),
_norm_layer(feats[0], **self.kwargs),
_act_layer(**self.kwargs),
conv3x3(feats[0], feats[0], stride=1),
_norm_layer(feats[0], **self.kwargs),
_act_layer(**self.kwargs),
)
encoders = []
for idx in range(num_layers):
encoders.append(
self._make_layer(feats[idx],
feats[idx + 1],
block[0],
layers[idx],
stride=stride[idx],
groups=groups[idx],
dilation=dilation[idx],
dropout=dropout[idx],
**self.kwargs))
self.encoders = nn.ModuleList(encoders)
ups = []
decoders = []
for idx in range(num_layers, 0, -1):
align_corners = True if 'trilinear' in up_mode else None
ups.append(
nn.Sequential(
_up_layer(in_chs=feats[idx],
mode=up_mode,
align_corners=align_corners,
**kwargs),
conv1x1(feats[idx], feats[idx - 1]),
_norm_layer(feats[idx - 1], **self.kwargs),
_act_layer(**self.kwargs),
))
decoders.append(
self._make_layer(feats[idx - 1] * 2,
feats[idx - 1],
block[1],
1,
stride=1,
groups=1,
dilation=1,
dropout=dropout[-idx],
**self.kwargs))
self.ups = nn.ModuleList(ups)
self.decoders = nn.ModuleList(decoders)
if isinstance(out_chs, int):
self.predict = nn.Conv3d(feats[0],
out_chs,
kernel_size=1,
stride=1,
padding=0,
bias=True)
else:
predicts = []
for o in out_chs:
predicts.append(_predict_head(feats[0], o, **self.kwargs))
self.predict = nn.ModuleList(predicts)
def __init__(self,
input_size,
input_channels,
device,
control_point_spacing=(10, 10, 10),
bspline_order=3,
max_displacement=0.2):
super(BSplineSTN3D, self).__init__()
# Cuda params
self.device = device
self.dtype = torch.cuda.float if (self.device
== 'cuda') else torch.float
self.order = bspline_order
self.max_disp = max_displacement
self.input_size = input_size
self.control_point_spacing = np.array(control_point_spacing)
self.stride = self.control_point_spacing.astype(dtype=int).tolist()
area = self.control_point_spacing[0] * self.control_point_spacing[
1] * self.control_point_spacing[2]
self.area = area.astype(float)
cp_grid_shape = np.ceil(
np.divide(self.input_size,
self.control_point_spacing)).astype(dtype=int)
# new image size after convolution
self.inner_image_size = np.multiply(
self.control_point_spacing,
cp_grid_shape) - (self.control_point_spacing - 1)
# add one control point at each side
cp_grid_shape = cp_grid_shape + 2
# image size with additional control points
self.new_image_size = np.multiply(
self.control_point_spacing,
cp_grid_shape) - (self.control_point_spacing - 1)
# center image between control points
image_size_diff = self.inner_image_size - self.input_size
image_size_diff_floor = np.floor(
(np.abs(image_size_diff) / 2)) * np.sign(image_size_diff)
crop_start = image_size_diff_floor + np.remainder(
image_size_diff, 2) * np.sign(image_size_diff)
self.crop_start = crop_start.astype(dtype=int)
self.crop_end = image_size_diff_floor.astype(dtype=int)
self.cp_grid_shape = [3] + cp_grid_shape.tolist()
self.num_control_points = np.prod(self.cp_grid_shape)
self.kernel = self.bspline_kernel_3d(order=self.order).expand(
3, *((np.ones(3 + 1, dtype=int) * -1).tolist()))
self.kernel_size = np.asarray(self.kernel.size())[2:]
self.padding = ((self.kernel_size - 1) / 2).astype(dtype=int).tolist()
# Network params
num_features = torch.prod(
((((torch.tensor(input_size) - 4) / 2 - 4) / 2) - 4) / 2)
self.conv1 = nn.Conv3d(input_channels, 8,
kernel_size=5).to(self.device)
self.conv2 = nn.Conv3d(8, 16, kernel_size=5).to(self.device)
self.conv3 = nn.Conv3d(16, 32, kernel_size=5).to(self.device)
self.fc = nn.Linear(32 * num_features,
self.num_control_points).to(self.device)
def __init__(self):
super(PALNet, self).__init__()
print('PALNet: depth and TSDF stream')
# ---- depth 2D CNN
depth_out = 8
self.res2d_1_1 = nn.Conv2d(1, depth_out, 1, 1, 0) # reduction
self.res2d_1_2 = nn.Sequential(
nn.Conv2d(1, 4, 1, 1, 0),
nn.ReLU(inplace=True),
# nn.Conv2d(4, 4, 3, 1, 1),
nn.Conv2d(4, 4, 3, 1, 1, 1), # dilated
nn.ReLU(inplace=True),
nn.Conv2d(4, 8, 1, 1, 0),
)
self.res2d_2 = nn.Sequential(
nn.Conv2d(8, 4, 1, 1, 0),
nn.ReLU(inplace=True),
# nn.Conv2d(4, 4, 3, 1, 1),
nn.Conv2d(4, 4, 3, 1, 2, 2), # dilated
nn.ReLU(inplace=True),
nn.Conv2d(4, 8, 1, 1, 0),
)
self.project_layer = Project2Dto3D(240, 144,
240) # w=240, h=144, d=240
# 3D CNN
stride1 = 1
stride2 = 2
self.a_pool1 = nn.Conv3d(8, 16, 7, stride2, 3)
self.a_res3d_1_1 = nn.Conv3d(16, 32, 1, 1, 0, bias=False) # reduction
self.a_res3d_1_2 = nn.Sequential(nn.Conv3d(16, 8, 1, 1, 0),
nn.ReLU(inplace=True),
nn.Conv3d(8, 8, 3, 1, 1, 1),
nn.ReLU(inplace=True),
nn.Conv3d(8, 32, 1, 1, 0))
in_channel_3d = 1 # ---- tsdf
self.b_pool1 = nn.Conv3d(in_channel_3d, 16, 7, stride2, 3)
self.b_res3d_1_1 = nn.Conv3d(16, 32, 1, 1, 0, bias=False)
self.b_res3d_1_2 = nn.Sequential(
nn.Conv3d(16, 8, 1, 1, 0),
nn.ReLU(inplace=True),
# nn.Conv3d(8, 8, 3, 1, 1),
nn.Conv3d(8, 8, 3, 1, 1, 1),
nn.ReLU(inplace=True),
nn.Conv3d(8, 32, 1, 1, 0))
# stride = 2
self.a_pool2 = nn.MaxPool3d(kernel_size=2, stride=2)
self.b_pool2 = nn.MaxPool3d(kernel_size=2, stride=2)
# self.pool2 = nn.Conv3d(32, 32, 3, stride, 1)
self.a_res3d_2_1 = nn.Conv3d(32, 64, 1, stride1, 0, bias=False)
self.a_res3d_2_2 = nn.Sequential(
nn.Conv3d(32, 16, 1, stride1, 0),
nn.ReLU(inplace=True),
# nn.Conv3d(16, 16, 3, 1, 1),
nn.Conv3d(16, 16, 3, 1, 2, 2),
nn.ReLU(inplace=True),
nn.Conv3d(16, 64, 1, 1, 0),
)
self.b_res3d_2_1 = nn.Conv3d(32, 64, 1, stride1, 0, bias=False)
self.b_res3d_2_2 = nn.Sequential(
nn.Conv3d(32, 16, 1, stride1, 0),
nn.ReLU(inplace=True),
# nn.Conv3d(16, 16, 3, 1, 1),
nn.Conv3d(16, 16, 3, 1, 2, 2),
nn.ReLU(inplace=True),
nn.Conv3d(16, 64, 1, 1, 0),
)
# self.cat = nn.Conv3d(32, 64, 1, 1, 0)
# -------------1/4
self.res3d_1 = nn.Sequential(
nn.Conv3d(64, 32, 1, 1, 0),
nn.ReLU(inplace=True),
nn.Conv3d(32, 32, 3, 1, 2, 2),
nn.ReLU(inplace=True),
nn.Conv3d(32, 64, 1, 1, 0),
)
self.res3d_2 = nn.Sequential(
nn.Conv3d(64, 32, 1, 1, 0),
nn.ReLU(inplace=True),
nn.Conv3d(32, 32, 3, 1, 3, 3),
nn.ReLU(inplace=True),
nn.Conv3d(32, 64, 1, 1, 0),
)
self.res3d_3 = nn.Sequential(
nn.Conv3d(64, 32, 1, 1, 0),
nn.ReLU(inplace=True),
nn.Conv3d(32, 32, 3, 1, 5, 5),
nn.ReLU(inplace=True),
nn.Conv3d(32, 64, 1, 1, 0),
)
self.conv4_1 = nn.Conv3d(256, 128, 1, 1, 0)
# self.relu4_1 = nn.ReLU(inplace=True)
self.conv4_2 = nn.Conv3d(128, 128, 1, 1, 0)
# self.relu4_2 = nn.ReLU(inplace=True)
self.fc12 = nn.Conv3d(128, 12, 1, 1,
0) # C_NUM = 12, number of classes is 12
# self.softmax = nn.Softmax(dim=1) # pytorch 0.3.0
# self.logsoftmax = nn.LogSoftmax(dim=1)
# ---- weights init
for m in self.modules():
if isinstance(m, nn.Conv3d):
n = m.kernel_size[0] * m.kernel_size[1] * m.kernel_size[
2] * m.out_channels
# nn.init.xavier_normal(m.weight.data, gain=math.sqrt(2. / n))
# nn.init.xavier_uniform(m.weight.data, gain=math.sqrt(2. / n))
nn.init.xavier_uniform_(m.weight.data) # gain=1
# nn.init.constant(m.bias.data, 0)
nn.init.normal_(self.conv4_1.weight.data, mean=0, std=0.1)
nn.init.normal_(self.conv4_2.weight.data, mean=0, std=0.01)
nn.init.normal_(self.fc12.weight.data, mean=0, std=0.01)
def __init__(self):
super(Net, self).__init__()
# The first few layers consumes the most memory, so use simple convolution to save memory.
# Call these layers preBlock, i.e., before the residual blocks of later layers.
self.preBlock = nn.Sequential(
nn.Conv3d(1, 24, kernel_size=3, padding=1), nn.BatchNorm3d(24),
nn.ReLU(inplace=True), nn.Conv3d(24, 24, kernel_size=3, padding=1),
nn.BatchNorm3d(24), nn.ReLU(inplace=True))
# 3 poolings, each pooling downsamples the feature map by a factor 2.
# 3 groups of blocks. The first block of each group has one pooling.
num_blocks_forw = [2, 2, 3, 3]
num_blocks_back = [3, 3]
self.featureNum_forw = [24, 32, 64, 64, 64]
self.featureNum_back = [128, 64, 64]
for i in range(len(num_blocks_forw)):
blocks = []
for j in range(num_blocks_forw[i]):
if j == 0:
blocks.append(
PostRes(self.featureNum_forw[i],
self.featureNum_forw[i + 1]))
else:
blocks.append(
PostRes(self.featureNum_forw[i + 1],
self.featureNum_forw[i + 1]))
setattr(self, 'forw' + str(i + 1), nn.Sequential(*blocks))
for i in range(len(num_blocks_back)):
blocks = []
for j in range(num_blocks_back[i]):
if j == 0:
if i == 0:
addition = 3
else:
addition = 0
blocks.append(
PostRes(
self.featureNum_back[i + 1] +
self.featureNum_forw[i + 2] + addition,
self.featureNum_back[i]))
else:
blocks.append(
PostRes(self.featureNum_back[i],
self.featureNum_back[i]))
setattr(self, 'back' + str(i + 2), nn.Sequential(*blocks))
self.maxpool1 = nn.MaxPool3d(kernel_size=2,
stride=2,
return_indices=True)
self.maxpool2 = nn.MaxPool3d(kernel_size=2,
stride=2,
return_indices=True)
self.maxpool3 = nn.MaxPool3d(kernel_size=2,
stride=2,
return_indices=True)
self.maxpool4 = nn.MaxPool3d(kernel_size=2,
stride=2,
return_indices=True)
self.unmaxpool1 = nn.MaxUnpool3d(kernel_size=2, stride=2)
self.unmaxpool2 = nn.MaxUnpool3d(kernel_size=2, stride=2)
self.path1 = nn.Sequential(
nn.ConvTranspose3d(64, 64, kernel_size=2, stride=2), SCse(64),
nn.BatchNorm3d(64), nn.ReLU(inplace=True))
self.path2 = nn.Sequential(
nn.ConvTranspose3d(64, 64, kernel_size=2, stride=2), SCse(64),
nn.BatchNorm3d(64), nn.ReLU(inplace=True))
self.drop = nn.Dropout3d(p=0.5, inplace=False)
self.output1 = nn.Sequential(
nn.Conv3d(self.featureNum_back[0], 64, kernel_size=1),
nn.ReLU(),
#nn.Dropout3d(p = 0.3),
nn.Conv3d(64, 5 * len(config['anchors']), kernel_size=1))
self.output2 = nn.Sequential(
nn.Conv3d(self.featureNum_back[0], 64, kernel_size=1),
nn.ReLU(),
#nn.Dropout3d(p = 0.3),
nn.Conv3d(64, 5 * len(config['anchors']), kernel_size=1))
self.output3 = nn.Sequential(
nn.Conv3d(self.featureNum_back[0], 64, kernel_size=1),
nn.ReLU(),
#nn.Dropout3d(p = 0.3),
nn.Conv3d(64, 5 * len(config['anchors']), kernel_size=1))
self.output4 = nn.Sequential(
nn.Conv3d(self.featureNum_back[0], 64, kernel_size=1),
nn.ReLU(),
#nn.Dropout3d(p = 0.3),
nn.Conv3d(64, 5 * len(config['anchors']), kernel_size=1))
def __init__(self, block, args):
self.inplanes = 64
super(ResNetFPN, self).__init__()
self.MODEL_TYPE = args.MODEL_TYPE
num_blocks = args.model_perms
non_local_inds = args.non_local_inds
model_3d_layers = args.model_3d_layers
self.num_blocks = num_blocks
self.non_local_inds = non_local_inds
self.model_3d_layers = model_3d_layers
self.conv1 = nn.Conv3d(3,
64,
kernel_size=(1, 7, 7),
stride=(1, 2, 2),
padding=(0, 3, 3),
bias=False)
self.bn1 = nn.BatchNorm3d(64)
self.relu = nn.ReLU(inplace=True)
self.pool1 = nn.MaxPool3d(kernel_size=(1, 3, 3),
stride=(1, 2, 2),
padding=(0, 0, 0))
self.layer_names = []
self.layer1 = self._make_layer(block,
64,
num_blocks[0],
temp_kernals=model_3d_layers[0],
nl_inds=non_local_inds[0])
self.MODEL_TYPE = args.MODEL_TYPE
if args.model_subtype in [
'C2D', 'RCN', 'CLSTM', 'RCLSTM', 'CGRU', 'RCGRU'
]:
self.pool2 = None
else:
self.pool2 = nn.MaxPool3d(kernel_size=(2, 1, 1),
stride=(2, 1, 1),
padding=(0, 0, 0))
self.layer2 = self._make_layer(block,
128,
num_blocks[1],
stride=2,
temp_kernals=model_3d_layers[1],
nl_inds=non_local_inds[1])
self.layer3 = self._make_layer(block,
256,
num_blocks[2],
stride=2,
temp_kernals=model_3d_layers[2],
nl_inds=non_local_inds[2])
self.layer4 = self._make_layer(block,
512,
num_blocks[3],
stride=2,
temp_kernals=model_3d_layers[3],
nl_inds=non_local_inds[3])
#self.avgpool = nn.AvgPool2d(7, stride=1)
#self.fc = nn.Linear(512 * block.expansion, num_classes)
self.conv6 = conv3x3(512 * block.expansion, 256, stride=2,
padding=1) # P6
self.conv7 = conv3x3(256, 256, stride=2, padding=1) # P7
self.ego_lateral = conv3x3(512 * block.expansion,
256,
stride=2,
padding=0)
self.avg_pool = nn.AdaptiveAvgPool3d((None, 1, 1))
self.lateral_layer1 = conv1x1(512 * block.expansion, 256)
self.lateral_layer2 = conv1x1(256 * block.expansion, 256)
self.lateral_layer3 = conv1x1(128 * block.expansion, 256)
self.corr_layer1 = conv3x3(256, 256, stride=1, padding=1) # P4
self.corr_layer2 = conv3x3(256, 256, stride=1, padding=1) # P4
self.corr_layer3 = conv3x3(256, 256, stride=1, padding=1) # P3
for m in self.modules():
if isinstance(m, nn.Conv2d):
torch.nn.init.kaiming_uniform_(m.weight, a=1)
if hasattr(m.bias, 'data'):
torch.nn.init.constant_(m.bias, 0)
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def __init__(self, in_channels, out_channels, final_sigmoid, basic_module, f_maps=64, layer_order='gcr',
num_groups=8, num_levels=4, is_segmentation=True, testing=False,
conv_kernel_size=3, pool_kernel_size=2, conv_padding=1, **kwargs):
super(Abstract3DUNet, self).__init__()
self.testing = testing
if isinstance(f_maps, int):
f_maps = number_of_features_per_level(f_maps, num_levels=num_levels)
# create encoder path consisting of Encoder modules. Depth of the encoder is equal to `len(f_maps)`
encoders = []
for i, out_feature_num in enumerate(f_maps):
if i == 0:
encoder = Encoder(in_channels, out_feature_num,
apply_pooling=False, # skip pooling in the firs encoder
basic_module=basic_module,
conv_layer_order=layer_order,
conv_kernel_size=conv_kernel_size,
num_groups=num_groups,
padding=conv_padding)
else:
# TODO: adapt for anisotropy in the data, i.e. use proper pooling kernel to make the data isotropic after 1-2 pooling operations
encoder = Encoder(f_maps[i - 1], out_feature_num,
basic_module=basic_module,
conv_layer_order=layer_order,
conv_kernel_size=conv_kernel_size,
num_groups=num_groups,
pool_kernel_size=pool_kernel_size,
padding=conv_padding)
encoders.append(encoder)
self.encoders = nn.ModuleList(encoders)
# create decoder path consisting of the Decoder modules. The length of the decoder is equal to `len(f_maps) - 1`
decoders = []
reversed_f_maps = list(reversed(f_maps))
for i in range(len(reversed_f_maps) - 1):
if basic_module == DoubleConv:
in_feature_num = reversed_f_maps[i] + reversed_f_maps[i + 1]
else:
in_feature_num = reversed_f_maps[i]
out_feature_num = reversed_f_maps[i + 1]
# TODO: if non-standard pooling was used, make sure to use correct striding for transpose conv
# currently strides with a constant stride: (2, 2, 2)
decoder = Decoder(in_feature_num, out_feature_num,
basic_module=basic_module,
conv_layer_order=layer_order,
conv_kernel_size=conv_kernel_size,
num_groups=num_groups,
padding=conv_padding)
decoders.append(decoder)
self.decoders = nn.ModuleList(decoders)
# in the last layer a 1×1 convolution reduces the number of output
# channels to the number of labels
self.final_conv = nn.Conv3d(f_maps[0], out_channels, 1)
if is_segmentation:
# semantic segmentation problem
if final_sigmoid:
self.final_activation = nn.Sigmoid()
else:
self.final_activation = nn.Softmax(dim=1)
else:
# regression problem
self.final_activation = None
def __init__(self, img_sz, opt=None):
super(VoxelMorphCVPR2018, self).__init__()
self.is_train = opt['tsk_set'][('train', False, 'if is in train mode')]
opt_voxelmorph = opt['tsk_set']['reg']['morph_cvpr']
self.load_trained_affine_net = opt_voxelmorph[(
'load_trained_affine_net', False,
'if true load_trained_affine_net; if false, the affine network is not initialized'
)]
self.using_affine_init = opt_voxelmorph[(
"using_affine_init", False,
"deploy affine network before the nonparametric network")]
self.affine_init_path = opt_voxelmorph[(
'affine_init_path', '', "the path of pretrained affine model")]
self.affine_refine_step = opt_voxelmorph[(
'affine_refine_step', 5,
"the multi-step num in affine refinement")]
self.initial_reg_factor = opt_voxelmorph[(
'initial_reg_factor', 1., 'initial regularization factor')]
self.min_reg_factor = opt_voxelmorph[(
'min_reg_factor', 1., 'minimum of regularization factor')]
enc_filters = [16, 32, 32, 32, 32]
#dec_filters = [32, 32, 32, 8, 8]
dec_filters = [32, 32, 32, 32, 32, 16, 16]
self.enc_filter = enc_filters
self.dec_filter = dec_filters
input_channel = 2
output_channel = 3
self.input_channel = 2
self.output_channel = 3
self.img_sz = img_sz
self.spacing = 1. / (np.array(img_sz) - 1)
self.loss_fn = None #NCCLoss()
self.epoch = -1
self.print_count = 0
def set_cur_epoch(self, cur_epoch=-1):
""" set current epoch"""
self.epoch = cur_epoch
if self.using_affine_init:
self.init_affine_net(opt)
self.id_transform = None
else:
self.id_transform = gen_identity_map(self.img_sz, 1.0).cuda()
print("Attention, the affine net is not used")
self.encoders = nn.ModuleList()
self.decoders = nn.ModuleList()
self.bilinear = Bilinear(zero_boundary=True)
for i in range(len(enc_filters)):
if i == 0:
self.encoders.append(
convBlock(input_channel,
enc_filters[i],
stride=1,
bias=True))
else:
self.encoders.append(
convBlock(enc_filters[i - 1],
enc_filters[i],
stride=2,
bias=True))
self.decoders.append(
convBlock(enc_filters[-1], dec_filters[0], stride=1, bias=True))
self.decoders.append(
convBlock(dec_filters[0] + enc_filters[3],
dec_filters[1],
stride=1,
bias=True))
self.decoders.append(
convBlock(dec_filters[1] + enc_filters[2],
dec_filters[2],
stride=1,
bias=True))
self.decoders.append(
convBlock(dec_filters[2] + enc_filters[1],
dec_filters[3],
stride=1,
bias=True))
self.decoders.append(
convBlock(dec_filters[3], dec_filters[4], stride=1, bias=True))
self.decoders.append(
convBlock(dec_filters[4] + enc_filters[0],
dec_filters[5],
stride=1,
bias=True))
self.decoders.append(
convBlock(dec_filters[5], dec_filters[6], stride=1, bias=True))
self.flow = nn.Conv3d(dec_filters[-1],
output_channel,
kernel_size=3,
stride=1,
padding=1,
bias=True)
def __init__(self, num_classes, pretrained=False):
super(C3D, self).__init__()
self.attn_conv2_channel = nn.Conv3d(128,
128,
kernel_size=(8, 28, 28),
padding=(0, 0, 0))
self.attn_conv2_channel_conv = nn.Conv3d(256,
128,
kernel_size=(3, 3, 3),
padding=(1, 1, 1))
self.attn_conv2_temp = nn.Conv3d(128,
1,
kernel_size=(3, 28, 28),
padding=(1, 0, 0))
self.attn_conv2_temp_conv = nn.Conv3d(128,
128,
kernel_size=(9, 3, 3),
padding=(0, 1, 1))
self.attn_conv4_channel = nn.Conv3d(512,
512,
kernel_size=(2, 7, 7),
padding=(0, 0, 0))
self.attn_conv4_channel_conv = nn.Conv3d(1024,
512,
kernel_size=(3, 3, 3),
padding=(1, 1, 1))
self.attn_conv4_temp = nn.Conv3d(512,
1,
kernel_size=(3, 7, 7),
padding=(1, 0, 0))
self.attn_conv4_temp_conv = nn.Conv3d(512,
512,
kernel_size=(3, 3, 3),
padding=(0, 1, 1))
self.conv1 = nn.Conv3d(3, 64, kernel_size=(3, 3, 3), padding=(1, 1, 1))
self.pool1 = nn.MaxPool3d(kernel_size=(1, 2, 2), stride=(1, 2, 2))
self.conv2 = nn.Conv3d(64,
128,
kernel_size=(3, 3, 3),
padding=(1, 1, 1))
self.pool2 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2))
self.conv3a = nn.Conv3d(128,
256,
kernel_size=(3, 3, 3),
padding=(1, 1, 1))
self.conv3b = nn.Conv3d(256,
256,
kernel_size=(3, 3, 3),
padding=(1, 1, 1))
self.pool3 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2))
self.conv4a = nn.Conv3d(256,
512,
kernel_size=(3, 3, 3),
padding=(1, 1, 1))
self.conv4b = nn.Conv3d(512,
512,
kernel_size=(3, 3, 3),
padding=(1, 1, 1))
self.pool4 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2))
self.conv5a = nn.Conv3d(512,
512,
kernel_size=(3, 3, 3),
padding=(1, 1, 1))
self.conv5b = nn.Conv3d(512,
512,
kernel_size=(3, 3, 3),
padding=(1, 1, 1))
self.pool5 = nn.MaxPool3d(kernel_size=(2, 2, 2),
stride=(2, 2, 2),
padding=(0, 1, 1))
self.fc6 = nn.Linear(8192, 4096)
self.fc7 = nn.Linear(4096, 4096)
self.fc8 = nn.Linear(4096, num_classes)
# self.fc8 = nn.Linear(4096, 300)
# self.fc9 = nn.Linear(300, 51)
self.dropout = nn.Dropout(p=0.5)
self.relu = nn.ReLU()
self.batchnorm1 = nn.BatchNorm3d(256)
self.batchnorm2 = nn.BatchNorm3d(512)
self.full_batchnorm1 = nn.BatchNorm1d(4096)
self.full_batchnorm2 = nn.BatchNorm1d(4096)
self.__init_weight()
if pretrained:
self.__load_pretrained_weights()
def __init__(self,
temporal_dim,
boundary_ratio,
num_samples,
num_samples_per_bin,
feat_dim,
soft_nms_alpha,
soft_nms_low_threshold,
soft_nms_high_threshold,
post_process_top_k,
loss_cls=dict(type='BMNLoss'),
hidden_dim_1d=256,
hidden_dim_2d=128,
hidden_dim_3d=512):
super(BaseLocalizer, self).__init__()
self.tscale = temporal_dim
self.boundary_ratio = boundary_ratio
self.num_samples = num_samples
self.num_samples_per_bin = num_samples_per_bin
self.feat_dim = feat_dim
self.soft_nms_alpha = soft_nms_alpha
self.soft_nms_low_threshold = soft_nms_low_threshold
self.soft_nms_high_threshold = soft_nms_high_threshold
self.post_process_top_k = post_process_top_k
self.loss_cls = build_loss(loss_cls)
self.hidden_dim_1d = hidden_dim_1d
self.hidden_dim_2d = hidden_dim_2d
self.hidden_dim_3d = hidden_dim_3d
self._get_interp1d_mask()
# Base Module
self.x_1d_b = nn.Sequential(
nn.Conv1d(
self.feat_dim,
self.hidden_dim_1d,
kernel_size=3,
padding=1,
groups=4), nn.ReLU(inplace=True),
nn.Conv1d(
self.hidden_dim_1d,
self.hidden_dim_1d,
kernel_size=3,
padding=1,
groups=4), nn.ReLU(inplace=True))
# Temporal Evaluation Module
self.x_1d_s = nn.Sequential(
nn.Conv1d(
self.hidden_dim_1d,
self.hidden_dim_1d,
kernel_size=3,
padding=1,
groups=4), nn.ReLU(inplace=True),
nn.Conv1d(self.hidden_dim_1d, 1, kernel_size=1), nn.Sigmoid())
self.x_1d_e = nn.Sequential(
nn.Conv1d(
self.hidden_dim_1d,
self.hidden_dim_1d,
kernel_size=3,
padding=1,
groups=4), nn.ReLU(inplace=True),
nn.Conv1d(self.hidden_dim_1d, 1, kernel_size=1), nn.Sigmoid())
# Proposal Evaluation Module
self.x_1d_p = nn.Sequential(
nn.Conv1d(
self.hidden_dim_1d,
self.hidden_dim_1d,
kernel_size=3,
padding=1), nn.ReLU(inplace=True))
self.x_3d_p = nn.Sequential(
nn.Conv3d(
self.hidden_dim_1d,
self.hidden_dim_3d,
kernel_size=(self.num_samples, 1, 1)), nn.ReLU(inplace=True))
self.x_2d_p = nn.Sequential(
nn.Conv2d(self.hidden_dim_3d, self.hidden_dim_2d, kernel_size=1),
nn.ReLU(inplace=True),
nn.Conv2d(
self.hidden_dim_2d,
self.hidden_dim_2d,
kernel_size=3,
padding=1), nn.ReLU(inplace=True),
nn.Conv2d(
self.hidden_dim_2d,
self.hidden_dim_2d,
kernel_size=3,
padding=1), nn.ReLU(inplace=True),
nn.Conv2d(self.hidden_dim_2d, 2, kernel_size=1), nn.Sigmoid())
self.anchors_tmins, self.anchors_tmaxs = self._temporal_anchors(
-0.5, 1.5)
self.match_map = self._match_map()
self.bm_mask = self._get_bm_mask()
def __init__(self,
block,
layers,
c_in=3,
num_classes=1000,
zero_init_residual=False,
groups=1,
width_per_group=26,
scale=4,
replace_stride_with_dilation=None,
norm_layer=None):
super(Res2Net3D, self).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm3d
self._norm_layer = norm_layer
self.inplanes = 64
self.dilation = 1
if replace_stride_with_dilation is None:
# each element in the tuple indicates if we should replace
# the 2x2 stride with a dilated convolution instead
replace_stride_with_dilation = [False, False, False]
if len(replace_stride_with_dilation) != 3:
raise ValueError("replace_stride_with_dilation should be None "
"or a 3-element tuple, got {}".format(
replace_stride_with_dilation))
self.groups = groups
self.base_width = width_per_group
self.scale = scale
self.expansion = 4
#self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3,
# bias=False)
#modify stem
#stem = []
sizes = [c_in, 32, 64, 64] #modified per Grankin
#for i in range(3):
# stem.append(conv_layer(sizes[i], sizes[i+1], stride=2 if i==0 else 1))
#stem (initial entry layers)
self.conv1 = conv_layer(c_in, sizes[1], stride=2)
self.conv2 = conv_layer(sizes[1], sizes[2])
self.conv3 = conv_layer(sizes[2], sizes[3])
self.maxpool = nn.MaxPool3d(kernel_size=3, stride=2, padding=1)
#nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.layer1 = self._make_layer(block, 64, layers[0])
self.layer2 = self._make_layer(block,
128,
layers[1],
stride=2,
dilate=replace_stride_with_dilation[0])
self.layer3 = self._make_layer(block,
256,
layers[2],
stride=2,
dilate=replace_stride_with_dilation[1])
self.layer4 = self._make_layer(block,
512,
layers[3],
stride=2,
dilate=replace_stride_with_dilation[2])
self.scale = 1
self.layerup1 = self._make_uplayer(block, 2048, layers[3])
self.layerup2 = self._make_uplayer(block, 512, layers[2])
self.layerup3 = self._make_uplayer(block, 512, layers[1])
self.layerup4 = self._make_uplayer(block, 128, layers[0])
self.layerup5 = self._make_uplayer(block, 32, 2)
self.final_conv = nn.Sequential(
nn.Conv3d(8, 8, 3, padding=2, dilation=2), Mish(),
nn.BatchNorm3d(8), nn.Conv3d(8, 1, 1))
# self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
# self.fc = nn.Linear(512 * block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight,
mode='fan_out',
nonlinearity='relu')
elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
# Zero-initialize the last BN in each residual branch,
# so that the residual branch starts with zeros, and each residual block behaves like an identity.
# This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
if zero_init_residual:
for m in self.modules():
if isinstance(m, Bottle2neck):
nn.init.constant_(m.bn3.weight, 0)
elif isinstance(m, BasicBlock):
nn.init.constant_(m.bn2.weight, 0)
def conv(in_channels, out_channels, kernel_size=3, stride=1, bias=False):
padding = utils.pad_size(kernel_size, 'same')
return nn.Conv3d(in_channels, out_channels, kernel_size=kernel_size,
stride=stride, padding=padding, bias=bias)
def __init__(self, channel):
super(SpatialAttention3d, self).__init__()
self.squeeze = nn.Conv3d(channel, 1, kernel_size=1, bias=False)
self.sigmoid = nn.Sigmoid()
