def __init__(self): super(CNNScore, self).__init__() self.conv1 = nn.Conv3d(16, 96, kernel_size=1, stride=2) self.fire2_squeeze = nn.Conv3d(96, 16, kernel_size=1) self.fire2_expand1 = nn.Conv3d(16, 64, kernel_size=1) # Padding = (k-1)/2 where k is the kernel size self.fire2_expand2 = nn.Conv3d(16, 64, kernel_size=3, padding=1) self.fire3_squeeze = nn.Conv3d(128, 16, kernel_size=1) self.fire3_expand1 = nn.Conv3d(16, 64, kernel_size=1) self.fire3_expand2 = nn.Conv3d(16, 64, kernel_size=3, padding=1) self.fire4_squeeze = nn.Conv3d(128, 32, kernel_size=1) self.fire4_expand1 = nn.Conv3d(32, 128, kernel_size=1) self.fire4_expand2 = nn.Conv3d(32, 128, kernel_size=3, padding=1) self.pool = nn.MaxPool3d(kernel_size=3, stride=2) self.fire5_squeeze = nn.Conv3d(256, 32, kernel_size=1) self.fire5_expand1 = nn.Conv3d(32, 128, kernel_size=1) self.fire5_expand2 = nn.Conv3d(32, 128, kernel_size=3, padding=1) self.fire6_squeeze = nn.Conv3d(256, 48, kernel_size=1) self.fire6_expand1 = nn.Conv3d(48, 192, kernel_size=1) self.fire6_expand2 = nn.Conv3d(48, 192, kernel_size=3, padding=1) self.fire7_squeeze = nn.Conv3d(384, 48, kernel_size=1) self.fire7_expand1 = nn.Conv3d(48, 192, kernel_size=1) self.fire7_expand2 = nn.Conv3d(48, 192, kernel_size=3, padding=1) self.fire8_squeeze = nn.Conv3d(384, 64, kernel_size=1) self.fire8_expand1 = nn.Conv3d(64, 256, kernel_size=1) self.fire8_expand2 = nn.Conv3d(64, 256, kernel_size=3, padding=1) self.avg_pool = nn.AvgPool3d(kernel_size=3, padding=1) self.dense1 = nn.Linear(512*2*2*2, 1)

def __init__(self, in_ch, out_ch, se=False, norm='bn'): super(inconv, self).__init__() self.conv1 = nn.Conv3d(in_ch, out_ch, kernel_size=(1,3,3), padding=(0,1,1), bias=False) self.relu = nn.ReLU(inplace=True) self.conv2 = SEBasicBlock(out_ch, out_ch, kernel_size=(1,3,3), norm=norm)

def __init__(self): super(vgg16, self).__init__() self.block1_output = nn.Sequential( nn.Conv3d(3, 63, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv3d(64, 64, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.MaxPool3d(kernel_size=2, stride=2)) self.block2_output = nn.Sequential( nn.Conv3d(64, 128, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv3d(128, 128, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.MaxPool3d(kernel_size=2, stride=2)) self.block3_output = nn.Sequential( nn.Conv3d(128, 256, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv3d(256, 256, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv3d(256, 256, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.MaxPool3d(kernel_size=2, stride=2)) self.block4_output = nn.Sequential( nn.Conv3d(256, 512, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv3d(512, 512, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv3d(512, 512, kernel_size=3, padding=1), nn.ReLU(inplace=True)) self.block4_3output = nn.Sequential( nn.MaxPool3d(kernel_size=2, stride=2)) self.block5_output = nn.Sequential( nn.Conv3d(512, 512, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv3d(512, 512, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv3d(512, 512, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.MaxPool3d(kernel_size=2, stride=2)) self.block6_output = nn.Sequential( nn.Conv3d(512, 1024, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.MaxPool3d(kernel_size=2, stride=2)) self.block7_output = nn.Sequential( nn.Conv3d(1024, 1024, kernel_size=1, padding=0), nn.ReLU(inplace=True)) self.block7_1_output = nn.Sequential( nn.MaxPool3d(kernel_size=2, striide=2)) self.block8_output = nn.Sequential( nn.Conv3d(512, 128, kernel_size=1, padding=0), nn.ReLU(inplace=True), nn.Conv3d(128, 256, kernel_size=3, padding=1), nn.ReLU(inplace=True)) self.block8_2_output = nn.Sequential( nn.MaxPool3d(kernel_size=2, striide=2)) self.block9_output = nn.Sequential( nn.Conv3d(256, 128, kernel_size=1, padding=0), nn.ReLU(inplace=True), nn.Conv3d(128, 256, kernel_size=3, padding=1), nn.ReLU(inplace=True)) self.block9_2_output = nn.Sequential( nn.MaxPool3d(kernel_size=2, striide=2)) self.block10_output = nn.Sequential( nn.Conv3d(256, 128, kernel_size=1, padding=0), nn.ReLU(inplace=True), nn.Conv3d(128, 256, kernel_size=3, padding=1), nn.ReLU(inplace=True), ) self.block10_2_output = nn.Sequential( nn.MaxPool3d(kernel_size=2, striide=2)) self.block11_output = nn.Sequential( nn.Conv3d(256, 128, kernel_size=1, padding=0), nn.ReLU(inplace=True), nn.Conv3d(128, 256, kernel_size=3, padding=1), nn.ReLU(inplace=True)) self.block11_2_output = nn.Sequential( nn.MaxPool3d(kernel_size=2, striide=2))

def conv1x1x1(in_planes, out_planes, stride=1): return nn.Conv3d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)

def __init__(self, img_sz, opt=None): super(VoxelMorphMICCAI2019, self).__init__() self.is_train = opt['tsk_set'][('train', False, 'if is in train mode')] opt_voxelmorph = opt['tsk_set']['reg']['morph_miccai'] self.load_trained_affine_net = opt_voxelmorph[( 'load_trained_affine_net', False, 'if true load_trained_affine_net; if false, the affine network is not initialized' )] self.affine_refine_step = opt_voxelmorph[( 'affine_refine_step', 5, "the multi-step num in affine refinement")] self.using_affine_init = opt_voxelmorph[( "using_affine_init", False, "deploy affine network before the nonparametric network")] self.affine_init_path = opt_voxelmorph[( 'affine_init_path', '', "the path of pretrained affine model")] enc_filters = [16, 32, 32, 32, 32] #dec_filters = [32, 32, 32, 8, 8] dec_filters = [32, 32, 32, 32, 16] self.enc_filter = enc_filters self.dec_filter = dec_filters input_channel = 2 output_channel = 3 self.input_channel = input_channel self.output_channel = output_channel self.img_sz = img_sz self.low_res_img_sz = [int(x / 2) for x in img_sz] self.spacing = 1. / (np.array(img_sz) - 1) self.int_steps = 7 self.image_sigma = opt_voxelmorph[('image_sigma', 0.02, 'image_sigma')] self.prior_lambda = opt_voxelmorph[('lambda_factor_in_vmr', 50, 'lambda_factor_in_vmr')] self.prior_lambda_mean = opt_voxelmorph[('lambda_mean_factor_in_vmr', 50, 'lambda_mean_factor_in_vmr')] self.flow_vol_shape = self.low_res_img_sz self.D = self._degree_matrix(self.flow_vol_shape) self.D = (self.D).cuda() # 1, 96, 40,40 3' self.loss_fn = None if self.using_affine_init: self.init_affine_net(opt) self.id_transform = None else: self.id_transform = gen_identity_map(self.img_sz, 1.0).cuda() self.id_transform = self.id_transform.view( [1] + list(self.id_transform.shape)) print("Attention, the affine net is not used") """to compatiable to the mesh setting in voxel morph""" self.low_res_id_transform = gen_identity_map(self.img_sz, 0.5, normalized=False).cuda() self.encoders = nn.ModuleList() self.decoders = nn.ModuleList() #self.bilinear = Bilinear(zero_boundary=True) self.bilinear = stn_nd.STN_ND_BCXYZ(np.array([1., 1., 1.]), zero_boundary=True) self.bilinear_img = Bilinear(zero_boundary=True) for i in range(len(enc_filters)): if i == 0: self.encoders.append( convBlock(input_channel, enc_filters[i], stride=1, bias=True)) else: self.encoders.append( convBlock(enc_filters[i - 1], enc_filters[i], stride=2, bias=True)) self.decoders.append( convBlock(enc_filters[-1], dec_filters[0], stride=1, bias=True)) self.decoders.append( convBlock(dec_filters[0] + enc_filters[3], dec_filters[1], stride=1, bias=True)) self.decoders.append( convBlock(dec_filters[1] + enc_filters[2], dec_filters[2], stride=1, bias=True)) self.decoders.append( convBlock(dec_filters[2] + enc_filters[1], dec_filters[3], stride=1, bias=True)) self.decoders.append( convBlock(dec_filters[3], dec_filters[4], stride=1, bias=True)) self.flow_mean = nn.Conv3d(dec_filters[-1], output_channel, kernel_size=3, stride=1, padding=1, bias=True) self.flow_sigma = nn.Conv3d(dec_filters[-1], output_channel, kernel_size=3, stride=1, padding=1, bias=True) self.flow_mean.weight.data.normal_(0., 1e-5) self.flow_sigma.weight.data.normal_(0., 1e-10) self.flow_sigma.bias.data = torch.Tensor([-10] * 3) self.print_count = 0

def __init__(self, n_features=80, log_transform=True, colored=True): super().__init__() self.log_transform = log_transform self.channels = 5 if colored else 2 # Encoder self.enc1 = nn.Conv3d(self.channels, n_features, 4, stride=2, padding=1) self.bn1 = nn.BatchNorm3d(n_features) self.enc2 = nn.Conv3d(n_features, 2 * n_features, 4, stride=2, padding=1) self.bn2 = nn.BatchNorm3d(2 * n_features) self.enc3 = nn.Conv3d(2 * n_features, 4 * n_features, 4, stride=2, padding=1) self.bn3 = nn.BatchNorm3d(4 * n_features) self.enc4 = nn.Conv3d(4 * n_features, 8 * n_features, 4, stride=1, padding=0) self.bn4 = nn.BatchNorm3d(8 * n_features) # Bottleneck self.fc1 = nn.Linear(8 * n_features, 8 * n_features) self.bn5 = nn.BatchNorm1d(8 * n_features) self.fc2 = nn.Linear(8 * n_features, 8 * n_features) self.bn6 = nn.BatchNorm1d(8 * n_features) # Decoder self.dec1 = nn.ConvTranspose3d(2 * 8 * n_features, 4 * n_features, 4, stride=1, padding=0) self.dbn1 = nn.BatchNorm3d(4 * n_features) self.dec2 = nn.ConvTranspose3d(2 * 4 * n_features, 2 * n_features, 4, stride=2, padding=1) self.dbn2 = nn.BatchNorm3d(2 * n_features) self.dec3 = nn.ConvTranspose3d(2 * 2 * n_features, n_features, 4, stride=2, padding=1) self.dbn3 = nn.BatchNorm3d(n_features) self.dec4 = nn.ConvTranspose3d(2 * n_features, self.channels - 1, 4, stride=2, padding=1)

def __init__(self): super(PhysNet, self).__init__() self.ConvBlock1 = nn.Sequential( nn.Conv3d(3, 16, [1, 5, 5], stride=1, padding=[0, 2, 2]), nn.BatchNorm3d(16), nn.ReLU(inplace=True), ) ####################### self.ConvBlock2 = nn.Sequential( nn.Conv3d(16, 32, [1, 3, 3], stride=1, padding=[0, 1, 1]), nn.BatchNorm3d(32), nn.ReLU(inplace=True), ) self.ConvBlock3 = nn.Sequential( nn.Conv3d(32, 32, [3, 1, 1], stride=1, padding=[1, 0, 0]), nn.BatchNorm3d(32), nn.ReLU(inplace=True), ) self.ConvBlock4 = nn.Sequential( nn.Conv3d(32, 32, [1, 3, 3], stride=1, padding=[0, 1, 1]), nn.BatchNorm3d(32), nn.ReLU(inplace=True), ) self.ConvBlock5 = nn.Sequential( nn.Conv3d(32, 32, [3, 1, 1], stride=1, padding=[1, 0, 0]), nn.BatchNorm3d(32), nn.ReLU(inplace=True), ) ######################## self.ConvBlock6 = nn.Sequential( nn.Conv3d(32, 64, [1, 3, 3], stride=1, padding=[0, 1, 1]), nn.BatchNorm3d(64), nn.ReLU(inplace=True), ) self.ConvBlock7 = nn.Sequential( nn.Conv3d(64, 64, [3, 1, 1], stride=1, padding=[1, 0, 0]), nn.BatchNorm3d(64), nn.ReLU(inplace=True), ) self.ConvBlock8 = nn.Sequential( nn.Conv3d(64, 64, [1, 3, 3], stride=1, padding=[0, 1, 1]), nn.BatchNorm3d(64), nn.ReLU(inplace=True), ) self.ConvBlock9 = nn.Sequential( nn.Conv3d(64, 64, [3, 1, 1], stride=1, padding=[1, 0, 0]), nn.BatchNorm3d(64), nn.ReLU(inplace=True), ) ######################## self.ConvBlock10 = nn.Sequential( nn.Conv3d(64, 64, [1, 3, 3], stride=1, padding=[0, 1, 1]), nn.BatchNorm3d(64), nn.ReLU(inplace=True), ) self.ConvBlock11 = nn.Sequential( nn.Conv3d(64, 64, [3, 1, 1], stride=1, padding=[1, 0, 0]), nn.BatchNorm3d(64), nn.ReLU(inplace=True), ) self.ConvBlock12 = nn.Sequential( nn.Conv3d(64, 64, [1, 3, 3], stride=1, padding=[0, 1, 1]), nn.BatchNorm3d(64), nn.ReLU(inplace=True), ) self.ConvBlock13 = nn.Sequential( nn.Conv3d(64, 64, [3, 1, 1], stride=1, padding=[1, 0, 0]), nn.BatchNorm3d(64), nn.ReLU(inplace=True), ) ######################## self.ConvBlock14 = nn.Sequential( # padding?? padding=[0, 2, 2] nn.Conv3d(64, 64, [1, 3, 3], stride=1, padding=[0, 1, 1]), nn.BatchNorm3d(64), nn.ReLU(inplace=True), ) self.ConvBlock15 = nn.Sequential( nn.Conv3d(64, 64, [3, 1, 1], stride=1, padding=[1, 0, 0]), nn.BatchNorm3d(64), nn.ReLU(inplace=True), ) self.ConvBlock16 = nn.Conv3d(64, 1, [1, 1, 1], stride=1, padding=0) self.AvgpoolSpa1 = nn.AvgPool3d((1, 2, 2), stride=(1, 2, 2)) self.AvgpoolSpa2 = nn.AvgPool3d((1, 7, 7), stride=(1, 2, 2)) self.MaxpoolSpa = nn.MaxPool3d((1, 2, 2), stride=(1, 2, 2))

def __init__(self, in_dims): super(MatchHead, self).__init__() self.feature = VoxRes(in_dims) self.dims_align = nn.Conv3d(64, 1, 1, 1, 0)

def __init__(self, in_dims, emd_dims=32): super(EmbHead, self).__init__() self.feature = VoxRes(in_dims) self.dims_align = nn.Conv3d(64, emd_dims, 1, 1, 0)

def __init__(self, opt): super(VI_2D_Decoder_3, self).__init__() self.opt = opt dv = 2 if self.opt.double_size else 1 ### decoder self.dc0 = GatedConvolution(128, 128, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False) self.dc1 = GatedConvolution(128, 128, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False) #### UPCONV self.dc1_1 = GatedUpConvolution( (1, opt.crop_size // 4 // dv, opt.crop_size // 4 // dv), 128, 96, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False) self.dc2_1 = GatedConvolution(96 + 96, 96, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False) self.dc2_bt1 = GatedConvolution(96, 96, kernel_size=(1, 3, 3), stride=(1, 1, 1), dilation=(1, 2, 2), padding=(0, 2, 2), bias=False) self.dc2_bt2 = GatedConvolution(96, 96, kernel_size=(1, 3, 3), stride=(1, 1, 1), dilation=(1, 4, 4), padding=(0, 4, 4), bias=False) self.dc2_bt3 = GatedConvolution(96, 96, kernel_size=(1, 3, 3), stride=(1, 1, 1), dilation=(1, 8, 8), padding=(0, 8, 8), bias=False) #### UPCONV self.dc2_2 = GatedUpConvolution( (1, opt.crop_size // 2 // dv, opt.crop_size // 2 // dv), 96, 64, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False) self.dc3_1 = GatedConvolution(64 + 64, 64, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False) self.dc3_2 = GatedConvolution(64, 64, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False) #### UPCONV self.dc4 = GatedUpConvolution( (1, opt.crop_size // dv, opt.crop_size // dv), 64, 32, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False) if self.opt.double_size: self.upsample = nn.Upsample(size=(1, opt.crop_size, opt.crop_size), mode='trilinear') self.dc5 = GatedConvolution(32, 16, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False) self.dc6 = nn.Conv3d(16, 3, kernel_size=(1, 3, 3), stride=(1, 1, 1), padding=(0, 1, 1), bias=False) for m in self.modules(): if isinstance(m, nn.Conv3d) or isinstance(m, nn.Conv2d): m.weight = nn.init.kaiming_normal_(m.weight, mode='fan_out') elif isinstance(m, nn.BatchNorm3d) or isinstance( m, nn.BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_()

def __init__(self, num_classes=12): super(SSC_RGBD_GRFNet, self).__init__() print('SSC_RGBD_GRFNet.') # --- depth c_in, c, c_out, dilation, residual = 1, 4, 8, 1, True self.dep_feature2d = nn.Sequential( nn.Conv2d(c_in, c_out, 1, 1, 0), # reduction BottleneckDDR2d(c_out, c, c_out, dilation=dilation, residual=residual), BottleneckDDR2d(c_out, c, c_out, dilation=dilation, residual=residual), ) self.project_layer_dep = Project2Dto3D(240, 144, 240) # w=240, h=144, d=240 self.dep_feature3d = nn.Sequential( DownsampleBlock3d(8, 16), BottleneckDDR3d(c_in=16, c=8, c_out=16, dilation=1, residual=True), DownsampleBlock3d(16, 64), # nn.MaxPool3d(kernel_size=2, stride=2) BottleneckDDR3d(c_in=64, c=16, c_out=64, dilation=1, residual=True), ) # --- RGB c_in, c, c_out, dilation, residual = 3, 4, 8, 1, True self.rgb_feature2d = nn.Sequential( nn.Conv2d(c_in, c_out, 1, 1, 0), # reduction BottleneckDDR2d(c_out, c, c_out, dilation=dilation, residual=residual), BottleneckDDR2d(c_out, c, c_out, dilation=dilation, residual=residual), ) self.project_layer_rgb = Project2Dto3D(240, 144, 240) # w=240, h=144, d=240 self.rgb_feature3d = nn.Sequential( DownsampleBlock3d(8, 16), BottleneckDDR3d(c_in=16, c=8, c_out=16, dilation=1, residual=True), DownsampleBlock3d(16, 64), # nn.MaxPool3d(kernel_size=2, stride=2) BottleneckDDR3d(c_in=64, c=16, c_out=64, dilation=1, residual=True), ) # -------------1/4 ck = 64 c = ck // 4 # --- RGB self.res3d_1r = BottleneckDDR3d(c_in=ck, c=c, c_out=ck, dilation=2, residual=True) self.res3d_2r = BottleneckDDR3d(c_in=ck, c=c, c_out=ck, dilation=3, residual=True) self.res3d_3r = BottleneckDDR3d(c_in=ck, c=c, c_out=ck, dilation=5, residual=True) # --- Depth self.res3d_1d = BottleneckDDR3d(c_in=ck, c=c, c_out=ck, dilation=2, residual=True) self.res3d_2d = BottleneckDDR3d(c_in=ck, c=c, c_out=ck, dilation=3, residual=True) self.res3d_3d = BottleneckDDR3d(c_in=ck, c=c, c_out=ck, dilation=5, residual=True) # self.lstm = DDRConv3dLSTMCell(input_channels=128, hidden_channels=64, kernel_size=(3, 3, 3), bias=True) self.gru = Conv3dGRUCell(input_channels=64, hidden_channels=64, kernel_size=3, bias=True) self.aspp = DDR_ASPP3d(c_in=ck, c=16, c_out=64) self.conv_out = nn.Sequential(nn.Conv3d(320, 160, 1, 1, 0), nn.ReLU(inplace=True), nn.Conv3d(160, num_classes, 1, 1, 0)) # ---- weights init for m in self.modules(): if isinstance(m, nn.Conv3d): n = m.kernel_size[0] * m.kernel_size[1] * m.kernel_size[ 2] * m.out_channels # nn.init.xavier_normal(m.weight.data, gain=math.sqrt(2. / n)) # nn.init.xavier_uniform(m.weight.data, gain=math.sqrt(2. / n)) nn.init.xavier_uniform_(m.weight.data) # gain=1 # nn.init.constant(m.bias.data, 0) if isinstance(m, nn.Conv2d): nn.init.normal_(m.weight.data, mean=0, std=0.1)

def __init__(self, nInputChannels=3, n_classes=21, os=16, pretrained=False, _print=True, final_sigmoid=False, hidden_layers=128, device=None): if _print: print("Constructing DeepLabv3+ model...") print("Number of classes: {}".format(n_classes)) print("Output stride: {}".format(os)) print("Number of Input Channels: {}".format(nInputChannels)) super(DeepLabv3_plus_gcn_3d, self).__init__() self.device = device # Atrous Conv self.xception_features = Xception(nInputChannels, os, pretrained) # ASPP if os == 16: rates = [1, 6, 12, 18] elif os == 8: rates = [1, 12, 24, 36] raise NotImplementedError else: raise NotImplementedError self.aspp1 = ASPP_module_rate0(1024, 128, rate=rates[0]) self.aspp2 = ASPP_module(1024, 128, rate=rates[1]) self.aspp3 = ASPP_module(1024, 128, rate=rates[2]) self.aspp4 = ASPP_module(1024, 128, rate=rates[3]) self.relu = nn.ReLU() self.global_avg_pool = nn.Sequential(nn.AdaptiveAvgPool3d((1, 1, 1)), nn.Conv3d(1024, 128, 1, stride=1, bias=False), nn.BatchNorm3d(128), nn.ReLU() ) self.concat_projection_conv1 = nn.Conv3d(640, 128, 1, bias=False) self.concat_projection_bn1 = nn.BatchNorm3d(128) # adopt [1x1, 48] for channel reduction. self.feature_projection_conv1 = nn.Conv3d(128, 24, 1, bias=False) self.feature_projection_bn1 = nn.BatchNorm3d(24) # self.decoder = nn.Sequential(Decoder_module(152, 128), # Decoder_module(128, 128) # ) # # self.featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=128, hidden_layers=hidden_layers, # nodes=n_classes) # self.graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers) # self.graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers) # self.graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers) # # self.graph_2_fea = gcn.Graph_to_Featuremaps_savemem(input_channels=128, # output_channels=128, # hidden_layers=hidden_layers, # nodes=n_classes) # # self.skip_conv = nn.Sequential(*[nn.Conv3d(128, 128, kernel_size=1), # nn.ReLU(True)]) # # self.semantic = nn.Conv3d(128, n_classes, kernel_size=1, stride=1) self.decoder = nn.Sequential(Decoder_module(152, 256), Decoder_module(256, 256) ) self.featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=256, hidden_layers=hidden_layers, nodes=n_classes) self.graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers) self.graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers) self.graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers) self.graph_2_fea = gcn.Graph_to_Featuremaps_savemem(input_channels=256, output_channels=256, hidden_layers=hidden_layers, nodes=n_classes) self.skip_conv = nn.Sequential(*[nn.Conv3d(256, 256, kernel_size=1), nn.ReLU(True)]) self.semantic = nn.Conv3d(256, n_classes, kernel_size=1, stride=1) if final_sigmoid: self.final_activation = nn.Sigmoid() else: self.final_activation = nn.Softmax(dim=1)

def __init__(self, inplanes=3, os=16, pretrained=False): super(Xception, self).__init__() if os == 16: entry_block3_stride = (1, 2, 2) middle_block_rate = 1 exit_block_rates = (1, 2) elif os == 8: entry_block3_stride = 1 middle_block_rate = 2 exit_block_rates = (2, 4) else: raise NotImplementedError # Entry flow self.conv1 = nn.Conv3d(inplanes, 16, 3, stride=(1,2,2), padding=1, bias=False) self.bn1 = nn.BatchNorm3d(16) self.relu = nn.ReLU(inplace=True) self.conv2 = nn.Conv3d(16, 32, 3, stride=1, padding=1, bias=False) self.bn2 = nn.BatchNorm3d(32) self.block1 = Block(32, 64, reps=2, stride=(1,2,2), start_with_relu=False, grow_first=True) self.block2 = Block2(64, 128, reps=2, stride=(1,2,2), start_with_relu=True, grow_first=True) self.block3 = Block(128, 364, reps=2, stride=entry_block3_stride, start_with_relu=True, grow_first=True) # Middle flow # self.block4 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block5 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block6 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block7 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block8 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block9 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block10 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block11 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block12 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block13 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block14 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block15 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block16 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block17 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block18 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # self.block19 = Block(728, 728, reps=3, stride=1, dilation=middle_block_rate, start_with_relu=True, grow_first=True) # Exit flow self.block20 = Block(364, 512, reps=2, stride=1, dilation=exit_block_rates[0], start_with_relu=True, grow_first=False, is_last=True) self.conv3 = SeparableConv3d_aspp(512, 768, 3, stride=1, dilation=exit_block_rates[1],padding=exit_block_rates[1]) # self.bn3 = nn.BatchNorm3d(1536) self.conv4 = SeparableConv3d_aspp(768, 768, 3, stride=1, dilation=exit_block_rates[1],padding=exit_block_rates[1]) # self.bn4 = nn.BatchNorm3d(1536) self.conv5 = SeparableConv3d_aspp(768, 1024, 3, stride=1, dilation=exit_block_rates[1],padding=exit_block_rates[1]) # self.bn5 = nn.BatchNorm3d(2048) # Init weights # self.__init_weight() # Load pretrained model if pretrained: self.__load_xception_pretrained()

def __init__(self, inplanes, planes, kernel_size=3, stride=1, padding=0, dilation=1, bias=False): super(SeparableConv3d, self).__init__() self.conv1 = nn.Conv3d(inplanes, inplanes, kernel_size, stride, padding, dilation, groups=inplanes, bias=bias) self.pointwise = nn.Conv3d(inplanes, planes, 1, 1, 0, 1, 1, bias=bias)

def __init__(self, growth_rate=12, shared_block_config=(8, 8), MC_block_config=(8, 8), CA_block_config=(8, 8), compression=0.5, num_init_features=32, bn_size=4, drop_rate=0.2, num_classes_MC=1, num_classes_CA=1, efficient=False): super(DenseNet, self).__init__() assert 0 < compression <= 1, 'compression of densenet should be between 0 and 1' # First convolution self.shared_blocks = nn.Sequential(OrderedDict([ ('conv0', nn.Conv3d(1, num_init_features, kernel_size=5, stride=1, padding=2, bias=False)), ])) # Shared blocks num_features = num_init_features for i, num_layers in enumerate(shared_block_config): block = _DenseBlock( num_layers=num_layers, num_input_features=num_features, bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate, efficient=efficient, ) self.shared_blocks.add_module('denseblock%d' % (i + 1), block) num_features = num_features + num_layers * growth_rate # if i != len(shared_block_config) - 1: if True: trans = _Transition_no_pool(num_input_features=num_features, num_output_features=int(num_features * compression)) self.shared_blocks.add_module('transition%d' % (i + 1), trans) num_features = int(num_features * compression) num_shared_features = num_features self.MC_blocks=nn.Sequential() num_features=num_shared_features for i, num_layers in enumerate(MC_block_config): block = _DenseBlock( num_layers=num_layers, num_input_features=num_features, bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate, efficient=efficient, ) self.MC_blocks.add_module('denseblock_%d' % (i + 1), block) num_features = num_features + num_layers * growth_rate if i != len(CA_block_config) - 1: trans = _Transition(num_input_features=num_features, num_output_features=int(num_features * compression)) self.MC_blocks.add_module('transition_%d' % (i + 1), trans) num_features = int(num_features * compression) # Final batch norm self.MC_blocks.add_module('norm_final', nn.BatchNorm3d(num_features)) # Final ReLU self.MC_blocks.add_module('relu_final', nn.ReLU(inplace=True)) # Final pool self.MC_blocks.add_module('pool_final', nn.AdaptiveAvgPool3d(1)) num_features_MC = num_features self.MC_classifier = nn.Linear(num_features_MC, num_classes_MC) self.CA_blocks=nn.Sequential() num_features=num_shared_features for i, num_layers in enumerate(CA_block_config): block = _DenseBlock( num_layers=num_layers, num_input_features=num_features, bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate, efficient=efficient, ) self.CA_blocks.add_module('denseblock_%d' % (i + 1), block) num_features = num_features + num_layers * growth_rate if i != len(CA_block_config) - 1: trans = _Transition(num_input_features=num_features, num_output_features=int(num_features * compression)) self.CA_blocks.add_module('transition_%d' % (i + 1), trans) num_features = int(num_features * compression) # Final batch norm self.CA_blocks.add_module('norm_final', nn.BatchNorm3d(num_features)) # Final ReLU self.CA_blocks.add_module('relu_final', nn.ReLU(inplace=True)) # Final pool self.CA_blocks.add_module('pool_final', nn.AdaptiveAvgPool3d(1)) num_features_CA = num_features self.CA_classifier = nn.Linear(num_features_CA, num_classes_CA)

def conv3x3x3(in_planes, out_planes, stride=1): # 3x3x3 convolution with padding return nn.Conv3d(in_planes,out_planes, kernel_size=3, stride=stride, padding=1, bias=False)

def __init__(self, num_input_features, num_output_features): super(_Transition_no_pool, self).__init__() self.add_module('norm', nn.BatchNorm3d(num_input_features)) self.add_module('relu', nn.ReLU(inplace=True)) self.add_module('conv', nn.Conv3d(num_input_features, num_output_features, kernel_size=1, stride=1, bias=False))

def __init__(self, kernel_size = 4, stride = 2, padding = 1, full_conv_limit = 4, just_conv_limit = 1, cube_edge = 128, ch_mult = 2, conv_bias = False, leakyrelu_const = 0.01): super(Encoder, self).__init__() print("Encoder = encoder_v02.py") """ input: batch_size * channels * cube_edge * cube_edge * cube_edge output: batch_size * (channel_multiplier * channels) BatchNorm is also added. Conv3D arguments=in_channels,out_channels,kernel_size,stride,padding """ """ Convolutional Layers """ self.embed_cube_edge = cube_edge conv_net = nn.Sequential() channels = 1 layer = 1 """ Convolutions with BatchNorm """ while layer <= full_conv_limit: conv_net.add_module("Conv_{0}".format(layer), nn.Conv3d(channels, channels * ch_mult, kernel_size = kernel_size, stride = stride, padding = padding, bias = conv_bias)) conv_net.add_module("BatchNorm_{0}".format(layer), nn.BatchNorm3d(channels * ch_mult)) conv_net.add_module("leakyrelu_{0}".format(layer), nn.LeakyReLU(leakyrelu_const, inplace = True)) self.embed_cube_edge = calculate_conv_output_dim(D = self.embed_cube_edge, K = kernel_size, P = padding, S = stride) channels = channels * ch_mult layer = layer + 1 """ Convolutions without BatchNorm """ while layer <= just_conv_limit: conv_net.add_module("Conv_{0}".format(layer), nn.Conv3d(channels, channels * ch_mult, kernel_size = kernel_size, stride = stride, padding = padding, bias = conv_bias)) channels = channels * ch_mult layer = layer + 1 # conv_net.add_module("Conv_{0}".format(layer), # nn.Conv3d(channels, # channels * ch_mult, # kernel_size = 4, # stride = 2, # padding = 1, # bias = conv_bias)) # channels = channels * ch_mult self.conv_net = conv_net self.channels = channels self.embed_cube_edge = calculate_conv_output_dim(D = self.embed_cube_edge, K = kernel_size, P = padding, S = stride)

def __init__(self, in_chs, out_chs, block, feats, layers, stride=2, groups=1, dilation=1, dropout=None, **kwargs): assert isinstance(out_chs, (int, tuple)) super(UNet, self).__init__() up_mode = kwargs.pop('up_mode') if 'up_mode' in kwargs else 'tconv' self.kwargs = kwargs num_layers = len(feats) - 1 layers = _repeat(layers, num_layers) stride = _repeat(stride, num_layers) groups = _repeat(groups, num_layers) dilation = _repeat(dilation, num_layers) dropout = _repeat(dropout, num_layers * 2) if type(block) != tuple: block = (block, block) self.stem = nn.Sequential( conv3x3(in_chs, feats[0], stride=1), _norm_layer(feats[0], **self.kwargs), _act_layer(**self.kwargs), conv3x3(feats[0], feats[0], stride=1), _norm_layer(feats[0], **self.kwargs), _act_layer(**self.kwargs), conv3x3(feats[0], feats[0], stride=1), _norm_layer(feats[0], **self.kwargs), _act_layer(**self.kwargs), ) encoders = [] for idx in range(num_layers): encoders.append( self._make_layer(feats[idx], feats[idx + 1], block[0], layers[idx], stride=stride[idx], groups=groups[idx], dilation=dilation[idx], dropout=dropout[idx], **self.kwargs)) self.encoders = nn.ModuleList(encoders) ups = [] decoders = [] for idx in range(num_layers, 0, -1): align_corners = True if 'trilinear' in up_mode else None ups.append( nn.Sequential( _up_layer(in_chs=feats[idx], mode=up_mode, align_corners=align_corners, **kwargs), conv1x1(feats[idx], feats[idx - 1]), _norm_layer(feats[idx - 1], **self.kwargs), _act_layer(**self.kwargs), )) decoders.append( self._make_layer(feats[idx - 1] * 2, feats[idx - 1], block[1], 1, stride=1, groups=1, dilation=1, dropout=dropout[-idx], **self.kwargs)) self.ups = nn.ModuleList(ups) self.decoders = nn.ModuleList(decoders) if isinstance(out_chs, int): self.predict = nn.Conv3d(feats[0], out_chs, kernel_size=1, stride=1, padding=0, bias=True) else: predicts = [] for o in out_chs: predicts.append(_predict_head(feats[0], o, **self.kwargs)) self.predict = nn.ModuleList(predicts)

def __init__(self, input_size, input_channels, device, control_point_spacing=(10, 10, 10), bspline_order=3, max_displacement=0.2): super(BSplineSTN3D, self).__init__() # Cuda params self.device = device self.dtype = torch.cuda.float if (self.device == 'cuda') else torch.float self.order = bspline_order self.max_disp = max_displacement self.input_size = input_size self.control_point_spacing = np.array(control_point_spacing) self.stride = self.control_point_spacing.astype(dtype=int).tolist() area = self.control_point_spacing[0] * self.control_point_spacing[ 1] * self.control_point_spacing[2] self.area = area.astype(float) cp_grid_shape = np.ceil( np.divide(self.input_size, self.control_point_spacing)).astype(dtype=int) # new image size after convolution self.inner_image_size = np.multiply( self.control_point_spacing, cp_grid_shape) - (self.control_point_spacing - 1) # add one control point at each side cp_grid_shape = cp_grid_shape + 2 # image size with additional control points self.new_image_size = np.multiply( self.control_point_spacing, cp_grid_shape) - (self.control_point_spacing - 1) # center image between control points image_size_diff = self.inner_image_size - self.input_size image_size_diff_floor = np.floor( (np.abs(image_size_diff) / 2)) * np.sign(image_size_diff) crop_start = image_size_diff_floor + np.remainder( image_size_diff, 2) * np.sign(image_size_diff) self.crop_start = crop_start.astype(dtype=int) self.crop_end = image_size_diff_floor.astype(dtype=int) self.cp_grid_shape = [3] + cp_grid_shape.tolist() self.num_control_points = np.prod(self.cp_grid_shape) self.kernel = self.bspline_kernel_3d(order=self.order).expand( 3, *((np.ones(3 + 1, dtype=int) * -1).tolist())) self.kernel_size = np.asarray(self.kernel.size())[2:] self.padding = ((self.kernel_size - 1) / 2).astype(dtype=int).tolist() # Network params num_features = torch.prod( ((((torch.tensor(input_size) - 4) / 2 - 4) / 2) - 4) / 2) self.conv1 = nn.Conv3d(input_channels, 8, kernel_size=5).to(self.device) self.conv2 = nn.Conv3d(8, 16, kernel_size=5).to(self.device) self.conv3 = nn.Conv3d(16, 32, kernel_size=5).to(self.device) self.fc = nn.Linear(32 * num_features, self.num_control_points).to(self.device)

def __init__(self): super(PALNet, self).__init__() print('PALNet: depth and TSDF stream') # ---- depth 2D CNN depth_out = 8 self.res2d_1_1 = nn.Conv2d(1, depth_out, 1, 1, 0) # reduction self.res2d_1_2 = nn.Sequential( nn.Conv2d(1, 4, 1, 1, 0), nn.ReLU(inplace=True), # nn.Conv2d(4, 4, 3, 1, 1), nn.Conv2d(4, 4, 3, 1, 1, 1), # dilated nn.ReLU(inplace=True), nn.Conv2d(4, 8, 1, 1, 0), ) self.res2d_2 = nn.Sequential( nn.Conv2d(8, 4, 1, 1, 0), nn.ReLU(inplace=True), # nn.Conv2d(4, 4, 3, 1, 1), nn.Conv2d(4, 4, 3, 1, 2, 2), # dilated nn.ReLU(inplace=True), nn.Conv2d(4, 8, 1, 1, 0), ) self.project_layer = Project2Dto3D(240, 144, 240) # w=240, h=144, d=240 # 3D CNN stride1 = 1 stride2 = 2 self.a_pool1 = nn.Conv3d(8, 16, 7, stride2, 3) self.a_res3d_1_1 = nn.Conv3d(16, 32, 1, 1, 0, bias=False) # reduction self.a_res3d_1_2 = nn.Sequential(nn.Conv3d(16, 8, 1, 1, 0), nn.ReLU(inplace=True), nn.Conv3d(8, 8, 3, 1, 1, 1), nn.ReLU(inplace=True), nn.Conv3d(8, 32, 1, 1, 0)) in_channel_3d = 1 # ---- tsdf self.b_pool1 = nn.Conv3d(in_channel_3d, 16, 7, stride2, 3) self.b_res3d_1_1 = nn.Conv3d(16, 32, 1, 1, 0, bias=False) self.b_res3d_1_2 = nn.Sequential( nn.Conv3d(16, 8, 1, 1, 0), nn.ReLU(inplace=True), # nn.Conv3d(8, 8, 3, 1, 1), nn.Conv3d(8, 8, 3, 1, 1, 1), nn.ReLU(inplace=True), nn.Conv3d(8, 32, 1, 1, 0)) # stride = 2 self.a_pool2 = nn.MaxPool3d(kernel_size=2, stride=2) self.b_pool2 = nn.MaxPool3d(kernel_size=2, stride=2) # self.pool2 = nn.Conv3d(32, 32, 3, stride, 1) self.a_res3d_2_1 = nn.Conv3d(32, 64, 1, stride1, 0, bias=False) self.a_res3d_2_2 = nn.Sequential( nn.Conv3d(32, 16, 1, stride1, 0), nn.ReLU(inplace=True), # nn.Conv3d(16, 16, 3, 1, 1), nn.Conv3d(16, 16, 3, 1, 2, 2), nn.ReLU(inplace=True), nn.Conv3d(16, 64, 1, 1, 0), ) self.b_res3d_2_1 = nn.Conv3d(32, 64, 1, stride1, 0, bias=False) self.b_res3d_2_2 = nn.Sequential( nn.Conv3d(32, 16, 1, stride1, 0), nn.ReLU(inplace=True), # nn.Conv3d(16, 16, 3, 1, 1), nn.Conv3d(16, 16, 3, 1, 2, 2), nn.ReLU(inplace=True), nn.Conv3d(16, 64, 1, 1, 0), ) # self.cat = nn.Conv3d(32, 64, 1, 1, 0) # -------------1/4 self.res3d_1 = nn.Sequential( nn.Conv3d(64, 32, 1, 1, 0), nn.ReLU(inplace=True), nn.Conv3d(32, 32, 3, 1, 2, 2), nn.ReLU(inplace=True), nn.Conv3d(32, 64, 1, 1, 0), ) self.res3d_2 = nn.Sequential( nn.Conv3d(64, 32, 1, 1, 0), nn.ReLU(inplace=True), nn.Conv3d(32, 32, 3, 1, 3, 3), nn.ReLU(inplace=True), nn.Conv3d(32, 64, 1, 1, 0), ) self.res3d_3 = nn.Sequential( nn.Conv3d(64, 32, 1, 1, 0), nn.ReLU(inplace=True), nn.Conv3d(32, 32, 3, 1, 5, 5), nn.ReLU(inplace=True), nn.Conv3d(32, 64, 1, 1, 0), ) self.conv4_1 = nn.Conv3d(256, 128, 1, 1, 0) # self.relu4_1 = nn.ReLU(inplace=True) self.conv4_2 = nn.Conv3d(128, 128, 1, 1, 0) # self.relu4_2 = nn.ReLU(inplace=True) self.fc12 = nn.Conv3d(128, 12, 1, 1, 0) # C_NUM = 12, number of classes is 12 # self.softmax = nn.Softmax(dim=1) # pytorch 0.3.0 # self.logsoftmax = nn.LogSoftmax(dim=1) # ---- weights init for m in self.modules(): if isinstance(m, nn.Conv3d): n = m.kernel_size[0] * m.kernel_size[1] * m.kernel_size[ 2] * m.out_channels # nn.init.xavier_normal(m.weight.data, gain=math.sqrt(2. / n)) # nn.init.xavier_uniform(m.weight.data, gain=math.sqrt(2. / n)) nn.init.xavier_uniform_(m.weight.data) # gain=1 # nn.init.constant(m.bias.data, 0) nn.init.normal_(self.conv4_1.weight.data, mean=0, std=0.1) nn.init.normal_(self.conv4_2.weight.data, mean=0, std=0.01) nn.init.normal_(self.fc12.weight.data, mean=0, std=0.01)

def __init__(self): super(Net, self).__init__() # The first few layers consumes the most memory, so use simple convolution to save memory. # Call these layers preBlock, i.e., before the residual blocks of later layers. self.preBlock = nn.Sequential( nn.Conv3d(1, 24, kernel_size=3, padding=1), nn.BatchNorm3d(24), nn.ReLU(inplace=True), nn.Conv3d(24, 24, kernel_size=3, padding=1), nn.BatchNorm3d(24), nn.ReLU(inplace=True)) # 3 poolings, each pooling downsamples the feature map by a factor 2. # 3 groups of blocks. The first block of each group has one pooling. num_blocks_forw = [2, 2, 3, 3] num_blocks_back = [3, 3] self.featureNum_forw = [24, 32, 64, 64, 64] self.featureNum_back = [128, 64, 64] for i in range(len(num_blocks_forw)): blocks = [] for j in range(num_blocks_forw[i]): if j == 0: blocks.append( PostRes(self.featureNum_forw[i], self.featureNum_forw[i + 1])) else: blocks.append( PostRes(self.featureNum_forw[i + 1], self.featureNum_forw[i + 1])) setattr(self, 'forw' + str(i + 1), nn.Sequential(*blocks)) for i in range(len(num_blocks_back)): blocks = [] for j in range(num_blocks_back[i]): if j == 0: if i == 0: addition = 3 else: addition = 0 blocks.append( PostRes( self.featureNum_back[i + 1] + self.featureNum_forw[i + 2] + addition, self.featureNum_back[i])) else: blocks.append( PostRes(self.featureNum_back[i], self.featureNum_back[i])) setattr(self, 'back' + str(i + 2), nn.Sequential(*blocks)) self.maxpool1 = nn.MaxPool3d(kernel_size=2, stride=2, return_indices=True) self.maxpool2 = nn.MaxPool3d(kernel_size=2, stride=2, return_indices=True) self.maxpool3 = nn.MaxPool3d(kernel_size=2, stride=2, return_indices=True) self.maxpool4 = nn.MaxPool3d(kernel_size=2, stride=2, return_indices=True) self.unmaxpool1 = nn.MaxUnpool3d(kernel_size=2, stride=2) self.unmaxpool2 = nn.MaxUnpool3d(kernel_size=2, stride=2) self.path1 = nn.Sequential( nn.ConvTranspose3d(64, 64, kernel_size=2, stride=2), SCse(64), nn.BatchNorm3d(64), nn.ReLU(inplace=True)) self.path2 = nn.Sequential( nn.ConvTranspose3d(64, 64, kernel_size=2, stride=2), SCse(64), nn.BatchNorm3d(64), nn.ReLU(inplace=True)) self.drop = nn.Dropout3d(p=0.5, inplace=False) self.output1 = nn.Sequential( nn.Conv3d(self.featureNum_back[0], 64, kernel_size=1), nn.ReLU(), #nn.Dropout3d(p = 0.3), nn.Conv3d(64, 5 * len(config['anchors']), kernel_size=1)) self.output2 = nn.Sequential( nn.Conv3d(self.featureNum_back[0], 64, kernel_size=1), nn.ReLU(), #nn.Dropout3d(p = 0.3), nn.Conv3d(64, 5 * len(config['anchors']), kernel_size=1)) self.output3 = nn.Sequential( nn.Conv3d(self.featureNum_back[0], 64, kernel_size=1), nn.ReLU(), #nn.Dropout3d(p = 0.3), nn.Conv3d(64, 5 * len(config['anchors']), kernel_size=1)) self.output4 = nn.Sequential( nn.Conv3d(self.featureNum_back[0], 64, kernel_size=1), nn.ReLU(), #nn.Dropout3d(p = 0.3), nn.Conv3d(64, 5 * len(config['anchors']), kernel_size=1))

def __init__(self, block, args): self.inplanes = 64 super(ResNetFPN, self).__init__() self.MODEL_TYPE = args.MODEL_TYPE num_blocks = args.model_perms non_local_inds = args.non_local_inds model_3d_layers = args.model_3d_layers self.num_blocks = num_blocks self.non_local_inds = non_local_inds self.model_3d_layers = model_3d_layers self.conv1 = nn.Conv3d(3, 64, kernel_size=(1, 7, 7), stride=(1, 2, 2), padding=(0, 3, 3), bias=False) self.bn1 = nn.BatchNorm3d(64) self.relu = nn.ReLU(inplace=True) self.pool1 = nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 0, 0)) self.layer_names = [] self.layer1 = self._make_layer(block, 64, num_blocks[0], temp_kernals=model_3d_layers[0], nl_inds=non_local_inds[0]) self.MODEL_TYPE = args.MODEL_TYPE if args.model_subtype in [ 'C2D', 'RCN', 'CLSTM', 'RCLSTM', 'CGRU', 'RCGRU' ]: self.pool2 = None else: self.pool2 = nn.MaxPool3d(kernel_size=(2, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0)) self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2, temp_kernals=model_3d_layers[1], nl_inds=non_local_inds[1]) self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2, temp_kernals=model_3d_layers[2], nl_inds=non_local_inds[2]) self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2, temp_kernals=model_3d_layers[3], nl_inds=non_local_inds[3]) #self.avgpool = nn.AvgPool2d(7, stride=1) #self.fc = nn.Linear(512 * block.expansion, num_classes) self.conv6 = conv3x3(512 * block.expansion, 256, stride=2, padding=1) # P6 self.conv7 = conv3x3(256, 256, stride=2, padding=1) # P7 self.ego_lateral = conv3x3(512 * block.expansion, 256, stride=2, padding=0) self.avg_pool = nn.AdaptiveAvgPool3d((None, 1, 1)) self.lateral_layer1 = conv1x1(512 * block.expansion, 256) self.lateral_layer2 = conv1x1(256 * block.expansion, 256) self.lateral_layer3 = conv1x1(128 * block.expansion, 256) self.corr_layer1 = conv3x3(256, 256, stride=1, padding=1) # P4 self.corr_layer2 = conv3x3(256, 256, stride=1, padding=1) # P4 self.corr_layer3 = conv3x3(256, 256, stride=1, padding=1) # P3 for m in self.modules(): if isinstance(m, nn.Conv2d): torch.nn.init.kaiming_uniform_(m.weight, a=1) if hasattr(m.bias, 'data'): torch.nn.init.constant_(m.bias, 0) elif isinstance(m, nn.BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_()

def __init__(self, in_channels, out_channels, final_sigmoid, basic_module, f_maps=64, layer_order='gcr', num_groups=8, num_levels=4, is_segmentation=True, testing=False, conv_kernel_size=3, pool_kernel_size=2, conv_padding=1, **kwargs): super(Abstract3DUNet, self).__init__() self.testing = testing if isinstance(f_maps, int): f_maps = number_of_features_per_level(f_maps, num_levels=num_levels) # create encoder path consisting of Encoder modules. Depth of the encoder is equal to `len(f_maps)` encoders = [] for i, out_feature_num in enumerate(f_maps): if i == 0: encoder = Encoder(in_channels, out_feature_num, apply_pooling=False, # skip pooling in the firs encoder basic_module=basic_module, conv_layer_order=layer_order, conv_kernel_size=conv_kernel_size, num_groups=num_groups, padding=conv_padding) else: # TODO: adapt for anisotropy in the data, i.e. use proper pooling kernel to make the data isotropic after 1-2 pooling operations encoder = Encoder(f_maps[i - 1], out_feature_num, basic_module=basic_module, conv_layer_order=layer_order, conv_kernel_size=conv_kernel_size, num_groups=num_groups, pool_kernel_size=pool_kernel_size, padding=conv_padding) encoders.append(encoder) self.encoders = nn.ModuleList(encoders) # create decoder path consisting of the Decoder modules. The length of the decoder is equal to `len(f_maps) - 1` decoders = [] reversed_f_maps = list(reversed(f_maps)) for i in range(len(reversed_f_maps) - 1): if basic_module == DoubleConv: in_feature_num = reversed_f_maps[i] + reversed_f_maps[i + 1] else: in_feature_num = reversed_f_maps[i] out_feature_num = reversed_f_maps[i + 1] # TODO: if non-standard pooling was used, make sure to use correct striding for transpose conv # currently strides with a constant stride: (2, 2, 2) decoder = Decoder(in_feature_num, out_feature_num, basic_module=basic_module, conv_layer_order=layer_order, conv_kernel_size=conv_kernel_size, num_groups=num_groups, padding=conv_padding) decoders.append(decoder) self.decoders = nn.ModuleList(decoders) # in the last layer a 1×1 convolution reduces the number of output # channels to the number of labels self.final_conv = nn.Conv3d(f_maps[0], out_channels, 1) if is_segmentation: # semantic segmentation problem if final_sigmoid: self.final_activation = nn.Sigmoid() else: self.final_activation = nn.Softmax(dim=1) else: # regression problem self.final_activation = None

def __init__(self, img_sz, opt=None): super(VoxelMorphCVPR2018, self).__init__() self.is_train = opt['tsk_set'][('train', False, 'if is in train mode')] opt_voxelmorph = opt['tsk_set']['reg']['morph_cvpr'] self.load_trained_affine_net = opt_voxelmorph[( 'load_trained_affine_net', False, 'if true load_trained_affine_net; if false, the affine network is not initialized' )] self.using_affine_init = opt_voxelmorph[( "using_affine_init", False, "deploy affine network before the nonparametric network")] self.affine_init_path = opt_voxelmorph[( 'affine_init_path', '', "the path of pretrained affine model")] self.affine_refine_step = opt_voxelmorph[( 'affine_refine_step', 5, "the multi-step num in affine refinement")] self.initial_reg_factor = opt_voxelmorph[( 'initial_reg_factor', 1., 'initial regularization factor')] self.min_reg_factor = opt_voxelmorph[( 'min_reg_factor', 1., 'minimum of regularization factor')] enc_filters = [16, 32, 32, 32, 32] #dec_filters = [32, 32, 32, 8, 8] dec_filters = [32, 32, 32, 32, 32, 16, 16] self.enc_filter = enc_filters self.dec_filter = dec_filters input_channel = 2 output_channel = 3 self.input_channel = 2 self.output_channel = 3 self.img_sz = img_sz self.spacing = 1. / (np.array(img_sz) - 1) self.loss_fn = None #NCCLoss() self.epoch = -1 self.print_count = 0 def set_cur_epoch(self, cur_epoch=-1): """ set current epoch""" self.epoch = cur_epoch if self.using_affine_init: self.init_affine_net(opt) self.id_transform = None else: self.id_transform = gen_identity_map(self.img_sz, 1.0).cuda() print("Attention, the affine net is not used") self.encoders = nn.ModuleList() self.decoders = nn.ModuleList() self.bilinear = Bilinear(zero_boundary=True) for i in range(len(enc_filters)): if i == 0: self.encoders.append( convBlock(input_channel, enc_filters[i], stride=1, bias=True)) else: self.encoders.append( convBlock(enc_filters[i - 1], enc_filters[i], stride=2, bias=True)) self.decoders.append( convBlock(enc_filters[-1], dec_filters[0], stride=1, bias=True)) self.decoders.append( convBlock(dec_filters[0] + enc_filters[3], dec_filters[1], stride=1, bias=True)) self.decoders.append( convBlock(dec_filters[1] + enc_filters[2], dec_filters[2], stride=1, bias=True)) self.decoders.append( convBlock(dec_filters[2] + enc_filters[1], dec_filters[3], stride=1, bias=True)) self.decoders.append( convBlock(dec_filters[3], dec_filters[4], stride=1, bias=True)) self.decoders.append( convBlock(dec_filters[4] + enc_filters[0], dec_filters[5], stride=1, bias=True)) self.decoders.append( convBlock(dec_filters[5], dec_filters[6], stride=1, bias=True)) self.flow = nn.Conv3d(dec_filters[-1], output_channel, kernel_size=3, stride=1, padding=1, bias=True)

def __init__(self, num_classes, pretrained=False): super(C3D, self).__init__() self.attn_conv2_channel = nn.Conv3d(128, 128, kernel_size=(8, 28, 28), padding=(0, 0, 0)) self.attn_conv2_channel_conv = nn.Conv3d(256, 128, kernel_size=(3, 3, 3), padding=(1, 1, 1)) self.attn_conv2_temp = nn.Conv3d(128, 1, kernel_size=(3, 28, 28), padding=(1, 0, 0)) self.attn_conv2_temp_conv = nn.Conv3d(128, 128, kernel_size=(9, 3, 3), padding=(0, 1, 1)) self.attn_conv4_channel = nn.Conv3d(512, 512, kernel_size=(2, 7, 7), padding=(0, 0, 0)) self.attn_conv4_channel_conv = nn.Conv3d(1024, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1)) self.attn_conv4_temp = nn.Conv3d(512, 1, kernel_size=(3, 7, 7), padding=(1, 0, 0)) self.attn_conv4_temp_conv = nn.Conv3d(512, 512, kernel_size=(3, 3, 3), padding=(0, 1, 1)) self.conv1 = nn.Conv3d(3, 64, kernel_size=(3, 3, 3), padding=(1, 1, 1)) self.pool1 = nn.MaxPool3d(kernel_size=(1, 2, 2), stride=(1, 2, 2)) self.conv2 = nn.Conv3d(64, 128, kernel_size=(3, 3, 3), padding=(1, 1, 1)) self.pool2 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2)) self.conv3a = nn.Conv3d(128, 256, kernel_size=(3, 3, 3), padding=(1, 1, 1)) self.conv3b = nn.Conv3d(256, 256, kernel_size=(3, 3, 3), padding=(1, 1, 1)) self.pool3 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2)) self.conv4a = nn.Conv3d(256, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1)) self.conv4b = nn.Conv3d(512, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1)) self.pool4 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2)) self.conv5a = nn.Conv3d(512, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1)) self.conv5b = nn.Conv3d(512, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1)) self.pool5 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=(0, 1, 1)) self.fc6 = nn.Linear(8192, 4096) self.fc7 = nn.Linear(4096, 4096) self.fc8 = nn.Linear(4096, num_classes) # self.fc8 = nn.Linear(4096, 300) # self.fc9 = nn.Linear(300, 51) self.dropout = nn.Dropout(p=0.5) self.relu = nn.ReLU() self.batchnorm1 = nn.BatchNorm3d(256) self.batchnorm2 = nn.BatchNorm3d(512) self.full_batchnorm1 = nn.BatchNorm1d(4096) self.full_batchnorm2 = nn.BatchNorm1d(4096) self.__init_weight() if pretrained: self.__load_pretrained_weights()

def __init__(self, temporal_dim, boundary_ratio, num_samples, num_samples_per_bin, feat_dim, soft_nms_alpha, soft_nms_low_threshold, soft_nms_high_threshold, post_process_top_k, loss_cls=dict(type='BMNLoss'), hidden_dim_1d=256, hidden_dim_2d=128, hidden_dim_3d=512): super(BaseLocalizer, self).__init__() self.tscale = temporal_dim self.boundary_ratio = boundary_ratio self.num_samples = num_samples self.num_samples_per_bin = num_samples_per_bin self.feat_dim = feat_dim self.soft_nms_alpha = soft_nms_alpha self.soft_nms_low_threshold = soft_nms_low_threshold self.soft_nms_high_threshold = soft_nms_high_threshold self.post_process_top_k = post_process_top_k self.loss_cls = build_loss(loss_cls) self.hidden_dim_1d = hidden_dim_1d self.hidden_dim_2d = hidden_dim_2d self.hidden_dim_3d = hidden_dim_3d self._get_interp1d_mask() # Base Module self.x_1d_b = nn.Sequential( nn.Conv1d( self.feat_dim, self.hidden_dim_1d, kernel_size=3, padding=1, groups=4), nn.ReLU(inplace=True), nn.Conv1d( self.hidden_dim_1d, self.hidden_dim_1d, kernel_size=3, padding=1, groups=4), nn.ReLU(inplace=True)) # Temporal Evaluation Module self.x_1d_s = nn.Sequential( nn.Conv1d( self.hidden_dim_1d, self.hidden_dim_1d, kernel_size=3, padding=1, groups=4), nn.ReLU(inplace=True), nn.Conv1d(self.hidden_dim_1d, 1, kernel_size=1), nn.Sigmoid()) self.x_1d_e = nn.Sequential( nn.Conv1d( self.hidden_dim_1d, self.hidden_dim_1d, kernel_size=3, padding=1, groups=4), nn.ReLU(inplace=True), nn.Conv1d(self.hidden_dim_1d, 1, kernel_size=1), nn.Sigmoid()) # Proposal Evaluation Module self.x_1d_p = nn.Sequential( nn.Conv1d( self.hidden_dim_1d, self.hidden_dim_1d, kernel_size=3, padding=1), nn.ReLU(inplace=True)) self.x_3d_p = nn.Sequential( nn.Conv3d( self.hidden_dim_1d, self.hidden_dim_3d, kernel_size=(self.num_samples, 1, 1)), nn.ReLU(inplace=True)) self.x_2d_p = nn.Sequential( nn.Conv2d(self.hidden_dim_3d, self.hidden_dim_2d, kernel_size=1), nn.ReLU(inplace=True), nn.Conv2d( self.hidden_dim_2d, self.hidden_dim_2d, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv2d( self.hidden_dim_2d, self.hidden_dim_2d, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv2d(self.hidden_dim_2d, 2, kernel_size=1), nn.Sigmoid()) self.anchors_tmins, self.anchors_tmaxs = self._temporal_anchors( -0.5, 1.5) self.match_map = self._match_map() self.bm_mask = self._get_bm_mask()

def __init__(self, block, layers, c_in=3, num_classes=1000, zero_init_residual=False, groups=1, width_per_group=26, scale=4, replace_stride_with_dilation=None, norm_layer=None): super(Res2Net3D, self).__init__() if norm_layer is None: norm_layer = nn.BatchNorm3d self._norm_layer = norm_layer self.inplanes = 64 self.dilation = 1 if replace_stride_with_dilation is None: # each element in the tuple indicates if we should replace # the 2x2 stride with a dilated convolution instead replace_stride_with_dilation = [False, False, False] if len(replace_stride_with_dilation) != 3: raise ValueError("replace_stride_with_dilation should be None " "or a 3-element tuple, got {}".format( replace_stride_with_dilation)) self.groups = groups self.base_width = width_per_group self.scale = scale self.expansion = 4 #self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, # bias=False) #modify stem #stem = [] sizes = [c_in, 32, 64, 64] #modified per Grankin #for i in range(3): # stem.append(conv_layer(sizes[i], sizes[i+1], stride=2 if i==0 else 1)) #stem (initial entry layers) self.conv1 = conv_layer(c_in, sizes[1], stride=2) self.conv2 = conv_layer(sizes[1], sizes[2]) self.conv3 = conv_layer(sizes[2], sizes[3]) self.maxpool = nn.MaxPool3d(kernel_size=3, stride=2, padding=1) #nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.layer1 = self._make_layer(block, 64, layers[0]) self.layer2 = self._make_layer(block, 128, layers[1], stride=2, dilate=replace_stride_with_dilation[0]) self.layer3 = self._make_layer(block, 256, layers[2], stride=2, dilate=replace_stride_with_dilation[1]) self.layer4 = self._make_layer(block, 512, layers[3], stride=2, dilate=replace_stride_with_dilation[2]) self.scale = 1 self.layerup1 = self._make_uplayer(block, 2048, layers[3]) self.layerup2 = self._make_uplayer(block, 512, layers[2]) self.layerup3 = self._make_uplayer(block, 512, layers[1]) self.layerup4 = self._make_uplayer(block, 128, layers[0]) self.layerup5 = self._make_uplayer(block, 32, 2) self.final_conv = nn.Sequential( nn.Conv3d(8, 8, 3, padding=2, dilation=2), Mish(), nn.BatchNorm3d(8), nn.Conv3d(8, 1, 1)) # self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) # self.fc = nn.Linear(512 * block.expansion, num_classes) for m in self.modules(): if isinstance(m, nn.Conv2d): nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') elif isinstance(m, nn.BatchNorm2d): nn.init.constant_(m.weight, 1) nn.init.constant_(m.bias, 0) # Zero-initialize the last BN in each residual branch, # so that the residual branch starts with zeros, and each residual block behaves like an identity. # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 if zero_init_residual: for m in self.modules(): if isinstance(m, Bottle2neck): nn.init.constant_(m.bn3.weight, 0) elif isinstance(m, BasicBlock): nn.init.constant_(m.bn2.weight, 0)

def conv(in_channels, out_channels, kernel_size=3, stride=1, bias=False): padding = utils.pad_size(kernel_size, 'same') return nn.Conv3d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias)

def __init__(self, channel): super(SpatialAttention3d, self).__init__() self.squeeze = nn.Conv3d(channel, 1, kernel_size=1, bias=False) self.sigmoid = nn.Sigmoid()