def build_activation(activation, params, init=0): if activation == 'prelu' or activation is None: return nn.PReLU(params, init=init) else: return activation
def __init__(self, nIn): super().__init__() self.bn = nn.BatchNorm2d(nIn, eps=1e-3) self.acti = nn.PReLU(nIn)
def __init__(self, in_dim: int, out_dim: int): super().__init__() self.projecting_factor = 4 self.n_kernels = 16 from lib.models.modules.enet import (BottleNeckDownSampling, BottleNeckNormal, BottleNeckDownSamplingDilatedConv, BottleNeckNormal_Asym, BottleNeckDownSamplingDilatedConvLast, BottleNeckUpSampling, upSampleConv) # Initial self.conv0 = nn.Conv2d(in_dim, 15, kernel_size=3, stride=2, padding=1) self.maxpool0 = nn.MaxPool2d(2, return_indices=True) # First group self.bottleNeck1_0 = BottleNeckDownSampling(self.n_kernels, self.projecting_factor, self.n_kernels * 4) self.bottleNeck1_1 = BottleNeckNormal(self.n_kernels * 4, self.n_kernels * 4, self.projecting_factor, 0.01) self.bottleNeck1_2 = BottleNeckNormal(self.n_kernels * 4, self.n_kernels * 4, self.projecting_factor, 0.01) self.bottleNeck1_3 = BottleNeckNormal(self.n_kernels * 4, self.n_kernels * 4, self.projecting_factor, 0.01) self.bottleNeck1_4 = BottleNeckNormal(self.n_kernels * 4, self.n_kernels * 4, self.projecting_factor, 0.01) # Second group self.bottleNeck2_0 = BottleNeckDownSampling(self.n_kernels * 4, self.projecting_factor, self.n_kernels * 8) self.bottleNeck2_1 = BottleNeckNormal(self.n_kernels * 8, self.n_kernels * 8, self.projecting_factor, 0.1) self.bottleNeck2_2 = BottleNeckDownSamplingDilatedConv(self.n_kernels * 8, self.projecting_factor, self.n_kernels * 8, 2) self.bottleNeck2_3 = BottleNeckNormal_Asym(self.n_kernels * 8, self.n_kernels * 8, self.projecting_factor, 0.1) self.bottleNeck2_4 = BottleNeckDownSamplingDilatedConv(self.n_kernels * 8, self.projecting_factor, self.n_kernels * 8, 4) self.bottleNeck2_5 = BottleNeckNormal(self.n_kernels * 8, self.n_kernels * 8, self.projecting_factor, 0.1) self.bottleNeck2_6 = BottleNeckDownSamplingDilatedConv(self.n_kernels * 8, self.projecting_factor, self.n_kernels * 8, 8) self.bottleNeck2_7 = BottleNeckNormal_Asym(self.n_kernels * 8, self.n_kernels * 8, self.projecting_factor, 0.1) self.bottleNeck2_8 = BottleNeckDownSamplingDilatedConv(self.n_kernels * 8, self.projecting_factor, self.n_kernels * 8, 16) # Third group self.bottleNeck3_1 = BottleNeckNormal(self.n_kernels * 8, self.n_kernels * 8, self.projecting_factor, 0.1) self.bottleNeck3_2 = BottleNeckDownSamplingDilatedConv(self.n_kernels * 8, self.projecting_factor, self.n_kernels * 8, 2) self.bottleNeck3_3 = BottleNeckNormal_Asym(self.n_kernels * 8, self.n_kernels * 8, self.projecting_factor, 0.1) self.bottleNeck3_4 = BottleNeckDownSamplingDilatedConv(self.n_kernels * 8, self.projecting_factor, self.n_kernels * 8, 4) self.bottleNeck3_5 = BottleNeckNormal(self.n_kernels * 8, self.n_kernels * 8, self.projecting_factor, 0.1) self.bottleNeck3_6 = BottleNeckDownSamplingDilatedConv(self.n_kernels * 8, self.projecting_factor, self.n_kernels * 8, 8) self.bottleNeck3_7 = BottleNeckNormal_Asym(self.n_kernels * 8, self.n_kernels * 8, self.projecting_factor, 0.1) self.bottleNeck3_8 = BottleNeckDownSamplingDilatedConvLast(self.n_kernels * 8, self.projecting_factor, self.n_kernels * 4, 16) # ### Decoding path #### # Unpooling 1 self.unpool_0 = nn.MaxUnpool2d(2) self.bottleNeck_Up_1_0 = BottleNeckUpSampling(self.n_kernels * 8, self.projecting_factor, self.n_kernels * 4) self.PReLU_Up_1 = nn.PReLU() self.bottleNeck_Up_1_1 = BottleNeckNormal(self.n_kernels * 4, self.n_kernels * 4, self.projecting_factor, 0.1) self.bottleNeck_Up_1_2 = BottleNeckNormal(self.n_kernels * 4, self.n_kernels, self.projecting_factor, 0.1) # Unpooling 2 self.unpool_1 = nn.MaxUnpool2d(2) self.bottleNeck_Up_2_1 = BottleNeckUpSampling(self.n_kernels * 2, self.projecting_factor, self.n_kernels) self.bottleNeck_Up_2_2 = BottleNeckNormal(self.n_kernels, self.n_kernels, self.projecting_factor, 0.1) self.PReLU_Up_2 = nn.PReLU() # Unpooling Last self.deconv3 = upSampleConv(self.n_kernels, self.n_kernels) self.out_025 = nn.Conv2d(self.n_kernels * 8, out_dim, kernel_size=3, stride=1, padding=1) self.out_05 = nn.Conv2d(self.n_kernels, out_dim, kernel_size=3, stride=1, padding=1) self.final = nn.Conv2d(self.n_kernels, out_dim, kernel_size=1)
def __init__( self, input_dim: int, num_spk: int = 1, rnn_layer: int = 2, rnn_units: int = 256, masking_mode: str = "E", use_clstm: bool = True, bidirectional: bool = False, use_cbn: bool = False, kernel_size: int = 5, kernel_num: List[int] = [32, 64, 128, 256, 256, 256], use_builtin_complex: bool = True, use_noise_mask: bool = False, ): """DCCRN separator. Args: input_dim (int): input dimension。 num_spk (int, optional): number of speakers. Defaults to 1. rnn_layer (int, optional): number of lstm layers in the crn. Defaults to 2. rnn_units (int, optional): rnn units. Defaults to 128. masking_mode (str, optional): usage of the estimated mask. Defaults to "E". use_clstm (bool, optional): whether use complex LSTM. Defaults to False. bidirectional (bool, optional): whether use BLSTM. Defaults to False. use_cbn (bool, optional): whether use complex BN. Defaults to False. kernel_size (int, optional): convolution kernel size. Defaults to 5. kernel_num (list, optional): output dimension of each layer of the encoder. use_builtin_complex (bool, optional): torch.complex if True, else ComplexTensor. use_noise_mask (bool, optional): whether to estimate the mask of noise. """ super().__init__() self.use_builtin_complex = use_builtin_complex self._num_spk = num_spk self.use_noise_mask = use_noise_mask if masking_mode not in ["C", "E", "R"]: raise ValueError("Unsupported masking mode: %s" % masking_mode) # Network config self.rnn_units = rnn_units self.hidden_layers = rnn_layer self.kernel_size = kernel_size self.kernel_num = [2] + kernel_num self.masking_mode = masking_mode self.use_clstm = use_clstm fac = 2 if bidirectional else 1 self.encoder = nn.ModuleList() self.decoder = nn.ModuleList() for idx in range(len(self.kernel_num) - 1): self.encoder.append( nn.Sequential( ComplexConv2d( self.kernel_num[idx], self.kernel_num[idx + 1], kernel_size=(self.kernel_size, 2), stride=(2, 1), padding=(2, 1), ), nn.BatchNorm2d(self.kernel_num[idx + 1]) if not use_cbn else ComplexBatchNorm(self.kernel_num[idx + 1]), nn.PReLU(), )) hidden_dim = (input_dim - 1 + 2**(len(self.kernel_num) - 1) - 1) // (2**(len(self.kernel_num) - 1)) hidden_dim = hidden_dim if hidden_dim > 0 else 1 if self.use_clstm: rnns = [] for idx in range(rnn_layer): rnns.append( NavieComplexLSTM( input_size=hidden_dim * self.kernel_num[-1] if idx == 0 else self.rnn_units * fac, hidden_size=self.rnn_units, bidirectional=bidirectional, batch_first=False, projection_dim=hidden_dim * self.kernel_num[-1] if idx == rnn_layer - 1 else None, )) self.enhance = nn.Sequential(*rnns) else: self.enhance = nn.LSTM( input_size=hidden_dim * self.kernel_num[-1], hidden_size=self.rnn_units, num_layers=2, dropout=0.0, bidirectional=bidirectional, batch_first=False, ) self.tranform = nn.Linear(self.rnn_units * fac, hidden_dim * self.kernel_num[-1]) for idx in range(len(self.kernel_num) - 1, 0, -1): if idx != 1: self.decoder.append( nn.Sequential( ComplexConvTranspose2d( self.kernel_num[idx] * 2, self.kernel_num[idx - 1], kernel_size=(self.kernel_size, 2), stride=(2, 1), padding=(2, 0), output_padding=(1, 0), ), nn.BatchNorm2d(self.kernel_num[idx - 1]) if not use_cbn else ComplexBatchNorm(self.kernel_num[idx - 1]), nn.PReLU(), )) else: self.decoder.append( nn.Sequential( ComplexConvTranspose2d( self.kernel_num[idx] * 2, self.kernel_num[idx - 1] * (self._num_spk + 1) if self.use_noise_mask else self.kernel_num[idx - 1] * self._num_spk, kernel_size=(self.kernel_size, 2), stride=(2, 1), padding=(2, 0), output_padding=(1, 0), ), )) self.flatten_parameters()
def __init__(self, in_channels, hidden_channels): super(Encoder, self).__init__() self.conv = GCNConv(in_channels, hidden_channels, cached=True) self.prelu = nn.PReLU(hidden_channels)
def __init__(self, depth, num_output=10572, num_features=512, margin_inner_product_type='quadruple', rate=0.): super(wcSphereNet, self).__init__() if depth == 4: layers = [0, 0, 0, 0] elif depth == 10: layers = [0, 1, 2, 0] elif depth == 20: layers = [1, 2, 4, 1] elif depth == 38: layers = [2, 4, 8, 2] elif depth == 64: layers = [3, 8, 16, 3] else: assert False, "invalid depth: %d, only support: 4, 10, 20, 38, 64" % depth self.rate = rate self.base = 1000. # self.gamma = 0.000003 # self.power = 45 self.gamma = 0.12 self.power = 1. self.depth = depth block = wcSphereBlock # define network structure self.conv1 = nn.Conv2d(3, 30, kernel_size=3, stride=2, padding=1) # self.relu1 = nn.ReLU() self.relu1 = nn.PReLU(30) self.layer1 = self._make_layer(block, 30, layers[0]) self.conv2 = prune.wcConv2d(30, 93, kernel_size=3, stride=2, padding=1, rate=rate) # self.relu2 = nn.ReLU() self.relu2 = nn.PReLU(93) self.layer2 = self._make_layer(block, 93, layers[1], stride=2) self.conv3 = prune.wcConv2d(93, 242, kernel_size=3, stride=2, padding=1, rate=rate) # self.relu3 = nn.ReLU() self.relu3 = nn.PReLU(242) self.layer3 = self._make_layer(block, 242, layers[2], stride=2) self.conv4 = prune.wcConv2d(242, 256, kernel_size=3, stride=2, padding=1, rate=rate) # self.relu4 = nn.ReLU() self.relu4 = nn.PReLU(256) self.layer4 = self._make_layer(block, 256, layers[3], stride=2) # self.pooling = nn.MaxPool2d(kernel_size=(7, 6)) self.fc = nn.Linear(256 * 7 * 6, num_features) # self.fc = nn.Linear(512, num_features) self.margin_inner_product_type = margin_inner_product_type if margin_inner_product_type == 'single': margin_inner_product_type = 1 elif margin_inner_product_type == 'double': margin_inner_product_type = 2 elif margin_inner_product_type == 'triple': margin_inner_product_type = 3 elif margin_inner_product_type == 'quadruple': margin_inner_product_type = 4 else: print('Unknown margin type.') self.margin_linear = MarginLinear( num_output=num_output, num_features=num_features, margin_inner_product_type=margin_inner_product_type) self._init_weight()
def __init__(self, num_filters, kernel_size=3, res_scale=1, expand=3, bias=True, padding_type=None, activation='relu', normalization='weight'): super(ResidualBlock_A, self).__init__() self.res_scale = res_scale conv_padding = 0 pd = None if padding_type == 'reflect': pd = nn.ReflectionPad2d elif padding_type == 'replicate': pd = nn.ReplicationPad2d else: conv_padding = kernel_size // 2 conv1 = nn.Conv2d(num_filters, num_filters*expand, kernel_size, 1, conv_padding, bias=bias) conv2 = nn.Conv2d(num_filters*expand, num_filters, kernel_size, 1, conv_padding, bias=bias) norm = None if normalization == 'batch': norm = nn.BatchNorm2d elif normalization == 'instance': norm = nn.InstanceNorm2d elif normalization == 'spectral': conv1 = SNConv2d(num_filters, num_filters*expand, kernel_size, 1, conv_padding, bias=bias) conv2 = SNConv2d(num_filters*expand, num_filters, kernel_size, 1, conv_padding, bias=bias) elif norm == 'weight': def wn(x): return nn.utils.weight_norm(x) conv1 = wn(conv1) conv2 = wn(conv2) elif norm == 'weight-batch': norm = nn.BatchNorm2d def wn(x): return nn.utils.weight_norm(x) conv1 = wn(conv1) conv2 = wn(conv2) act = None if activation == 'relu': act = nn.ReLU(True) elif activation == 'prelu': act = nn.PReLU() elif activation == 'lrelu': act = nn.LeakyReLU(0.2, True) elif activation == 'tanh': act = nn.Tanh() elif activation == 'sigmoid': act = nn.Sigmoid() body = [] if pd is not None: body.append(pd(kernel_size//2)) body.append(conv1) if norm is not None: body.append(norm(num_filters*expand)) if act is not None: body.append(act) if pd is not None: body.append(pd(kernel_size//2)) body.append(conv2) if norm is not None: body.append(norm(num_filters)) self.body = nn.Sequential(*body)
def __init__(self, in_channels, out_channels, dilation=2, reduction=16, skip_connect=True, downsample=False, conv_cfg=None, norm_cfg=dict(type='BN', requires_grad=True), act_cfg=dict(type='PReLU'), with_cp=False): super(ContextGuidedBlock, self).__init__() self.with_cp = with_cp self.downsample = downsample channels = out_channels if downsample else out_channels // 2 if 'type' in act_cfg and act_cfg['type'] == 'PReLU': act_cfg['num_parameters'] = channels kernel_size = 3 if downsample else 1 stride = 2 if downsample else 1 padding = (kernel_size - 1) // 2 self.conv1x1 = ConvModule( in_channels, channels, kernel_size, stride, padding, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.f_loc = build_conv_layer( conv_cfg, channels, channels, kernel_size=3, padding=1, groups=channels, bias=False) self.f_sur = build_conv_layer( conv_cfg, channels, channels, kernel_size=3, padding=dilation, groups=channels, dilation=dilation, bias=False) self.bn = build_norm_layer(norm_cfg, 2 * channels)[1] self.activate = nn.PReLU(2 * channels) if downsample: self.bottleneck = build_conv_layer( conv_cfg, 2 * channels, out_channels, kernel_size=1, bias=False) self.skip_connect = skip_connect and not downsample self.f_glo = GlobalContextExtractor(out_channels, reduction, with_cp)
def __init__(self, embedding_net, n_classes): super(ClassificationNet, self).__init__() self.embedding_net = embedding_net self.n_classes = n_classes self.nonlinear = nn.PReLU() self.fc1 = nn.Linear(2, n_classes)
def __init__(self, Sampler, ModelSettings): super().__init__() # init args self.num_features_dict = Sampler.num_features_dict self.embed_dim = eval(ModelSettings['embed_dim']) dnn_dim_list = eval(ModelSettings['dnn_dim_list']) self.page_layer = ModelSettings['page_layer'] self.remove_nan = eval(ModelSettings['remove_nan']) mha_head_num = eval(ModelSettings['mha_head_num']) # init model layer self._build_embedding_layer( self.num_features_dict) # build embeeding and cnt_*_fts self.ad_embed_dim = self.cnt_fts_dict['ad_embed'] * self.embed_dim self.tad_embed_dim = ( self.cnt_fts_dict['ad_embed'] - 2) * self.embed_dim # remove is_click and page_click_num self.qy_embed_dim = self.cnt_fts_dict['qy_embed'] * self.embed_dim self.adq_embed_dim = self.ad_embed_dim + self.qy_embed_dim if self.page_layer == 'dynamic_page': alpha_input_dim = self.ad_embed_dim + self.tad_embed_dim alpha_dim_list = eval(ModelSettings['alpha_dim_list']) self.page_net = nn.ModuleDict({ 'gru': nn.GRU(self.ad_embed_dim, self.ad_embed_dim, num_layers=1), 'target_to_adq': nn.Sequential(nn.Linear(self.tad_embed_dim, self.ad_embed_dim), nn.ReLU()), 'din1': Attention(self.ad_embed_dim, ModelSettings), 'alpha1': MultiLayerPerceptron(alpha_input_dim, alpha_dim_list, dropout=0, activation=nn.ReLU(), output_layer=True), 'target_to_pq': nn.Sequential( # nn.Linear(self.tad_embed_dim, self.adq_embed_dim), nn.ReLU() nn.Linear(self.tad_embed_dim, self.ad_embed_dim), nn.ReLU()), 'mha2': nn.MultiheadAttention(self.adq_embed_dim, num_heads=mha_head_num), 'din2': Attention(self.ad_embed_dim, ModelSettings), # 'din2': Attention(self.adq_embed_dim, ModelSettings), 'alpha2': MultiLayerPerceptron(alpha_input_dim, alpha_dim_list, dropout=0, activation=nn.ReLU(), output_layer=True), }) else: raise ValueError('unknow PIN page layer name: ', self.page_layer) self.atten_net = torch.nn.MultiheadAttention(self.ad_embed_dim, num_heads=mha_head_num) dnn_input_dim = self.cnt_fts_dict[ 'user'] * self.embed_dim + self.tad_embed_dim + self.ad_embed_dim # dnn_input_dim = self.cnt_fts_dict['user']*self.embed_dim + self.tad_embed_dim + self.ad_embed_dim + self.qy_embed_dim self.dnn_net = nn.ModuleDict({ # 'dnn_input_bn': nn.BatchNorm1d(dnn_input_dim), 'dnn': MultiLayerPerceptron(dnn_input_dim, dnn_dim_list, dropout=0, activation=nn.PReLU(), output_layer=False) }) self.logits_linear = nn.Linear(dnn_dim_list[-1], 1) self.init_weights()
def __init__(self, in_channels=3, num_channels=(32, 64, 128), num_blocks=(3, 21), dilations=(2, 4), reductions=(8, 16), conv_cfg=None, norm_cfg=dict(type='BN', requires_grad=True), act_cfg=dict(type='PReLU'), norm_eval=False, with_cp=False, pretrained=None, init_cfg=None): super(CGNet, self).__init__(init_cfg) assert not (init_cfg and pretrained), \ 'init_cfg and pretrained cannot be setting at the same time' if isinstance(pretrained, str): warnings.warn('DeprecationWarning: pretrained is a deprecated, ' 'please use "init_cfg" instead') self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) elif pretrained is None: if init_cfg is None: self.init_cfg = [ dict(type='Kaiming', layer=['Conv2d', 'Linear']), dict( type='Constant', val=1, layer=['_BatchNorm', 'GroupNorm']), dict(type='Constant', val=0, layer='PReLU') ] else: raise TypeError('pretrained must be a str or None') self.in_channels = in_channels self.num_channels = num_channels assert isinstance(self.num_channels, tuple) and len( self.num_channels) == 3 self.num_blocks = num_blocks assert isinstance(self.num_blocks, tuple) and len(self.num_blocks) == 2 self.dilations = dilations assert isinstance(self.dilations, tuple) and len(self.dilations) == 2 self.reductions = reductions assert isinstance(self.reductions, tuple) and len(self.reductions) == 2 self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.act_cfg = act_cfg if 'type' in self.act_cfg and self.act_cfg['type'] == 'PReLU': self.act_cfg['num_parameters'] = num_channels[0] self.norm_eval = norm_eval self.with_cp = with_cp cur_channels = in_channels self.stem = nn.ModuleList() for i in range(3): self.stem.append( ConvModule( cur_channels, num_channels[0], 3, 2 if i == 0 else 1, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) cur_channels = num_channels[0] self.inject_2x = InputInjection(1) # down-sample for Input, factor=2 self.inject_4x = InputInjection(2) # down-sample for Input, factor=4 cur_channels += in_channels self.norm_prelu_0 = nn.Sequential( build_norm_layer(norm_cfg, cur_channels)[1], nn.PReLU(cur_channels)) # stage 1 self.level1 = nn.ModuleList() for i in range(num_blocks[0]): self.level1.append( ContextGuidedBlock( cur_channels if i == 0 else num_channels[1], num_channels[1], dilations[0], reductions[0], downsample=(i == 0), conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg, with_cp=with_cp)) # CG block cur_channels = 2 * num_channels[1] + in_channels self.norm_prelu_1 = nn.Sequential( build_norm_layer(norm_cfg, cur_channels)[1], nn.PReLU(cur_channels)) # stage 2 self.level2 = nn.ModuleList() for i in range(num_blocks[1]): self.level2.append( ContextGuidedBlock( cur_channels if i == 0 else num_channels[2], num_channels[2], dilations[1], reductions[1], downsample=(i == 0), conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg, with_cp=with_cp)) # CG block cur_channels = 2 * num_channels[2] self.norm_prelu_2 = nn.Sequential( build_norm_layer(norm_cfg, cur_channels)[1], nn.PReLU(cur_channels))
def encode(self, x): m = nn.PReLU() x = m(self.f(x)) x = self.f2(x) return x
def __init__(self, in_channels, up_scale): super(UpsampleBLock, self).__init__() self.conv = nn.Conv2d(in_channels, in_channels * up_scale ** 2, kernel_size=3, padding=1) self.pixel_shuffle = nn.PixelShuffle(up_scale) self.prelu = nn.PReLU()
def __init__(self, pof=False): super(Gines, self).__init__() self.pof = pof # Input Channel, Channel, Kernel Size, Stride, Padding self.vgg19 = nn.Sequential( OrderedDict([ ('conv1_1', nn.Conv2d(3, 64, 3, 1, 1)), ('relu1_1', nn.ReLU()), ('conv1_2', nn.Conv2d(64, 64, 3, 1, 1)), ('relu1_2', nn.ReLU()), ('pool1_stage1', nn.MaxPool2d(2, 2)), ('conv2_1', nn.Conv2d(64, 128, 3, 1, 1)), ('relu2_1', nn.ReLU()), ('conv2_2', nn.Conv2d(128, 128, 3, 1, 1)), ('relu2_2', nn.ReLU()), ('pool2_stage1', nn.MaxPool2d(2, 2)), ('conv3_1', nn.Conv2d(128, 256, 3, 1, 1)), ('relu3_1', nn.ReLU()), ('conv3_2', nn.Conv2d(256, 256, 3, 1, 1)), ('relu3_2', nn.ReLU()), ('conv3_3', nn.Conv2d(256, 256, 3, 1, 1)), ('relu3_3', nn.ReLU()), ('conv3_4', nn.Conv2d(256, 256, 3, 1, 1)), ('relu3_4', nn.ReLU()), ('pool3_stage1', nn.MaxPool2d(2, 2)), ('conv4_1', nn.Conv2d(256, 512, 3, 1, 1)), ('relu4_1', nn.ReLU()), ('conv4_2', nn.Conv2d(512, 512, 3, 1, 1)), ('prelu4_2', nn.PReLU(512)), ('conv4_3_CPM', nn.Conv2d(512, 256, 3, 1, 1)), ('prelu4_3_CPM', nn.PReLU(256)), ('conv4_4_CPM', nn.Conv2d(256, TOTAL_FM, 3, 1, 1)), ('prelu4_4_CPM', nn.PReLU(128)), ])) self.pafA = ABlock3x3_Extended(TOTAL_FM, TOTAL_PAFS, Depth=96, SubDepth=256) self.pafB = ABlock3x3_Extended(TOTAL_FM + TOTAL_PAFS, TOTAL_PAFS, Depth=256, SubDepth=512) self.pafC = ABlock3x3_Extended(TOTAL_FM + TOTAL_PAFS, TOTAL_PAFS, Depth=256, SubDepth=512) self.hmNetwork = ABlock3x3_Extended(TOTAL_FM + TOTAL_PAFS, TOTAL_HMS, Depth=192, SubDepth=512) # POF Networks if self.pof: self.pofA = ABlock3x3_Extended(TOTAL_FM + TOTAL_PAFS, TOTAL_POFS, Depth=128, SubDepth=256) self.pofB = ABlock3x3_Extended(TOTAL_FM + TOTAL_PAFS + TOTAL_POFS, TOTAL_POFS, Depth=256, SubDepth=512) self.load_vgg()
def __init__(self, channels, up_scale): super(UpSampleBlock, self).__init__() self.__conv = nn.Conv2d(channels, channels * up_scale ** 2, kernel_size=3, padding=1) self.__pixel_shuffler = nn.PixelShuffle(up_scale) self.__prelu = nn.PReLU()
def __init__(self, num_inputs, num_action, dev ): super(Critic,self).__init__() if cnn_enable: size=7*7*64 self.feature = nn.Sequential( nn.Conv2d(num_inputs,64,8,stride= 4),nn.PReLU(), nn.Conv2d(64,64,4,stride=2),nn.PReLU(), nn.Conv2d(64,64,3,stride=1),nn.PReLU(), Flatten(), nn.Linear(size,120),nn.PReLU(), ) self.feature_action = nn.Sequential( nn.Linear(num_action,8),nn.PReLU(), ) else : self.feature = nn.Sequential( nn.Linear(s_dim,120),nn.PReLU(), ) self.feature_action = nn.Sequential( nn.Linear(num_action,8),nn.PReLU(), ) self.lstm_size = 128 self.lstm = nn.LSTMCell(self.lstm_size, self.lstm_size) # self.advantage = nn.Sequential( # nn.Linear(self.lstm_size,128),nn.PReLU(), # nn.Linear(128,128),nn.PReLU(), # nn.Linear(128,num_outputs), # ) self.value = nn.Sequential( nn.Linear(self.lstm_size,128),nn.PReLU(), nn.Linear(128,128),nn.PReLU(), nn.Linear(128,1), ) # self.iadvantage = nn.Sequential( # nn.Linear(self.lstm_size,128),nn.PReLU(), # nn.Linear(128,128),nn.PReLU(), # nn.Linear(128,num_outputs), # ) self.ivalue = nn.Sequential( nn.Linear(self.lstm_size,128),nn.PReLU(), nn.Linear(128,128),nn.PReLU(), nn.Linear(128,1), ) self.hx = None self.cx = None self.dev = dev
def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, group, affine=True, options="D"): super(Reactblock, self).__init__() norm_layer = nn.BatchNorm2d self.stride = stride self.inplanes = C_in self.planes = C_out self.group = int(C_in // 6) # react sign self.move11 = LearnableBias(C_in) self.binary_3x3 = nn.Conv2d(C_in, C_in, kernel_size=kernel_size, dilation=dilation, stride=stride, padding=padding, groups=self.group, bias=False) self.bn1 = norm_layer(C_in, affine=affine) self.shuffle = ShuffleBlock(self.group) self.shortcut = nn.Sequential() if stride != 1: if options == "A": self.shortcut = LambdaLayer(lambda x: F.pad( x[:, :, ::2, ::2], (0, 0, 0, 0, C_in // 4, C_in // 4), "constant", 0)) elif options == "B": self.shortcut = nn.Sequential( Layer.Conv2d_1w1a(C_in, C_in, kernel_size=1, stride=stride, bias=False), nn.BatchNorm2d(C_in, affine=affine)) elif options == "C": self.shortcut = nn.Sequential( nn.MaxPool2d(kernel_size=3, stride=2, padding=1), ) else: self.shortcut = nn.Sequential( nn.AvgPool2d(kernel_size=3, stride=2, padding=1), ) # react prelu self.move12 = LearnableBias(C_in) self.prelu1 = nn.PReLU(C_in) self.move13 = LearnableBias(C_in) # react sign self.move21 = LearnableBias(C_in) self.binary_pw = Layer.Conv2d_1w1a(C_in, C_out, kernel_size=1, stride=1, bias=False) self.bn2 = norm_layer(C_out, affine=affine) self.move22 = LearnableBias(C_out) self.prelu2 = nn.PReLU(C_out) self.move23 = LearnableBias(C_out)
def __init__(self,num_inputs): super(RND,self).__init__() if cnn_enable: size=7*7*64 self.target = nn.Sequential( nn.Conv2d(num_inputs,64,8,stride= 4),nn.PReLU(), nn.Conv2d(64,64,4,stride=2),nn.PReLU(), nn.Conv2d(64,64,3,stride=1),nn.PReLU(), Flatten(), nn.Linear(size,128),nn.PReLU(), nn.Linear(128,128), ) self.predictor = nn.Sequential( nn.Conv2d(num_inputs,64,8,stride= 4),nn.PReLU(), nn.Conv2d(64,64,4,stride=2),nn.PReLU(), nn.Conv2d(64,64,3,stride=1),nn.PReLU(), Flatten(), nn.Linear(size,128),nn.PReLU(), nn.Linear(128,128),nn.PReLU(), nn.Linear(128,128), ) else : self.target = nn.Sequential( nn.Linear(s_dim,128),nn.PReLU(), nn.Linear(128,128), ) self.predictor = nn.Sequential( nn.Linear(s_dim,128),nn.PReLU(), nn.Linear(128,128),nn.PReLU(), nn.Linear(128,128), ) for m in self.modules(): if isinstance(m,nn.Linear): nn.init.orthogonal_(m.weight,np.sqrt(2)) m.bias.data.zero_() for param in self.target.parameters(): param.requires_grad =False
def __init__(self, args): super(BioPDSN, self).__init__() #self.device = args.device #data args self.dfPath = args.dfPath self.df = None self.trainDF = None self.validateDF = None #train args self.batch_size = args.batch_size self.num_workers = args.num_workers self.lr = args.lr #self.momentum = args.momentum #self.weight_decay = args.weight_decay self.num_class = args.num_class #loss criterion self.loss_cls = nn.CrossEntropyLoss().to(self.device) self.loss_diff = nn.L1Loss(reduction='mean').to(self.device) #model args self.imageShape = [int(x) for x in args.input_size.split(',')] self.features_shape = args.embedding_size #nets self.classifier = MarginCosineProduct(self.features_shape, self.num_class) ''' if(args.use_mtcnn == "False"): self.use_mtcnn = False else: self.use_mtcnn = True if(self.use_mtcnn): self.mtcnn = MTCNN(image_size=self.imageShape[1], min_face_size=80, device = self.device, post_process=args.mtcnn_norm, keep_all=args.keep_all) else: self.mtcnn = None ''' self.resnet = Resnet(args) # Mask Generator self.sia = nn.Sequential( #nn.BatchNorm2d(filter_list[4]), nn.Conv2d(self.features_shape, 512, kernel_size=3, stride=1, padding=1, bias=False), nn.PReLU(self.features_shape), nn.BatchNorm2d(self.features_shape), nn.Sigmoid(), ) self.fc = nn.Sequential( nn.BatchNorm1d(self.features_shape * 7 * 7), #nn.Dropout(p=0), nn.Linear(self.features_shape * 7 * 7, self.features_shape), nn.BatchNorm1d(self.features_shape), ) # Weight initialization for m in self.modules(): if (isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear)): nn.init.xavier_uniform_(m.weight) if m.bias is not None: nn.init.constant_(m.bias, 0.0) elif (isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.BatchNorm1d)): nn.init.constant_(m.weight, 1) nn.init.constant_(m.bias, 0)
def __init__(self, block, layers, dropout=0, num_features=512, zero_init_residual=False, groups=1, width_per_group=64, replace_stride_with_dilation=None): super(IResNet, self).__init__() self.inplanes = 64 self.dilation = 1 if replace_stride_with_dilation is None: replace_stride_with_dilation = [False, False, False] if len(replace_stride_with_dilation) != 3: raise ValueError("replace_stride_with_dilation should be None " "or a 3-element tuple, got {}".format( replace_stride_with_dilation)) self.groups = groups self.base_width = width_per_group self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False) self.bn1 = nn.BatchNorm2d(self.inplanes, eps=1e-05) self.prelu = nn.PReLU(self.inplanes) self.layer1 = self._make_layer(block, 64, layers[0], stride=2) self.layer2 = self._make_layer(block, 128, layers[1], stride=2, dilate=replace_stride_with_dilation[0]) self.layer3 = self._make_layer(block, 256, layers[2], stride=2, dilate=replace_stride_with_dilation[1]) self.layer4 = self._make_layer(block, 512, layers[3], stride=2, dilate=replace_stride_with_dilation[2]) self.bn2 = nn.BatchNorm2d( 512 * block.expansion, eps=1e-05, ) self.dropout = nn.Dropout(p=dropout, inplace=True) self.fc = nn.Linear(512 * block.expansion * self.fc_scale, num_features) self.features = nn.BatchNorm1d(num_features, eps=1e-05) nn.init.constant_(self.features.weight, 1.0) self.features.weight.requires_grad = False for m in self.modules(): if isinstance(m, nn.Conv2d): nn.init.normal_(m.weight, 0, 0.1) elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): nn.init.constant_(m.weight, 1) nn.init.constant_(m.bias, 0) if zero_init_residual: for m in self.modules(): if isinstance(m, IBasicBlock): nn.init.constant_(m.bn2.weight, 0)
def __init__(self, classnum=10574, feature=False): super(sphere20a, self).__init__() self.classnum = classnum print('classnum: ', classnum) self.feature = feature #input = B*3*112*96 self.conv1_1 = nn.Conv2d(3, 64, 3, 2, 1) #=>B*64*56*48 self.relu1_1 = nn.PReLU(64) self.conv1_2 = nn.Conv2d(64, 64, 3, 1, 1) self.relu1_2 = nn.PReLU(64) self.conv1_3 = nn.Conv2d(64, 64, 3, 1, 1) self.relu1_3 = nn.PReLU(64) self.conv2_1 = nn.Conv2d(64, 128, 3, 2, 1) #=>B*128*28*24 self.relu2_1 = nn.PReLU(128) self.conv2_2 = nn.Conv2d(128, 128, 3, 1, 1) self.relu2_2 = nn.PReLU(128) self.conv2_3 = nn.Conv2d(128, 128, 3, 1, 1) self.relu2_3 = nn.PReLU(128) self.conv2_4 = nn.Conv2d(128, 128, 3, 1, 1) #=>B*128*28*24 self.relu2_4 = nn.PReLU(128) self.conv2_5 = nn.Conv2d(128, 128, 3, 1, 1) self.relu2_5 = nn.PReLU(128) self.conv3_1 = nn.Conv2d(128, 256, 3, 2, 1) #=>B*256*14*12 self.relu3_1 = nn.PReLU(256) self.conv3_2 = nn.Conv2d(256, 256, 3, 1, 1) self.relu3_2 = nn.PReLU(256) self.conv3_3 = nn.Conv2d(256, 256, 3, 1, 1) self.relu3_3 = nn.PReLU(256) self.conv3_4 = nn.Conv2d(256, 256, 3, 1, 1) #=>B*256*14*12 self.relu3_4 = nn.PReLU(256) self.conv3_5 = nn.Conv2d(256, 256, 3, 1, 1) self.relu3_5 = nn.PReLU(256) self.conv3_6 = nn.Conv2d(256, 256, 3, 1, 1) #=>B*256*14*12 self.relu3_6 = nn.PReLU(256) self.conv3_7 = nn.Conv2d(256, 256, 3, 1, 1) self.relu3_7 = nn.PReLU(256) self.conv3_8 = nn.Conv2d(256, 256, 3, 1, 1) #=>B*256*14*12 self.relu3_8 = nn.PReLU(256) self.conv3_9 = nn.Conv2d(256, 256, 3, 1, 1) self.relu3_9 = nn.PReLU(256) self.conv4_1 = nn.Conv2d(256, 512, 3, 2, 1) #=>B*512*7*6 self.relu4_1 = nn.PReLU(512) self.conv4_2 = nn.Conv2d(512, 512, 3, 1, 1) self.relu4_2 = nn.PReLU(512) self.conv4_3 = nn.Conv2d(512, 512, 3, 1, 1) self.relu4_3 = nn.PReLU(512) self.fc5 = nn.Linear(512 * 7 * 6, 512) self.fc6 = AngleLinear(512, self.classnum)
def get_activation(self, i, o): return nn.Sequential( nn.PReLU(), nn.BatchNorm1d(o), )
def __init__(self, use_spect=False): super(Generator, self).__init__() input_dim = config.latent_dim + config.au_dim # input_dim = 256 + 17 + 1 # input_dim = 256 + 17 self.conv_blocks = nn.Sequential( # [-1, z + cc + dc, 1, 1] -> [-1, 1024, 4, 4] spectral_norm(nn.ConvTranspose2d(input_dim, 2048, 4, 1, 0), use_spect), nn.InstanceNorm2d(2048), # AdaptiveInstanceNorm2d(2048), nn.PReLU(), spectral_norm(nn.ConvTranspose2d(2048, 2048, 3, 1, 1), use_spect), nn.InstanceNorm2d(2048), # AdaptiveInstanceNorm2d(2048), nn.PReLU(), spectral_norm(nn.ConvTranspose2d(2048, 1024, 4, 2, 1), use_spect), nn.InstanceNorm2d(1024), # AdaptiveInstanceNorm2d(1024), nn.PReLU(), spectral_norm(nn.ConvTranspose2d(1024, 1024, 3, 1, 1), use_spect), nn.InstanceNorm2d(1024), # AdaptiveInstanceNorm2d(1024), nn.PReLU(), spectral_norm(nn.ConvTranspose2d(1024, 512, 4, 2, 1), use_spect), nn.InstanceNorm2d(512), # AdaptiveInstanceNorm2d(512), nn.PReLU(), spectral_norm(nn.ConvTranspose2d(512, 512, 3, 1, 1), use_spect), nn.InstanceNorm2d(512), # AdaptiveInstanceNorm2d(512), nn.PReLU(), # [-1, 512, 8, 8] spectral_norm(nn.ConvTranspose2d(512, 256, 4, 2, 1), use_spect), nn.InstanceNorm2d(256), # AdaptiveInstanceNorm2d(256), nn.PReLU(), spectral_norm(nn.ConvTranspose2d(256, 256, 3, 1, 1), use_spect), nn.InstanceNorm2d(256), # AdaptiveInstanceNorm2d(256), nn.PReLU(), # [-1, 256, 16, 16] spectral_norm(nn.ConvTranspose2d(256, 128, 4, 2, 1), use_spect), nn.InstanceNorm2d(128), # AdaptiveInstanceNorm2d(128), nn.PReLU(), spectral_norm(nn.ConvTranspose2d(128, 128, 3, 1, 1), use_spect), nn.InstanceNorm2d(128), # AdaptiveInstanceNorm2d(128), nn.PReLU(), # [-1, 128, 32, 32] spectral_norm(nn.ConvTranspose2d(128, 64, 4, 2, 1), use_spect), nn.InstanceNorm2d(64), # AdaptiveInstanceNorm2d(64), nn.PReLU(), spectral_norm(nn.ConvTranspose2d(64, 64, 3, 1, 1), use_spect), nn.InstanceNorm2d(64), # AdaptiveInstanceNorm2d(64), nn.PReLU(), spectral_norm(nn.ConvTranspose2d(64, 3, 1, 1, 0), use_spect), nn.Tanh())
def __init__(self, input_dim, output_dim, kernel_size, stride, padding=0, norm='none', activation='relu', pad_type='zero', coord_conv=False): super(Conv2dBlock, self).__init__() self.use_bias = True # initialize padding if pad_type == 'reflect': self.pad = nn.ReflectionPad2d(padding) elif pad_type == 'replicate': self.pad = nn.ReplicationPad2d(padding) elif pad_type == 'zero': self.pad = nn.ZeroPad2d(padding) else: assert 0, "Unsupported padding type: {}".format(pad_type) # initialize normalization norm_dim = output_dim if norm == 'bn': self.norm = nn.BatchNorm2d(norm_dim) elif norm == 'in': #self.norm = nn.InstanceNorm2d(norm_dim, track_running_stats=True) self.norm = nn.InstanceNorm2d(norm_dim) elif norm == 'ln': self.norm = LayerNorm(norm_dim) elif norm == 'adain': self.norm = AdaptiveInstanceNorm2d(norm_dim) elif norm == 'none': self.norm = None else: assert 0, "Unsupported normalization: {}".format(norm) # initialize activation if activation == 'relu': self.activation = nn.ReLU(inplace=True) elif activation == 'lrelu': self.activation = nn.LeakyReLU(0.2, inplace=True) elif activation == 'prelu': self.activation = nn.PReLU() elif activation == 'selu': self.activation = nn.SELU(inplace=True) elif activation == 'tanh': self.activation = nn.Tanh() elif activation == 'none': self.activation = None else: assert 0, "Unsupported activation: {}".format(activation) # initialize convolution if coord_conv: self.conv = nn.Conv2d(input_dim, output_dim, kernel_size, stride, bias=self.use_bias) else: self.conv = CoordConv(input_dim, output_dim, kernel_size=kernel_size, stride=stride, bias=self.use_bias)
def ELUCons(elu, nchan): if elu: return nn.ELU(inplace=True) else: return nn.PReLU(nchan)
def __init__(self, hidden_dim): super(Encoder, self).__init__() self.conv = GCNConv(dataset.num_features, hidden_dim) self.prelu = nn.PReLU(hidden_dim)
def __init__(self, ch_in, ch_out): super().__init__() self.f = nn.Sequential( nn.PReLU(), nn.Conv2d(ch_in, ch_out, kernel_size=3, stride=2, padding=1), nn.PReLU(), nn.Conv2d(ch_out, ch_out, kernel_size=3, padding=1))
def __init__(self, dilation, in_channels, out_channels, down_flag, relu=False, projection_ratio=4, p=0.1): super().__init__() # Define class variables self.in_channels = in_channels self.out_channels = out_channels self.dilation = dilation self.down_flag = down_flag if down_flag: self.stride = 2 self.reduced_depth = int(in_channels // projection_ratio) else: self.stride = 1 self.reduced_depth = int(out_channels // projection_ratio) if relu: activation = nn.ReLU() else: activation = nn.PReLU() self.maxpool = nn.MaxPool2d(kernel_size=2, stride=2, padding=0, return_indices=True) self.dropout = nn.Dropout2d(p=p) self.conv1 = nn.Conv2d(in_channels=self.in_channels, out_channels=self.reduced_depth, kernel_size=1, stride=1, padding=0, bias=False, dilation=1) self.prelu1 = activation self.conv2 = nn.Conv2d(in_channels=self.reduced_depth, out_channels=self.reduced_depth, kernel_size=3, stride=self.stride, padding=self.dilation, bias=True, dilation=self.dilation) self.prelu2 = activation self.conv3 = nn.Conv2d(in_channels=self.reduced_depth, out_channels=self.out_channels, kernel_size=1, stride=1, padding=0, bias=False, dilation=1) self.prelu3 = activation self.batchnorm = nn.BatchNorm2d(self.reduced_depth) self.batchnorm2 = nn.BatchNorm2d(self.out_channels)
def __init__(self, delay_frame=31, in_dim=1, enc_dim=512, window=32, shift=16, block_dim=128, block_kernel=3, hidden_dim=512, num_blocks=8, num_repeats=3, non_casual=5, casual=True): super(Model, self).__init__() self.delay_frame = delay_frame self.window = window self.shift = shift self.enc_dim = enc_dim self.conv_1 = nn.Conv1d(in_dim, enc_dim, kernel_size=window, bias=False, stride=shift) if casual: self.norm_1 = ChannelWiseLayerNorm(enc_dim, eps=1e-8) else: self.norm_1 = nn.GroupNorm(1, enc_dim, eps=1e-8) self.conv_2 = nn.Conv1d(enc_dim, block_dim, kernel_size=1) self.conv_blocks = nn.ModuleList() count = 0 for i in range(num_repeats): for j in range(num_blocks): if count < non_casual: self.conv_blocks.append( ConvBlock(block_dim, hidden_dim, kernel_size=block_kernel, padding=2**j, dilation=2**j)) else: self.conv_blocks.append( ConvBlock(block_dim, hidden_dim, kernel_size=block_kernel, padding=2**j, dilation=2**j, casual=True)) count += 1 self.block_output = nn.Sequential( nn.PReLU(), nn.Conv1d(block_dim, enc_dim * 2, kernel_size=1)) self.mask_nl = nn.Sigmoid() self.decoder_noise = nn.ConvTranspose1d(enc_dim, 1, kernel_size=window, bias=False, stride=shift) self.decoder_speech = nn.ConvTranspose1d(enc_dim, 1, kernel_size=window, bias=False, stride=shift)
def replace_relu_with_prelu(self) : id_relu = [1,3,6,8,11,13,15,18,20,22]; for i in id_relu : self.tmp[i] = nn.PReLU(self.tmp[i-1].out_channels);