def __init__(self, dict_dim, emb_dim, class_dim, padding_idx=0, cnn_dim=128, filter_sizes=[1, 2, 3], hidden_size=96, conv_layer_activation=nn.Tanh()): super(TextCNNLayer, self).__init__() self.dict_dim = dict_dim self.emb_dim = emb_dim self.class_dim = class_dim self.padding_idx = padding_idx self.cnn_dim = cnn_dim self.filter_sizes = filter_sizes self.hidden_size = hidden_size self.conv_layer_activation = conv_layer_activation self.embedding = paddle.nn.Embedding(self.dict_dim, self.emb_dim, padding_idx=self.padding_idx) self.convs = [ nn.Conv2D(in_channels=1, out_channels=self.cnn_dim, kernel_size=(i, self.emb_dim)) for i in self.filter_sizes ] self.projection_layer = paddle.nn.Linear(in_features=self.cnn_dim * len(self.filter_sizes), out_features=self.hidden_size) self.output_layer = paddle.nn.Linear(in_features=self.hidden_size, out_features=self.class_dim)
def __init__(self) -> None: super().__init__() self.A = nn.Sequential(Conv2DNormLReLU(3, 32, 7, padding=3), Conv2DNormLReLU(32, 64, stride=2), Conv2DNormLReLU(64, 64)) self.B = nn.Sequential(Conv2DNormLReLU(64, 128, stride=2), Conv2DNormLReLU(128, 128), Conv2DNormLReLU(128, 128)) self.C = nn.Sequential(InvertedresBlock(128, 2, 256), InvertedresBlock(256, 2, 256), InvertedresBlock(256, 2, 256), InvertedresBlock(256, 2, 256), Conv2DNormLReLU(256, 128)) self.D = nn.Sequential(nn.Upsample(scale_factor=2, mode='bilinear'), Conv2DNormLReLU(128, 128), Conv2DNormLReLU(128, 128)) self.E = nn.Sequential(nn.Upsample(scale_factor=2, mode='bilinear'), Conv2DNormLReLU(128, 64), Conv2DNormLReLU(64, 64), Conv2DNormLReLU(64, 32, 7, padding=3)) self.out = nn.Sequential(nn.Conv2D(32, 3, 1, bias_attr=False), nn.Tanh())
def __init__(self, input_nc=3, output_nc=3, ngf=64): super(UnetGenerator, self).__init__() self.down1 = nn.Conv2D(input_nc, ngf, kernel_size=4, stride=2, padding=1) self.down2 = Downsample(ngf, ngf*2) self.down3 = Downsample(ngf*2, ngf*4) self.down4 = Downsample(ngf*4, ngf*8) self.down5 = Downsample(ngf*8, ngf*8) self.down6 = Downsample(ngf*8, ngf*8) self.down7 = Downsample(ngf*8, ngf*8) self.center = Downsample(ngf*8, ngf*8) self.up7 = Upsample(ngf*8, ngf*8, use_dropout=True) self.up6 = Upsample(ngf*8*2, ngf*8, use_dropout=True) self.up5 = Upsample(ngf*8*2, ngf*8, use_dropout=True) self.up4 = Upsample(ngf*8*2, ngf*8) self.up3 = Upsample(ngf*8*2, ngf*4) self.up2 = Upsample(ngf*4*2, ngf*2) self.up1 = Upsample(ngf*2*2, ngf) self.output_block = nn.Sequential( nn.ReLU(), nn.Conv2DTranspose(ngf*2, output_nc, kernel_size=4, stride=2, padding=1), nn.Tanh() )
def __init__(self, latent_dim, output_nc, size=64, ngf=64): """Construct a Deep Convolutional generator Args: latent_dim (int) -- the number of latent dimension output_nc (int) -- the number of channels in output images size (int) -- size of output tensor ngf (int) -- the number of filters in the last conv layer Refer to https://arxiv.org/abs/1511.06434 """ super(DeepConvGenerator, self).__init__() self.latent_dim = latent_dim self.ngf = ngf self.init_size = size // 4 self.l1 = nn.Sequential( nn.Linear(latent_dim, ngf * 2 * self.init_size**2)) self.conv_blocks = nn.Sequential( nn.BatchNorm2D(ngf * 2), nn.Upsample(scale_factor=2), nn.Conv2D(ngf * 2, ngf * 2, 3, stride=1, padding=1), nn.BatchNorm2D(ngf * 2, 0.2), nn.LeakyReLU(0.2), nn.Upsample(scale_factor=2), nn.Conv2D(ngf * 2, ngf, 3, stride=1, padding=1), nn.BatchNorm2D(ngf, 0.2), nn.LeakyReLU(0.2), nn.Conv2D(ngf, output_nc, 3, stride=1, padding=1), nn.Tanh(), )
def __init__(self, z_dim, channels_img, features_g): super(Generator, self).__init__() self.gen = nn.Sequential( # Input: N x z_dim x 1 x 1 self._block(z_dim, features_g * 64, 4, 1, 0), # N x f_g x 4 x 4 self._block(features_g * 64, features_g * 32, 4, 2, 1), # N x f_g x 8 x 8 self._block(features_g * 32, features_g * 16, 4, 2, 1), # N x f_g x 16 x 16 self._block(features_g * 16, features_g * 8, 4, 2, 1), # N x f_g x 32 x 32 self._block(features_g * 8, features_g * 4, 4, 2, 1), # N x f_g x 64 x 64 self._block(features_g * 4, features_g * 2, 4, 2, 1), # N x f_g x 128 x 128 nn.Conv2DTranspose( features_g * 2, channels_img, kernel_size=4, stride=2, padding=1, bias_attr=False, weight_attr=paddle.ParamAttr(initializer=conv_initializer())), nn.Tanh() # [-1, 1] )
def __init__(self, emb_dim, num_filter, ngram_filter_sizes=(2, 3, 4, 5), conv_layer_activation=nn.Tanh(), output_dim=None, **kwargs): super().__init__() self._emb_dim = emb_dim self._num_filter = num_filter self._ngram_filter_sizes = ngram_filter_sizes self._activation = conv_layer_activation self._output_dim = output_dim self.convs = paddle.nn.LayerList([ nn.Conv2D(in_channels=1, out_channels=self._num_filter, kernel_size=(i, self._emb_dim), **kwargs) for i in self._ngram_filter_sizes ]) maxpool_output_dim = self._num_filter * len(self._ngram_filter_sizes) if self._output_dim: self.projection_layer = nn.Linear(maxpool_output_dim, self._output_dim) else: self.projection_layer = None self._output_dim = maxpool_output_dim
def __init__(self, bond_dim, hidden_dim, dropout, activation=F.relu): super(DomainAttentionLayer, self).__init__() self.attn_fc = nn.Linear(2 * bond_dim, hidden_dim) self.attn_out = nn.Linear(hidden_dim, 1, bias_attr=False) self.feat_drop = nn.Dropout(p=dropout) self.attn_drop = nn.Dropout(p=dropout) self.tanh = nn.Tanh() self.activation = activation
def __init__( self, vocab_size=30000, embedding_size=128, hidden_size=768, num_hidden_layers=12, num_hidden_groups=1, num_attention_heads=12, intermediate_size=3072, inner_group_num=1, hidden_act="gelu", hidden_dropout_prob=0, attention_probs_dropout_prob=0, max_position_embeddings=512, type_vocab_size=2, initializer_range=0.02, layer_norm_eps=1e-12, pad_token_id=0, bos_token_id=2, eos_token_id=3, add_pooling_layer=True, ): super(AlbertModel, self).__init__() self.initializer_range = initializer_range self.num_hidden_layers = num_hidden_layers self.embeddings = AlbertEmbeddings( vocab_size, embedding_size, hidden_dropout_prob, max_position_embeddings, type_vocab_size, layer_norm_eps, pad_token_id, ) self.encoder = AlbertTransformer( embedding_size, hidden_size, num_hidden_layers, num_hidden_groups, num_attention_heads, intermediate_size, inner_group_num, hidden_act, hidden_dropout_prob, attention_probs_dropout_prob, layer_norm_eps, ) if add_pooling_layer: self.pooler = nn.Linear(hidden_size, hidden_size) self.pooler_activation = nn.Tanh() else: self.pooler = None self.pooler_activation = None self.init_weights()
def __init__(self): super(Generator, self).__init__() self.gen = nn.Sequential( nn.Conv2DTranspose(100, 64 * 4, 4, 1, 0, bias_attr=False), nn.BatchNorm2D(64 * 4), nn.ReLU(True), nn.Conv2DTranspose(64 * 4, 64 * 2, 4, 2, 1, bias_attr=False), nn.BatchNorm2D(64 * 2), nn.ReLU(True), nn.Conv2DTranspose(64 * 2, 64, 4, 2, 1, bias_attr=False), nn.BatchNorm2D(64), nn.ReLU(True), nn.Conv2DTranspose(64, 1, 4, 2, 1, bias_attr=False), nn.Tanh())
def __init__(self, hidden_size, sequence_length, max_predictions_per_seq, pool_act="tanh"): super(IpuBertPooler, self).__init__() self.dense = nn.Linear(hidden_size, hidden_size) self.activation = nn.Tanh() self.pool_act = pool_act self.sequence_length = sequence_length self.max_predictions_per_seq = max_predictions_per_seq self.hidden_size = hidden_size
def __init__(self, in_channels, attention_channels): super(GatedAttentionLayer, self).__init__() self.att = nn.Sequential( nn.Linear(in_channels, attention_channels), # V nn.Tanh(), # tanh(V * H_t) nn.Linear(attention_channels, 1) ) self.gate = nn.Sequential( nn.Linear(in_channels, attention_channels), # U nn.Sigmoid() # sigm(U * H_t) ) # W_t * [tanh(V * H_t) * sigm(U * H_t)] self.w_t = nn.Linear(attention_channels, 1)
def __init__(self, channels, attention_channels=128, global_context=True): super().__init__() self.eps = 1e-12 self.global_context = global_context if global_context: self.tdnn = TDNNBlock(channels * 3, attention_channels, 1, 1) else: self.tdnn = TDNNBlock(channels, attention_channels, 1, 1) self.tanh = nn.Tanh() self.conv = Conv1d(in_channels=attention_channels, out_channels=channels, kernel_size=1)
def __init__(self): super(Generator, self).__init__() def block(in_feat, out_feat, normalize=True): layers = [nn.Linear(in_feat, out_feat)] if normalize: layers.append(nn.BatchNorm1D(out_feat, 0.8)) layers.append(nn.LeakyReLU(0.2)) return layers self.model = nn.Sequential( *block(opt.latent_dim, 128, normalize=False), *block(128, 256), *block(256, 512), *block(512, 1024), nn.Linear(1024, int(np.prod(img_shape))), nn.Tanh())
def __init__(self, ): super(Generator, self).__init__() self.gen = nn.Sequential( # input is Z, [B, 100, 1, 1] -> [B, 64 * 4, 4, 4] nn.Conv2DTranspose(100, 64 * 4, 4, 1, 0, bias_attr=False), nn.BatchNorm2D(64 * 4), nn.ReLU(True), # state size. [B, 64 * 4, 4, 4] -> [B, 64 * 2, 8, 8] nn.Conv2DTranspose(64 * 4, 64 * 2, 4, 2, 1, bias_attr=False), nn.BatchNorm2D(64 * 2), nn.ReLU(True), # state size. [B, 64 * 2, 8, 8] -> [B, 64, 16, 16] nn.Conv2DTranspose(64 * 2, 64, 4, 2, 1, bias_attr=False), nn.BatchNorm2D(64), nn.ReLU(True), # state size. [B, 64, 16, 16] -> [B, 1, 32, 32] nn.Conv2DTranspose(64, 1, 4, 2, 1, bias_attr=False), nn.Tanh())
def train(): device = paddle.set_device('cpu') # or 'gpu' net = nn.Sequential(nn.Flatten(1), nn.Linear(784, 200), nn.Tanh(), nn.Linear(200, 10)) # inputs and labels are not required for dynamic graph. input = InputSpec([None, 784], 'float32', 'x') label = InputSpec([None, 1], 'int64', 'label') model = paddle.Model(net, input, label) optim = paddle.optimizer.SGD(learning_rate=1e-3, parameters=model.parameters()) model.prepare(optim, paddle.nn.CrossEntropyLoss(), paddle.metric.Accuracy()) data = paddle.vision.datasets.MNIST(mode='train') model.fit(data, epochs=2, batch_size=32, verbose=1)
def __init__(self, vocab_size, type_size, max_position_seq_len, num_layers, n_head, hidden_size, attn_dropout, act_dropout): super(NSP, self).__init__() self.n_head = n_head self.hidden_size = hidden_size self.word_embedding_layer = nn.Embedding(vocab_size, hidden_size) self.sent_embedding_layer = nn.Embedding(type_size, hidden_size) self.pos_embedding_layer = nn.Embedding(max_position_seq_len, hidden_size) encoder_layer = nn.TransformerEncoderLayer( hidden_size, n_head, hidden_size * 4, act_dropout, 'gelu', attn_dropout, act_dropout, 'True') encoder_norm = nn.LayerNorm(hidden_size) self.encoder = nn.TransformerEncoder(encoder_layer, num_layers, encoder_norm) self.fc1 = nn.Linear(hidden_size, hidden_size) self.fc2 = nn.Linear(hidden_size, 2) self.dropout_layer = nn.Dropout(act_dropout) self.tanh_layer = nn.Tanh() self.softmax = nn.Softmax()
def __init__(self, hidden_size): super(ErnieMPooler, self).__init__() self.dense = nn.Linear(hidden_size, hidden_size) self.activation = nn.Tanh()
def __init__(self, config: Callable[..., None]) -> None: super().__init__() self.dense = nn.Linear(config.hidden_size, config.hidden_size) self.activation = nn.Tanh()
def __init__(self, hidden_size, cls_token_idx=-1): super(ErnieDocPooler, self).__init__() self.dense = nn.Linear(hidden_size, hidden_size) self.activation = nn.Tanh() self.cls_token_idx = cls_token_idx
def __init__(self, input_channel, output_nc, ngf=64, norm_type='instance', use_dropout=False, n_blocks=9, padding_type='reflect'): super(MobileResnetGenerator, self).__init__() norm_layer = build_norm_layer(norm_type) if type(norm_layer) == functools.partial: use_bias = norm_layer.func == InstanceNorm else: use_bias = norm_layer == InstanceNorm self.model = nn.LayerList([ nn.ReflectionPad2d([3, 3, 3, 3]), nn.Conv2d(input_channel, int(ngf), kernel_size=7, padding=0, bias_attr=use_bias), norm_layer(ngf), nn.ReLU() ]) n_downsampling = 2 for i in range(n_downsampling): mult = 2**i self.model.extend([ nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1, bias_attr=use_bias), norm_layer(ngf * mult * 2), nn.ReLU() ]) mult = 2**n_downsampling for i in range(n_blocks): self.model.extend([ MobileResnetBlock(ngf * mult, ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias) ]) for i in range(n_downsampling): mult = 2**(n_downsampling - i) output_size = (i + 1) * 128 self.model.extend([ nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2), kernel_size=3, stride=2, padding=1, output_padding=1, bias_attr=use_bias), norm_layer(int(ngf * mult / 2)), nn.ReLU() ]) self.model.extend([nn.ReflectionPad2d([3, 3, 3, 3])]) self.model.extend( [nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)]) self.model.extend([nn.Tanh()])
def __init__(self, hidden_size, with_pool): super(LayoutXLMPooler, self).__init__() self.dense = nn.Linear(hidden_size, hidden_size) self.activation = nn.Tanh() self.with_pool = with_pool
def __init__(self, input_nc, output_nc, ngf=64, norm_type='instance', use_dropout=False, n_blocks=6, padding_type='reflect'): """Construct a Resnet-based generator Args: input_nc (int) -- the number of channels in input images output_nc (int) -- the number of channels in output images ngf (int) -- the number of filters in the last conv layer norm_layer -- normalization layer use_dropout (bool) -- if use dropout layers n_blocks (int) -- the number of ResNet blocks padding_type (str) -- the name of padding layer in conv layers: reflect | replicate | zero """ assert (n_blocks >= 0) super(ResnetGenerator, self).__init__() norm_layer = build_norm_layer(norm_type) if type(norm_layer) == functools.partial: use_bias = norm_layer.func == nn.InstanceNorm else: use_bias = norm_layer == nn.InstanceNorm model = [ ReflectionPad2d(3), nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0, bias_attr=use_bias), norm_layer(ngf), nn.ReLU() ] n_downsampling = 2 for i in range(n_downsampling): # add downsampling layers mult = 2**i model += [ nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1, bias_attr=use_bias), norm_layer(ngf * mult * 2), nn.ReLU() ] mult = 2**n_downsampling for i in range(n_blocks): # add ResNet blocks model += [ ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias) ] for i in range(n_downsampling): # add upsampling layers mult = 2**(n_downsampling - i) model += [ nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2), kernel_size=3, stride=2, padding=1, output_padding=1, bias_attr=use_bias), norm_layer(int(ngf * mult / 2)), nn.ReLU() ] model += [ReflectionPad2d(3)] model += [nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)] model += [nn.Tanh()] self.model = nn.Sequential(*model)
def __init__(self, hidden_size, weight_attr=None): super(PPMiniLMPooler, self).__init__() self.dense = nn.Linear(hidden_size, hidden_size, weight_attr=weight_attr) self.activation = nn.Tanh()
def __init__(self, hidden_size, pool_act="tanh"): super(BertPooler, self).__init__() self.dense = nn.Linear(hidden_size, hidden_size) self.activation = nn.Tanh() self.pool_act = pool_act
def __init__(self, outer_nc, inner_nc, input_nc=None, submodule=None, outermost=False, innermost=False, norm_layer=nn.BatchNorm, use_dropout=False): """Construct a Unet submodule with skip connections. Parameters: outer_nc (int) -- the number of filters in the outer conv layer inner_nc (int) -- the number of filters in the inner conv layer input_nc (int) -- the number of channels in input images/features submodule (UnetSkipConnectionBlock) -- previously defined submodules outermost (bool) -- if this module is the outermost module innermost (bool) -- if this module is the innermost module norm_layer -- normalization layer use_dropout (bool) -- if use dropout layers. """ super(UnetSkipConnectionBlock, self).__init__() self.outermost = outermost if type(norm_layer) == functools.partial: use_bias = norm_layer.func == nn.InstanceNorm2D else: use_bias = norm_layer == nn.InstanceNorm2D if input_nc is None: input_nc = outer_nc downconv = nn.Conv2D(input_nc, inner_nc, kernel_size=4, stride=2, padding=1, bias_attr=use_bias) downrelu = nn.LeakyReLU(0.2) downnorm = norm_layer(inner_nc) uprelu = nn.ReLU() upnorm = norm_layer(outer_nc) if outermost: upconv = nn.Conv2DTranspose(inner_nc * 2, outer_nc, kernel_size=4, stride=2, padding=1) down = [downconv] up = [uprelu, upconv, nn.Tanh()] model = down + [submodule] + up elif innermost: upconv = nn.Conv2DTranspose(inner_nc, outer_nc, kernel_size=4, stride=2, padding=1, bias_attr=use_bias) down = [downrelu, downconv] up = [uprelu, upconv, upnorm] model = down + up else: upconv = nn.Conv2DTranspose(inner_nc * 2, outer_nc, kernel_size=4, stride=2, padding=1, bias_attr=use_bias) down = [downrelu, downconv, downnorm] up = [uprelu, upconv, upnorm] if use_dropout: model = down + [submodule] + up + [nn.Dropout(0.5)] else: model = down + [submodule] + up self.model = nn.Sequential(*model)
def __init__(self, input_nc, output_nc, ngf=64, n_blocks=6, img_size=256, light=False, norm_type='instance'): assert (n_blocks >= 0) super(ResnetUGATITGenerator, self).__init__() self.input_nc = input_nc self.output_nc = output_nc self.ngf = ngf self.n_blocks = n_blocks self.img_size = img_size self.light = light norm_layer = build_norm_layer(norm_type) DownBlock = [] DownBlock += [ nn.Pad2D(padding=[3, 3, 3, 3], mode="reflect"), nn.Conv2D(input_nc, ngf, kernel_size=7, stride=1, padding=0, bias_attr=False), norm_layer(ngf), nn.ReLU() ] # Down-Sampling n_downsampling = 2 for i in range(n_downsampling): mult = 2**i DownBlock += [ nn.Pad2D(padding=[1, 1, 1, 1], mode="reflect"), nn.Conv2D(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=0, bias_attr=False), norm_layer(ngf * mult * 2), nn.ReLU() ] # Down-Sampling Bottleneck mult = 2**n_downsampling for i in range(n_blocks): DownBlock += [ ResnetBlock(ngf * mult, use_bias=False, norm_layer=norm_layer) ] # Class Activation Map self.gap_fc = nn.Linear(ngf * mult, 1, bias_attr=False) self.gmp_fc = nn.Linear(ngf * mult, 1, bias_attr=False) self.conv1x1 = nn.Conv2D(ngf * mult * 2, ngf * mult, kernel_size=1, stride=1, bias_attr=True) self.relu = nn.ReLU() # Gamma, Beta block if self.light: FC = [ nn.Linear(ngf * mult, ngf * mult, bias_attr=False), nn.ReLU(), nn.Linear(ngf * mult, ngf * mult, bias_attr=False), nn.ReLU() ] else: FC = [ nn.Linear(img_size // mult * img_size // mult * ngf * mult, ngf * mult, bias_attr=False), nn.ReLU(), nn.Linear(ngf * mult, ngf * mult, bias_attr=False), nn.ReLU() ] self.gamma = nn.Linear(ngf * mult, ngf * mult, bias_attr=False) self.beta = nn.Linear(ngf * mult, ngf * mult, bias_attr=False) # Up-Sampling Bottleneck for i in range(n_blocks): setattr(self, 'UpBlock1_' + str(i + 1), ResnetAdaILNBlock(ngf * mult, use_bias=False)) # Up-Sampling UpBlock2 = [] for i in range(n_downsampling): mult = 2**(n_downsampling - i) UpBlock2 += [ nn.Upsample(scale_factor=2, mode='nearest'), nn.Pad2D(padding=[1, 1, 1, 1], mode="reflect"), nn.Conv2D(ngf * mult, int(ngf * mult / 2), kernel_size=3, stride=1, padding=0, bias_attr=False), ILN(int(ngf * mult / 2)), nn.ReLU() ] UpBlock2 += [ nn.Pad2D(padding=[3, 3, 3, 3], mode="reflect"), nn.Conv2D(ngf, output_nc, kernel_size=7, stride=1, padding=0, bias_attr=False), nn.Tanh() ] self.DownBlock = nn.Sequential(*DownBlock) self.FC = nn.Sequential(*FC) self.UpBlock2 = nn.Sequential(*UpBlock2)
def func_test_layer_str(self): module = nn.ELU(0.2) self.assertEqual(str(module), 'ELU(alpha=0.2)') module = nn.CELU(0.2) self.assertEqual(str(module), 'CELU(alpha=0.2)') module = nn.GELU(True) self.assertEqual(str(module), 'GELU(approximate=True)') module = nn.Hardshrink() self.assertEqual(str(module), 'Hardshrink(threshold=0.5)') module = nn.Hardswish(name="Hardswish") self.assertEqual(str(module), 'Hardswish(name=Hardswish)') module = nn.Tanh(name="Tanh") self.assertEqual(str(module), 'Tanh(name=Tanh)') module = nn.Hardtanh(name="Hardtanh") self.assertEqual(str(module), 'Hardtanh(min=-1.0, max=1.0, name=Hardtanh)') module = nn.PReLU(1, 0.25, name="PReLU", data_format="NCHW") self.assertEqual( str(module), 'PReLU(num_parameters=1, data_format=NCHW, init=0.25, dtype=float32, name=PReLU)' ) module = nn.ReLU() self.assertEqual(str(module), 'ReLU()') module = nn.ReLU6() self.assertEqual(str(module), 'ReLU6()') module = nn.SELU() self.assertEqual( str(module), 'SELU(scale=1.0507009873554805, alpha=1.6732632423543772)') module = nn.LeakyReLU() self.assertEqual(str(module), 'LeakyReLU(negative_slope=0.01)') module = nn.Sigmoid() self.assertEqual(str(module), 'Sigmoid()') module = nn.Hardsigmoid() self.assertEqual(str(module), 'Hardsigmoid()') module = nn.Softplus() self.assertEqual(str(module), 'Softplus(beta=1, threshold=20)') module = nn.Softshrink() self.assertEqual(str(module), 'Softshrink(threshold=0.5)') module = nn.Softsign() self.assertEqual(str(module), 'Softsign()') module = nn.Swish() self.assertEqual(str(module), 'Swish()') module = nn.Tanhshrink() self.assertEqual(str(module), 'Tanhshrink()') module = nn.ThresholdedReLU() self.assertEqual(str(module), 'ThresholdedReLU(threshold=1.0)') module = nn.LogSigmoid() self.assertEqual(str(module), 'LogSigmoid()') module = nn.Softmax() self.assertEqual(str(module), 'Softmax(axis=-1)') module = nn.LogSoftmax() self.assertEqual(str(module), 'LogSoftmax(axis=-1)') module = nn.Maxout(groups=2) self.assertEqual(str(module), 'Maxout(groups=2, axis=1)') module = nn.Linear(2, 4, name='linear') self.assertEqual( str(module), 'Linear(in_features=2, out_features=4, dtype=float32, name=linear)' ) module = nn.Upsample(size=[12, 12]) self.assertEqual( str(module), 'Upsample(size=[12, 12], mode=nearest, align_corners=False, align_mode=0, data_format=NCHW)' ) module = nn.UpsamplingNearest2D(size=[12, 12]) self.assertEqual( str(module), 'UpsamplingNearest2D(size=[12, 12], data_format=NCHW)') module = nn.UpsamplingBilinear2D(size=[12, 12]) self.assertEqual( str(module), 'UpsamplingBilinear2D(size=[12, 12], data_format=NCHW)') module = nn.Bilinear(in1_features=5, in2_features=4, out_features=1000) self.assertEqual( str(module), 'Bilinear(in1_features=5, in2_features=4, out_features=1000, dtype=float32)' ) module = nn.Dropout(p=0.5) self.assertEqual(str(module), 'Dropout(p=0.5, axis=None, mode=upscale_in_train)') module = nn.Dropout2D(p=0.5) self.assertEqual(str(module), 'Dropout2D(p=0.5, data_format=NCHW)') module = nn.Dropout3D(p=0.5) self.assertEqual(str(module), 'Dropout3D(p=0.5, data_format=NCDHW)') module = nn.AlphaDropout(p=0.5) self.assertEqual(str(module), 'AlphaDropout(p=0.5)') module = nn.Pad1D(padding=[1, 2], mode='constant') self.assertEqual( str(module), 'Pad1D(padding=[1, 2], mode=constant, value=0.0, data_format=NCL)') module = nn.Pad2D(padding=[1, 0, 1, 2], mode='constant') self.assertEqual( str(module), 'Pad2D(padding=[1, 0, 1, 2], mode=constant, value=0.0, data_format=NCHW)' ) module = nn.ZeroPad2D(padding=[1, 0, 1, 2]) self.assertEqual(str(module), 'ZeroPad2D(padding=[1, 0, 1, 2], data_format=NCHW)') module = nn.Pad3D(padding=[1, 0, 1, 2, 0, 0], mode='constant') self.assertEqual( str(module), 'Pad3D(padding=[1, 0, 1, 2, 0, 0], mode=constant, value=0.0, data_format=NCDHW)' ) module = nn.CosineSimilarity(axis=0) self.assertEqual(str(module), 'CosineSimilarity(axis=0, eps=1e-08)') module = nn.Embedding(10, 3, sparse=True) self.assertEqual(str(module), 'Embedding(10, 3, sparse=True)') module = nn.Conv1D(3, 2, 3) self.assertEqual(str(module), 'Conv1D(3, 2, kernel_size=[3], data_format=NCL)') module = nn.Conv1DTranspose(2, 1, 2) self.assertEqual( str(module), 'Conv1DTranspose(2, 1, kernel_size=[2], data_format=NCL)') module = nn.Conv2D(4, 6, (3, 3)) self.assertEqual(str(module), 'Conv2D(4, 6, kernel_size=[3, 3], data_format=NCHW)') module = nn.Conv2DTranspose(4, 6, (3, 3)) self.assertEqual( str(module), 'Conv2DTranspose(4, 6, kernel_size=[3, 3], data_format=NCHW)') module = nn.Conv3D(4, 6, (3, 3, 3)) self.assertEqual( str(module), 'Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)') module = nn.Conv3DTranspose(4, 6, (3, 3, 3)) self.assertEqual( str(module), 'Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)') module = nn.PairwiseDistance() self.assertEqual(str(module), 'PairwiseDistance(p=2.0)') module = nn.InstanceNorm1D(2) self.assertEqual(str(module), 'InstanceNorm1D(num_features=2, epsilon=1e-05)') module = nn.InstanceNorm2D(2) self.assertEqual(str(module), 'InstanceNorm2D(num_features=2, epsilon=1e-05)') module = nn.InstanceNorm3D(2) self.assertEqual(str(module), 'InstanceNorm3D(num_features=2, epsilon=1e-05)') module = nn.GroupNorm(num_channels=6, num_groups=6) self.assertEqual( str(module), 'GroupNorm(num_groups=6, num_channels=6, epsilon=1e-05)') module = nn.LayerNorm([2, 2, 3]) self.assertEqual( str(module), 'LayerNorm(normalized_shape=[2, 2, 3], epsilon=1e-05)') module = nn.BatchNorm1D(1) self.assertEqual( str(module), 'BatchNorm1D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCL)' ) module = nn.BatchNorm2D(1) self.assertEqual( str(module), 'BatchNorm2D(num_features=1, momentum=0.9, epsilon=1e-05)') module = nn.BatchNorm3D(1) self.assertEqual( str(module), 'BatchNorm3D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCDHW)' ) module = nn.SyncBatchNorm(2) self.assertEqual( str(module), 'SyncBatchNorm(num_features=2, momentum=0.9, epsilon=1e-05)') module = nn.LocalResponseNorm(size=5) self.assertEqual( str(module), 'LocalResponseNorm(size=5, alpha=0.0001, beta=0.75, k=1.0)') module = nn.AvgPool1D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'AvgPool1D(kernel_size=2, stride=2, padding=0)') module = nn.AvgPool2D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'AvgPool2D(kernel_size=2, stride=2, padding=0)') module = nn.AvgPool3D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'AvgPool3D(kernel_size=2, stride=2, padding=0)') module = nn.MaxPool1D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'MaxPool1D(kernel_size=2, stride=2, padding=0)') module = nn.MaxPool2D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'MaxPool2D(kernel_size=2, stride=2, padding=0)') module = nn.MaxPool3D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'MaxPool3D(kernel_size=2, stride=2, padding=0)') module = nn.AdaptiveAvgPool1D(output_size=16) self.assertEqual(str(module), 'AdaptiveAvgPool1D(output_size=16)') module = nn.AdaptiveAvgPool2D(output_size=3) self.assertEqual(str(module), 'AdaptiveAvgPool2D(output_size=3)') module = nn.AdaptiveAvgPool3D(output_size=3) self.assertEqual(str(module), 'AdaptiveAvgPool3D(output_size=3)') module = nn.AdaptiveMaxPool1D(output_size=16, return_mask=True) self.assertEqual( str(module), 'AdaptiveMaxPool1D(output_size=16, return_mask=True)') module = nn.AdaptiveMaxPool2D(output_size=3, return_mask=True) self.assertEqual(str(module), 'AdaptiveMaxPool2D(output_size=3, return_mask=True)') module = nn.AdaptiveMaxPool3D(output_size=3, return_mask=True) self.assertEqual(str(module), 'AdaptiveMaxPool3D(output_size=3, return_mask=True)') module = nn.SimpleRNNCell(16, 32) self.assertEqual(str(module), 'SimpleRNNCell(16, 32)') module = nn.LSTMCell(16, 32) self.assertEqual(str(module), 'LSTMCell(16, 32)') module = nn.GRUCell(16, 32) self.assertEqual(str(module), 'GRUCell(16, 32)') module = nn.PixelShuffle(3) self.assertEqual(str(module), 'PixelShuffle(upscale_factor=3)') module = nn.SimpleRNN(16, 32, 2) self.assertEqual( str(module), 'SimpleRNN(16, 32, num_layers=2\n (0): RNN(\n (cell): SimpleRNNCell(16, 32)\n )\n (1): RNN(\n (cell): SimpleRNNCell(32, 32)\n )\n)' ) module = nn.LSTM(16, 32, 2) self.assertEqual( str(module), 'LSTM(16, 32, num_layers=2\n (0): RNN(\n (cell): LSTMCell(16, 32)\n )\n (1): RNN(\n (cell): LSTMCell(32, 32)\n )\n)' ) module = nn.GRU(16, 32, 2) self.assertEqual( str(module), 'GRU(16, 32, num_layers=2\n (0): RNN(\n (cell): GRUCell(16, 32)\n )\n (1): RNN(\n (cell): GRUCell(32, 32)\n )\n)' ) module1 = nn.Sequential( ('conv1', nn.Conv2D(1, 20, 5)), ('relu1', nn.ReLU()), ('conv2', nn.Conv2D(20, 64, 5)), ('relu2', nn.ReLU())) self.assertEqual( str(module1), 'Sequential(\n '\ '(conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n '\ '(relu1): ReLU()\n '\ '(conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n '\ '(relu2): ReLU()\n)' ) module2 = nn.Sequential( nn.Conv3DTranspose(4, 6, (3, 3, 3)), nn.AvgPool3D(kernel_size=2, stride=2, padding=0), nn.Tanh(name="Tanh"), module1, nn.Conv3D(4, 6, (3, 3, 3)), nn.MaxPool3D(kernel_size=2, stride=2, padding=0), nn.GELU(True)) self.assertEqual( str(module2), 'Sequential(\n '\ '(0): Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n '\ '(1): AvgPool3D(kernel_size=2, stride=2, padding=0)\n '\ '(2): Tanh(name=Tanh)\n '\ '(3): Sequential(\n (conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n (relu1): ReLU()\n'\ ' (conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n (relu2): ReLU()\n )\n '\ '(4): Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n '\ '(5): MaxPool3D(kernel_size=2, stride=2, padding=0)\n '\ '(6): GELU(approximate=True)\n)' )
def __init__(self, hidden_size): super(BertPooler, self).__init__() self.weight_attr = paddle.ParamAttr( initializer=paddle.fluid.initializer.ConstantInitializer(value=0.01)) self.dense = nn.Linear(hidden_size, hidden_size, weight_attr=self.weight_attr) self.activation = nn.Tanh()
def __init__(self, ngf=32, img_size=256, n_blocks=4, light=True): super(ResnetGenerator, self).__init__() self.light = light self.n_blocks = n_blocks DownBlock = [] DownBlock += [ nn.Pad2D([3, 3, 3, 3], 'reflect'), nn.Conv2D(3, ngf, kernel_size=7, stride=1, bias_attr=False), nn.InstanceNorm2D(ngf, weight_attr=False, bias_attr=False), nn.ReLU() ] DownBlock += [HourGlass(ngf, ngf), HourGlass(ngf, ngf)] # Down-Sampling n_downsampling = 2 for i in range(n_downsampling): mult = 2**i DownBlock += [ nn.Pad2D([1, 1, 1, 1], 'reflect'), nn.Conv2D(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, bias_attr=False), nn.InstanceNorm2D(ngf * mult * 2, weight_attr=False, bias_attr=False), nn.ReLU() ] # Encoder Bottleneck mult = 2**n_downsampling for i in range(n_blocks): setattr(self, 'EncodeBlock' + str(i + 1), ResnetBlock(ngf * mult)) # Class Activation Map self.gap_fc = nn.Linear(ngf * mult, 1, bias_attr=False) self.gmp_fc = nn.Linear(ngf * mult, 1, bias_attr=False) self.conv1x1 = nn.Conv2D(ngf * mult * 2, ngf * mult, kernel_size=1, stride=1) self.relu = nn.ReLU() # Gamma, Beta block FC = [] if self.light: FC += [ nn.Linear(ngf * mult, ngf * mult, bias_attr=False), nn.ReLU(), nn.Linear(ngf * mult, ngf * mult, bias_attr=False), nn.ReLU() ] else: FC += [ nn.Linear(img_size // mult * img_size // mult * ngf * mult, ngf * mult, bias_attr=False), nn.ReLU(), nn.Linear(ngf * mult, ngf * mult, bias_attr=False), nn.ReLU() ] # Decoder Bottleneck mult = 2**n_downsampling for i in range(n_blocks): setattr(self, 'DecodeBlock' + str(i + 1), ResnetSoftAdaLINBlock(ngf * mult)) # Up-Sampling UpBlock = [] for i in range(n_downsampling): mult = 2**(n_downsampling - i) UpBlock += [ nn.Upsample(scale_factor=2), nn.Pad2D([1, 1, 1, 1], 'reflect'), nn.Conv2D(ngf * mult, ngf * mult // 2, kernel_size=3, stride=1, bias_attr=False), LIN(ngf * mult // 2), nn.ReLU() ] UpBlock += [HourGlass(ngf, ngf), HourGlass(ngf, ngf, False)] UpBlock += [ nn.Pad2D([3, 3, 3, 3], 'reflect'), nn.Conv2D(3, 3, kernel_size=7, stride=1, bias_attr=False), nn.Tanh() ] self.DownBlock = nn.Sequential(*DownBlock) self.FC = nn.Sequential(*FC) self.UpBlock = nn.Sequential(*UpBlock)
def __init__(self, use_tanh: bool = True, load_checkpoint: str = None): super(UserGuidedColorization, self).__init__() self.input_nc = 4 self.output_nc = 2 # Conv1 model1 = ( Conv2D(self.input_nc, 64, 3, 1, 1), nn.ReLU(), Conv2D(64, 64, 3, 1, 1), nn.ReLU(), nn.BatchNorm(64), ) # Conv2 model2 = ( Conv2D(64, 128, 3, 1, 1), nn.ReLU(), Conv2D(128, 128, 3, 1, 1), nn.ReLU(), nn.BatchNorm(128), ) # Conv3 model3 = ( Conv2D(128, 256, 3, 1, 1), nn.ReLU(), Conv2D(256, 256, 3, 1, 1), nn.ReLU(), Conv2D(256, 256, 3, 1, 1), nn.ReLU(), nn.BatchNorm(256), ) # Conv4 model4 = ( Conv2D(256, 512, 3, 1, 1), nn.ReLU(), Conv2D(512, 512, 3, 1, 1), nn.ReLU(), Conv2D(512, 512, 3, 1, 1), nn.ReLU(), nn.BatchNorm(512), ) # Conv5 model5 = ( Conv2D(512, 512, 3, 1, 2, 2), nn.ReLU(), Conv2D(512, 512, 3, 1, 2, 2), nn.ReLU(), Conv2D(512, 512, 3, 1, 2, 2), nn.ReLU(), nn.BatchNorm(512), ) # Conv6 model6 = ( Conv2D(512, 512, 3, 1, 2, 2), nn.ReLU(), Conv2D(512, 512, 3, 1, 2, 2), nn.ReLU(), Conv2D(512, 512, 3, 1, 2, 2), nn.ReLU(), nn.BatchNorm(512), ) # Conv7 model7 = ( Conv2D(512, 512, 3, 1, 1), nn.ReLU(), Conv2D(512, 512, 3, 1, 1), nn.ReLU(), Conv2D(512, 512, 3, 1, 1), nn.ReLU(), nn.BatchNorm(512), ) # Conv8 model8up = (Conv2DTranspose(512, 256, kernel_size=4, stride=2, padding=1), ) model3short8 = (Conv2D(256, 256, 3, 1, 1), ) model8 = ( nn.ReLU(), Conv2D(256, 256, 3, 1, 1), nn.ReLU(), Conv2D(256, 256, 3, 1, 1), nn.ReLU(), nn.BatchNorm(256), ) # Conv9 model9up = (Conv2DTranspose(256, 128, kernel_size=4, stride=2, padding=1), ) model2short9 = (Conv2D( 128, 128, 3, 1, 1, ), ) model9 = (nn.ReLU(), Conv2D(128, 128, 3, 1, 1), nn.ReLU(), nn.BatchNorm(128)) # Conv10 model10up = (Conv2DTranspose(128, 128, kernel_size=4, stride=2, padding=1), ) model1short10 = (Conv2D(64, 128, 3, 1, 1), ) model10 = (nn.ReLU(), Conv2D(128, 128, 3, 1, 1), nn.LeakyReLU(negative_slope=0.2)) model_class = (Conv2D(256, 529, 1), ) if use_tanh: model_out = (Conv2D(128, 2, 1, 1, 0, 1), nn.Tanh()) else: model_out = (Conv2D(128, 2, 1, 1, 0, 1), ) self.model1 = nn.Sequential(*model1) self.model2 = nn.Sequential(*model2) self.model3 = nn.Sequential(*model3) self.model4 = nn.Sequential(*model4) self.model5 = nn.Sequential(*model5) self.model6 = nn.Sequential(*model6) self.model7 = nn.Sequential(*model7) self.model8up = nn.Sequential(*model8up) self.model8 = nn.Sequential(*model8) self.model9up = nn.Sequential(*model9up) self.model9 = nn.Sequential(*model9) self.model10up = nn.Sequential(*model10up) self.model10 = nn.Sequential(*model10) self.model3short8 = nn.Sequential(*model3short8) self.model2short9 = nn.Sequential(*model2short9) self.model1short10 = nn.Sequential(*model1short10) self.model_class = nn.Sequential(*model_class) self.model_out = nn.Sequential(*model_out) self.set_config() if load_checkpoint is not None: self.model_dict = paddle.load(load_checkpoint) self.set_dict(self.model_dict) print("load custom checkpoint success") else: checkpoint = os.path.join(self.directory, 'user_guided.pdparams') self.model_dict = paddle.load(checkpoint) self.set_dict(self.model_dict) print("load pretrained checkpoint success")