def __init__(self, ernie_gram, num_classes=2, dropout=None):
super(ErnieGramForSequenceClassification, self).__init__()
self.num_classes = num_classes
self.ernie_gram = ernie_gram # allow ernie gram to be config
self.dropout = nn.Dropout(dropout if dropout is not None else self.
ernie_gram.config["hidden_dropout_prob"])
self.classifier = nn.Linear(self.ernie_gram.config["hidden_size"],
num_classes)
self.apply(self.init_weights)
def __init__(self, cin, cout, groups, dropout_prob):
super().__init__()
self.conv1d = nn.Conv1D(in_channels=cin,
out_channels=cout,
kernel_size=1,
groups=groups)
self.layernorm = SqueezeBertLayerNorm(cout)
self.dropout = nn.Dropout(dropout_prob)
def __init__(self, squeezebert, num_classes=2, dropout=None):
super().__init__()
self.num_classes = num_classes
self.squeezebert = squeezebert
self.dropout = nn.Dropout(dropout if dropout is not None else self.
squeezebert.config["hidden_dropout_prob"])
self.classifier = nn.Linear(self.squeezebert.config["hidden_size"],
num_classes)
self.apply(self.init_weights)
def __init__(self, skep, num_classes=2, dropout=None):
super(SkepSequenceModel, self).__init__()
self.num_classes = num_classes
self.skep = skep # allow skep to be config
self.dropout = nn.Dropout(dropout if dropout is not None else
self.skep.config["hidden_dropout_prob"])
self.classifier = nn.Linear(self.skep.config["hidden_size"],
num_classes)
self.apply(self.init_weights)
def __init__(self, roberta, num_classes=2, dropout=None):
super(RobertaForSequenceClassification, self).__init__()
self.num_classes = num_classes
self.roberta = roberta # allow roberta to be config
self.dropout = nn.Dropout(dropout if dropout is not None else self.
roberta.config["hidden_dropout_prob"])
self.classifier = nn.Linear(self.roberta.config["hidden_size"],
num_classes)
self.apply(self.init_weights)
def __init__(self,
img_size=224,
patch_size=16,
in_chans=3,
embed_dim=768,
depth=12,
num_heads=12,
mlp_ratio=4,
qkv_bias=False,
qk_scale=None,
drop_rate=0.,
attn_drop_rate=0.,
drop_path_rate=0.,
norm_layer='nn.LayerNorm',
epsilon=1e-5,
final_norm=False,
pretrained=None,
**args):
super().__init__()
self.img_size = img_size
self.embed_dim = embed_dim
self.patch_embed = PatchEmbed(img_size=img_size,
patch_size=patch_size,
in_chans=in_chans,
embed_dim=embed_dim)
self.pos_w = self.patch_embed.num_patches_in_w
self.pos_h = self.patch_embed.num_patches_in_h
self.pos_embed = self.create_parameter(
shape=(1, self.pos_w * self.pos_h + 1, embed_dim),
default_initializer=paddle.nn.initializer.TruncatedNormal(std=.02))
self.cls_token = self.create_parameter(
shape=(1, 1, embed_dim),
default_initializer=paddle.nn.initializer.Constant(value=0.))
self.pos_drop = nn.Dropout(p=drop_rate)
dpr = np.linspace(0, drop_path_rate, depth)
self.blocks = nn.LayerList([
Block(dim=embed_dim,
num_heads=num_heads,
mlp_ratio=mlp_ratio,
qkv_bias=qkv_bias,
qk_scale=qk_scale,
drop=drop_rate,
attn_drop=attn_drop_rate,
drop_path=dpr[i],
norm_layer=norm_layer,
epsilon=epsilon) for i in range(depth)
])
self.final_norm = final_norm
if self.final_norm:
self.norm = eval(norm_layer)(embed_dim, epsilon=epsilon)
self.pretrained = pretrained
self.init_weight()
def __init__(self, layoutlm, num_classes=2, dropout=None):
super(LayoutLMForTokenClassification, self).__init__()
self.num_classes = num_classes
self.layoutlm = layoutlm
self.dropout = nn.Dropout(dropout if dropout is not None else self.
layoutlm.config["hidden_dropout_prob"])
self.classifier = nn.Linear(self.layoutlm.config["hidden_size"],
num_classes)
self.classifier.apply(self.init_weights)
def __init__(self, params_file, batch_size, sent_embedding_dim,
num_labels):
super(ELMoBowTextClassification, self).__init__()
self._elmo = get_elmo_layer(params_file, batch_size, trainable=True)
word_embedding_dim = self._elmo.embedding_dim
self._fc1 = nn.Linear(word_embedding_dim, sent_embedding_dim)
self._fc2 = nn.Linear(sent_embedding_dim, num_labels)
self._dropout = nn.Dropout(p=0.5)
def __init__(self, in_feats, hidden, out_feats, num_hops, n_layers, dropout, input_drop):
super(SIGN, self).__init__()
self.dropout = nn.Dropout(dropout)
self.prelu = nn.PReLU()
self.inception_ffs = nn.LayerList()
self.input_drop = input_drop
for i in range(num_hops):
self.inception_ffs.append(FeedForwardNet(in_feats, hidden, hidden, n_layers, dropout))
self.project = FeedForwardNet(num_hops * hidden, hidden, out_feats, n_layers, dropout)
def __init__(self, nezha, num_classes=2, dropout=None):
super(NeZhaForTokenClassification, self).__init__()
self.num_classes = num_classes
self.nezha = nezha
self.dropout = nn.Dropout(dropout if dropout is not None else
self.nezha.config["hidden_dropout_prob"])
self.classifier = nn.Linear(self.nezha.config["hidden_size"],
num_classes)
self.apply(self.init_weights)
def __init__(self, ernie, num_classes=2, dropout=None):
super(ErnieForTokenClassification, self).__init__()
self.num_classes = num_classes
self.ernie = ernie # allow ernie to be config
self.dropout = nn.Dropout(dropout if dropout is not None else
self.ernie.config["hidden_dropout_prob"])
self.classifier = nn.Linear(self.ernie.config["hidden_size"],
num_classes)
self.apply(self.init_weights)
def __init__(self,
input_dim: int,
inner_dim: int,
num_classes: int,
pooler_dropout: float):
super().__init__()
self.dense = nn.Linear(input_dim, inner_dim)
self.dropout = nn.Dropout(p=pooler_dropout)
self.out_proj = nn.Linear(inner_dim, num_classes)
def __init__(self, roformerv2, num_classes=2, dropout=None):
super(RoFormerv2ForTokenClassification, self).__init__()
self.num_classes = num_classes
self.roformerv2 = roformerv2 # allow roformerv2 to be config
self.dropout = nn.Dropout(dropout if dropout is not None else
self.roformerv2.config["hidden_dropout_prob"])
self.classifier = nn.Linear(self.roformerv2.config["hidden_size"],
num_classes)
self.apply(self.init_weights)
def __init__(self, gau_alpha, num_classes=2, dropout=None):
super(GAUAlphaForTokenClassification, self).__init__()
self.num_classes = num_classes
self.gau_alpha = gau_alpha # allow gau_alpha to be config
self.dropout = nn.Dropout(dropout if dropout is not None else self.
gau_alpha.config["hidden_dropout_prob"])
self.classifier = nn.Linear(self.gau_alpha.config["hidden_size"],
num_classes)
self.apply(self.init_weights)
def __init__(
self,
classifier_dropout_prob,
hidden_size,
num_labels,
):
super(AlbertSOPHead, self).__init__()
self.dropout = nn.Dropout(classifier_dropout_prob)
self.classifier = nn.Linear(hidden_size, num_labels)
def __init__(self, bert, num_classes=2, dropout=None):
super(BertForTokenClassification, self).__init__()
self.num_classes = num_classes
self.bert = bert # allow bert to be config
self.dropout = nn.Dropout(dropout if dropout is not None else self.
bert.config["hidden_dropout_prob"])
self.classifier = nn.Linear(self.bert.config["hidden_size"],
num_classes)
self.apply(self.init_weights)
def __init__(self, pretrained_model, dropout=None, rdrop_coef=0.0):
super().__init__()
self.ptm = pretrained_model
self.dropout = nn.Dropout(dropout if dropout is not None else 0.1)
# num_labels = 2 (similar or dissimilar)
self.classifier = nn.Linear(self.ptm.config["hidden_size"], 2)
self.rdrop_coef = rdrop_coef
self.rdrop_loss = ppnlp.losses.RDropLoss()
def __init__(self, bigbird, num_classes=None):
super(BigBirdForSequenceClassification, self).__init__()
self.bigbird = bigbird
if num_classes is None:
num_classes = self.bigbird.config["num_labels"]
self.linear = nn.Linear(self.bigbird.config["hidden_size"], num_classes)
self.dropout = nn.Dropout(
self.bigbird.config['hidden_dropout_prob'], mode="upscale_in_train")
self.apply(self.init_weights)
def __init__(self,
hidden_size,
hidden_dropout_prob,
intermediate_size,
layer_norm_eps=1e-12):
super(RemBertOutput, self).__init__()
self.dense = nn.Linear(intermediate_size, hidden_size)
self.layer_norm = nn.LayerNorm(hidden_size, epsilon=layer_norm_eps)
self.dropout = nn.Dropout(hidden_dropout_prob)
def __init__(self,
n_inputs,
n_outputs,
kernel_size,
stride,
dilation,
padding,
dropout=0.2):
super(TemporalBlock, self).__init__()
self.conv1 = weight_norm(
nn.Conv1D(
n_inputs,
n_outputs,
kernel_size,
stride=stride,
padding=padding,
dilation=dilation))
# Chomp1d is used to make sure the network is causal.
# We pad by (k-1)*d on the two sides of the input for convolution,
# and then use Chomp1d to remove the (k-1)*d output elements on the right.
self.chomp1 = Chomp1d(padding)
self.relu1 = nn.ReLU()
self.dropout1 = nn.Dropout(dropout)
self.conv2 = weight_norm(
nn.Conv1D(
n_outputs,
n_outputs,
kernel_size,
stride=stride,
padding=padding,
dilation=dilation))
self.chomp2 = Chomp1d(padding)
self.relu2 = nn.ReLU()
self.dropout2 = nn.Dropout(dropout)
self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1,
self.dropout1, self.conv2, self.chomp2,
self.relu2, self.dropout2)
self.downsample = nn.Conv1D(n_inputs, n_outputs,
1) if n_inputs != n_outputs else None
self.relu = nn.ReLU()
self.init_weights()
def __init__(self,
d_model,
nhead,
dim_feedforward,
dropout=0.1,
activation="gelu",
attn_dropout=None,
act_dropout=None,
normalize_before=True,
weight_attr=None,
bias_attr=None,
topo=None):
self._config = locals()
self._config.pop("self")
self._config.pop("__class__", None) # py3
super(TransformerDecoderLayer, self).__init__()
attn_dropout = dropout if attn_dropout is None else attn_dropout
act_dropout = dropout if act_dropout is None else act_dropout
self.normalize_before = normalize_before
weight_attrs = _convert_param_attr_to_list(weight_attr, 3)
bias_attrs = _convert_param_attr_to_list(bias_attr, 3)
self.self_attn = MultiHeadAttention(d_model,
nhead,
dropout=attn_dropout,
weight_attr=weight_attrs[0],
bias_attr=bias_attrs[0],
topo=topo)
self.linear1 = nn.Linear(d_model,
dim_feedforward,
weight_attrs[2],
bias_attr=bias_attrs[2])
self.linear2 = nn.Linear(dim_feedforward,
d_model,
weight_attrs[2],
bias_attr=bias_attrs[2])
self.norm1 = nn.LayerNorm(d_model, epsilon=1e-5)
self.norm2 = nn.LayerNorm(d_model, epsilon=1e-5)
self.dropout1 = nn.Dropout(dropout, mode="upscale_in_train")
self.dropout2 = nn.Dropout(act_dropout, mode="upscale_in_train")
self.activation = getattr(F, activation)
def __init__(self,
config,
last_channel,
scale=1.0,
num_classes=1000,
with_pool=True):
super().__init__()
self.config = config
self.scale = scale
self.last_channel = last_channel
self.num_classes = num_classes
self.with_pool = with_pool
self.firstconv_in_channels = config[0].in_channels
self.lastconv_in_channels = config[-1].in_channels
self.lastconv_out_channels = self.lastconv_in_channels * 6
norm_layer = partial(nn.BatchNorm2D, epsilon=0.001, momentum=0.99)
self.conv = ConvNormActivation(in_channels=3,
out_channels=self.firstconv_in_channels,
kernel_size=3,
stride=2,
padding=1,
groups=1,
activation_layer=nn.Hardswish,
norm_layer=norm_layer)
self.blocks = nn.Sequential(*[
InvertedResidual(in_channels=cfg.in_channels,
expanded_channels=cfg.expanded_channels,
out_channels=cfg.out_channels,
filter_size=cfg.kernel,
stride=cfg.stride,
use_se=cfg.use_se,
activation_layer=cfg.activation_layer,
norm_layer=norm_layer) for cfg in self.config
])
self.lastconv = ConvNormActivation(
in_channels=self.lastconv_in_channels,
out_channels=self.lastconv_out_channels,
kernel_size=1,
stride=1,
padding=0,
groups=1,
norm_layer=norm_layer,
activation_layer=nn.Hardswish)
if with_pool:
self.avgpool = nn.AdaptiveAvgPool2D(1)
if num_classes > 0:
self.classifier = nn.Sequential(
nn.Linear(self.lastconv_out_channels, self.last_channel),
nn.Hardswish(), nn.Dropout(p=0.2),
nn.Linear(self.last_channel, num_classes))
def __init__(self, in_channels=3, out_channels=3, kernel_size=3, *args):
super(Block, self).__init__()
self.nn = nn.Sequential(
nn.Conv1D(in_channels=in_channels,
out_channels=out_channels,
kernel_size=kernel_size,
padding=0,
bias_attr=False),
nn.BatchNorm1D(num_features=out_channels), nn.LeakyReLU(.2),
nn.Dropout(p=.2))
def __init__(self,
dim,
hidden_dim,
num_heads=8,
qkv_bias=False,
attn_drop=0.,
proj_drop=0.):
super().__init__()
self.hidden_dim = hidden_dim
self.num_heads = num_heads
head_dim = hidden_dim // num_heads
self.head_dim = head_dim
self.scale = head_dim**-0.5
self.qk = nn.Linear(dim, hidden_dim * 2, bias_attr=qkv_bias)
self.v = nn.Linear(dim, dim, bias_attr=qkv_bias)
self.attn_drop = nn.Dropout(attn_drop)
self.proj = nn.Linear(dim, dim)
self.proj_drop = nn.Dropout(proj_drop)
def __init__(self, num_layers, input_size, hidden_size):
super(Seq2SeqDecoderCell, self).__init__()
self.dropout = nn.Dropout(0.2)
self.lstm_cells = nn.LayerList([
nn.LSTMCell(input_size=input_size +
hidden_size if i == 0 else hidden_size,
hidden_size=hidden_size) for i in range(num_layers)
])
self.attention_layer = AttentionLayer(hidden_size)
def __init__(self, features, num_classes=100, with_pool=False):
super(VGG, self).__init__()
self.features = features
self.num_classes = num_classes
self.with_pool = with_pool
if with_pool:
self.avgpool = nn.AdaptiveAvgPool2D((7, 7))
if num_classes > 0:
self.classifier = nn.Sequential(
nn.Linear(512, 4096),
nn.ReLU(),
nn.Dropout(),
nn.Linear(4096, 4096),
nn.ReLU(),
nn.Dropout(),
nn.Linear(4096, num_classes),
)
def __init__(self, vocab_size, hidden_size, max_position_size,
dropout_ratio):
"""
Initialization
"""
super(EnhancedEmbedding, self).__init__()
self.word_embedding = nn.Embedding(vocab_size, hidden_size)
self.position_embedding = nn.Embedding(max_position_size, hidden_size)
self.LayerNorm = LayerNorm(hidden_size)
self.dropout = nn.Dropout(dropout_ratio)
def __init__(self, albert, classifier_dropout_prob=0, num_classes=2):
super(AlbertForSequenceClassification, self).__init__()
self.num_classes = num_classes
self.transformer = albert
self.dropout = nn.Dropout(classifier_dropout_prob)
self.classifier = nn.Linear(self.transformer.config["hidden_size"],
self.num_classes)
self.init_weights()
def __init__(self, tinybert, num_classes=2, dropout=None):
super(TinyBertForSequenceClassification, self).__init__()
self.tinybert = tinybert
self.num_classes = num_classes
self.dropout = nn.Dropout(dropout if dropout is not None else self.
tinybert.config["hidden_dropout_prob"])
self.classifier = nn.Linear(self.tinybert.config["hidden_size"],
num_classes)
self.activation = nn.ReLU()
self.apply(self.init_weights)
def __init__(self, vocab_size, layer_size, block_size, embedding_dropout,
embedding_size, num_attention_heads, attention_dropout,
residual_dropout):
super(GPT2Model, self).__init__()
self.word_embeddings = nn.Embedding(vocab_size, embedding_size)
self.position_embeddings = nn.Embedding(block_size, embedding_size)
self.emb_drop = nn.Dropout(embedding_dropout)
self.transformer = Transformer(layer_size, embedding_size,
num_attention_heads, attention_dropout,
residual_dropout)
