def __init__(self, ernie_gram, num_classes=2, dropout=None): super(ErnieGramForSequenceClassification, self).__init__() self.num_classes = num_classes self.ernie_gram = ernie_gram # allow ernie gram to be config self.dropout = nn.Dropout(dropout if dropout is not None else self. ernie_gram.config["hidden_dropout_prob"]) self.classifier = nn.Linear(self.ernie_gram.config["hidden_size"], num_classes) self.apply(self.init_weights)
def __init__(self, cin, cout, groups, dropout_prob): super().__init__() self.conv1d = nn.Conv1D(in_channels=cin, out_channels=cout, kernel_size=1, groups=groups) self.layernorm = SqueezeBertLayerNorm(cout) self.dropout = nn.Dropout(dropout_prob)
def __init__(self, squeezebert, num_classes=2, dropout=None): super().__init__() self.num_classes = num_classes self.squeezebert = squeezebert self.dropout = nn.Dropout(dropout if dropout is not None else self. squeezebert.config["hidden_dropout_prob"]) self.classifier = nn.Linear(self.squeezebert.config["hidden_size"], num_classes) self.apply(self.init_weights)
def __init__(self, skep, num_classes=2, dropout=None): super(SkepSequenceModel, self).__init__() self.num_classes = num_classes self.skep = skep # allow skep to be config self.dropout = nn.Dropout(dropout if dropout is not None else self.skep.config["hidden_dropout_prob"]) self.classifier = nn.Linear(self.skep.config["hidden_size"], num_classes) self.apply(self.init_weights)
def __init__(self, roberta, num_classes=2, dropout=None): super(RobertaForSequenceClassification, self).__init__() self.num_classes = num_classes self.roberta = roberta # allow roberta to be config self.dropout = nn.Dropout(dropout if dropout is not None else self. roberta.config["hidden_dropout_prob"]) self.classifier = nn.Linear(self.roberta.config["hidden_size"], num_classes) self.apply(self.init_weights)
def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4, qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer='nn.LayerNorm', epsilon=1e-5, final_norm=False, pretrained=None, **args): super().__init__() self.img_size = img_size self.embed_dim = embed_dim self.patch_embed = PatchEmbed(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) self.pos_w = self.patch_embed.num_patches_in_w self.pos_h = self.patch_embed.num_patches_in_h self.pos_embed = self.create_parameter( shape=(1, self.pos_w * self.pos_h + 1, embed_dim), default_initializer=paddle.nn.initializer.TruncatedNormal(std=.02)) self.cls_token = self.create_parameter( shape=(1, 1, embed_dim), default_initializer=paddle.nn.initializer.Constant(value=0.)) self.pos_drop = nn.Dropout(p=drop_rate) dpr = np.linspace(0, drop_path_rate, depth) self.blocks = nn.LayerList([ Block(dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, epsilon=epsilon) for i in range(depth) ]) self.final_norm = final_norm if self.final_norm: self.norm = eval(norm_layer)(embed_dim, epsilon=epsilon) self.pretrained = pretrained self.init_weight()
def __init__(self, layoutlm, num_classes=2, dropout=None): super(LayoutLMForTokenClassification, self).__init__() self.num_classes = num_classes self.layoutlm = layoutlm self.dropout = nn.Dropout(dropout if dropout is not None else self. layoutlm.config["hidden_dropout_prob"]) self.classifier = nn.Linear(self.layoutlm.config["hidden_size"], num_classes) self.classifier.apply(self.init_weights)
def __init__(self, params_file, batch_size, sent_embedding_dim, num_labels): super(ELMoBowTextClassification, self).__init__() self._elmo = get_elmo_layer(params_file, batch_size, trainable=True) word_embedding_dim = self._elmo.embedding_dim self._fc1 = nn.Linear(word_embedding_dim, sent_embedding_dim) self._fc2 = nn.Linear(sent_embedding_dim, num_labels) self._dropout = nn.Dropout(p=0.5)
def __init__(self, in_feats, hidden, out_feats, num_hops, n_layers, dropout, input_drop): super(SIGN, self).__init__() self.dropout = nn.Dropout(dropout) self.prelu = nn.PReLU() self.inception_ffs = nn.LayerList() self.input_drop = input_drop for i in range(num_hops): self.inception_ffs.append(FeedForwardNet(in_feats, hidden, hidden, n_layers, dropout)) self.project = FeedForwardNet(num_hops * hidden, hidden, out_feats, n_layers, dropout)
def __init__(self, nezha, num_classes=2, dropout=None): super(NeZhaForTokenClassification, self).__init__() self.num_classes = num_classes self.nezha = nezha self.dropout = nn.Dropout(dropout if dropout is not None else self.nezha.config["hidden_dropout_prob"]) self.classifier = nn.Linear(self.nezha.config["hidden_size"], num_classes) self.apply(self.init_weights)
def __init__(self, ernie, num_classes=2, dropout=None): super(ErnieForTokenClassification, self).__init__() self.num_classes = num_classes self.ernie = ernie # allow ernie to be config self.dropout = nn.Dropout(dropout if dropout is not None else self.ernie.config["hidden_dropout_prob"]) self.classifier = nn.Linear(self.ernie.config["hidden_size"], num_classes) self.apply(self.init_weights)
def __init__(self, input_dim: int, inner_dim: int, num_classes: int, pooler_dropout: float): super().__init__() self.dense = nn.Linear(input_dim, inner_dim) self.dropout = nn.Dropout(p=pooler_dropout) self.out_proj = nn.Linear(inner_dim, num_classes)
def __init__(self, roformerv2, num_classes=2, dropout=None): super(RoFormerv2ForTokenClassification, self).__init__() self.num_classes = num_classes self.roformerv2 = roformerv2 # allow roformerv2 to be config self.dropout = nn.Dropout(dropout if dropout is not None else self.roformerv2.config["hidden_dropout_prob"]) self.classifier = nn.Linear(self.roformerv2.config["hidden_size"], num_classes) self.apply(self.init_weights)
def __init__(self, gau_alpha, num_classes=2, dropout=None): super(GAUAlphaForTokenClassification, self).__init__() self.num_classes = num_classes self.gau_alpha = gau_alpha # allow gau_alpha to be config self.dropout = nn.Dropout(dropout if dropout is not None else self. gau_alpha.config["hidden_dropout_prob"]) self.classifier = nn.Linear(self.gau_alpha.config["hidden_size"], num_classes) self.apply(self.init_weights)
def __init__( self, classifier_dropout_prob, hidden_size, num_labels, ): super(AlbertSOPHead, self).__init__() self.dropout = nn.Dropout(classifier_dropout_prob) self.classifier = nn.Linear(hidden_size, num_labels)
def __init__(self, bert, num_classes=2, dropout=None): super(BertForTokenClassification, self).__init__() self.num_classes = num_classes self.bert = bert # allow bert to be config self.dropout = nn.Dropout(dropout if dropout is not None else self. bert.config["hidden_dropout_prob"]) self.classifier = nn.Linear(self.bert.config["hidden_size"], num_classes) self.apply(self.init_weights)
def __init__(self, pretrained_model, dropout=None, rdrop_coef=0.0): super().__init__() self.ptm = pretrained_model self.dropout = nn.Dropout(dropout if dropout is not None else 0.1) # num_labels = 2 (similar or dissimilar) self.classifier = nn.Linear(self.ptm.config["hidden_size"], 2) self.rdrop_coef = rdrop_coef self.rdrop_loss = ppnlp.losses.RDropLoss()
def __init__(self, bigbird, num_classes=None): super(BigBirdForSequenceClassification, self).__init__() self.bigbird = bigbird if num_classes is None: num_classes = self.bigbird.config["num_labels"] self.linear = nn.Linear(self.bigbird.config["hidden_size"], num_classes) self.dropout = nn.Dropout( self.bigbird.config['hidden_dropout_prob'], mode="upscale_in_train") self.apply(self.init_weights)
def __init__(self, hidden_size, hidden_dropout_prob, intermediate_size, layer_norm_eps=1e-12): super(RemBertOutput, self).__init__() self.dense = nn.Linear(intermediate_size, hidden_size) self.layer_norm = nn.LayerNorm(hidden_size, epsilon=layer_norm_eps) self.dropout = nn.Dropout(hidden_dropout_prob)
def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.2): super(TemporalBlock, self).__init__() self.conv1 = weight_norm( nn.Conv1D( n_inputs, n_outputs, kernel_size, stride=stride, padding=padding, dilation=dilation)) # Chomp1d is used to make sure the network is causal. # We pad by (k-1)*d on the two sides of the input for convolution, # and then use Chomp1d to remove the (k-1)*d output elements on the right. self.chomp1 = Chomp1d(padding) self.relu1 = nn.ReLU() self.dropout1 = nn.Dropout(dropout) self.conv2 = weight_norm( nn.Conv1D( n_outputs, n_outputs, kernel_size, stride=stride, padding=padding, dilation=dilation)) self.chomp2 = Chomp1d(padding) self.relu2 = nn.ReLU() self.dropout2 = nn.Dropout(dropout) self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, self.dropout1, self.conv2, self.chomp2, self.relu2, self.dropout2) self.downsample = nn.Conv1D(n_inputs, n_outputs, 1) if n_inputs != n_outputs else None self.relu = nn.ReLU() self.init_weights()
def __init__(self, d_model, nhead, dim_feedforward, dropout=0.1, activation="gelu", attn_dropout=None, act_dropout=None, normalize_before=True, weight_attr=None, bias_attr=None, topo=None): self._config = locals() self._config.pop("self") self._config.pop("__class__", None) # py3 super(TransformerDecoderLayer, self).__init__() attn_dropout = dropout if attn_dropout is None else attn_dropout act_dropout = dropout if act_dropout is None else act_dropout self.normalize_before = normalize_before weight_attrs = _convert_param_attr_to_list(weight_attr, 3) bias_attrs = _convert_param_attr_to_list(bias_attr, 3) self.self_attn = MultiHeadAttention(d_model, nhead, dropout=attn_dropout, weight_attr=weight_attrs[0], bias_attr=bias_attrs[0], topo=topo) self.linear1 = nn.Linear(d_model, dim_feedforward, weight_attrs[2], bias_attr=bias_attrs[2]) self.linear2 = nn.Linear(dim_feedforward, d_model, weight_attrs[2], bias_attr=bias_attrs[2]) self.norm1 = nn.LayerNorm(d_model, epsilon=1e-5) self.norm2 = nn.LayerNorm(d_model, epsilon=1e-5) self.dropout1 = nn.Dropout(dropout, mode="upscale_in_train") self.dropout2 = nn.Dropout(act_dropout, mode="upscale_in_train") self.activation = getattr(F, activation)
def __init__(self, config, last_channel, scale=1.0, num_classes=1000, with_pool=True): super().__init__() self.config = config self.scale = scale self.last_channel = last_channel self.num_classes = num_classes self.with_pool = with_pool self.firstconv_in_channels = config[0].in_channels self.lastconv_in_channels = config[-1].in_channels self.lastconv_out_channels = self.lastconv_in_channels * 6 norm_layer = partial(nn.BatchNorm2D, epsilon=0.001, momentum=0.99) self.conv = ConvNormActivation(in_channels=3, out_channels=self.firstconv_in_channels, kernel_size=3, stride=2, padding=1, groups=1, activation_layer=nn.Hardswish, norm_layer=norm_layer) self.blocks = nn.Sequential(*[ InvertedResidual(in_channels=cfg.in_channels, expanded_channels=cfg.expanded_channels, out_channels=cfg.out_channels, filter_size=cfg.kernel, stride=cfg.stride, use_se=cfg.use_se, activation_layer=cfg.activation_layer, norm_layer=norm_layer) for cfg in self.config ]) self.lastconv = ConvNormActivation( in_channels=self.lastconv_in_channels, out_channels=self.lastconv_out_channels, kernel_size=1, stride=1, padding=0, groups=1, norm_layer=norm_layer, activation_layer=nn.Hardswish) if with_pool: self.avgpool = nn.AdaptiveAvgPool2D(1) if num_classes > 0: self.classifier = nn.Sequential( nn.Linear(self.lastconv_out_channels, self.last_channel), nn.Hardswish(), nn.Dropout(p=0.2), nn.Linear(self.last_channel, num_classes))
def __init__(self, in_channels=3, out_channels=3, kernel_size=3, *args): super(Block, self).__init__() self.nn = nn.Sequential( nn.Conv1D(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, padding=0, bias_attr=False), nn.BatchNorm1D(num_features=out_channels), nn.LeakyReLU(.2), nn.Dropout(p=.2))
def __init__(self, dim, hidden_dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.): super().__init__() self.hidden_dim = hidden_dim self.num_heads = num_heads head_dim = hidden_dim // num_heads self.head_dim = head_dim self.scale = head_dim**-0.5 self.qk = nn.Linear(dim, hidden_dim * 2, bias_attr=qkv_bias) self.v = nn.Linear(dim, dim, bias_attr=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Linear(dim, dim) self.proj_drop = nn.Dropout(proj_drop)
def __init__(self, num_layers, input_size, hidden_size): super(Seq2SeqDecoderCell, self).__init__() self.dropout = nn.Dropout(0.2) self.lstm_cells = nn.LayerList([ nn.LSTMCell(input_size=input_size + hidden_size if i == 0 else hidden_size, hidden_size=hidden_size) for i in range(num_layers) ]) self.attention_layer = AttentionLayer(hidden_size)
def __init__(self, features, num_classes=100, with_pool=False): super(VGG, self).__init__() self.features = features self.num_classes = num_classes self.with_pool = with_pool if with_pool: self.avgpool = nn.AdaptiveAvgPool2D((7, 7)) if num_classes > 0: self.classifier = nn.Sequential( nn.Linear(512, 4096), nn.ReLU(), nn.Dropout(), nn.Linear(4096, 4096), nn.ReLU(), nn.Dropout(), nn.Linear(4096, num_classes), )
def __init__(self, vocab_size, hidden_size, max_position_size, dropout_ratio): """ Initialization """ super(EnhancedEmbedding, self).__init__() self.word_embedding = nn.Embedding(vocab_size, hidden_size) self.position_embedding = nn.Embedding(max_position_size, hidden_size) self.LayerNorm = LayerNorm(hidden_size) self.dropout = nn.Dropout(dropout_ratio)
def __init__(self, albert, classifier_dropout_prob=0, num_classes=2): super(AlbertForSequenceClassification, self).__init__() self.num_classes = num_classes self.transformer = albert self.dropout = nn.Dropout(classifier_dropout_prob) self.classifier = nn.Linear(self.transformer.config["hidden_size"], self.num_classes) self.init_weights()
def __init__(self, tinybert, num_classes=2, dropout=None): super(TinyBertForSequenceClassification, self).__init__() self.tinybert = tinybert self.num_classes = num_classes self.dropout = nn.Dropout(dropout if dropout is not None else self. tinybert.config["hidden_dropout_prob"]) self.classifier = nn.Linear(self.tinybert.config["hidden_size"], num_classes) self.activation = nn.ReLU() self.apply(self.init_weights)
def __init__(self, vocab_size, layer_size, block_size, embedding_dropout, embedding_size, num_attention_heads, attention_dropout, residual_dropout): super(GPT2Model, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, embedding_size) self.position_embeddings = nn.Embedding(block_size, embedding_size) self.emb_drop = nn.Dropout(embedding_dropout) self.transformer = Transformer(layer_size, embedding_size, num_attention_heads, attention_dropout, residual_dropout)