def __init__(self, in_dim): super().__init__() self.channle_in = in_dim self.query_conv = nn.Conv1D( in_channels=in_dim, out_channels=in_dim // 2, kernel_size=1) self.key_conv = nn.Conv1D( in_channels=in_dim, out_channels=in_dim // 2, kernel_size=1) self.value_conv_vis = nn.Conv1D( in_channels=in_dim, out_channels=in_dim, kernel_size=1) self.value_conv_word = nn.Conv1D( in_channels=in_dim, out_channels=in_dim, kernel_size=1) self.softmax_vis = nn.Softmax(axis=-1) self.softmax_word = nn.Softmax(axis=-2)
def __init__(self, num_attention_heads, attention_probs_dropout_prob, cin, q_groups=1, k_groups=1, v_groups=1): super().__init__() if cin % num_attention_heads != 0: raise ValueError( f"cin ({cin}) is not a multiple of the number of attention heads ({num_attention_heads})" ) self.num_attention_heads = num_attention_heads self.attention_head_size = int(cin / num_attention_heads) self.all_head_size = self.num_attention_heads * self.attention_head_size self.query = nn.Conv1D(in_channels=cin, out_channels=cin, kernel_size=1, groups=q_groups) self.key = nn.Conv1D(in_channels=cin, out_channels=cin, kernel_size=1, groups=k_groups) self.value = nn.Conv1D(in_channels=cin, out_channels=cin, kernel_size=1, groups=v_groups) self.dropout = nn.Dropout(attention_probs_dropout_prob) self.softmax = nn.Softmax(axis=-1) self.matmul_qk = MatMulWrapper() self.matmul_qkv = MatMulWrapper()
def __init__(self, attention_layer, vocab_size, num_classes, emb_dim=128, lstm_hidden_size=196, fc_hidden_size=96, lstm_layers=1, dropout_rate=0.0, padding_idx=0): super().__init__() self.padding_idx = padding_idx self.embedder = nn.Embedding(num_embeddings=vocab_size, embedding_dim=emb_dim, padding_idx=padding_idx) self.bilstm = nn.LSTM(input_size=emb_dim, hidden_size=lstm_hidden_size, num_layers=lstm_layers, dropout=dropout_rate, direction='bidirect') self.attention = attention_layer if isinstance(attention_layer, SelfAttention): self.fc = nn.Linear(lstm_hidden_size, fc_hidden_size) elif isinstance(attention_layer, SelfInteractiveAttention): self.fc = nn.Linear(lstm_hidden_size * 2, fc_hidden_size) else: raise RuntimeError("Unknown attention type %s." % attention_layer.__class__.__name__) self.output_layer = nn.Linear(fc_hidden_size, num_classes) self.softmax = nn.Softmax(axis=1)
def __init__(self, dim=32): super(Attention, self).__init__() self.dim = dim self.q_layer = nn.Linear(dim, dim, bias_attr=False) self.k_layer = nn.Linear(dim, dim, bias_attr=False) self.v_layer = nn.Linear(dim, dim, bias_attr=False) self.softmax = nn.Softmax(1)
def __init__(self, num_classes=10): super(ImperativeLenet, self).__init__() self.features = nn.Sequential( nn.Conv2D( in_channels=1, out_channels=6, kernel_size=3, stride=1, padding=1, bias_attr=False), nn.BatchNorm2D(6), nn.ReLU(), nn.MaxPool2D( kernel_size=2, stride=2), nn.Conv2D( in_channels=6, out_channels=16, kernel_size=5, stride=1, padding=0), nn.BatchNorm2D(16), nn.PReLU(), nn.MaxPool2D( kernel_size=2, stride=2)) self.fc = nn.Sequential( nn.Linear( in_features=400, out_features=120), nn.LeakyReLU(), nn.Linear( in_features=120, out_features=84), nn.Sigmoid(), nn.Linear( in_features=84, out_features=num_classes), nn.Softmax())
def forward(self, x, H, W): B, N, C = x.shape q = self.q(x).reshape([B, N, self.num_heads, C // self.num_heads]).transpose([0, 2, 1, 3]) if self.sr_ratio > 1: x_ = x.transpose([0, 2, 1]).reshape([B, C, H, W]) tmp_n = H * W // self.sr_ratio**2 x_ = self.sr(x_).reshape([B, C, tmp_n]).transpose([0, 2, 1]) x_ = self.norm(x_) kv = self.kv(x_).reshape( [B, tmp_n, 2, self.num_heads, C // self.num_heads]).transpose([2, 0, 3, 1, 4]) else: kv = self.kv(x).reshape( [B, N, 2, self.num_heads, C // self.num_heads]).transpose([2, 0, 3, 1, 4]) k, v = kv[0], kv[1] attn = paddle.matmul(q, k.transpose([0, 1, 3, 2])) * self.scale attn = nn.Softmax(axis=-1)(attn) attn = self.attn_drop(attn) x = paddle.matmul(attn, v).transpose([0, 2, 1, 3]).reshape([B, N, C]) x = self.proj(x) x = self.proj_drop(x) return x
def __init__(self, dnn_units=[8, 64, 16], dnn_activation='sigmoid', weight_normalization=False, name=None): super().__init__() self.dnn_units = dnn_units self.dnn_activation = 'sigmoid' self.weight_normalization = weight_normalization self.name = name layer_list = [] #bn_list = [] for i in range(len(dnn_units) - 1): dnn_layer = nn.Linear(in_features=self.dnn_units[i] if i != 0 else self.dnn_units[i] * 4, out_features=self.dnn_units[i + 1], weight_attr=self._weight_init()) self.add_sublayer(self.name + f'linear_{i}', dnn_layer) layer_list.append(dnn_layer) #layer_list.append(copy.deepcopy(dnn_layer)) #bn_layer = nn.BatchNorm(50) #self.add_sublayer(self.name + f'bn_{i}', bn_layer) #bn_list.append(bn_layer) #bn_list.append(copy.deepcopy(bn_layer)) #self.bn_layer = nn.LayerList(bn_list) self.layers = nn.LayerList(layer_list) self.dnn = nn.Linear(self.dnn_units[-1], 1, weight_attr=self._weight_init()) self.activation = nn.Sigmoid() self.soft = nn.Softmax()
def main(args): # define model model = paddlevision.models.__dict__[args.model]( pretrained=args.pretrained, num_classes=args.num_classes) model = nn.Sequential(model, nn.Softmax()) model.eval() # define transforms eval_transforms = ClassificationPresetEval(args.resize_size, args.crop_size) with open(args.img_path, 'rb') as f: img = Image.open(f).convert('RGB') img = eval_transforms(img) img = paddle.to_tensor(img) img = img.expand([1] + img.shape) output = model(img).numpy()[0] class_id = output.argmax() prob = output[class_id] print(f"class_id: {class_id}, prob: {prob}") return output
def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.): super().__init__() self.dim = dim self.window_size = window_size # Wh, Ww self.num_heads = num_heads head_dim = dim // num_heads self.scale = qk_scale or head_dim ** -0.5 # define a parameter table of relative position bias relative_position_bias_table = self.create_parameter( shape=((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads), default_initializer=nn.initializer.Constant(value=0)) # 2*Wh-1 * 2*Ww-1, nH self.add_parameter("relative_position_bias_table", relative_position_bias_table) # get pair-wise relative position index for each token inside the window coords_h = paddle.arange(self.window_size[0]) coords_w = paddle.arange(self.window_size[1]) coords = paddle.stack(paddle.meshgrid([coords_h, coords_w])) # 2, Wh, Ww coords_flatten = paddle.flatten(coords, 1) # 2, Wh*Ww relative_coords = coords_flatten.unsqueeze(-1) - coords_flatten.unsqueeze(1) # 2, Wh*Ww, Wh*Ww relative_coords = relative_coords.transpose([1, 2, 0]) # Wh*Ww, Wh*Ww, 2 relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 relative_coords[:, :, 1] += self.window_size[1] - 1 relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 self.relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww self.register_buffer("relative_position_index", self.relative_position_index) self.qkv = nn.Linear(dim, dim * 3, bias_attr=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) self.proj = nn.Linear(dim, dim) self.proj_drop = nn.Dropout(proj_drop) self.softmax = nn.Softmax(axis=-1)
def forward(self, hidden_states, attention_mask): """Self-Attention block.""" temp_q = self.q(hidden_states) temp_k = self.k(hidden_states) temp_v = self.v(hidden_states) q_layer = self.score_transpose(temp_q) k_layer = self.score_transpose(temp_k) v_layer = self.score_transpose(temp_v) attention_score = paddle.matmul(q_layer, k_layer.transpose(-1, -2)) attention_score = attention_score / math.sqrt(self.head_size) attention_score = attention_score + attention_mask attention_prob = nn.Softmax(axis=-1)(attention_score) attention_prob = self.dropout(attention_prob) attention_layer = paddle.matmul(attention_prob, v_layer) attention_layer = attention_layer.permute(0, 2, 1, 3).contiguous() temp_attention_layer = attention_layer.size()[:-2] + [ self.all_head_size ] attention_map = attention_layer.view(*temp_attention_layer) return attention_map
def __init__(self, block, layers, num_filters, feature_dim, encoder_type='SAP', n_mels=40, log_input=True, **kwargs): super(ResNetSE, self).__init__() print('Embedding size is %d, encoder %s.' % (feature_dim, encoder_type)) self.inplanes = num_filters[0] self.encoder_type = encoder_type self.n_mels = n_mels self.log_input = log_input self.conv1 = nn.Conv2D(1, num_filters[0], kernel_size=3, stride=1, padding=1) self.relu = nn.ReLU() self.bn1 = nn.BatchNorm2D(num_filters[0]) self.layer1 = self._make_layer(block, num_filters[0], layers[0]) self.layer2 = self._make_layer(block, num_filters[1], layers[1], stride=(2, 2)) self.layer3 = self._make_layer(block, num_filters[2], layers[2], stride=(2, 2)) self.layer4 = self._make_layer(block, num_filters[3], layers[3], stride=(2, 2)) outmap_size = int(self.n_mels / 8) self.attention = nn.Sequential( nn.Conv1D(num_filters[3] * outmap_size, 128, kernel_size=1), nn.ReLU(), nn.BatchNorm1D(128), nn.Conv1D(128, num_filters[3] * outmap_size, kernel_size=1), nn.Softmax(axis=2), ) if self.encoder_type == "SAP": out_dim = num_filters[3] * outmap_size elif self.encoder_type == "ASP": out_dim = num_filters[3] * outmap_size * 2 else: raise ValueError('Undefined encoder') self.fc = nn.Linear(out_dim, feature_dim)
def __init__(self, roberta, num_classes=2, dropout=None): super(RobertaForSequenceClassification, self).__init__() self.num_classes = num_classes self.roberta = roberta # allow roberta to be config self.dropout = nn.Dropout(dropout if dropout is not None else self. roberta.config["hidden_dropout_prob"]) self.classifier = nn.Linear(self.roberta.config["hidden_size"], num_classes) self.softmax = nn.Softmax() self.apply(self.init_weights)
def __init__(self, act=None, axis=-1): super().__init__() if act is not None: assert act in ["softmax", "sigmoid"] if act == "softmax": self.act = nn.Softmax(axis=axis) elif act == "sigmoid": self.act = nn.Sigmoid() else: self.act = None
def __init__(self, in_channels, ds=8, activation=nn.ReLU): super(BAM, self).__init__() self.key_channel = in_channels //8 self.activation = activation self.ds = ds self.pool = nn.AvgPool2D(self.ds) self.query_conv = nn.Conv2D(in_channels=in_channels, out_channels=in_channels // 8, kernel_size=1) self.key_conv = nn.Conv2D(in_channels=in_channels, out_channels=in_channels // 8, kernel_size=1) self.value_conv = nn.Conv2D(in_channels=in_channels, out_channels=in_channels, kernel_size=1) self.gamma = nn.ParameterList([paddle.create_parameter(shape=[1], dtype='float32', default_initializer=nn.initializer.Constant(value=0))]) self.softmax = nn.Softmax(axis=-1)
def __init__(self, bond_dim, hidden_dim, num_angle): super(PiPoolLayer, self).__init__() self.bond_dim = bond_dim self.num_angle = num_angle self.num_type = 4 * 9 fc_in_dim = num_angle * bond_dim self.fc_1 = DenseLayer(fc_in_dim, hidden_dim, activation=F.relu, bias=True) self.fc_2 = nn.Linear(hidden_dim, 1, bias_attr=False) self.softmax = nn.Softmax(axis=1)
def __init__(self, act=None): super().__init__() if act is not None: assert act in ["softmax", "sigmoid"] if act == "softmax": self.act = nn.Softmax(axis=-1) elif act == "sigmoid": self.act = nn.Sigmoid() else: self.act = None self.jskl_loss = KLJSLoss(mode="js")
def __init__(self, num_classes=10, classifier_activation='softmax'): super(LeNetDygraph, self).__init__() self.num_classes = num_classes self.features = nn.Sequential(nn.Conv2d(1, 6, 3, stride=1, padding=1), nn.ReLU(), nn.Pool2D(2, 'max', 2), nn.Conv2d(6, 16, 5, stride=1, padding=0), nn.ReLU(), nn.Pool2D(2, 'max', 2)) if num_classes > 0: self.fc = nn.Sequential(nn.Linear(400, 120), nn.Linear(120, 84), nn.Linear(84, 10), nn.Softmax()) #Todo: accept any activation
def __init__(self, act='softmax', axis=-1, reduction='mean'): super().__init__() assert act in ['softmax', 'sigmoid', None] self.reduction = reduction if act == 'softmax': self.act = nn.Softmax(axis=axis) elif act == 'sigmoid': self.act = nn.Sigmoid() else: self.act = None
def build_mlp(hidden_size: int, num_hidden_layers: int, output_size: int): FC_layers = [] FC_layers.extend([ nn.Linear(in_features=hidden_size, out_features=output_size), nn.ReLU() ]) for _ in range(num_hidden_layers - 1): FC_layers.extend([ nn.Linear(in_features=output_size, out_features=output_size), nn.ReLU() ]) FC_layers.append(nn.Softmax()) # 最后一层使用softmax return nn.Sequential(*FC_layers)
def __init__(self, stride=1, padding=0, dilation=1, groups=1, padding_mode="zeros"): super(Conv2D1, self).__init__() self.conv = nn.Conv2D( 3, 6, 3, stride=stride, padding=padding, dilation=dilation, groups=groups, padding_mode=padding_mode, ) self.softmax = nn.Softmax()
def export(args): model = paddlevision.models.__dict__[args.model]( pretrained=args.pretrained, num_classes=args.num_classes) model = nn.Sequential(model, nn.Softmax()) model.eval() model = paddle.jit.to_static( model, input_spec=[ InputSpec(shape=[None, 3, args.img_size, args.img_size], dtype='float32') ]) paddle.jit.save(model, os.path.join(args.save_inference_dir, "inference")) print(f"inference model has been saved into {args.save_inference_dir}")
def __init__(self, nsp_reader, num_layers, n_head, hidden_size, vocab_size=8001, type_size=2, latent_type_size=20, max_position_seq_len=256, act_dropout=0.1, attn_dropout=0.1, max_dec_len=64, min_dec_len=1, topk=10): super(Plato2InferModel, self).__init__() self.nsp_reader = nsp_reader self.num_layers = num_layers self.latent_type_size = latent_type_size self.max_dec_len = max_dec_len self.min_dec_len = min_dec_len self.topk = topk self.unk_id = 0 self.bos_id = 1 self.eos_id = 2 self.mask_id = 8000 self.after_eos = paddle.ones([vocab_size]) * -1e9 self.after_eos[self.eos_id] = 0 self.is_cn = False self.batch_size = 1 self.latent_weight = paddle.create_parameter( [hidden_size, latent_type_size], 'float32') self.plato2_encoder = Plato2Encoder( vocab_size, type_size, max_position_seq_len, num_layers, n_head, hidden_size, attn_dropout, act_dropout) self.logits_fc_layer = nn.Linear(hidden_size, hidden_size) self.logits_layer_norm = nn.LayerNorm(hidden_size) self.logits_bias = paddle.create_parameter( [vocab_size], 'float32', is_bias=True) self.nsp_predictor = NSP(vocab_size, type_size, max_position_seq_len, num_layers, n_head, hidden_size, attn_dropout, act_dropout) self.gelu_layer = nn.GELU() self.softmax = nn.Softmax()
def __init__(self, inplanes, use_scale=True, **kwargs): planes = inplanes // 2 self.use_scale = use_scale super(NonLocal, self).__init__(inplanes) self.t = nn.Conv2D(inplanes, planes, kernel_size=1, stride=1, bias_attr=True) self.p = nn.Conv2D(inplanes, planes, kernel_size=1, stride=1, bias_attr=True) self.g = nn.Conv2D(inplanes, planes, kernel_size=1, stride=1, bias_attr=True) self.softmax = nn.Softmax(axis=2) self.z = nn.Conv2D(planes, inplanes, kernel_size=1, stride=1, bias_attr=True) self.bn = nn.BatchNorm2D(inplanes)
def __init__(self, ernie, pinyin_vocab_size, pad_pinyin_id=0): super(ErnieForCSC, self).__init__() self.ernie = ernie emb_size = self.ernie.config["hidden_size"] hidden_size = self.ernie.config["hidden_size"] vocab_size = self.ernie.config["vocab_size"] self.pad_token_id = self.ernie.config["pad_token_id"] self.pinyin_vocab_size = pinyin_vocab_size self.pad_pinyin_id = pad_pinyin_id self.pinyin_embeddings = nn.Embedding(self.pinyin_vocab_size, emb_size, padding_idx=pad_pinyin_id) self.detection_layer = nn.Linear(hidden_size, 2) self.correction_layer = nn.Linear(hidden_size, vocab_size) self.softmax = nn.Softmax()
def __init__(self, in_channels, ratio): super().__init__() self.conv_mask = nn.Conv2D(in_channels=in_channels, out_channels=1, kernel_size=1) self.softmax = nn.Softmax(axis=2) inter_channels = int(in_channels * ratio) self.channel_add_conv = nn.Sequential( nn.Conv2D(in_channels=in_channels, out_channels=inter_channels, kernel_size=1), nn.LayerNorm(normalized_shape=[inter_channels, 1, 1]), nn.ReLU(), nn.Conv2D(in_channels=inter_channels, out_channels=in_channels, kernel_size=1))
def __init__(self, config, model, use_multilabel): super().__init__() self.base_model = model # we should choose a final model to export if isinstance(self.base_model, DistillationModel): self.infer_model_name = config["infer_model_name"] else: self.infer_model_name = None self.infer_output_key = config.get("infer_output_key", None) if self.infer_output_key == "features" and isinstance( self.base_model, RecModel): self.base_model.head = IdentityHead() if use_multilabel: self.out_act = nn.Sigmoid() else: if config.get("infer_add_softmax", True): self.out_act = nn.Softmax(axis=-1) else: self.out_act = None
def __init__(self, in_channels=3, out_classes=5, hid=64, num=64): super(FallNet, self).__init__() self.cnn0 = Block(in_channels, hid, 7, 0) self.cnn1 = Block(hid, hid, 5, 0) self.cnn2 = Block(hid, hid, 3, 0) self.cnn3 = Block(hid, hid, 1, 0) self.avg = nn.AdaptiveAvgPool1D(output_size=num) # self.rnn0 = nn.LSTM(input_size=145, hidden_size=num, dropout=.2, num_layers=3) self.rnn0 = nn.GRU(input_size=145, hidden_size=num, num_layers=1, dropout=0.2) self.rnn1 = Block(hid, hid, 1, 0) self.rnn2 = Block(hid, 4, 3, 0) self.cls = nn.Sequential( nn.Linear(in_features=1016, out_features=128), nn.Dropout(p=.2), nn.Linear(in_features=128, out_features=out_classes), nn.Softmax(axis=1))
def forward(self, x, H, W): B, N, C = x.shape h_group, w_group = H // self.ws, W // self.ws total_groups = h_group * w_group x = x.reshape([B, h_group, self.ws, w_group, self.ws, C]).transpose([0, 1, 3, 2, 4, 5]) qkv = self.qkv(x).reshape([ B, total_groups, self.ws**2, 3, self.num_heads, C // self.num_heads ]).transpose([3, 0, 1, 4, 2, 5]) q, k, v = qkv[0], qkv[1], qkv[2] attn = paddle.matmul(q, k.transpose([0, 1, 2, 4, 3])) * self.scale attn = nn.Softmax(axis=-1)(attn) attn = self.attn_drop(attn) attn = paddle.matmul(attn, v).transpose([0, 1, 3, 2, 4]).reshape( [B, h_group, w_group, self.ws, self.ws, C]) x = attn.transpose([0, 1, 3, 2, 4, 5]).reshape([B, N, C]) x = self.proj(x) x = self.proj_drop(x) return x
def forward( self, query: paddle.Tensor, key: paddle.Tensor, value: paddle.Tensor, attn_mask: Optional[paddle.Tensor] = None, ) -> Tuple[paddle.Tensor, paddle.Tensor]: r""" Args: query: [batch, num_attention_heads, len_query, dim_query] key: [batch, num_attention_heads, len_key, dim_key] value: [batch, num_attention_heads, len_value, dim_value] attn_mask: [batch, num_attention_heads, len_query, len_key] """ attention = paddle.matmul(query, key.transpose((0, 1, 3, 2))) attention = attention / math.sqrt(query.shape[-1]) if attn_mask is not None: attention = attention + attn_mask attention = nn.Softmax(axis=-1)(attention) attention = self.dropout(attention) context = paddle.matmul(attention, value) return context, attention
def __init__(self, vocab_size, num_classes, emb_dim=128, padding_idx=0, lstm_hidden_size=198, direction='forward', lstm_layers=1, dropout_rate=0.0, pooling_type=None, fc_hidden_size=96): super().__init__() self.direction = direction self.embedder = nn.Embedding(num_embeddings=vocab_size, embedding_dim=emb_dim, padding_idx=padding_idx) # self.lstm_encoder = nlp.seq2vec.LSTMEncoder(emb_dim, # lstm_hidden_size, # num_layers=lstm_layers, # direction=direction, # dropout=dropout_rate, # pooling_type=pooling_type) self.lstm_layer = nn.LSTM(input_size=emb_dim, hidden_size=lstm_hidden_size, num_layers=lstm_layers, direction=direction, dropout=dropout_rate) self.fc = nn.Linear( lstm_hidden_size * (2 if direction == 'bidirect' else 1), fc_hidden_size) self.output_layer = nn.Linear(fc_hidden_size, num_classes) self.softmax = nn.Softmax(axis=1)