def __init__(self, layer, N, relative_pos_emb, num_heads): super(Encoder, self).__init__() self.layers = clones(layer, N) self.norm = nn.LayerNorm(layer.size) self.relative_pos_emb = relative_pos_emb self.num_heads = num_heads
def __init__(self, size, self_attn, src_attn, feed_forward, dropout): super(DecoderLayer, self).__init__() self.size = size self.self_attn = self_attn self.src_attn = src_attn self.feed_forward = feed_forward self.sublayer = utils.clones(utils.SublayerConnection(size, dropout), 3)
def __init__(self, h, d_model, dropout=0.1): super(MultiHeadedAttention, self).__init__() assert d_model % h == 0 self.d_k = d_model // h self.h = h self.linears = utils.clones(nn.Linear(d_model, d_model), 4) self.attn = None self.dropout = nn.Dropout(p=dropout)
def __init__(self, layer, N): """ Initialization :param layer: SubLayer -- subEncoder :param N: the number of subEncoder """ super(Encoder, self).__init__() self.layers = utils.clones(layer, N) self.norm = utils.LayerNorm(layer.size)
def __init__(self, model_dim, head_count, dropout=0.1): super(MultiHeadAttn, self).__init__() self.model_dim = model_dim assert model_dim % head_count == 0 self.dim_per_head = model_dim // head_count self.head_count = head_count self.linear_layers = clones(nn.Linear(model_dim, model_dim), 4) self.dropout = nn.Dropout(dropout) self.soft_max = nn.Softmax(dim=-1)
def __init__(self, d_model, k, num_heads, num_features, dropout=0.0): """ 生成相对位置信息编码 :param d_model: 词向量维度 :param k: 相对位置窗口大小 :param dropout: """ super(RelativePositionEmbedding, self).__init__() self.d_model = d_model self.k = k self.num_features = num_features self.num_heads = num_heads assert self.num_heads % self.num_features == 0 self.dropout = nn.Dropout(dropout) self.emb_list = clones( nn.Embedding(2 * k + 2, d_model * 2, padding_idx=0), num_features)
def __init__(self, layer, N): super(Decoder, self).__init__() self.layers = utils.clones(layer, N) self.norm = utils.LayerNorm(layer.size)
def __init__(self, size, self_attn, feed_forward, dropout): super(EncoderLayer, self).__init__() self.self_attn = self_attn self.feed_forward = feed_forward self.sublayer = clones(SublayerConnection(size, dropout), 2) self.size = size