def __init__(self, layer, N, relative_pos_emb, num_heads):
super(Encoder, self).__init__()
self.layers = clones(layer, N)
self.norm = nn.LayerNorm(layer.size)
self.relative_pos_emb = relative_pos_emb
self.num_heads = num_heads
def __init__(self, size, self_attn, src_attn, feed_forward, dropout):
super(DecoderLayer, self).__init__()
self.size = size
self.self_attn = self_attn
self.src_attn = src_attn
self.feed_forward = feed_forward
self.sublayer = utils.clones(utils.SublayerConnection(size, dropout), 3)
def __init__(self, h, d_model, dropout=0.1):
super(MultiHeadedAttention, self).__init__()
assert d_model % h == 0
self.d_k = d_model // h
self.h = h
self.linears = utils.clones(nn.Linear(d_model, d_model), 4)
self.attn = None
self.dropout = nn.Dropout(p=dropout)
def __init__(self, layer, N):
"""
Initialization
:param layer: SubLayer -- subEncoder
:param N: the number of subEncoder
"""
super(Encoder, self).__init__()
self.layers = utils.clones(layer, N)
self.norm = utils.LayerNorm(layer.size)
def __init__(self, model_dim, head_count, dropout=0.1):
super(MultiHeadAttn, self).__init__()
self.model_dim = model_dim
assert model_dim % head_count == 0
self.dim_per_head = model_dim // head_count
self.head_count = head_count
self.linear_layers = clones(nn.Linear(model_dim, model_dim), 4)
self.dropout = nn.Dropout(dropout)
self.soft_max = nn.Softmax(dim=-1)
def __init__(self, d_model, k, num_heads, num_features, dropout=0.0):
"""
生成相对位置信息编码
:param d_model: 词向量维度
:param k: 相对位置窗口大小
:param dropout:
"""
super(RelativePositionEmbedding, self).__init__()
self.d_model = d_model
self.k = k
self.num_features = num_features
self.num_heads = num_heads
assert self.num_heads % self.num_features == 0
self.dropout = nn.Dropout(dropout)
self.emb_list = clones(
nn.Embedding(2 * k + 2, d_model * 2, padding_idx=0), num_features)
def __init__(self, layer, N):
super(Decoder, self).__init__()
self.layers = utils.clones(layer, N)
self.norm = utils.LayerNorm(layer.size)
def __init__(self, size, self_attn, feed_forward, dropout):
super(EncoderLayer, self).__init__()
self.self_attn = self_attn
self.feed_forward = feed_forward
self.sublayer = clones(SublayerConnection(size, dropout), 2)
self.size = size
