def __init__(self, size, self_attn, src_attn, feed_forward, dropout):
super(DecoderLayer, self).__init__()
self.size = size
self.self_attn = self_attn
self.src_attn = src_attn
self.feed_forward = feed_forward
self.sublayer = clones(SublayerConnection(size, dropout), 3)
def __init__(self, size: int, self_attn: MultiHeadedAttention,
feed_forward: PositionwiseFeedForward,
dropout: float) -> None:
super().__init__()
self.self_attn = self_attn
self.feed_forward = feed_forward
self.sublayer = clones(SublayerConnection(size, dropout), 2)
self.size = size
def __init__(self, h: int, d_model: int, dropout: float = 0.1) -> None:
super().__init__()
assert d_model % h == 0
# We assume d_v always equals d_k
self.d_k = d_model // h
self.h = h
self.linears = clones(nn.Linear(d_model, d_model), 4)
self.attn = None
self.dropout = nn.Dropout(p=dropout)
def __init__(self, h, d_model, dropout=0.1):
""" Take in model size and the number of heads. """
super(MultiHeadedAttention, self).__init__()
assert d_model % h == 0
# Assume d_v always equals d_k
self.d_k = d_model // h
self.h = h
self.linears = clones(nn.Linear(d_model, d_model), 4)
self.attn = None
self.dropout = nn.Dropout(p=dropout)
def __init__(self, layer: EncoderLayer, n: int) -> None:
super().__init__()
self.layers = clones(layer, n)
self.norm = LayerNorm(layer.size)
def __init__(self, layer, N):
super(Decoder, self).__init__()
self.layers = clones(layer, N)
self.norm = Norm(layer.size)