def __init__(self, h, d_model, attn_p=0.1):
super(HierarchicalMultiHeadAttention, self).__init__()
self.h = h
self.d = d_model
assert d_model % h == 0
self.d_head = d_model // h
# first attention layer for states
self.fc_query = Bottle(Linear(d_model, h * self.d_head, bias=False))
self.fc_key = Bottle(Linear(d_model, h * self.d_head, bias=False))
self.fc_value = Bottle(Linear(d_model, h * self.d_head, bias=False))
# second attention for layers
self.fc_query_2 = Bottle(Linear(d_model, h * self.d_head, bias=False))
#~ self.fc_key_2 = Bottle(Linear(d_model, h*self.d_head, bias=False))
#~ self.fc_value_2 = Bottle(Linear(d_model, h*self.d_head, bias=False))
# for output
self.fc_concat = Bottle(Linear(h * self.d_head, d_model, bias=False))
self.fc_concat_2 = Bottle(Linear(d_model, d_model, bias=False))
self.sm = nn.Softmax(dim=-1)
self.sm_2 = nn.Softmax(dim=-1)
#~ self.attn_dropout = nn.Dropout(attn_p)
self.attn_dropout = StaticDropout(attn_p)
self.attn_dropout_2 = StaticDropout(attn_p)
def __init__(self,
d_model,
dropout_p,
sequence='nda',
static=True,
elementwise_affine=True):
super(PrePostProcessing, self).__init__()
self.d_model = d_model
self.dropout_p = dropout_p
self.steps = list(sequence)
if onmt.Constants.residual_type == 'gated':
# gated residual
# initialize k with one
self.k = nn.Parameter(torch.ones(1))
if 'n' in self.steps:
ln = nn.LayerNorm((self.d_model, ),
elementwise_affine=elementwise_affine)
#~ ln.weight.data.fill_(1)
self.layer_norm = Bottle(ln)
if 'd' in self.steps:
if static:
self.dropout = StaticDropout(self.dropout_p)
else:
self.dropout = nn.Dropout(self.dropout_p, inplace=False)
def __init__(self, h, d_model, attn_p=0.1, static=True, share=3):
super(MultiHeadAttention, self).__init__()
self.h = h
self.d = d_model
self.share = share
assert d_model % h == 0
self.d_head = d_model // h #D.S: d_head = d_v, d_k
#D.S. fc_query is fully conntected layer to produce the Linear combination of W_q * x_i = q_i for given word embedding x_i
self.fc_query = Bottle(
Linear(d_model, h * self.d_head, bias=False)
) #D.S: Bottle (Mask for skipping unnecesarry computations)
self.fc_key = Bottle(
Linear(d_model, h * self.d_head, bias=False)
) #D.S. Params Linear(d_in, d_out, bias=True, nonlinearity='linear'):
self.fc_value = Bottle(Linear(d_model, h * self.d_head, bias=False))
self.attention_out = onmt.Constants.attention_out #TODO: Constant not existing??
#D.S: Concat all outputs of heads to output of size d_model which is the output of encoder/decoder sublayer
self.fc_concat = Bottle(Linear(h * self.d_head, d_model, bias=False))
self.sm = nn.Softmax(dim=-1) #D.S: Apply softmax on last dimension
if static:
self.attn_dropout = StaticDropout(attn_p)
else:
self.attn_dropout = nn.Dropout(attn_p)
def __init__(self,
h,
d_model,
attn_p=0.1,
static=True,
share=3,
limit_rhs_steps=None):
super(MultiHeadAttention, self).__init__()
self.h = h
self.d = d_model
self.share = share
assert d_model % h == 0
self.d_head = d_model // h
self.fc_query = Bottle(Linear(d_model, h * self.d_head, bias=False))
self.fc_key = Bottle(Linear(d_model, h * self.d_head, bias=False))
self.fc_value = Bottle(Linear(d_model, h * self.d_head, bias=False))
self.fc_concat = Bottle(Linear(h * self.d_head, d_model, bias=False))
self.sm = nn.Softmax(dim=-1)
if static:
self.attn_dropout = StaticDropout(attn_p)
else:
self.attn_dropout = nn.Dropout(attn_p)
self.limit_rhs_steps = limit_rhs_steps
def __init__(self, d_model, d_ff, p, static=True):
super(FeedForward, self).__init__()
self.d_model = d_model
self.d_ff = d_ff
self.fc_1 = Linear(d_model, d_ff, nonlinearity="relu")
self.fc_2 = Linear(d_ff, d_model)
if static:
self.dropout = StaticDropout(p)
else:
self.dropout = nn.Dropout(p)
def __init__(self, h, d_model, attn_p=0.1, static=True):
super(MultiHeadAttention, self).__init__()
self.h = h
self.d = d_model
assert d_model % h == 0
self.d_head = d_model // h
self.fc_query = Bottle(Linear(d_model, h * self.d_head, bias=False))
self.fc_key = Bottle(Linear(d_model, h * self.d_head, bias=False))
self.fc_value = Bottle(Linear(d_model, h * self.d_head, bias=False))
self.attention_out = onmt.Constants.attention_out
self.fc_concat = Bottle(Linear(h * self.d_head, d_model, bias=False))
self.sm = nn.Softmax(dim=-1)
if static:
self.attn_dropout = StaticDropout(attn_p)
else:
self.attn_dropout = nn.Dropout(attn_p)