def __init__(self,
isize,
fhsize=None,
dropout=0.0,
attn_drop=0.0,
num_head=8,
ahsize=None,
**kwargs):
_ahsize = isize if ahsize is None else ahsize
_fhsize = _ahsize * 4 if fhsize is None else fhsize
super(DecoderLayer, self).__init__(isize,
fhsize=_fhsize,
dropout=dropout,
attn_drop=attn_drop,
num_head=num_head,
ahsize=_ahsize,
**kwargs)
self.cross_attn = ResCrossAttn(
isize,
_ahsize,
num_head=num_head,
dropout=attn_drop,
norm_residual=self.cross_attn.norm_residual)
self.ff = PositionwiseFF(isize, _fhsize, dropout)
self.scff = ResidueCombiner(isize, 2, _fhsize)
self.drop, self.layer_normer1 = self.self_attn.drop, self.self_attn.drop.normer
self.self_attn = self.self_attn.net
def __init__(self,
isize,
fhsize=None,
dropout=0.0,
attn_drop=0.0,
num_head=8,
ahsize=None,
**kwargs):
_ahsize = isize if ahsize is None else ahsize
_fhsize = _ahsize * 4 if fhsize is None else fhsize
super(DecoderLayer, self).__init__(isize,
fhsize=_fhsize,
dropout=dropout,
attn_drop=attn_drop,
num_head=num_head,
ahsize=_ahsize,
**kwargs)
self.cattn = CrossAttn(isize,
_ahsize,
isize,
num_head,
dropout=attn_drop)
self.ff = PositionwiseFF(isize, _fhsize, dropout)
self.scff = ResidueCombiner(isize, 2, _fhsize, dropout)
def __init__(self,
isize,
fhsize=None,
dropout=0.0,
attn_drop=0.0,
num_head=8,
ahsize=None):
_ahsize = isize if ahsize is None else ahsize
_fhsize = _ahsize * 4 if fhsize is None else fhsize
super(EncoderLayer, self).__init__(isize, _fhsize, dropout, attn_drop,
num_head, _ahsize)
self.cattn = CrossAttn(isize,
_ahsize,
isize,
num_head,
dropout=attn_drop)
self.scff = ResidueCombiner(isize, 2, _fhsize, dropout)