def __init__(self,
isize,
fhsize=None,
dropout=0.0,
attn_drop=0.0,
num_head=8,
ahsize=None):
super(EncoderLayer, self).__init__()
_ahsize = isize if ahsize is None else ahsize
_fhsize = _ahsize * 4 if fhsize is None else fhsize
self.attn = SelfAttn(isize,
_ahsize,
isize,
num_head,
dropout=attn_drop)
self.ff = PositionwiseFF(isize, _fhsize, dropout)
self.layer_normer = nn.LayerNorm(isize, eps=1e-06)
self.drop = Dropout(dropout, inplace=True) if dropout > 0.0 else None
def __init__(self,
isize,
fhsize=None,
dropout=0.0,
attn_drop=0.0,
num_head=8,
ahsize=None,
**kwargs):
_ahsize = isize if ahsize is None else ahsize
_fhsize = _ahsize * 4 if fhsize is None else fhsize
super(DecoderLayer, self).__init__(isize,
fhsize=_fhsize,
dropout=dropout,
attn_drop=attn_drop,
num_head=num_head,
ahsize=_ahsize,
**kwargs)
self.cattn = CrossAttn(isize,
_ahsize,
isize,
num_head,
dropout=attn_drop)
self.ff = PositionwiseFF(isize, _fhsize, dropout)
self.scff = ResidueCombiner(isize, 2, _fhsize, dropout)
def __init__(self,
isize,
fhsize=None,
dropout=0.0,
attn_drop=0.0,
num_head=8,
ahsize=None,
**kwargs):
_ahsize = isize if ahsize is None else ahsize
_fhsize = _ahsize * 4 if fhsize is None else fhsize
super(DecoderLayer, self).__init__(isize,
fhsize=_fhsize,
dropout=dropout,
attn_drop=attn_drop,
num_head=num_head,
ahsize=_ahsize,
**kwargs)
self.cross_attn = ResCrossAttn(
isize,
_ahsize,
num_head=num_head,
dropout=attn_drop,
norm_residual=self.cross_attn.norm_residual)
self.ff = PositionwiseFF(isize, _fhsize, dropout)
self.scff = ResidueCombiner(isize, 2, _fhsize)
self.drop, self.layer_normer1 = self.self_attn.drop, self.self_attn.drop.normer
self.self_attn = self.self_attn.net
def __init__(self,
isize,
fhsize=None,
dropout=0.0,
attn_drop=0.0,
num_head=8,
ahsize=None,
**kwargs):
_ahsize = isize if ahsize is None else ahsize
_fhsize = _ahsize * 4 if fhsize is None else fhsize
super(EncoderLayer, self).__init__(isize,
fhsize=_fhsize,
dropout=dropout,
attn_drop=attn_drop,
num_head=num_head,
ahsize=_ahsize,
**kwargs)
self.attn = ResSelfAttn(isize,
_ahsize,
num_head=num_head,
dropout=attn_drop)
self.ff = PositionwiseFF(isize, _fhsize, dropout)
def __init__(self, isize, fhsize=None, dropout=0.0, attn_drop=0.0, num_head=8, ahsize=None): _ahsize = isize if ahsize is None else ahsize _fhsize = _ahsize * 4 if fhsize is None else fhsize super(EncoderLayer, self).__init__(isize, _fhsize, dropout, attn_drop, num_head, _ahsize) self.ff = PositionwiseFF(isize, _fhsize, dropout)
def __init__(self,
isize,
fhsize=None,
dropout=0.0,
attn_drop=0.0,
num_head=8,
ahsize=None):
super(DecoderLayer, self).__init__()
_ahsize = isize if ahsize is None else ahsize
_fhsize = _ahsize * 4 if fhsize is None else fhsize
self.self_attn = SelfAttn(isize,
_ahsize,
isize,
num_head,
dropout=attn_drop)
self.cross_attn = CrossAttn(isize,
_ahsize,
isize,
num_head,
dropout=attn_drop)
self.ff = PositionwiseFF(isize, _fhsize, dropout)
#self.scff = FtFFN(isize, _fhsize, dropout)
self.scff = FullResidueCombiner(isize, 2, _fhsize)
self.layer_normer1 = nn.LayerNorm(isize, eps=1e-06)
self.layer_normer2 = nn.LayerNorm(isize, eps=1e-06)
if dropout > 0:
self.d1 = Dropout(dropout, inplace=True)
self.d2 = Dropout(dropout, inplace=True)
else:
self.d1 = None
self.d2 = None