def __init__(self, isize, fhsize=None, dropout=0.0, attn_drop=0.0, num_head=8, ahsize=None, **kwargs): _ahsize = isize if ahsize is None else ahsize _fhsize = _ahsize * 4 if fhsize is None else fhsize super(DecoderLayer, self).__init__(isize, fhsize=_fhsize, dropout=dropout, attn_drop=attn_drop, num_head=num_head, ahsize=_ahsize, **kwargs) self.cattn = CrossAttn(isize, _ahsize, isize, num_head, dropout=attn_drop) self.ff = PositionwiseFF(isize, _fhsize, dropout) self.scff = ResidueCombiner(isize, 2, _fhsize, dropout)
def __init__(self, isize, fhsize=None, dropout=0.0, attn_drop=0.0, num_head=8, ahsize=None, **kwargs): _ahsize = isize if ahsize is None else ahsize super(DecoderLayer, self).__init__(isize, fhsize=fhsize, dropout=dropout, attn_drop=attn_drop, num_head=num_head, ahsize=_ahsize, **kwargs) self.cross_attn_mt = CrossAttn(isize, _ahsize, isize, num_head=num_head, dropout=attn_drop) self.layer_normer3 = nn.LayerNorm( isize, eps=ieps_ln_default, elementwise_affine=enable_ln_parameters)
def __init__(self, isize, fhsize=None, dropout=0.0, attn_drop=0.0, num_head=8, ahsize=None): _ahsize = isize if ahsize is None else ahsize _fhsize = _ahsize * 4 if fhsize is None else fhsize super(EncoderLayer, self).__init__(isize, _fhsize, dropout, attn_drop, num_head, _ahsize) self.cattn = CrossAttn(isize, _ahsize, isize, num_head, dropout=attn_drop) self.scff = ResidueCombiner(isize, 2, _fhsize, dropout)
def __init__(self, isize, fhsize=None, dropout=0.0, attn_drop=0.0, num_head=8, ahsize=None): super(DecoderLayer, self).__init__() _ahsize = isize if ahsize is None else ahsize _fhsize = _ahsize * 4 if fhsize is None else fhsize self.self_attn = SelfAttn(isize, _ahsize, isize, num_head, dropout=attn_drop) self.cross_attn = CrossAttn(isize, _ahsize, isize, num_head, dropout=attn_drop) self.ff = PositionwiseFF(isize, _fhsize, dropout) #self.scff = FtFFN(isize, _fhsize, dropout) self.scff = FullResidueCombiner(isize, 2, _fhsize) self.layer_normer1 = nn.LayerNorm(isize, eps=1e-06) self.layer_normer2 = nn.LayerNorm(isize, eps=1e-06) if dropout > 0: self.d1 = Dropout(dropout, inplace=True) self.d2 = Dropout(dropout, inplace=True) else: self.d1 = None self.d2 = None