def __init__(self, opt, death_rate=0.0): super(ConformerEncoderLayer, self).__init__() # FFN -> SelfAttention -> Conv -> FFN # PreNorm self.opt = opt self.variational = opt.variational_dropout self.death_rate = death_rate self.dropout = opt.dropout self.ffn_scale = 0.5 self.preprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) self.attn = RelativeSelfMultiheadAttn(opt.model_size, opt.n_heads, opt.attn_dropout) self.preprocess_mcr_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.mcr_feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, activation='swish') self.preprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, activation='swish') # there is batch norm inside convolution already # so no need for layer norm? self.preprocess_conv = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_conv = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) self.conv = ConformerConvBlock(opt.model_size, opt.conv_kernel, activation='swish')
def __init__(self, opt, death_rate=0.0): super(DecoderLayer, self).__init__() self.ignore_source = opt.ignore_source self.variational = opt.variational_dropout self.death_rate = death_rate self.fast_self_attention = opt.fast_self_attention self.macaron = opt.macaron self.ffn_scale = 0.5 if self.macaron else 1 if self.macaron: self.preprocess_mcr_ffn = preprocessing(opt.rezero, opt.model_size, opt.dropout, sequence='n') self.postprocess_mcr_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) self.mcr_feedforward = PositionWiseFeedForward(opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, activation=opt.ffn_activation, glu=opt.ffn_glu) self.preprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) if opt.fast_self_attention: self.multihead_tgt = SelfMultiheadAttn(opt.model_size, opt.n_heads, opt.attn_dropout) else: self.multihead_tgt = MultiHeadAttention(opt.n_heads, opt.model_size, attn_p=opt.attn_dropout, share=1) if not self.ignore_source: self.preprocess_src_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_src_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) if not opt.fast_xattention: self.multihead_src = MultiHeadAttention(opt.n_heads, opt.model_size, attn_p=opt.attn_dropout, share=2) else: self.multihead_src = EncdecMultiheadAttn(opt.n_heads, opt.model_size, opt.attn_dropout) self.preprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) if not opt.fast_feed_forward: feedforward = FeedForward(opt.model_size, opt.inner_size, opt.dropout, variational=self.variational) self.feedforward = Bottle(feedforward) else: self.feedforward = PositionWiseFeedForward(opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, activation=opt.ffn_activation, glu=opt.ffn_glu)
def __init__(self, opt, death_rate=0.0, **kwargs): super(EncoderLayer, self).__init__() self.variational = opt.variational_dropout self.death_rate = death_rate self.fast_self_attention = opt.fast_self_attention self.preprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) self.preprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) if opt.fast_self_attention: self.multihead = SelfMultiheadAttn(opt.model_size, opt.n_heads, opt.attn_dropout) else: self.multihead = MultiHeadAttention(opt.n_heads, opt.model_size, attn_p=opt.attn_dropout, share=1) if not opt.fast_feed_forward: feedforward = FeedForward(opt.model_size, opt.inner_size, opt.dropout, variational=self.variational) self.feedforward = Bottle(feedforward) else: self.feedforward = PositionWiseFeedForward(opt.model_size, opt.inner_size, opt.dropout, variational=self.variational)
def __init__(self, opt, death_rate=0.0, **kwargs): super().__init__() self.variational = opt.variational_dropout self.death_rate = death_rate self.fast_self_attention = opt.fast_self_attention self.preprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) self.preprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) d_head = opt.model_size // opt.n_heads self.adaptive_type = opt.adaptive self.factor_size = opt.layers # this model defaults as fast relative self attention if self.adaptive_type == 'universal': self.multihead = RelativeSelfMultiheadAttn(opt.model_size, opt.n_heads, opt.attn_dropout) self.feedforward = PositionWiseFeedForward(opt.model_size, opt.inner_size, opt.dropout, variational=self.variational) else: self.multihead = AdaptiveRelativeAttn(opt.model_size, opt.n_heads, self.factor_size, opt.attn_dropout) self.feedforward = AdaptiveFeedForward(opt.model_size, opt.inner_size, self.factor_size, opt.dropout, variational=self.variational)
def __init__(self, opt): super().__init__() self.layer_norm = nn.LayerNorm((opt.model_size, ), elementwise_affine=True) self.residual_dropout = opt.residual_dropout if opt.residual_dropout >= 0 else opt.dropout self.ffn_dropout = opt.ffn_dropout if opt.ffn_dropout >= 0 else opt.dropout self.feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, self.ffn_dropout, variational=opt.variational_dropout, glu=opt.ffn_glu, activation=opt.ffn_activation) self.variational = opt.variational_dropout
def __init__(self, opt, death_rate=0.0, **kwargs): super(RelativeTransformerEncoderLayer, self).__init__() self.variational = opt.variational_dropout self.death_rate = death_rate self.fast_self_attention = opt.fast_self_attention self.preprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) self.preprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) d_head = opt.model_size // opt.n_heads if not self.fast_self_attention: self.multihead = RelPartialLearnableMultiHeadAttn( opt.n_heads, opt.model_size, d_head, dropatt=opt.attn_dropout) else: self.multihead = RelativeSelfMultiheadAttn(opt.model_size, opt.n_heads, opt.attn_dropout) print(opt.fast_feed_forward) if not opt.fast_feed_forward: feedforward = FeedForward(opt.model_size, opt.inner_size, opt.dropout, variational=self.variational) self.feedforward = Bottle(feedforward) else: self.feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational)
def __init__(self, opt, death_rate=0.0): super().__init__() self.ignore_source = opt.ignore_source self.variational = opt.variational_dropout self.death_rate = death_rate self.fast_self_attention = opt.fast_self_attention self.factor_size = opt.layers self.adaptive_type = opt.adaptive self.preprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) if not self.ignore_source: self.preprocess_src_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_src_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) if self.adaptive_type == 'universal': self.multihead_src = EncdecMultiheadAttn(opt.n_heads, opt.model_size, opt.attn_dropout) else: self.multihead_src = AdaptiveEncDecAttn(opt.n_heads, opt.model_size, self.factor_size, opt.attn_dropout) self.preprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) if self.adaptive_type == 'universal': self.multihead_tgt = RelativeSelfMultiheadAttn(opt.model_size, opt.n_heads, opt.attn_dropout) self.feedforward = PositionWiseFeedForward(opt.model_size, opt.inner_size, opt.dropout, variational=self.variational) else: self.multihead_tgt = AdaptiveRelativeAttn(opt.model_size, opt.n_heads, self.factor_size, opt.attn_dropout) self.feedforward = AdaptiveFeedForward(opt.model_size, opt.inner_size, self.factor_size, opt.dropout, variational=self.variational)
def __init__(self, opt, death_rate=0.0): super(RelativeTransformerDecoderLayer, self).__init__() self.ignore_source = opt.ignore_source self.variational = opt.variational_dropout self.death_rate = death_rate self.fast_self_attention = opt.fast_self_attention # self.lfv_multilingual = opt.lfv_multilingual self.preprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) if not self.ignore_source: self.preprocess_src_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_src_attn = PrePostProcessing( opt.model_size, opt.dropout, sequence='da', variational=self.variational) if opt.fast_xattention: self.multihead_src = EncdecMultiheadAttn( opt.n_heads, opt.model_size, opt.attn_dropout) else: self.multihead_src = MultiHeadAttention( opt.n_heads, opt.model_size, attn_p=opt.attn_dropout, share=2) self.preprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) d_head = opt.model_size // opt.n_heads if not self.fast_self_attention: self.multihead_tgt = RelPartialLearnableMultiHeadAttn( opt.n_heads, opt.model_size, d_head, dropatt=opt.attn_dropout) else: self.multihead_tgt = RelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout) if not opt.fast_feed_forward: feedforward = FeedForward(opt.model_size, opt.inner_size, opt.dropout, variational=self.variational) self.feedforward = Bottle(feedforward) else: self.feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational)
def __init__(self, opt, death_rate=0.0, lid_net=None): super(RelativeTransformerDecoderLayer, self).__init__() self.ignore_source = opt.ignore_source self.variational = opt.variational_dropout self.death_rate = death_rate self.mfw = opt.multilingual_factorized_weights self.mpw = opt.multilingual_partitioned_weights self.mln = opt.multilingual_layer_norm self.weight_drop = opt.weight_drop self.multilingual_adapter = opt.multilingual_adapter self.adapter_bottleneck_size = opt.adapter_bottleneck_size self.preprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n', multilingual=self.mln, n_languages=opt.n_languages) self.postprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) if not self.ignore_source: self.preprocess_src_attn = PrePostProcessing( opt.model_size, opt.dropout, sequence='n', multilingual=self.mln, n_languages=opt.n_languages) self.postprocess_src_attn = PrePostProcessing( opt.model_size, opt.dropout, sequence='da', variational=self.variational) if self.mfw: self.multihead_src = MFWEncdecMultiheadAttn( opt.n_heads, opt.model_size, opt.attn_dropout, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative, weight_drop=self.weight_drop, mfw_activation=opt.mfw_activation) elif self.mpw: self.multihead_src = MPEncdecMultiheadAttn( opt.n_heads, opt.model_size, opt.attn_dropout, factor_size=opt.mpw_factor_size) else: self.multihead_src = EncdecMultiheadAttn( opt.n_heads, opt.model_size, opt.attn_dropout) self.preprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n', multilingual=self.mln, n_languages=opt.n_languages) self.postprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) d_head = opt.model_size // opt.n_heads if self.mfw: self.feedforward = MFWPositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative, weight_drop=self.weight_drop, mfw_activation=opt.mfw_activation) self.multihead_tgt = MFWRelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative, weight_drop=self.weight_drop, mfw_activation=opt.mfw_activation) elif self.mpw: self.feedforward = MPPositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, factor_size=opt.mpw_factor_size) self.multihead_tgt = MPRelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout, factor_size=opt.mpw_factor_size) else: self.multihead_tgt = RelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout) self.feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational) self.lfv_multilingual = opt.lfv_multilingual if opt.lfv_multilingual: self.lid_net = lid_net self.lfv_mapper = nn.Linear(opt.bottleneck_size, opt.model_size) else: self.lid_net = None self.lfv_mapper = None if self.multilingual_adapter: from onmt.modules.multilingual_factorized.multilingual_adapters import MultilingualAdapter self.adapters = MultilingualAdapter(opt.model_size, opt.adapter_bottleneck_size, n_languages=opt.n_languages, dropout=opt.dropout)
def __init__(self, opt, death_rate=0.0, **kwargs): super(RelativeTransformerEncoderLayer, self).__init__() self.variational = opt.variational_dropout self.death_rate = death_rate self.fast_self_attention = opt.fast_self_attention self.depthwise_conv = opt.depthwise_conv self.mfw = opt.multilingual_factorized_weights self.mpw = opt.multilingual_partitioned_weights self.mln = opt.multilingual_layer_norm self.no_ffn = opt.no_ffn self.weight_drop = opt.weight_drop self.multilingual_adapter = opt.multilingual_adapter self.adapter_bottleneck_size = opt.adapter_bottleneck_size if self.mfw: assert not self.mpw, "[ERROR] factorized and partitioned weights cannot be used at the same time." self.preprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n', multilingual=self.mln, n_languages=opt.n_languages) self.postprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) if not self.no_ffn: self.preprocess_ffn = PrePostProcessing( opt.model_size, opt.dropout, sequence='n', multilingual=self.mln, n_languages=opt.n_languages) self.postprocess_ffn = PrePostProcessing( opt.model_size, opt.dropout, sequence='da', variational=self.variational) d_head = opt.model_size // opt.n_heads if self.mfw: if not self.no_ffn: self.feedforward = MFWPositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative, weight_drop=self.weight_drop, mfw_activation=opt.mfw_activation) self.multihead = MFWRelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative, weight_drop=self.weight_drop, mfw_activation=opt.mfw_activation) elif self.mpw: if not self.no_ffn: self.feedforward = MPPositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, factor_size=opt.mpw_factor_size) self.multihead = MPRelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout, factor_size=opt.mpw_factor_size) else: if not self.no_ffn: self.feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational) self.multihead = RelativeSelfMultiheadAttn(opt.model_size, opt.n_heads, opt.attn_dropout) if self.depthwise_conv: self.preprocess_conv = PrePostProcessing( opt.model_size, opt.dropout, sequence='n', multilingual=self.mln, n_languages=opt.n_languages) self.postprocess_conv = PrePostProcessing( opt.model_size, opt.dropout, sequence='da', variational=self.variational) self.depthwise_conv = ConformerConvBlock(opt.model_size, opt.conv_kernel, bias=True) else: self.depthwise_conv = None if self.multilingual_adapter: from onmt.modules.multilingual_factorized.multilingual_adapters import MultilingualAdapter self.adapters = MultilingualAdapter(opt.model_size, opt.adapter_bottleneck_size, n_languages=opt.n_languages, dropout=opt.dropout)
def __init__(self, opt, death_rate=0.0): super(RelativeTransformerDecoderLayer, self).__init__() self.ignore_source = opt.ignore_source self.variational = opt.variational_dropout self.death_rate = death_rate self.batch_ensemble = opt.batch_ensemble self.mfw = opt.multilingual_factorized_weights self.macaron = opt.macaron self.ffn_scale = 0.5 if self.macaron else 1 self.dropout = opt.dropout if self.macaron: self.preprocess_mcr_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_mcr_ffn = PrePostProcessing( opt.model_size, opt.dropout, sequence='da', variational=self.variational) if self.mfw: self.mcr_feedforward = MFWPositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative) else: self.mcr_feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational) self.preprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) if not self.ignore_source: self.preprocess_src_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_src_attn = PrePostProcessing( opt.model_size, opt.dropout, sequence='da', variational=self.variational) # if self.batch_ensemble > 0: # self.multihead_src = BEEncdecMultiheadAttn(opt.n_heads, opt.model_size, opt.attn_dropout, # ensemble=self.batch_ensemble) # else: if not self.mfw: self.multihead_src = EncdecMultiheadAttn( opt.n_heads, opt.model_size, opt.attn_dropout) else: self.multihead_src = MFWEncdecMultiheadAttn( opt.n_heads, opt.model_size, opt.attn_dropout, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative) self.preprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) d_head = opt.model_size // opt.n_heads if self.mfw: self.feedforward = MFWPositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative) self.multihead_tgt = MFWRelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative) else: self.feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational) self.multihead_tgt = RelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout)
def __init__(self, opt, death_rate=0.0, **kwargs): super(RelativeTransformerEncoderLayer, self).__init__() self.variational = opt.variational_dropout self.batch_ensemble = opt.batch_ensemble # self.multilingual_factorized_weights = opt.multilingual_factorized_weights self.death_rate = death_rate self.mfw = opt.multilingual_factorized_weights self.macaron = opt.macaron self.ffn_scale = 0.5 if self.macaron else 1 self.dropout = opt.dropout self.residual_dropout = opt.residual_dropout if opt.residual_dropout >= 0 else opt.dropout self.ffn_dropout = opt.ffn_dropout if opt.ffn_dropout >= 0 else opt.dropout self.rezero = opt.rezero self.absolute_position_encoding = opt.absolute_position_encoding self.learnable_pos = opt.learnable_position_encoding self.stochastic_sublayer = opt.stochastic_sublayer self.post_norm = opt.post_norm if self.macaron: self.preprocess_mcr_ffn = preprocessing(opt.rezero, opt.model_size, self.post_norm) self.postprocess_mcr_ffn = postprocessing(opt.rezero, opt.model_size, self.residual_dropout, self.variational, self.post_norm) if self.mfw: self.mcr_feedforward = MFWPositionWiseFeedForward( opt.model_size, opt.inner_size, self.ffn_dropout, variational=self.variational, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative, no_bias=opt.mfw_no_bias, activation=opt.ffn_activation, glu=opt.ffn_glu) else: self.mcr_feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, self.ffn_dropout, variational=self.variational, activation=opt.ffn_activation, glu=opt.ffn_glu) self.preprocess_attn = preprocessing(opt.rezero, opt.model_size, self.post_norm) self.postprocess_attn = postprocessing(opt.rezero, opt.model_size, self.residual_dropout, self.variational, self.post_norm) self.preprocess_ffn = preprocessing(opt.rezero, opt.model_size, self.post_norm) self.postprocess_ffn = postprocessing(opt.rezero, opt.model_size, self.residual_dropout, self.variational, self.post_norm) d_head = opt.model_size // opt.n_heads if self.mfw: self.feedforward = MFWPositionWiseFeedForward( opt.model_size, opt.inner_size, self.ffn_dropout, variational=self.variational, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative, no_bias=opt.mfw_no_bias, activation=opt.ffn_activation, glu=opt.ffn_glu) self.multihead = MFWRelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative, no_bias=opt.mfw_no_bias, ) else: self.feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, self.ffn_dropout, variational=self.variational, activation=opt.ffn_activation, glu=opt.ffn_glu) if not self.absolute_position_encoding: self.multihead = RelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout, learnable_pos=self.learnable_pos, max_pos=opt.max_pos_length) else: self.multihead = SelfMultiheadAttn(opt.model_size, opt.n_heads, opt.attn_dropout)
def __init__(self, opt, death_rate=0.0, lid_net=None): super(RelativeTransformerDecoderLayer, self).__init__() self.ignore_source = opt.ignore_source self.variational = opt.variational_dropout self.death_rate = death_rate self.mfw = opt.multilingual_factorized_weights self.macaron = opt.macaron self.ffn_scale = 0.5 if self.macaron else 1 self.preprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) if not self.ignore_source: self.preprocess_src_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_src_attn = PrePostProcessing( opt.model_size, opt.dropout, sequence='da', variational=self.variational) if self.mfw: self.multihead_src = MFWEncdecMultiheadAttn( opt.n_heads, opt.model_size, opt.attn_dropout, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative) else: self.multihead_src = EncdecMultiheadAttn( opt.n_heads, opt.model_size, opt.attn_dropout) self.preprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) d_head = opt.model_size // opt.n_heads if not self.mfw: self.multihead_tgt = RelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout) self.feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, activation=opt.activation) else: self.feedforward = MFWPositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative) self.multihead_tgt = MFWRelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative) self.lfv_multilingual = opt.lfv_multilingual if opt.lfv_multilingual: self.lid_net = lid_net self.lfv_mapper = nn.Linear(opt.bottleneck_size, opt.model_size) else: self.lid_net = None self.lfv_mapper = None
def __init__(self, opt, death_rate=0.0, **kwargs): super(RelativeTransformerEncoderLayer, self).__init__() self.variational = opt.variational_dropout self.death_rate = death_rate self.fast_self_attention = opt.fast_self_attention self.macaron = opt.macaron self.ffn_scale = 0.5 if self.macaron else 1 self.dynamic_conv = opt.dynamic_conv self.no_self_attention = opt.no_self_attention self.preprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='da', variational=self.variational) d_head = opt.model_size // opt.n_heads if self.dynamic_conv: from onmt.modules.convolution import DynamicConvolution self.preprocess_dconv = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_dconv = PrePostProcessing( opt.model_size, opt.dropout, sequence='da', variational=self.variational) self.dynamic_conv_layer = DynamicConvolution( opt.model_size, opt.conv_kernel, weight_softmax=True, num_heads=opt.n_heads, weight_dropout=opt.attn_dropout, bias=True) if self.macaron: self.preprocess_mcr_ffn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_mcr_ffn = PrePostProcessing( opt.model_size, opt.dropout, sequence='da', variational=self.variational) self.mcr_feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, activation=opt.activation) if not self.no_self_attention: self.preprocess_attn = PrePostProcessing(opt.model_size, opt.dropout, sequence='n') self.postprocess_attn = PrePostProcessing( opt.model_size, opt.dropout, sequence='da', variational=self.variational) self.multihead = RelativeSelfMultiheadAttn(opt.model_size, opt.n_heads, opt.attn_dropout) self.feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, opt.dropout, variational=self.variational, activation=opt.activation)
def __init__(self, opt, death_rate=0.0, lid_net=None): super(RelativeTransformerDecoderLayer, self).__init__() self.ignore_source = opt.ignore_source self.variational = opt.variational_dropout self.death_rate = death_rate self.mfw = opt.multilingual_factorized_weights self.mpw = opt.multilingual_partitioned_weights self.mln = opt.multilingual_layer_norm self.weight_drop = opt.weight_drop self.multilingual_adapter = opt.multilingual_adapter self.adapter_bottleneck_size = opt.adapter_bottleneck_size self.macaron = opt.macaron self.ffn_scale = 0.5 if self.macaron else 1 self.rezero = opt.rezero self.learnable_pos = opt.learnable_position_encoding self.residual_dropout = opt.residual_dropout if opt.residual_dropout >= 0 else opt.dropout self.ffn_dropout = opt.ffn_dropout if opt.ffn_dropout >= 0 else opt.dropout self.preprocess_attn = preprocessing(self.rezero, opt.model_size, 0.0, sequence='n', multilingual=self.mln, n_languages=opt.n_languages) self.postprocess_attn = PrePostProcessing( opt.model_size, self.residual_dropout, sequence='dz' if self.rezero else 'da', variational=self.variational) if self.macaron: self.preprocess_mcr_ffn = preprocessing( self.rezero, opt.model_size, 0.0, sequence='n', multilingual=self.mln, n_languages=opt.n_languages) self.postprocess_mcr_ffn = PrePostProcessing( opt.model_size, self.residual_dropout, sequence='dz' if self.rezero else 'da', variational=self.variational) if self.mfw: self.mcr_feedforward = MFWPositionWiseFeedForward( opt.model_size, opt.inner_size, self.ffn_dropoutt, variational=self.variational, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative, activation=opt.ffn_activation, glu=opt.ffn_glu) else: self.mcr_feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, self.ffn_dropout, variational=self.variational, activation=opt.ffn_activation, glu=opt.ffn_glu) if not self.ignore_source: self.preprocess_src_attn = preprocessing( self.rezero, opt.model_size, 0.0, sequence='n', multilingual=self.mln, n_languages=opt.n_languages) self.postprocess_src_attn = PrePostProcessing( opt.model_size, self.residual_dropout, sequence='dz' if self.rezero else 'da', variational=self.variational) if self.mfw: self.multihead_src = MFWEncdecMultiheadAttn( opt.n_heads, opt.model_size, opt.attn_dropout, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative, weight_drop=self.weight_drop, mfw_activation=opt.mfw_activation) elif self.mpw: self.multihead_src = MPEncdecMultiheadAttn( opt.n_heads, opt.model_size, opt.attn_dropout, factor_size=opt.mpw_factor_size) else: self.multihead_src = EncdecMultiheadAttn( opt.n_heads, opt.model_size, opt.attn_dropout) self.preprocess_ffn = preprocessing(self.rezero, opt.model_size, 0.0, sequence='n', multilingual=self.mln, n_languages=opt.n_languages) self.postprocess_ffn = PrePostProcessing( opt.model_size, self.residual_dropout, sequence='dz' if self.rezero else 'da', variational=self.variational) d_head = opt.model_size // opt.n_heads if self.mfw: self.feedforward = MFWPositionWiseFeedForward( opt.model_size, opt.inner_size, self.ffn_dropout, variational=self.variational, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative, weight_drop=self.weight_drop, mfw_activation=opt.mfw_activation, activation=opt.ffn_activation, glu=opt.ffn_glu) self.multihead_tgt = MFWRelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout, n_languages=opt.n_languages, rank=opt.mfw_rank, use_multiplicative=opt.mfw_multiplicative, weight_drop=self.weight_drop, mfw_activation=opt.mfw_activation) elif self.mpw: self.feedforward = MPPositionWiseFeedForward( opt.model_size, opt.inner_size, self.ffn_dropout, variational=self.variational, factor_size=opt.mpw_factor_size) self.multihead_tgt = MPRelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout, factor_size=opt.mpw_factor_size) else: self.multihead_tgt = RelativeSelfMultiheadAttn( opt.model_size, opt.n_heads, opt.attn_dropout, learnable_pos=self.learnable_pos, max_pos=opt.max_pos_length) self.feedforward = PositionWiseFeedForward( opt.model_size, opt.inner_size, self.ffn_dropout, variational=self.variational, activation=opt.ffn_activation, glu=opt.ffn_glu) # self.lfv_multilingual = opt.lfv_multilingual # # if opt.lfv_multilingual: # self.lid_net = lid_net # self.lfv_mapper = nn.Linear(opt.bottleneck_size, opt.model_size) # else: # self.lid_net = None # self.lfv_mapper = None if self.multilingual_adapter: from onmt.modules.multilingual_factorized.multilingual_adapters import MultilingualAdapter self.adapters = MultilingualAdapter(opt.model_size, opt.adapter_bottleneck_size, n_languages=opt.n_languages, dropout=opt.dropout)