def ResidualFeedForward(d_feature, d_feedforward, dropout, mode): """Residual feed-forward layer with normalization at start.""" return Residual( tl.LayerNorm(), tl.Dense(d_feedforward), tl.Relu(), tl.Dropout(rate=dropout, mode=mode), tl.Dense(d_feature), tl.Dropout(rate=dropout, mode=mode) )
def ResidualFeedForward(feature_depth, feedforward_depth, dropout, mode): """Residual feed-forward layer with normalization at start.""" return layers.Residual( layers.LayerNorm(), layers.Dense(feedforward_depth, kernel_initializer=layers.XavierUniformInitializer()), layers.Relu(), layers.Dropout(rate=dropout, mode=mode), layers.Dense(feature_depth, kernel_initializer=layers.XavierUniformInitializer()), layers.Dropout(rate=dropout, mode=mode) )