def ResidualFeedForward(d_feature, d_feedforward, dropout, mode):
"""Residual feed-forward layer with normalization at start."""
return Residual(
tl.LayerNorm(),
tl.Dense(d_feedforward),
tl.Relu(),
tl.Dropout(rate=dropout, mode=mode),
tl.Dense(d_feature),
tl.Dropout(rate=dropout, mode=mode)
)
def ResidualFeedForward(feature_depth,
feedforward_depth,
dropout,
mode):
"""Residual feed-forward layer with normalization at start."""
return layers.Residual(
layers.LayerNorm(),
layers.Dense(feedforward_depth,
kernel_initializer=layers.XavierUniformInitializer()),
layers.Relu(),
layers.Dropout(rate=dropout, mode=mode),
layers.Dense(feature_depth,
kernel_initializer=layers.XavierUniformInitializer()),
layers.Dropout(rate=dropout, mode=mode)
)