def __init__(self, d_model, kernel_size):
super(ConformerConvolution, self).__init__()
assert (kernel_size - 1) % 2 == 0
self.d_model = d_model
self.pointwise_conv1 = nn.Conv1d(in_channels=d_model,
out_channels=d_model * 2,
kernel_size=1,
stride=1,
padding=0,
bias=True)
self.depthwise_conv = nn.Conv1d(
in_channels=d_model,
out_channels=d_model,
kernel_size=kernel_size,
stride=1,
padding=(kernel_size - 1) // 2,
groups=d_model,
bias=True,
)
self.batch_norm = nn.BatchNorm1d(d_model)
self.activation = Swish()
self.pointwise_conv2 = nn.Conv1d(in_channels=d_model,
out_channels=d_model,
kernel_size=1,
stride=1,
padding=0,
bias=True)
def __init__(self, d_model, d_ff, dropout, activation=Swish()):
super(ConformerFeedForward, self).__init__()
self.linear1 = nn.Linear(d_model, d_ff)
self.activation = activation
self.dropout = nn.Dropout(p=dropout)
self.linear2 = nn.Linear(d_ff, d_model)