def __init__(self, d_model, kernel_size): super(ConformerConvolution, self).__init__() assert (kernel_size - 1) % 2 == 0 self.d_model = d_model self.pointwise_conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_model * 2, kernel_size=1, stride=1, padding=0, bias=True) self.depthwise_conv = nn.Conv1d( in_channels=d_model, out_channels=d_model, kernel_size=kernel_size, stride=1, padding=(kernel_size - 1) // 2, groups=d_model, bias=True, ) self.batch_norm = nn.BatchNorm1d(d_model) self.activation = Swish() self.pointwise_conv2 = nn.Conv1d(in_channels=d_model, out_channels=d_model, kernel_size=1, stride=1, padding=0, bias=True)
def __init__(self, d_model, d_ff, dropout, activation=Swish()): super(ConformerFeedForward, self).__init__() self.linear1 = nn.Linear(d_model, d_ff) self.activation = activation self.dropout = nn.Dropout(p=dropout) self.linear2 = nn.Linear(d_ff, d_model)