def batch_encode(self, iterator, dim=0, **kwargs) -> (torch.Tensor, torch.Tensor): """ :param iterator (iterator): Batch of text to encode. :param dim (int, optional): Dimension along which to concatenate tensors. :param **kwargs: Keyword arguments passed to 'encode'. Returns torch.Tensor, torch.Tensor: Encoded and padded batch of sequences; Original lengths of sequences. """ return stack_and_pad_tensors( Encoder.batch_encode(self, iterator, **kwargs), padding_index=self.padding_index, dim=dim, )
def test_encoder(): encoder = Encoder(enforce_reversible=True) encoder.encode('this is a test') encoder.decode('this is a test')