def test_tokenizer_with_special_tokens(self, special_tokens, expected): tokenizer = TokenEncoder(special_tokens=special_tokens) indices = tokenizer.fit_transform(['abcdefg'])[0] result = [tokenizer.id2token_[idx] for idx in indices] assert result == expected joined = tokenizer.inverse_transform([indices])[0] assert joined == 'abcdefg'
def test_tokenizer_with_special_tokens(self, special_tokens, expected): tokenizer = TokenEncoder(special_tokens=special_tokens) indices = tokenizer.fit_transform(['abcdefg'])[0] result = [tokenizer.id2token_[idx] for idx in indices] assert result == expected joined = tokenizer.inverse_transform([indices])[0] assert joined == 'abcdefg'
def test_tokenizer_encoded_and_decodes_simple_text(self): X = 'abcdef' tokenizer = TokenEncoder().fit(X) encoded = tokenizer.transform(['fedcba'])[0] assert len(encoded) == 6 assert all([type(idx) == int for idx in encoded]) decoded = tokenizer.inverse_transform([encoded])[0] assert decoded == 'fedcba'
def test_tokenizer_encoded_and_decodes_simple_text(self): X = 'abcdef' tokenizer = TokenEncoder().fit(X) encoded = tokenizer.transform(['fedcba'])[0] assert len(encoded) == 6 assert all([type(idx) == int for idx in encoded]) decoded = tokenizer.inverse_transform([encoded])[0] assert decoded == 'fedcba'
def test_tokenizer_with_special_special_token(self, line): tokenizer = TokenEncoder(special_tokens=['$']).fit(['hi']) encoded = tokenizer.transform([line]) decoded = tokenizer.inverse_transform(encoded)[0] assert decoded == line
def test_detokenize_with_space(self, line): tokenizer = TokenEncoder(separator=" ").fit([line]) encoded = tokenizer.transform([line]) decoded = tokenizer.inverse_transform(encoded)[0] assert decoded == line
def test_tokenizer_with_special_special_token(self, line): tokenizer = TokenEncoder(special_tokens=['$']).fit(['hi']) encoded = tokenizer.transform([line]) decoded = tokenizer.inverse_transform(encoded)[0] assert decoded == line
def test_detokenize_with_space(self, line): tokenizer = TokenEncoder(separator=" ").fit([line]) encoded = tokenizer.transform([line]) decoded = tokenizer.inverse_transform(encoded)[0] assert decoded == line