def test_tokenizer_with_special_tokens(self, special_tokens, expected):
tokenizer = TokenEncoder(special_tokens=special_tokens)
indices = tokenizer.fit_transform(['abcdefg'])[0]
result = [tokenizer.id2token_[idx] for idx in indices]
assert result == expected
joined = tokenizer.inverse_transform([indices])[0]
assert joined == 'abcdefg'
def test_tokenizer_with_special_tokens(self, special_tokens, expected):
tokenizer = TokenEncoder(special_tokens=special_tokens)
indices = tokenizer.fit_transform(['abcdefg'])[0]
result = [tokenizer.id2token_[idx] for idx in indices]
assert result == expected
joined = tokenizer.inverse_transform([indices])[0]
assert joined == 'abcdefg'
def test_tokenizer_encoded_and_decodes_simple_text(self):
X = 'abcdef'
tokenizer = TokenEncoder().fit(X)
encoded = tokenizer.transform(['fedcba'])[0]
assert len(encoded) == 6
assert all([type(idx) == int for idx in encoded])
decoded = tokenizer.inverse_transform([encoded])[0]
assert decoded == 'fedcba'
def test_tokenizer_encoded_and_decodes_simple_text(self):
X = 'abcdef'
tokenizer = TokenEncoder().fit(X)
encoded = tokenizer.transform(['fedcba'])[0]
assert len(encoded) == 6
assert all([type(idx) == int for idx in encoded])
decoded = tokenizer.inverse_transform([encoded])[0]
assert decoded == 'fedcba'
def test_tokenizer_creates_indices(self):
X = 'abcdef'
tokenizer = TokenEncoder().fit(X)
# num tokens + start token + end token
assert len(tokenizer.token2id_) == 8
expected_keys = set(
list('abcdef') + [tokenizer.start_token, tokenizer.end_token])
assert set(tokenizer.token2id_.keys()) == expected_keys
def test_tokenizer_wo_args(self, line, expected):
tokenizer = TokenEncoder().fit(line)
result = tokenizer.transform([line])[0]
result = [tokenizer.id2token_[idx] for idx in result]
assert result == expected
def test_detokenize_with_space(self, line):
tokenizer = TokenEncoder(separator=" ").fit([line])
encoded = tokenizer.transform([line])
decoded = tokenizer.inverse_transform(encoded)[0]
assert decoded == line
def test_tokenizer_split_on_space(self, line, expected):
tokenizer = TokenEncoder(separator=" ").fit([line])
encoded = tokenizer.transform([line])[0]
expected = [tokenizer.token2id_[token] for token in expected]
assert encoded == expected
def test_tokenizer_with_special_tokens_more_matches(self, line, expected):
tokenizer = TokenEncoder(special_tokens=['abc']).fit(ALPHABET)
encoded = tokenizer.transform([line])[0]
expected = [tokenizer.token2id_[token] for token in expected]
assert encoded == expected
def encoder(self):
encoder = TokenEncoder().fit(self.X)
return encoder
def test_tokenizer_split_on_space(self, line, expected):
tokenizer = TokenEncoder(separator=" ").fit([line])
encoded = tokenizer.transform([line])[0]
expected = [tokenizer.token2id_[token] for token in expected]
assert encoded == expected
def test_tokenizer_with_special_special_token(self, line):
tokenizer = TokenEncoder(special_tokens=['$']).fit(['hi'])
encoded = tokenizer.transform([line])
decoded = tokenizer.inverse_transform(encoded)[0]
assert decoded == line
def test_tokenizer_wo_args(self, line, expected):
tokenizer = TokenEncoder().fit(line)
result = tokenizer.transform([line])[0]
result = [tokenizer.id2token_[idx] for idx in result]
assert result == expected
def test_tokenizer_with_unsupported_separator(self):
with pytest.raises(NotImplementedError):
TokenEncoder(separator='and')
def test_detokenize_with_space(self, line):
tokenizer = TokenEncoder(separator=" ").fit([line])
encoded = tokenizer.transform([line])
decoded = tokenizer.inverse_transform(encoded)[0]
assert decoded == line
def encoder(X):
encoder = TokenEncoder().fit(X)
return encoder
def test_tokenizer_with_special_special_token(self, line):
tokenizer = TokenEncoder(special_tokens=['$']).fit(['hi'])
encoded = tokenizer.transform([line])
decoded = tokenizer.inverse_transform(encoded)[0]
assert decoded == line
def test_tokenizer_with_special_tokens_more_matches(self, line, expected):
tokenizer = TokenEncoder(special_tokens=['abc']).fit(ALPHABET)
encoded = tokenizer.transform([line])[0]
expected = [tokenizer.token2id_[token] for token in expected]
assert encoded == expected