def test_init(initialize_bart, initialize_t5):
bart_tokenizer, _ = initialize_bart
t5_tokenizer, _ = initialize_t5
_init(bart_tokenizer, 'facebook/bart-base')
_init(t5_tokenizer, 't5-base')
with pytest.raises(ValueError):
te.SplitterEncoder("not-supported-model")
def initialize_t5():
from transformers import T5Tokenizer
t5_tokenizer = T5Tokenizer.from_pretrained('t5-base')
t5_splitter = te.SplitterEncoder('t5-base')
return t5_tokenizer, t5_splitter
def _init(tokenizer, model_name):
splitter = te.SplitterEncoder(model_name)
assert type(splitter.tokenizer) is type(tokenizer)