def test_lm_tokenizer_edge_cases(
model_name,
model_weights,
texts,
expected_tokens,
expected_indices,
expected_num_token_ids,
):
if model_weights is None:
model_weights_config = {}
else:
model_weights_config = {"model_weights": model_weights}
transformers_config = {
**{
"model_name": model_name
},
**model_weights_config
}
transformers_nlp = HFTransformersNLP(transformers_config)
lm_tokenizer = LanguageModelTokenizer()
for text, gt_tokens, gt_indices, gt_num_indices in zip(
texts, expected_tokens, expected_indices, expected_num_token_ids):
message = Message.build(text=text)
transformers_nlp.process(message)
tokens = lm_tokenizer.tokenize(message, TEXT)
token_ids = message.get(LANGUAGE_MODEL_DOCS[TEXT])[TOKEN_IDS]
assert [t.text for t in tokens] == gt_tokens
assert [t.start for t in tokens] == [i[0] for i in gt_indices]
assert [t.end for t in tokens] == [i[1] for i in gt_indices]
assert len(token_ids) == gt_num_indices
def test_preserve_sentence_and_sequence_features_old_config():
attribute = "text"
message = Message.build("hi there")
transformers_nlp = HFTransformersNLP({
"model_name": "bert",
"model_weights": "bert-base-uncased"
})
transformers_nlp.process(message)
lm_tokenizer = LanguageModelTokenizer()
lm_tokenizer.process(message)
lm_featurizer = LanguageModelFeaturizer({"model_name": "gpt2"})
lm_featurizer.process(message)
message.set(LANGUAGE_MODEL_DOCS[attribute], None)
lm_docs = lm_featurizer._get_docs_for_batch([message],
attribute=attribute,
inference_mode=True)[0]
hf_docs = transformers_nlp._get_docs_for_batch([message],
attribute=attribute,
inference_mode=True)[0]
assert not (message.features[0].features
== lm_docs[SEQUENCE_FEATURES]).any()
assert not (message.features[1].features
== lm_docs[SENTENCE_FEATURES]).any()
assert (message.features[0].features == hf_docs[SEQUENCE_FEATURES]).all()
assert (message.features[1].features == hf_docs[SENTENCE_FEATURES]).all()
def test_lm_tokenizer_number_of_sub_tokens(text, expected_number_of_sub_tokens):
transformers_config = {"model_name": "bert"} # Test for one should be enough
transformers_nlp = HFTransformersNLP(transformers_config)
lm_tokenizer = LanguageModelTokenizer()
message = Message(text)
td = TrainingData([message])
transformers_nlp.train(td)
lm_tokenizer.train(td)
assert [
t.get(NUMBER_OF_SUB_TOKENS) for t in message.get(TOKENS_NAMES[TEXT])[:-1]
] == expected_number_of_sub_tokens
def test_lm_tokenizer_custom_intent_symbol(text, expected_tokens):
component_config = {"intent_tokenization_flag": True, "intent_split_symbol": "+"}
transformers_config = {"model_name": "bert"} # Test for one should be enough
transformers_nlp = HFTransformersNLP(transformers_config)
lm_tokenizer = LanguageModelTokenizer(component_config)
message = Message(text)
message.set(INTENT, text)
td = TrainingData([message])
transformers_nlp.train(td)
lm_tokenizer.train(td)
assert [t.text for t in message.get(TOKENS_NAMES[INTENT])] == expected_tokens
def test_log_deprecation_warning_with_old_config(caplog: LogCaptureFixture):
message = Message.build("hi there")
transformers_nlp = HFTransformersNLP(
{"model_name": "bert", "model_weights": "bert-base-uncased"}
)
transformers_nlp.process(message)
caplog.set_level(logging.DEBUG)
lm_tokenizer = LanguageModelTokenizer()
lm_tokenizer.process(message)
lm_featurizer = LanguageModelFeaturizer(skip_model_load=True)
caplog.clear()
with caplog.at_level(logging.DEBUG):
lm_featurizer.process(message)
assert "deprecated component HFTransformersNLP" in caplog.text
def test_lm_tokenizer_edge_cases(model_name, texts, expected_tokens,
expected_indices):
transformers_config = {"model_name": model_name}
transformers_nlp = HFTransformersNLP(transformers_config)
lm_tokenizer = LanguageModelTokenizer()
for text, gt_tokens, gt_indices in zip(texts, expected_tokens,
expected_indices):
message = Message.build(text=text)
transformers_nlp.process(message)
tokens = lm_tokenizer.tokenize(message, TEXT)
assert [t.text for t in tokens] == gt_tokens
assert [t.start for t in tokens] == [i[0] for i in gt_indices]
assert [t.end for t in tokens] == [i[1] for i in gt_indices]
from rasa.nlu.constants import (TEXT, SPACY_DOCS)
logger = logging_setup()
test_input = "Okay, pick up this yellow banana for me."
message = Message(test_input)
tk = WhitespaceTokenizer()
tokens = tk.tokenize(message, attribute=TEXT)
logger.info('Whitespace: {}'.format([t.text for t in tokens]))
tk = SpacyTokenizer()
message.set(SPACY_DOCS[TEXT], spacy_nlp(test_input))
tokens = tk.tokenize(message, attribute=TEXT)
logger.info('SpaCy: {}'.format([t.text for t in tokens]))
tk = MitieTokenizer()
tokens = tk.tokenize(message, attribute=TEXT)
logger.info('Mitie: {}'.format([t.text for t in tokens]))
tk = ConveRTTokenizer()
tokens = tk.tokenize(message, attribute=TEXT)
logger.info('ConveRT: {}'.format([t.text for t in tokens]))
tk = LanguageModelTokenizer()
transformers_nlp = HFTransformersNLP({"model_name": "bert"})
transformers_nlp.process(message)
tokens = tk.tokenize(message, attribute=TEXT)
logger.info('BERT: {}'.format([t.text for t in tokens]))
