def test_lm_tokenizer_edge_cases( model_name, model_weights, texts, expected_tokens, expected_indices, expected_num_token_ids, ): if model_weights is None: model_weights_config = {} else: model_weights_config = {"model_weights": model_weights} transformers_config = { **{ "model_name": model_name }, **model_weights_config } transformers_nlp = HFTransformersNLP(transformers_config) lm_tokenizer = LanguageModelTokenizer() for text, gt_tokens, gt_indices, gt_num_indices in zip( texts, expected_tokens, expected_indices, expected_num_token_ids): message = Message.build(text=text) transformers_nlp.process(message) tokens = lm_tokenizer.tokenize(message, TEXT) token_ids = message.get(LANGUAGE_MODEL_DOCS[TEXT])[TOKEN_IDS] assert [t.text for t in tokens] == gt_tokens assert [t.start for t in tokens] == [i[0] for i in gt_indices] assert [t.end for t in tokens] == [i[1] for i in gt_indices] assert len(token_ids) == gt_num_indices
def test_preserve_sentence_and_sequence_features_old_config(): attribute = "text" message = Message.build("hi there") transformers_nlp = HFTransformersNLP({ "model_name": "bert", "model_weights": "bert-base-uncased" }) transformers_nlp.process(message) lm_tokenizer = LanguageModelTokenizer() lm_tokenizer.process(message) lm_featurizer = LanguageModelFeaturizer({"model_name": "gpt2"}) lm_featurizer.process(message) message.set(LANGUAGE_MODEL_DOCS[attribute], None) lm_docs = lm_featurizer._get_docs_for_batch([message], attribute=attribute, inference_mode=True)[0] hf_docs = transformers_nlp._get_docs_for_batch([message], attribute=attribute, inference_mode=True)[0] assert not (message.features[0].features == lm_docs[SEQUENCE_FEATURES]).any() assert not (message.features[1].features == lm_docs[SENTENCE_FEATURES]).any() assert (message.features[0].features == hf_docs[SEQUENCE_FEATURES]).all() assert (message.features[1].features == hf_docs[SENTENCE_FEATURES]).all()
def test_lm_tokenizer_number_of_sub_tokens(text, expected_number_of_sub_tokens): transformers_config = {"model_name": "bert"} # Test for one should be enough transformers_nlp = HFTransformersNLP(transformers_config) lm_tokenizer = LanguageModelTokenizer() message = Message(text) td = TrainingData([message]) transformers_nlp.train(td) lm_tokenizer.train(td) assert [ t.get(NUMBER_OF_SUB_TOKENS) for t in message.get(TOKENS_NAMES[TEXT])[:-1] ] == expected_number_of_sub_tokens
def test_lm_tokenizer_custom_intent_symbol(text, expected_tokens): component_config = {"intent_tokenization_flag": True, "intent_split_symbol": "+"} transformers_config = {"model_name": "bert"} # Test for one should be enough transformers_nlp = HFTransformersNLP(transformers_config) lm_tokenizer = LanguageModelTokenizer(component_config) message = Message(text) message.set(INTENT, text) td = TrainingData([message]) transformers_nlp.train(td) lm_tokenizer.train(td) assert [t.text for t in message.get(TOKENS_NAMES[INTENT])] == expected_tokens
def test_log_deprecation_warning_with_old_config(caplog: LogCaptureFixture): message = Message.build("hi there") transformers_nlp = HFTransformersNLP( {"model_name": "bert", "model_weights": "bert-base-uncased"} ) transformers_nlp.process(message) caplog.set_level(logging.DEBUG) lm_tokenizer = LanguageModelTokenizer() lm_tokenizer.process(message) lm_featurizer = LanguageModelFeaturizer(skip_model_load=True) caplog.clear() with caplog.at_level(logging.DEBUG): lm_featurizer.process(message) assert "deprecated component HFTransformersNLP" in caplog.text
def test_lm_tokenizer_edge_cases(model_name, texts, expected_tokens, expected_indices): transformers_config = {"model_name": model_name} transformers_nlp = HFTransformersNLP(transformers_config) lm_tokenizer = LanguageModelTokenizer() for text, gt_tokens, gt_indices in zip(texts, expected_tokens, expected_indices): message = Message.build(text=text) transformers_nlp.process(message) tokens = lm_tokenizer.tokenize(message, TEXT) assert [t.text for t in tokens] == gt_tokens assert [t.start for t in tokens] == [i[0] for i in gt_indices] assert [t.end for t in tokens] == [i[1] for i in gt_indices]
from rasa.nlu.constants import (TEXT, SPACY_DOCS) logger = logging_setup() test_input = "Okay, pick up this yellow banana for me." message = Message(test_input) tk = WhitespaceTokenizer() tokens = tk.tokenize(message, attribute=TEXT) logger.info('Whitespace: {}'.format([t.text for t in tokens])) tk = SpacyTokenizer() message.set(SPACY_DOCS[TEXT], spacy_nlp(test_input)) tokens = tk.tokenize(message, attribute=TEXT) logger.info('SpaCy: {}'.format([t.text for t in tokens])) tk = MitieTokenizer() tokens = tk.tokenize(message, attribute=TEXT) logger.info('Mitie: {}'.format([t.text for t in tokens])) tk = ConveRTTokenizer() tokens = tk.tokenize(message, attribute=TEXT) logger.info('ConveRT: {}'.format([t.text for t in tokens])) tk = LanguageModelTokenizer() transformers_nlp = HFTransformersNLP({"model_name": "bert"}) transformers_nlp.process(message) tokens = tk.tokenize(message, attribute=TEXT) logger.info('BERT: {}'.format([t.text for t in tokens]))