def test_preserve_sentence_and_sequence_features_old_config(): attribute = "text" message = Message.build("hi there") transformers_nlp = HFTransformersNLP({ "model_name": "bert", "model_weights": "bert-base-uncased" }) transformers_nlp.process(message) lm_tokenizer = LanguageModelTokenizer() lm_tokenizer.process(message) lm_featurizer = LanguageModelFeaturizer({"model_name": "gpt2"}) lm_featurizer.process(message) message.set(LANGUAGE_MODEL_DOCS[attribute], None) lm_docs = lm_featurizer._get_docs_for_batch([message], attribute=attribute, inference_mode=True)[0] hf_docs = transformers_nlp._get_docs_for_batch([message], attribute=attribute, inference_mode=True)[0] assert not (message.features[0].features == lm_docs[SEQUENCE_FEATURES]).any() assert not (message.features[1].features == lm_docs[SENTENCE_FEATURES]).any() assert (message.features[0].features == hf_docs[SEQUENCE_FEATURES]).all() assert (message.features[1].features == hf_docs[SENTENCE_FEATURES]).all()
def test_lm_tokenizer_edge_cases( model_name, model_weights, texts, expected_tokens, expected_indices, expected_num_token_ids, ): if model_weights is None: model_weights_config = {} else: model_weights_config = {"model_weights": model_weights} transformers_config = { **{ "model_name": model_name }, **model_weights_config } transformers_nlp = HFTransformersNLP(transformers_config) lm_tokenizer = LanguageModelTokenizer() for text, gt_tokens, gt_indices, gt_num_indices in zip( texts, expected_tokens, expected_indices, expected_num_token_ids): message = Message.build(text=text) transformers_nlp.process(message) tokens = lm_tokenizer.tokenize(message, TEXT) token_ids = message.get(LANGUAGE_MODEL_DOCS[TEXT])[TOKEN_IDS] assert [t.text for t in tokens] == gt_tokens assert [t.start for t in tokens] == [i[0] for i in gt_indices] assert [t.end for t in tokens] == [i[1] for i in gt_indices] assert len(token_ids) == gt_num_indices
def test_attention_mask(actual_sequence_length: int, max_input_sequence_length: int, zero_start_index: int): component = HFTransformersNLP({"model_name": "bert"}, skip_model_load=True) attention_mask = component._compute_attention_mask( [actual_sequence_length], max_input_sequence_length) mask_ones = attention_mask[0][:zero_start_index] mask_zeros = attention_mask[0][zero_start_index:] assert np.all(mask_ones == 1) assert np.all(mask_zeros == 0)
def train_texts(texts: List[Text], model_name: Text, model_weights: Text) -> List[Message]: config = create_pretrained_transformers_config(model_name, model_weights) whitespace_tokenizer = WhitespaceTokenizer() transformer = HFTransformersNLP(config) messages = [Message.build(text=text) for text in texts] td = TrainingData(messages) whitespace_tokenizer.train(td) transformer.train(td) return messages
def process_texts(texts: List[Text], model_name: Text, model_weights: Text) -> List[Message]: config = create_pretrained_transformers_config(model_name, model_weights) whitespace_tokenizer = WhitespaceTokenizer() transformer = HFTransformersNLP(config) messages = [] for text in texts: message = Message.build(text=text) whitespace_tokenizer.process(message) transformer.process(message) messages.append(message) return messages
def test_log_longer_sequence(sequence_length: int, model_name: Text, should_overflow: bool, caplog): transformers_config = {"model_name": model_name} transformers_nlp = HFTransformersNLP(transformers_config) text = " ".join(["hi"] * sequence_length) message = Message(text) caplog.set_level(logging.DEBUG) transformers_nlp.process(message) if should_overflow: assert "hi hi hi" in caplog.text assert message.get("text_language_model_doc") is not None
def test_input_padding( token_ids: List[List[int]], max_sequence_length_model: int, resulting_length: int, padding_added: bool, ): component = HFTransformersNLP({"model_name": "bert"}, skip_model_load=True) component.pad_token_id = 0 padded_input = component._add_padding_to_batch(token_ids, max_sequence_length_model) assert len(padded_input[0]) == resulting_length if padding_added: original_length = len(token_ids[0]) assert np.all(np.array(padded_input[0][original_length:]) == 0)
def test_lm_featurizer_shape_values(model_name, texts, expected_shape, expected_sequence_vec, expected_cls_vec): transformers_config = {"model_name": model_name} transformers_nlp = HFTransformersNLP(transformers_config) lm_featurizer = LanguageModelFeaturizer() messages = [] for text in texts: messages.append(Message.build(text=text)) td = TrainingData(messages) transformers_nlp.train(td) lm_featurizer.train(td) for index in range(len(texts)): computed_sequence_vec, computed_sentence_vec = messages[ index].get_dense_features(TEXT, []) if computed_sequence_vec: computed_sequence_vec = computed_sequence_vec.features if computed_sentence_vec: computed_sentence_vec = computed_sentence_vec.features assert computed_sequence_vec.shape[0] == expected_shape[index][0] - 1 assert computed_sequence_vec.shape[1] == expected_shape[index][1] assert computed_sentence_vec.shape[0] == 1 assert computed_sentence_vec.shape[1] == expected_shape[index][1] # Look at the value of first dimension for a few starting timesteps assert np.allclose( computed_sequence_vec[:len(expected_sequence_vec[index]), 0], expected_sequence_vec[index], atol=1e-5, ) # Look at the first value of first five dimensions assert np.allclose(computed_sentence_vec[0][:5], expected_cls_vec[index], atol=1e-5) intent_sequence_vec, intent_sentence_vec = messages[ index].get_dense_features(INTENT, []) if intent_sequence_vec: intent_sequence_vec = intent_sequence_vec.features if intent_sentence_vec: intent_sentence_vec = intent_sentence_vec.features assert intent_sequence_vec is None assert intent_sentence_vec is None
def test_lm_tokenizer_number_of_sub_tokens(text, expected_number_of_sub_tokens): transformers_config = {"model_name": "bert"} # Test for one should be enough transformers_nlp = HFTransformersNLP(transformers_config) lm_tokenizer = LanguageModelTokenizer() message = Message(text) td = TrainingData([message]) transformers_nlp.train(td) lm_tokenizer.train(td) assert [ t.get(NUMBER_OF_SUB_TOKENS) for t in message.get(TOKENS_NAMES[TEXT])[:-1] ] == expected_number_of_sub_tokens
def test_log_deprecation_warning_with_old_config(caplog: LogCaptureFixture): message = Message.build("hi there") transformers_nlp = HFTransformersNLP( {"model_name": "bert", "model_weights": "bert-base-uncased"} ) transformers_nlp.process(message) caplog.set_level(logging.DEBUG) lm_tokenizer = LanguageModelTokenizer() lm_tokenizer.process(message) lm_featurizer = LanguageModelFeaturizer(skip_model_load=True) caplog.clear() with caplog.at_level(logging.DEBUG): lm_featurizer.process(message) assert "deprecated component HFTransformersNLP" in caplog.text
def test_lm_tokenizer_custom_intent_symbol(text, expected_tokens): component_config = {"intent_tokenization_flag": True, "intent_split_symbol": "+"} transformers_config = {"model_name": "bert"} # Test for one should be enough transformers_nlp = HFTransformersNLP(transformers_config) lm_tokenizer = LanguageModelTokenizer(component_config) message = Message(text) message.set(INTENT, text) td = TrainingData([message]) transformers_nlp.train(td) lm_tokenizer.train(td) assert [t.text for t in message.get(TOKENS_NAMES[INTENT])] == expected_tokens
def test_lm_tokenizer_edge_cases(model_name, texts, expected_tokens, expected_indices): transformers_config = {"model_name": model_name} transformers_nlp = HFTransformersNLP(transformers_config) lm_tokenizer = LanguageModelTokenizer() for text, gt_tokens, gt_indices in zip(texts, expected_tokens, expected_indices): message = Message.build(text=text) transformers_nlp.process(message) tokens = lm_tokenizer.tokenize(message, TEXT) assert [t.text for t in tokens] == gt_tokens assert [t.start for t in tokens] == [i[0] for i in gt_indices] assert [t.end for t in tokens] == [i[1] for i in gt_indices]
def test_lm_featurizer_shape_values(): model_name, texts, expected_shape, expected_sequence_vec, expected_cls_vec = samples[3] transformers_config = {"model_name": model_name} transformers_nlp = HFTransformersNLP(transformers_config) lm_featurizer = LanguageModelFeaturizer() messages = [] for text in texts: messages.append(Message.build(text=text)) td = TrainingData(messages) show_training_data(td) transformers_nlp.train(td) show_training_data(td) lm_featurizer.train(td) show_training_data(td) for index in range(len(texts)): computed_feature_vec = messages[index].get(DENSE_FEATURE_NAMES[TEXT]) computed_sequence_vec, computed_sentence_vec = ( computed_feature_vec[:-1], computed_feature_vec[-1], ) assert computed_feature_vec.shape == expected_shape[index] # Look at the value of first dimension for a few starting timesteps assert np.allclose( computed_sequence_vec[: len(expected_sequence_vec[index]), 0], expected_sequence_vec[index], atol=1e-5, ) # Look at the first value of first five dimensions assert np.allclose( computed_sentence_vec[:5], expected_cls_vec[index], atol=1e-5 ) intent_vec = messages[index].get(DENSE_FEATURE_NAMES[INTENT]) assert intent_vec is None
def test_long_sequences_extra_padding( sequence_embeddings: np.ndarray, actual_sequence_lengths: List[int], model_name: Text, padding_needed: bool, ): component = HFTransformersNLP({"model_name": model_name}, skip_model_load=True) modified_sequence_embeddings = component._add_extra_padding( sequence_embeddings, actual_sequence_lengths) if not padding_needed: assert np.all(modified_sequence_embeddings) == np.all( sequence_embeddings) else: assert modified_sequence_embeddings.shape[ 1] == actual_sequence_lengths[0] assert (modified_sequence_embeddings[0].shape[-1] == sequence_embeddings[0].shape[-1]) zero_embeddings = modified_sequence_embeddings[0][sequence_embeddings. shape[1]:] assert np.all(zero_embeddings == 0)
def test_hf_transformer_edge_cases( model_name, model_weights, texts, expected_tokens, expected_indices ): if model_weights is None: model_weights_config = {} else: model_weights_config = {"model_weights": model_weights} transformers_config = {**{"model_name": model_name}, **model_weights_config} hf_transformer = HFTransformersNLP(transformers_config) whitespace_tokenizer = WhitespaceTokenizer() for text, gt_tokens, gt_indices in zip(texts, expected_tokens, expected_indices): message = Message.build(text=text) tokens = whitespace_tokenizer.tokenize(message, TEXT) message.set(TOKENS_NAMES[TEXT], tokens) hf_transformer.process(message) assert [t.text for t in tokens] == gt_tokens assert [t.start for t in tokens] == [i[0] for i in gt_indices] assert [t.end for t in tokens] == [i[1] for i in gt_indices]
def test_lm_featurizer_shape_values(): model_name, texts, expected_shape, expected_sequence_vec, expected_cls_vec = samples[0] transformers_config = {"model_name": model_name} transformers_nlp_bert = HFTransformersNLP({"model_name": "bert"}) transformers_nlp_gpt = HFTransformersNLP({"model_name": "gpt"}) lm_featurizer = LanguageModelFeaturizer() messages = [] for text in texts: messages.append(Message.build(text=text)) td = TrainingData(messages) show_training_data(td) transformers_nlp_bert.train(td) show_training_data(td) transformers_nlp_gpt.train(td) show_training_data(td) lm_featurizer.train(td) show_training_data(td)
def test_sequence_length_overflow_train( input_sequence_length: int, model_name: Text, should_overflow: bool ): component = HFTransformersNLP({"model_name": model_name}, skip_model_load=True) message = Message.build(text=" ".join(["hi"] * input_sequence_length)) if should_overflow: with pytest.raises(RuntimeError): component._validate_sequence_lengths( [input_sequence_length], [message], "text", inference_mode=False ) else: component._validate_sequence_lengths( [input_sequence_length], [message], "text", inference_mode=False )
from rasa.nlu.constants import (TEXT, SPACY_DOCS) logger = logging_setup() test_input = "Okay, pick up this yellow banana for me." message = Message(test_input) tk = WhitespaceTokenizer() tokens = tk.tokenize(message, attribute=TEXT) logger.info('Whitespace: {}'.format([t.text for t in tokens])) tk = SpacyTokenizer() message.set(SPACY_DOCS[TEXT], spacy_nlp(test_input)) tokens = tk.tokenize(message, attribute=TEXT) logger.info('SpaCy: {}'.format([t.text for t in tokens])) tk = MitieTokenizer() tokens = tk.tokenize(message, attribute=TEXT) logger.info('Mitie: {}'.format([t.text for t in tokens])) tk = ConveRTTokenizer() tokens = tk.tokenize(message, attribute=TEXT) logger.info('ConveRT: {}'.format([t.text for t in tokens])) tk = LanguageModelTokenizer() transformers_nlp = HFTransformersNLP({"model_name": "bert"}) transformers_nlp.process(message) tokens = tk.tokenize(message, attribute=TEXT) logger.info('BERT: {}'.format([t.text for t in tokens]))