def test_convert_featurizer_process(): featurizer = ConveRTFeaturizer.create({}, RasaNLUModelConfig()) sentence = "Hey how are you today ?" message = Message(sentence) show_message(message) tokens = ConveRTTokenizer().tokenize(message, attribute=TEXT) tokens = Tokenizer.add_cls_token(tokens, attribute=TEXT) message.set(TOKENS_NAMES[TEXT], tokens) assert show_message(message, False) == { "tokens": ["hey", "how", "are", "you", "today", "__CLS__"], "text": "Hey how are you today ?" } featurizer.process(message) show_message(message) expected = np.array( [2.2636216, -0.26475656, -1.1358104, -0.49751878, -1.3946456]) expected_cls = np.array( [1.0251294, -0.04053932, -0.7018805, -0.82054937, -0.75054353]) vecs = message.get(DENSE_FEATURE_NAMES[TEXT]) assert len(tokens) == len(vecs) assert len(vecs) == 6 assert len(tokens) == 6 assert len(vecs[0]) == 1024 assert np.allclose(vecs[0][:5], expected, atol=1e-5) assert np.allclose(vecs[-1][:5], expected_cls, atol=1e-5)
def test_convert_featurizer_train(): featurizer = ConveRTFeaturizer.create({}, RasaNLUModelConfig()) sentence = "Hey how are you today ?" message = Message(sentence) message.set(RESPONSE, sentence) tokens = ConveRTTokenizer().tokenize(message, attribute=TEXT) tokens = Tokenizer.add_cls_token(tokens, attribute=TEXT) message.set(TOKENS_NAMES[TEXT], tokens) message.set(TOKENS_NAMES[RESPONSE], tokens) featurizer.train(TrainingData([message]), RasaNLUModelConfig()) expected = np.array( [2.2636216, -0.26475656, -1.1358104, -0.49751878, -1.3946456]) expected_cls = np.array( [1.0251294, -0.04053932, -0.7018805, -0.82054937, -0.75054353]) vecs = message.get(DENSE_FEATURE_NAMES[TEXT]) assert len(tokens) == len(vecs) assert np.allclose(vecs[0][:5], expected, atol=1e-5) assert np.allclose(vecs[-1][:5], expected_cls, atol=1e-5) vecs = message.get(DENSE_FEATURE_NAMES[RESPONSE]) assert len(tokens) == len(vecs) assert np.allclose(vecs[0][:5], expected, atol=1e-5) assert np.allclose(vecs[-1][:5], expected_cls, atol=1e-5) vecs = message.get(DENSE_FEATURE_NAMES[INTENT]) assert vecs is None
from rasa.nlu.featurizers.sparse_featurizer.count_vectors_featurizer import CountVectorsFeaturizer from rasa.nlu.constants import SPACY_DOCS, TEXT, DENSE_FEATURE_NAMES, TOKENS_NAMES, SPARSE_FEATURE_NAMES logger = logging_setup() featurizer = SpacyFeaturizer.create({}, RasaNLUModelConfig()) test_input = "Okay, pick up this yellow banana for me." message = Message(test_input) message.set(SPACY_DOCS[TEXT], spacy_nlp(test_input)) featurizer._set_spacy_features(message) vecs = message.get(DENSE_FEATURE_NAMES[TEXT]) logger.info("SpaCy: {}".format(vecs.shape)) message = Message(test_input) featurizer = MitieFeaturizer.create({}, RasaNLUModelConfig()) MitieTokenizer().process(message) tokens = message.get(TOKENS_NAMES[TEXT]) vecs = featurizer.features_for_tokens(tokens, mitie_feature_extractor) logger.info("Mitie: {}".format(vecs.shape)) featurizer = ConveRTFeaturizer.create({}, RasaNLUModelConfig()) tokens = ConveRTTokenizer().tokenize(message, attribute=TEXT) tokens = Tokenizer.add_cls_token(tokens, attribute=TEXT) message.set(TOKENS_NAMES[TEXT], tokens) featurizer.process(message) vecs = message.get(DENSE_FEATURE_NAMES[TEXT]) logger.info("ConveRT: {}".format(vecs.shape))