def test_count_vector_featurizer_using_tokens(tokens, expected): from rasa.nlu.featurizers.count_vectors_featurizer import CountVectorsFeaturizer ftr = CountVectorsFeaturizer({"token_pattern": r"(?u)\b\w+\b"}) # using empty string instead of real text string to make sure # count vector only can come from `tokens` feature. # using `message.text` can not get correct result tokens_feature = [Token(i, 0) for i in tokens] train_message = Message("") train_message.set("tokens", tokens_feature) # this is needed for a valid training example train_message.set("intent", "bla") data = TrainingData([train_message]) ftr.train(data) test_message = Message("") test_message.set("tokens", tokens_feature) ftr.process(test_message) assert np.all(test_message.get("text_features") == expected)
def test_count_vector_featurizer_persist_load(tmpdir): from rasa.nlu.featurizers.count_vectors_featurizer import CountVectorsFeaturizer # set non default values to config config = { "analyzer": "char", "token_pattern": r"(?u)\b\w+\b", "strip_accents": "ascii", "stop_words": "stop", "min_df": 2, "max_df": 3, "min_ngram": 2, "max_ngram": 3, "max_features": 10, "lowercase": False, } train_ftr = CountVectorsFeaturizer(config) sentence1 = "ababab 123 13xc лаомтгцу sfjv oö aà" sentence2 = "abababalidcn 123123 13xcdc лаомтгцу sfjv oö aà" train_message1 = Message(sentence1) train_message2 = Message(sentence2) # this is needed for a valid training example train_message1.set("intent", "bla") train_message2.set("intent", "bla") data = TrainingData([train_message1, train_message2]) train_ftr.train(data) # persist featurizer file_dict = train_ftr.persist("ftr", tmpdir.strpath) train_vect_params = train_ftr.vectorizer.get_params() # add trained vocabulary to vectorizer params train_vect_params.update({"vocabulary": train_ftr.vectorizer.vocabulary_}) # load featurizer meta = train_ftr.component_config.copy() meta.update(file_dict) test_ftr = CountVectorsFeaturizer.load(meta, tmpdir.strpath) test_vect_params = test_ftr.vectorizer.get_params() assert train_vect_params == test_vect_params test_message1 = Message(sentence1) test_ftr.process(test_message1) test_message2 = Message(sentence2) test_ftr.process(test_message2) # check that train features and test features after loading are the same assert np.all([ train_message1.get("text_features") == test_message1.get( "text_features"), train_message2.get("text_features") == test_message2.get( "text_features"), ])
def test_count_vector_featurizer(sentence, expected): from rasa.nlu.featurizers.count_vectors_featurizer import CountVectorsFeaturizer ftr = CountVectorsFeaturizer({"token_pattern": r"(?u)\b\w+\b"}) train_message = Message(sentence) # this is needed for a valid training example train_message.set("intent", "bla") data = TrainingData([train_message]) ftr.train(data) test_message = Message(sentence) ftr.process(test_message) assert np.all(test_message.get("text_features") == expected)
def test_count_vector_featurizer_char(sentence, expected): from rasa.nlu.featurizers.count_vectors_featurizer import CountVectorsFeaturizer ftr = CountVectorsFeaturizer({"min_ngram": 1, "max_ngram": 2, "analyzer": "char"}) train_message = Message(sentence) # this is needed for a valid training example train_message.set("intent", "bla") data = TrainingData([train_message]) ftr.train(data) test_message = Message(sentence) ftr.process(test_message) assert np.all(test_message.get("text_features") == expected)
def test_count_vector_featurizer_attribute_featurization( sentence, intent, response, intent_features, response_features): from rasa.nlu.featurizers.count_vectors_featurizer import CountVectorsFeaturizer ftr = CountVectorsFeaturizer({"token_pattern": r"(?u)\b\w+\b"}) train_message = Message(sentence) # this is needed for a valid training example train_message.set("intent", intent) train_message.set("response", response) data = TrainingData([train_message]) ftr.train(data) assert train_message.get("intent_features") == intent_features assert train_message.get("response_features") == response_features
def test_count_vector_featurizer_char_intent_featurizer(): from rasa.nlu.featurizers.count_vectors_featurizer import CountVectorsFeaturizer ftr = CountVectorsFeaturizer({ "min_ngram": 1, "max_ngram": 2, "analyzer": "char" }) td = training_data.load_data("data/examples/rasa/demo-rasa.json") ftr.train(td, config=None) intent_features_exist = np.array([ True if example.get("intent_features") is not None else False for example in td.intent_examples ]) # no intent features should have been set assert not any(intent_features_exist)
def test_count_vector_featurizer_shared_vocab( sentence, intent, response, text_features, intent_features, response_features ): from rasa.nlu.featurizers.count_vectors_featurizer import CountVectorsFeaturizer ftr = CountVectorsFeaturizer( {"token_pattern": r"(?u)\b\w+\b", "use_shared_vocab": True} ) train_message = Message(sentence) # this is needed for a valid training example train_message.set("intent", intent) train_message.set("response", response) data = TrainingData([train_message]) ftr.train(data) assert np.all(train_message.get("text_features") == text_features) assert np.all(train_message.get("intent_features") == intent_features) assert np.all(train_message.get("response_features") == response_features)
class IncrementalCVF(IncrementalComponent): name = "IncrementalCVF" """ Since this is a wrapper for the non-incremental CountVectorsFeaturizer to be used with our incremental EmbeddingIntentClassifier, we just need to take its provides, requires, and defaults. """ provides = CountVectorsFeaturizer.provides requires = CountVectorsFeaturizer.requires defaults = CountVectorsFeaturizer.defaults @classmethod def required_packages(cls) -> List[Text]: reqs = CountVectorsFeaturizer.required_packages() reqs.append("numpy") return reqs def __init__(self, component_config=None): super(IncrementalCVF, self).__init__(component_config) self.CVF = CountVectorsFeaturizer() # we don't have anything to clear since our featuers are storeed # in the Message, which the IncrementalInterpreter clears. def new_utterance(self) -> None: return def train(self, training_data: TrainingData, cfg: RasaNLUModelConfig = None, **kwargs: Any) -> None: return self.CVF.train(training_data, cfg, **kwargs) # Similar to Featurizer's _combine_with_existing_text_features # Except we are doing a vector sum instead of array stack. This # is because we're adding the new features of that word in particular # rather than entire utterances side by side. def _add_text_features(self, message, additional_features): if message.get("text_features") is not None: return np.add(message.get("text_features"), additional_features) else: return additional_features # On revoke, remove the word's features from the vector def _sub_text_features(self, message, to_sub): if message.get("text_features") is not None: return np.subtract(message.get("text_features"), to_sub) #else: # logger.error("Nothing in text features, cannot subtract") # assuming not using spacy_doc or tokens, so just setting message.text def process(self, message: Message, **kwargs: Any) -> None: iu_list = message.get('iu_list') last_iu = iu_list[-1] iu_word, iu_type = last_iu if iu_type == "add": bag = self.CVF.vectorizers[MESSAGE_TEXT_ATTRIBUTE].transform( [iu_word]).toarray().squeeze() return message.set("text_features", self._add_text_features(message, bag)) elif iu_type == "revoke": return self._revoke(message, iu_word) else: logger.error("incompatible iu type, expected 'add' or 'revoke'," " got '" + iu_type + "'") # TODO: can we just subtract the vector instead of # storing previous features? def _revoke(self, message, word): # revoke on empty should do nothing if message.get("text_features") is not None: return else: bag = self.CVF.vectorizers[MESSAGE_TEXT_ATTRIBUTE].transform( [word]).toarray().squeeze() return message.set("text_features", self._sub_text_features(message, bag)) def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: file_name = file_name + ".pkl" featurizer_file = os.path.join(model_dir, file_name) utils.json_pickle(featurizer_file, self) return {"file": file_name} @classmethod def load(cls, meta: Dict[Text, Any], model_dir: Text = None, model_metadata: Metadata = None, cached_component: Optional['IncrementalCVF'] = None, **kwargs: Any) -> 'IncrementalCVF': if model_dir and meta.get("file"): file_name = meta.get("file") featurizer_file = os.path.join(model_dir, file_name) return utils.json_unpickle(featurizer_file) else: logger.warning("Failed to load featurizer. Maybe path {} " "doesn't exist".format(os.path.abspath(model_dir))) return IncrementalCVF(meta)