def test_train_load_predict_loop( default_model_storage: ModelStorage, default_execution_context: ExecutionContext, mitie_model: MitieModel, mitie_tokenizer: MitieTokenizer, ): resource = Resource("mitie_classifier") component = MitieIntentClassifier.create( MitieIntentClassifier.get_default_config(), default_model_storage, resource, default_execution_context, ) training_data = rasa.shared.nlu.training_data.loading.load_data( "data/examples/rasa/demo-rasa.yml") # Tokenize message as classifier needs that mitie_tokenizer.process_training_data(training_data) component.train(training_data, mitie_model) component = MitieIntentClassifier.load( MitieIntentClassifier.get_default_config(), default_model_storage, resource, default_execution_context, ) test_message = Message({TEXT: "hi"}) mitie_tokenizer.process([test_message]) component.process([test_message], mitie_model) assert test_message.data[INTENT][INTENT_NAME_KEY] == "greet" assert test_message.data[INTENT][PREDICTED_CONFIDENCE_KEY] > 0
def test_mitie_featurizer_train( create: Callable[[Dict[Text, Any]], MitieFeaturizer], mitie_model: MitieModel, mitie_tokenizer: MitieTokenizer, ): featurizer = create({"alias": "mitie_featurizer"}) sentence = "Hey how are you today" message = Message(data={TEXT: sentence}) message.set(RESPONSE, sentence) message.set(INTENT, "intent") mitie_tokenizer.process_training_data(TrainingData([message])) featurizer.process_training_data(TrainingData([message]), mitie_model) expected = np.array([ 0.00000000e00, -5.12735510e00, 4.39929873e-01, -5.60760403e00, -8.26445103e00 ]) expected_cls = np.array( [0.0, -4.4551446, 0.26073121, -1.46632245, -1.84205751]) seq_vec, sen_vec = message.get_dense_features(TEXT, []) if seq_vec: seq_vec = seq_vec.features if sen_vec: sen_vec = sen_vec.features assert len(message.get(TOKENS_NAMES[TEXT])) == len(seq_vec) assert np.allclose(seq_vec[0][:5], expected, atol=1e-5) assert np.allclose(sen_vec[-1][:5], expected_cls, atol=1e-5) seq_vec, sen_vec = message.get_dense_features(RESPONSE, []) if seq_vec: seq_vec = seq_vec.features if sen_vec: sen_vec = sen_vec.features assert len(message.get(TOKENS_NAMES[RESPONSE])) == len(seq_vec) assert np.allclose(seq_vec[0][:5], expected, atol=1e-5) assert np.allclose(sen_vec[-1][:5], expected_cls, atol=1e-5) seq_vec, sen_vec = message.get_dense_features(INTENT, []) if seq_vec: seq_vec = seq_vec.features if sen_vec: sen_vec = sen_vec.features assert seq_vec is None assert sen_vec is None