async def test_eval_data( component_builder: ComponentBuilder, tmp_path: Path, project: Text, unpacked_trained_rasa_model: Text, ): config_path = os.path.join(project, "config.yml") data_importer = TrainingDataImporter.load_nlu_importer_from_config( config_path, training_data_paths=[ "data/examples/rasa/demo-rasa.yml", "data/examples/rasa/demo-rasa-responses.yml", ], ) _, nlu_model_directory = rasa.model.get_model_subdirectories( unpacked_trained_rasa_model ) interpreter = Interpreter.load(nlu_model_directory, component_builder) data = await data_importer.get_nlu_data() (intent_results, response_selection_results, entity_results) = get_eval_data( interpreter, data ) assert len(intent_results) == 46 assert len(response_selection_results) == 0 assert len(entity_results) == 46
def test_replacing_fallback_intent(): expected_intent = "greet" expected_confidence = 0.345 fallback_prediction = { INTENT: { INTENT_NAME_KEY: DEFAULT_NLU_FALLBACK_INTENT_NAME, PREDICTED_CONFIDENCE_KEY: 1, }, INTENT_RANKING_KEY: [ { INTENT_NAME_KEY: DEFAULT_NLU_FALLBACK_INTENT_NAME, PREDICTED_CONFIDENCE_KEY: 1, }, { INTENT_NAME_KEY: expected_intent, PREDICTED_CONFIDENCE_KEY: expected_confidence, }, {INTENT_NAME_KEY: "some", PREDICTED_CONFIDENCE_KEY: 0.1}, ], } interpreter = ConstantInterpreter(fallback_prediction) training_data = TrainingData( [Message.build("hi", "greet"), Message.build("bye", "bye")] ) intent_evaluations, _, _ = get_eval_data(interpreter, training_data) assert all( prediction.intent_prediction == expected_intent and prediction.confidence == expected_confidence for prediction in intent_evaluations )
async def test_eval_data(component_builder, tmpdir, project): _config = RasaNLUModelConfig({ "pipeline": [ { "name": "WhitespaceTokenizer" }, { "name": "CountVectorsFeaturizer" }, { "name": "DIETClassifier", "epochs": 2 }, { "name": "ResponseSelector", "epochs": 2 }, ], "language": "en", }) config_path = os.path.join(project, "config.yml") data_importer = TrainingDataImporter.load_nlu_importer_from_config( config_path, training_data_paths=[ "data/examples/rasa/demo-rasa.md", "data/examples/rasa/demo-rasa-responses.md", ], ) (_, _, persisted_path) = await train( _config, path=tmpdir.strpath, data=data_importer, component_builder=component_builder, persist_nlu_training_data=True, ) interpreter = Interpreter.load(persisted_path, component_builder) data = await data_importer.get_nlu_data() ( intent_results, response_selection_results, entity_results, ) = get_eval_data(interpreter, data) assert len(intent_results) == 46 assert len(response_selection_results) == 46 assert len(entity_results) == 46
def evaluate_update(repository_version, repository_authorization): evaluations = backend().request_backend_start_evaluation( repository_version, repository_authorization) training_examples = [] for evaluate in evaluations: training_examples.append( Message.build( text=evaluate.get("text"), intent=evaluate.get("intent"), entities=evaluate.get("entities"), )) test_data = TrainingData(training_examples=training_examples) interpreter = update_interpreters.get(repository_version, repository_authorization, rasa_version, use_cache=False) result = { "intent_evaluation": None, "entity_evaluation": None, "response_selection_evaluation": None, } intent_results, response_selection_results, entity_results, = get_eval_data( interpreter, test_data) if intent_results: result["intent_evaluation"] = evaluate_intents(intent_results) if entity_results: extractors = get_entity_extractors(interpreter) result["entity_evaluation"] = evaluate_entities( entity_results, extractors) intent_evaluation = result.get("intent_evaluation") entity_evaluation = result.get("entity_evaluation") merged_logs = merge_intent_entity_log(intent_evaluation, entity_evaluation) log = get_formatted_log(merged_logs) charts = plot_and_save_charts(repository_version, intent_results) evaluate_result = backend().request_backend_create_evaluate_results( { "repository_version": repository_version, "matrix_chart": charts.get("matrix_chart"), "confidence_chart": charts.get("confidence_chart"), "log": json.dumps(log), "intentprecision": intent_evaluation.get("precision"), "intentf1_score": intent_evaluation.get("f1_score"), "intentaccuracy": intent_evaluation.get("accuracy"), "entityprecision": entity_evaluation.get("precision"), "entityf1_score": entity_evaluation.get("f1_score"), "entityaccuracy": entity_evaluation.get("accuracy"), }, repository_authorization, ) intent_reports = intent_evaluation.get("report", {}) entity_reports = entity_evaluation.get("report", {}) for intent_key in intent_reports.keys(): if intent_key and intent_key not in excluded_itens: intent = intent_reports.get(intent_key) backend().request_backend_create_evaluate_results_intent( { "evaluate_id": evaluate_result.get("evaluate_id"), "precision": intent.get("precision"), "recall": intent.get("recall"), "f1_score": intent.get("f1-score"), "support": intent.get("support"), "intent_key": intent_key, }, repository_authorization, ) for entity_key in entity_reports.keys(): if entity_key and entity_key not in excluded_itens: # pragma: no cover entity = entity_reports.get(entity_key) backend().request_backend_create_evaluate_results_score( { "evaluate_id": evaluate_result.get("evaluate_id"), "repository_version": repository_version, "precision": entity.get("precision"), "recall": entity.get("recall"), "f1_score": entity.get("f1-score"), "support": entity.get("support"), "entity_key": entity_key, }, repository_authorization, ) return { "id": evaluate_result.get("evaluate_id"), "version": evaluate_result.get("evaluate_version"), "cross_validation": False }
def run_test_on_nlu(nlu_path: str, model_path: str): """ Run tests on stories. Args: nlu_path: path where nlu test data is present as YAML. model_path: Model path where model on which test has to be run is present. Returns: dictionary with evaluation results """ from rasa.model import get_model import rasa.shared.nlu.training_data.loading from rasa.nlu.model import Interpreter from rasa.nlu.test import ( remove_pretrained_extractors, get_eval_data, evaluate_intents, evaluate_response_selections, get_entity_extractors, ) from kairon import Utility unpacked_model = get_model(model_path) nlu_model = os.path.join(unpacked_model, "nlu") interpreter = Interpreter.load(nlu_model) interpreter.pipeline = remove_pretrained_extractors(interpreter.pipeline) test_data = rasa.shared.nlu.training_data.loading.load_data( nlu_path, interpreter.model_metadata.language ) result: Dict[Text, Optional[Dict]] = { "intent_evaluation": None, "entity_evaluation": None, "response_selection_evaluation": None, } (intent_results, response_selection_results, entity_results) = get_eval_data( interpreter, test_data ) if intent_results: successes = [] errors = [] result["intent_evaluation"] = evaluate_intents(intent_results, None, False, False, True) if result["intent_evaluation"].get('predictions'): del result["intent_evaluation"]['predictions'] del result["intent_evaluation"]['report'] for r in intent_results: if r.intent_target == r.intent_prediction: pass # successes.append({ # "text": r.message, # "intent": r.intent_target, # "intent_prediction": { # 'name': r.intent_prediction, # "confidence": r.confidence, # }, # }) else: errors.append({ "text": r.message, "intent": r.intent_target, "intent_prediction": { 'name': r.intent_prediction, "confidence": r.confidence, }, }) result["intent_evaluation"]['total_count'] = len(successes) + len(errors) result["intent_evaluation"]['success_count'] = len(successes) result["intent_evaluation"]['failure_count'] = len(errors) result["intent_evaluation"]['successes'] = successes result["intent_evaluation"]['errors'] = errors if response_selection_results: successes = [] errors = [] result["response_selection_evaluation"] = evaluate_response_selections( response_selection_results, None, False, False, True ) if result["response_selection_evaluation"].get('predictions'): del result["response_selection_evaluation"]['predictions'] del result["response_selection_evaluation"]['report'] for r in response_selection_results: if r.intent_response_key_prediction == r.intent_response_key_target: pass # successes.append({ # "text": r.message, # "intent_response_key_target": r.intent_response_key_target, # "intent_response_key_prediction": { # "name": r.intent_response_key_prediction, # "confidence": r.confidence, # }, # }) else: if not Utility.check_empty_string(r.intent_response_key_target): errors.append( { "text": r.message, "intent_response_key_target": r.intent_response_key_target, "intent_response_key_prediction": { "name": r.intent_response_key_prediction, "confidence": r.confidence, }, } ) result["response_selection_evaluation"]['total_count'] = len(successes) + len(errors) result["response_selection_evaluation"]['success_count'] = len(successes) result["response_selection_evaluation"]['failure_count'] = len(errors) result["response_selection_evaluation"]['successes'] = successes result["response_selection_evaluation"]['errors'] = errors if any(entity_results): extractors = get_entity_extractors(interpreter) result["entity_evaluation"] = ModelTester.__evaluate_entities(entity_results, extractors) return result