async def test_events_schema(
monkeypatch: MonkeyPatch, default_agent: Agent, config_path: Text
):
# this allows us to patch the printing part used in debug mode to collect the
# reported events
monkeypatch.setenv("RASA_TELEMETRY_DEBUG", "true")
monkeypatch.setenv("RASA_TELEMETRY_ENABLED", "true")
mock = Mock()
monkeypatch.setattr(telemetry, "print_telemetry_event", mock)
with open(TELEMETRY_EVENTS_JSON) as f:
schemas = json.load(f)["events"]
initial = asyncio.all_tasks()
# Generate all known backend telemetry events, and then use events.json to
# validate their schema.
training_data = TrainingDataImporter.load_from_config(config_path)
with telemetry.track_model_training(training_data, "rasa"):
await asyncio.sleep(1)
telemetry.track_telemetry_disabled()
telemetry.track_data_split(0.5, "nlu")
telemetry.track_validate_files(True)
telemetry.track_data_convert("yaml", "nlu")
telemetry.track_tracker_export(5, TrackerStore(domain=None), EventBroker())
telemetry.track_interactive_learning_start(True, False)
telemetry.track_server_start([CmdlineInput()], None, None, 42, True)
telemetry.track_project_init("tests/")
telemetry.track_shell_started("nlu")
telemetry.track_rasa_x_local()
telemetry.track_visualization()
telemetry.track_core_model_test(5, True, default_agent)
telemetry.track_nlu_model_test(TrainingData())
pending = asyncio.all_tasks() - initial
await asyncio.gather(*pending)
assert mock.call_count == 15
for args, _ in mock.call_args_list:
event = args[0]
# `metrics_id` automatically gets added to all event but is
# not part of the schema so we need to remove it before validation
del event["properties"]["metrics_id"]
jsonschema.validate(
instance=event["properties"], schema=schemas[event["event"]]
)
async def test(
stories: Text,
agent: "Agent",
max_stories: Optional[int] = None,
out_directory: Optional[Text] = None,
fail_on_prediction_errors: bool = False,
e2e: bool = False,
disable_plotting: bool = False,
successes: bool = False,
errors: bool = True,
) -> Dict[Text, Any]:
"""Run the evaluation of the stories, optionally plot the results.
Args:
stories: the stories to evaluate on
agent: the agent
max_stories: maximum number of stories to consider
out_directory: path to directory to results to
fail_on_prediction_errors: boolean indicating whether to fail on prediction
errors or not
e2e: boolean indicating whether to use end to end evaluation or not
disable_plotting: boolean indicating whether to disable plotting or not
successes: boolean indicating whether to write down successful predictions or
not
errors: boolean indicating whether to write down incorrect predictions or not
Returns:
Evaluation summary.
"""
from rasa.test import get_evaluation_metrics
generator = await _create_data_generator(stories, agent, max_stories, e2e)
completed_trackers = generator.generate_story_trackers()
story_evaluation, _ = await _collect_story_predictions(
completed_trackers, agent, fail_on_prediction_errors, e2e)
evaluation_store = story_evaluation.evaluation_store
with warnings.catch_warnings():
from sklearn.exceptions import UndefinedMetricWarning
warnings.simplefilter("ignore", UndefinedMetricWarning)
targets, predictions = evaluation_store.serialise()
if out_directory:
report, precision, f1, accuracy = get_evaluation_metrics(
targets, predictions, output_dict=True)
report_filename = os.path.join(out_directory, REPORT_STORIES_FILE)
rasa.shared.utils.io.dump_obj_as_json_to_file(
report_filename, report)
logger.info(f"Stories report saved to {report_filename}.")
else:
report, precision, f1, accuracy = get_evaluation_metrics(
targets, predictions, output_dict=True)
telemetry.track_core_model_test(len(generator.story_graph.story_steps),
e2e, agent)
_log_evaluation_table(
evaluation_store.action_targets,
"ACTION",
report,
precision,
f1,
accuracy,
story_evaluation.in_training_data_fraction,
include_report=False,
)
if not disable_plotting and out_directory:
_plot_story_evaluation(
evaluation_store.action_targets,
evaluation_store.action_predictions,
out_directory,
)
if errors and out_directory:
_log_stories(
story_evaluation.failed_stories,
os.path.join(out_directory, FAILED_STORIES_FILE),
)
if successes and out_directory:
_log_stories(
story_evaluation.successful_stories,
os.path.join(out_directory, SUCCESSFUL_STORIES_FILE),
)
return {
"report": report,
"precision": precision,
"f1": f1,
"accuracy": accuracy,
"actions": story_evaluation.action_list,
"in_training_data_fraction":
story_evaluation.in_training_data_fraction,
"is_end_to_end_evaluation": e2e,
}
async def test(
stories: Text,
agent: "Agent",
max_stories: Optional[int] = None,
out_directory: Optional[Text] = None,
fail_on_prediction_errors: bool = False,
e2e: bool = False,
disable_plotting: bool = False,
successes: bool = False,
errors: bool = True,
warnings: bool = True,
) -> Dict[Text, Any]:
"""Run the evaluation of the stories, optionally plot the results.
Args:
stories: the stories to evaluate on
agent: the agent
max_stories: maximum number of stories to consider
out_directory: path to directory to results to
fail_on_prediction_errors: boolean indicating whether to fail on prediction
errors or not
e2e: boolean indicating whether to use end to end evaluation or not
disable_plotting: boolean indicating whether to disable plotting or not
successes: boolean indicating whether to write down successful predictions or
not
errors: boolean indicating whether to write down incorrect predictions or not
warnings: boolean indicating whether to write down prediction warnings or not
Returns:
Evaluation summary.
"""
from rasa.model_testing import get_evaluation_metrics
generator = _create_data_generator(stories, agent, max_stories, e2e)
completed_trackers = generator.generate_story_trackers()
story_evaluation, _, entity_results = await _collect_story_predictions(
completed_trackers, agent, fail_on_prediction_errors, use_e2e=e2e)
evaluation_store = story_evaluation.evaluation_store
with pywarnings.catch_warnings():
from sklearn.exceptions import UndefinedMetricWarning
pywarnings.simplefilter("ignore", UndefinedMetricWarning)
targets, predictions = evaluation_store.serialise()
if out_directory:
report, precision, f1, action_accuracy = get_evaluation_metrics(
targets, predictions, output_dict=True)
# Add conversation level accuracy to story report.
num_failed = len(story_evaluation.failed_stories)
num_correct = len(story_evaluation.successful_stories)
num_warnings = len(story_evaluation.stories_with_warnings)
num_convs = num_failed + num_correct
if num_convs and isinstance(report, Dict):
conv_accuracy = num_correct / num_convs
report["conversation_accuracy"] = {
"accuracy": conv_accuracy,
"correct": num_correct,
"with_warnings": num_warnings,
"total": num_convs,
}
report_filename = os.path.join(out_directory, REPORT_STORIES_FILE)
rasa.shared.utils.io.dump_obj_as_json_to_file(
report_filename, report)
logger.info(f"Stories report saved to {report_filename}.")
else:
report, precision, f1, action_accuracy = get_evaluation_metrics(
targets, predictions, output_dict=True)
evaluate_entities(
entity_results,
POLICIES_THAT_EXTRACT_ENTITIES,
out_directory,
successes,
errors,
disable_plotting,
)
telemetry.track_core_model_test(len(generator.story_graph.story_steps),
e2e, agent)
_log_evaluation_table(
evaluation_store.action_targets,
"ACTION",
action_accuracy,
precision=precision,
f1=f1,
in_training_data_fraction=story_evaluation.in_training_data_fraction,
)
if not disable_plotting and out_directory:
_plot_story_evaluation(
evaluation_store.action_targets,
evaluation_store.action_predictions,
out_directory,
)
if errors and out_directory:
_log_stories(
story_evaluation.failed_stories,
os.path.join(out_directory, FAILED_STORIES_FILE),
"None of the test stories failed - all good!",
)
if successes and out_directory:
_log_stories(
story_evaluation.successful_stories,
os.path.join(out_directory, SUCCESSFUL_STORIES_FILE),
"None of the test stories succeeded :(",
)
if warnings and out_directory:
_log_stories(
story_evaluation.stories_with_warnings,
os.path.join(out_directory, STORIES_WITH_WARNINGS_FILE),
"No warnings for test stories",
)
return {
"report": report,
"precision": precision,
"f1": f1,
"accuracy": action_accuracy,
"actions": story_evaluation.action_list,
"in_training_data_fraction":
story_evaluation.in_training_data_fraction,
"is_end_to_end_evaluation": e2e,
}
