async def compare(models: Text, stories_file: Text, output: Text) -> None:
"""Evaluates multiple trained models on a test set."""
from rasa.core.agent import Agent
import rasa.nlu.utils as nlu_utils
from rasa.core import utils
num_correct = defaultdict(list)
for run in nlu_utils.list_subdirectories(models):
num_correct_run = defaultdict(list)
for model in sorted(nlu_utils.list_subdirectories(run)):
logger.info("Evaluating model {}".format(model))
agent = Agent.load(model)
completed_trackers = await _generate_trackers(stories_file, agent)
story_eval_store, no_of_stories = collect_story_predictions(
completed_trackers, agent)
failed_stories = story_eval_store.failed_stories
policy_name = "".join(
[i for i in os.path.basename(model) if not i.isdigit()])
num_correct_run[policy_name].append(no_of_stories -
len(failed_stories))
for k, v in num_correct_run.items():
num_correct[k].append(v)
utils.dump_obj_as_json_to_file(os.path.join(output, "results.json"),
num_correct)
def _list_models_in_dir(path):
if not path or not os.path.isdir(path):
return []
else:
return [
os.path.relpath(model, path)
for model in utils.list_subdirectories(path)
]
def test_train_core_compare(run_in_default_project):
temp_dir = os.getcwd()
write_yaml_file(
{
"language": "en",
"pipeline": "supervised_embeddings",
"policies": [{
"name": "KerasPolicy"
}],
},
"config_1.yml",
)
write_yaml_file(
{
"language": "en",
"pipeline": "supervised_embeddings",
"policies": [{
"name": "MemoizationPolicy"
}],
},
"config_2.yml",
)
run_in_default_project(
"train",
"core",
"-c",
"config_1.yml",
"config_2.yml",
"--stories",
"data/stories.md",
"--out",
"core_comparison_results",
"--runs",
"2",
"--percentages",
"25",
"75",
"--augmentation",
"5",
)
assert os.path.exists(os.path.join(temp_dir, "core_comparison_results"))
run_directories = list_subdirectories(
os.path.join(temp_dir, "core_comparison_results"))
assert len(run_directories) == 2
model_files = list_files(
os.path.join(temp_dir, "core_comparison_results", run_directories[0]))
assert len(model_files) == 4
assert model_files[0].endswith("tar.gz")
def _list_projects(path: Text) -> List[Text]:
"""List the projects in the path, ignoring hidden directories."""
return [os.path.basename(fn) for fn in utils.list_subdirectories(path)]