async def compare_models_in_dir(model_dir: Text, stories_file: Text,
output: Text) -> None:
"""Evaluate multiple trained models in a directory on a test set.
Args:
model_dir: path to directory that contains the models to evaluate
stories_file: path to the story file
output: output directory to store results to
"""
number_correct = defaultdict(list)
for run in io_utils.list_subdirectories(model_dir):
number_correct_in_run = defaultdict(list)
for model in sorted(io_utils.list_files(run)):
if not model.endswith("tar.gz"):
continue
# The model files are named like <config-name>PERCENTAGE_KEY<number>.tar.gz
# Remove the percentage key and number from the name to get the config name
config_name = os.path.basename(model).split(PERCENTAGE_KEY)[0]
number_of_correct_stories = await _evaluate_core_model(
model, stories_file)
number_correct_in_run[config_name].append(
number_of_correct_stories)
for k, v in number_correct_in_run.items():
number_correct[k].append(v)
io_utils.dump_obj_as_json_to_file(os.path.join(output, RESULTS_FILE),
number_correct)
async def compare_models_in_dir(model_dir: Text, stories_file: Text,
output: Text) -> None:
"""Evaluates multiple trained models in a directory on a test set."""
from rasa.core import utils
import rasa.utils.io as io_utils
number_correct = defaultdict(list)
for run in io_utils.list_subdirectories(model_dir):
number_correct_in_run = defaultdict(list)
for model in sorted(io_utils.list_files(run)):
if not model.endswith("tar.gz"):
continue
# The model files are named like <policy-name><number>.tar.gz
# Remove the number from the name to get the policy name
policy_name = "".join(
[i for i in os.path.basename(model) if not i.isdigit()])
number_of_correct_stories = await _evaluate_core_model(
model, stories_file)
number_correct_in_run[policy_name].append(
number_of_correct_stories)
for k, v in number_correct_in_run.items():
number_correct[k].append(v)
utils.dump_obj_as_json_to_file(os.path.join(output, RESULTS_FILE),
number_correct)
def test_train_core_compare(
run_in_default_project_without_models: Callable[..., RunResult]):
temp_dir = os.getcwd()
io_utils.write_yaml_file(
{
"language": "en",
"pipeline": "supervised_embeddings",
"policies": [{
"name": "KerasPolicy"
}],
},
"config_1.yml",
)
io_utils.write_yaml_file(
{
"language": "en",
"pipeline": "supervised_embeddings",
"policies": [{
"name": "MemoizationPolicy"
}],
},
"config_2.yml",
)
run_in_default_project_without_models(
"train",
"core",
"-c",
"config_1.yml",
"config_2.yml",
"--stories",
"data/stories.md",
"--out",
"core_comparison_results",
"--runs",
"2",
"--percentages",
"25",
"75",
"--augmentation",
"5",
)
assert os.path.exists(os.path.join(temp_dir, "core_comparison_results"))
run_directories = io_utils.list_subdirectories(
os.path.join(temp_dir, "core_comparison_results"))
assert len(run_directories) == 2
model_files = io_utils.list_files(
os.path.join(temp_dir, "core_comparison_results", run_directories[0]))
assert len(model_files) == 4
assert model_files[0].endswith("tar.gz")
