async def compare_models_in_dir(model_dir: Text, stories_file: Text, output: Text) -> None: """Evaluate multiple trained models in a directory on a test set. Args: model_dir: path to directory that contains the models to evaluate stories_file: path to the story file output: output directory to store results to """ number_correct = defaultdict(list) for run in io_utils.list_subdirectories(model_dir): number_correct_in_run = defaultdict(list) for model in sorted(io_utils.list_files(run)): if not model.endswith("tar.gz"): continue # The model files are named like <config-name>PERCENTAGE_KEY<number>.tar.gz # Remove the percentage key and number from the name to get the config name config_name = os.path.basename(model).split(PERCENTAGE_KEY)[0] number_of_correct_stories = await _evaluate_core_model( model, stories_file) number_correct_in_run[config_name].append( number_of_correct_stories) for k, v in number_correct_in_run.items(): number_correct[k].append(v) io_utils.dump_obj_as_json_to_file(os.path.join(output, RESULTS_FILE), number_correct)
async def compare_models_in_dir(model_dir: Text, stories_file: Text, output: Text) -> None: """Evaluates multiple trained models in a directory on a test set.""" from rasa.core import utils import rasa.utils.io as io_utils number_correct = defaultdict(list) for run in io_utils.list_subdirectories(model_dir): number_correct_in_run = defaultdict(list) for model in sorted(io_utils.list_files(run)): if not model.endswith("tar.gz"): continue # The model files are named like <policy-name><number>.tar.gz # Remove the number from the name to get the policy name policy_name = "".join( [i for i in os.path.basename(model) if not i.isdigit()]) number_of_correct_stories = await _evaluate_core_model( model, stories_file) number_correct_in_run[policy_name].append( number_of_correct_stories) for k, v in number_correct_in_run.items(): number_correct[k].append(v) utils.dump_obj_as_json_to_file(os.path.join(output, RESULTS_FILE), number_correct)
def test_train_core_compare( run_in_default_project_without_models: Callable[..., RunResult]): temp_dir = os.getcwd() io_utils.write_yaml_file( { "language": "en", "pipeline": "supervised_embeddings", "policies": [{ "name": "KerasPolicy" }], }, "config_1.yml", ) io_utils.write_yaml_file( { "language": "en", "pipeline": "supervised_embeddings", "policies": [{ "name": "MemoizationPolicy" }], }, "config_2.yml", ) run_in_default_project_without_models( "train", "core", "-c", "config_1.yml", "config_2.yml", "--stories", "data/stories.md", "--out", "core_comparison_results", "--runs", "2", "--percentages", "25", "75", "--augmentation", "5", ) assert os.path.exists(os.path.join(temp_dir, "core_comparison_results")) run_directories = io_utils.list_subdirectories( os.path.join(temp_dir, "core_comparison_results")) assert len(run_directories) == 2 model_files = io_utils.list_files( os.path.join(temp_dir, "core_comparison_results", run_directories[0])) assert len(model_files) == 4 assert model_files[0].endswith("tar.gz")