async def compare(models: Text, stories_file: Text, output: Text) -> None: """Evaluates multiple trained models on a test set.""" from rasa.core.agent import Agent import rasa_nlu.utils as nlu_utils from rasa.core import utils num_correct = defaultdict(list) for run in nlu_utils.list_subdirectories(models): num_correct_run = defaultdict(list) for model in sorted(nlu_utils.list_subdirectories(run)): logger.info("Evaluating model {}".format(model)) agent = Agent.load(model) completed_trackers = await _generate_trackers(stories_file, agent) story_eval_store, no_of_stories = \ collect_story_predictions(completed_trackers, agent) failed_stories = story_eval_store.failed_stories policy_name = ''.join( [i for i in os.path.basename(model) if not i.isdigit()]) num_correct_run[policy_name].append(no_of_stories - len(failed_stories)) for k, v in num_correct_run.items(): num_correct[k].append(v) utils.dump_obj_as_json_to_file(os.path.join(output, 'results.json'), num_correct)
def run_comparison_evaluation(models, stories, output): # type: (Text, Text, Text) -> None """Evaluates multiple trained models on a test set""" num_correct = defaultdict(list) for run in nlu_utils.list_subdirectories(models): num_correct_run = defaultdict(list) for model in sorted(nlu_utils.list_subdirectories(run)): logger.info("Evaluating model {}".format(model)) agent = Agent.load(model) completed_trackers = _generate_trackers(stories, agent) story_eval_store, no_of_stories = \ collect_story_predictions(completed_trackers, agent) failed_stories = story_eval_store.failed_stories policy_name = ''.join( [i for i in os.path.basename(model) if not i.isdigit()]) num_correct_run[policy_name].append(no_of_stories - len(failed_stories)) for k, v in num_correct_run.items(): num_correct[k].append(v) utils.dump_obj_as_json_to_file(os.path.join(output, 'results.json'), num_correct)
def _list_models_in_dir(path): if not path or not os.path.isdir(path): return [] else: return [ os.path.relpath(model, path) for model in utils.list_subdirectories(path) ]
def _list_projects(path): """List the projects in the path, ignoring hidden directories.""" return [os.path.basename(fn) for fn in utils.list_subdirectories(path)]
def _list_models_in_dir(path): if not path or not os.path.isdir(path): return [] else: return [os.path.relpath(model, path) for model in utils.list_subdirectories(path)]