async def compare(models: Text, stories_file: Text, output: Text) -> None:
"""Evaluates multiple trained models on a test set."""
from rasa.core.agent import Agent
import rasa_nlu.utils as nlu_utils
from rasa.core import utils
num_correct = defaultdict(list)
for run in nlu_utils.list_subdirectories(models):
num_correct_run = defaultdict(list)
for model in sorted(nlu_utils.list_subdirectories(run)):
logger.info("Evaluating model {}".format(model))
agent = Agent.load(model)
completed_trackers = await _generate_trackers(stories_file, agent)
story_eval_store, no_of_stories = \
collect_story_predictions(completed_trackers,
agent)
failed_stories = story_eval_store.failed_stories
policy_name = ''.join(
[i for i in os.path.basename(model) if not i.isdigit()])
num_correct_run[policy_name].append(no_of_stories -
len(failed_stories))
for k, v in num_correct_run.items():
num_correct[k].append(v)
utils.dump_obj_as_json_to_file(os.path.join(output, 'results.json'),
num_correct)
def run_comparison_evaluation(models, stories, output):
# type: (Text, Text, Text) -> None
"""Evaluates multiple trained models on a test set"""
num_correct = defaultdict(list)
for run in nlu_utils.list_subdirectories(models):
num_correct_run = defaultdict(list)
for model in sorted(nlu_utils.list_subdirectories(run)):
logger.info("Evaluating model {}".format(model))
agent = Agent.load(model)
completed_trackers = _generate_trackers(stories, agent)
story_eval_store, no_of_stories = \
collect_story_predictions(completed_trackers,
agent)
failed_stories = story_eval_store.failed_stories
policy_name = ''.join(
[i for i in os.path.basename(model) if not i.isdigit()])
num_correct_run[policy_name].append(no_of_stories -
len(failed_stories))
for k, v in num_correct_run.items():
num_correct[k].append(v)
utils.dump_obj_as_json_to_file(os.path.join(output, 'results.json'),
num_correct)
def _list_models_in_dir(path):
if not path or not os.path.isdir(path):
return []
else:
return [
os.path.relpath(model, path)
for model in utils.list_subdirectories(path)
]
def _list_projects(path):
"""List the projects in the path, ignoring hidden directories."""
return [os.path.basename(fn) for fn in utils.list_subdirectories(path)]
def _list_models_in_dir(path):
if not path or not os.path.isdir(path):
return []
else:
return [os.path.relpath(model, path)
for model in utils.list_subdirectories(path)]
def _list_projects(path):
"""List the projects in the path, ignoring hidden directories."""
return [os.path.basename(fn)
for fn in utils.list_subdirectories(path)]
