def test_evaluation_image_creation(tmpdir):
img_path = os.path.join(tmpdir.strpath, "evaltion.png")
run_story_evaluation(
story_file="examples/concerts/data/stories.md",
policy_model_path="examples/concerts/models/policy/init",
nlu_model_path=None,
out_file=img_path,
max_stories=None)
assert os.path.isfile(img_path)
assert imghdr.what(img_path) == "png"
def test_evaluation_image_creation(tmpdir, default_agent):
stories_path = os.path.join(tmpdir.strpath, "failed_stories.md")
img_path = os.path.join(tmpdir.strpath, "story_confmat.pdf")
run_story_evaluation(resource_name=DEFAULT_STORIES_FILE,
agent=default_agent,
out_directory=tmpdir.strpath,
max_stories=None,
use_e2e=False)
assert os.path.isfile(img_path)
assert os.path.isfile(stories_path)
def test_evaluation_image_creation(tmpdir, default_agent):
model_path = os.path.join(tmpdir.strpath, "model")
default_agent.persist(model_path)
img_path = os.path.join(tmpdir.strpath, "evaluation.png")
run_story_evaluation(resource_name=DEFAULT_STORIES_FILE,
policy_model_path=model_path,
nlu_model_path=None,
out_file=img_path,
max_stories=None)
assert os.path.isfile(img_path)
assert imghdr.what(img_path) == "png"
def test_evaluation_image_creation(tmpdir, default_agent):
stories_path = tmpdir.join("failed_stories.md").strpath
img_path = tmpdir.join("evaluation.png").strpath
run_story_evaluation(resource_name=DEFAULT_STORIES_FILE,
agent=default_agent,
out_file_plot=img_path,
max_stories=None,
out_file_stories=stories_path)
assert os.path.isfile(img_path)
assert imghdr.what(img_path) == "png"
assert os.path.isfile(stories_path)
def test_evaluation_image_creation(tmpdir, default_agent):
model_path = tmpdir.join("model").strpath
stories_path = tmpdir.join("failed_stories.md").strpath
img_path = tmpdir.join("evaluation.png").strpath
default_agent.persist(model_path)
run_story_evaluation(resource_name=DEFAULT_STORIES_FILE,
policy_model_path=model_path,
nlu_model_path=None,
out_file_plot=img_path,
max_stories=None,
out_file_stories=stories_path)
assert os.path.isfile(img_path)
assert imghdr.what(img_path) == "png"
assert os.path.isfile(stories_path)
def evaluate_stories():
"""Evaluate stories against the currently loaded model."""
tmp_file = rasa_nlu.utils.create_temporary_file(request.get_data(),
mode='w+b')
use_e2e = utils.bool_arg('e2e', default=False)
try:
evaluation = run_story_evaluation(tmp_file, agent, use_e2e=use_e2e)
return jsonify(evaluation)
except ValueError as e:
return error(400, "FailedEvaluation",
"Evaluation could not be created. Error: {}"
"".format(e))