def plot_curve(output: Text, no_stories: List[int]) -> None:
"""Plot the results from run_comparison_evaluation.
Args:
output: Output directory to save resulting plots to
no_stories: Number of stories per run
"""
import matplotlib.pyplot as plt
import numpy as np
from rasa.core import utils
ax = plt.gca()
# load results from file
data = utils.read_json_file(os.path.join(output, 'results.json'))
x = no_stories
# compute mean of all the runs for keras/embed policies
for label in data.keys():
if len(data[label]) == 0:
continue
mean = np.mean(data[label], axis=0)
std = np.std(data[label], axis=0)
ax.plot(x, mean, label=label, marker='.')
ax.fill_between(x, [m - s for m, s in zip(mean, std)],
[m + s for m, s in zip(mean, std)],
color='#6b2def',
alpha=0.2)
ax.legend(loc=4)
ax.set_xlabel("Number of stories present during training")
ax.set_ylabel("Number of correct test stories")
plt.savefig(os.path.join(output, 'model_comparison_graph.pdf'),
format='pdf')
plt.show()
def main():
from rasa.core.agent import Agent
from rasa.core.interpreter import NaturalLanguageInterpreter
from rasa.core.utils import AvailableEndpoints, set_default_subparser
import rasa.nlu.utils as nlu_utils
import rasa.core.cli
from rasa.core import utils
loop = asyncio.get_event_loop()
# Running as standalone python application
arg_parser = create_argument_parser()
set_default_subparser(arg_parser, "default")
cmdline_arguments = arg_parser.parse_args()
logging.basicConfig(level=cmdline_arguments.loglevel)
_endpoints = AvailableEndpoints.read_endpoints(cmdline_arguments.endpoints)
if cmdline_arguments.output:
nlu_utils.create_dir(cmdline_arguments.output)
if not cmdline_arguments.core:
raise ValueError(
"you must provide a core model directory to evaluate using -d / --core"
)
if cmdline_arguments.mode == "default":
_interpreter = NaturalLanguageInterpreter.create(
cmdline_arguments.nlu, _endpoints.nlu
)
_agent = Agent.load(cmdline_arguments.core, interpreter=_interpreter)
stories = loop.run_until_complete(
rasa.core.cli.train.stories_from_cli_args(cmdline_arguments)
)
loop.run_until_complete(
test(
stories,
_agent,
cmdline_arguments.max_stories,
cmdline_arguments.output,
cmdline_arguments.fail_on_prediction_errors,
cmdline_arguments.e2e,
)
)
elif cmdline_arguments.mode == "compare":
compare(
cmdline_arguments.core, cmdline_arguments.stories, cmdline_arguments.output
)
story_n_path = os.path.join(cmdline_arguments.core, "num_stories.json")
number_of_stories = utils.read_json_file(story_n_path)
plot_curve(cmdline_arguments.output, number_of_stories)
logger.info("Finished evaluation")
def test_core(
model: Optional[Text] = None,
stories: Optional[Text] = None,
endpoints: Optional[Text] = None,
output: Text = DEFAULT_RESULTS_PATH,
model_path: Optional[Text] = None,
kwargs: Optional[Dict] = None,
):
import rasa.core.test
import rasa.core.utils as core_utils
from rasa.nlu import utils as nlu_utils
from rasa.model import get_model
from rasa.core.interpreter import NaturalLanguageInterpreter
from rasa.core.agent import Agent
_endpoints = core_utils.AvailableEndpoints.read_endpoints(endpoints)
if kwargs is None:
kwargs = {}
if output:
nlu_utils.create_dir(output)
if os.path.isfile(model):
model_path = get_model(model)
if model_path:
# Single model: Normal evaluation
loop = asyncio.get_event_loop()
model_path = get_model(model)
core_path, nlu_path = get_model_subdirectories(model_path)
if os.path.exists(core_path) and os.path.exists(nlu_path):
_interpreter = NaturalLanguageInterpreter.create(nlu_path, _endpoints.nlu)
_agent = Agent.load(core_path, interpreter=_interpreter)
kwargs = minimal_kwargs(kwargs, rasa.core.test, ["stories", "agent"])
loop.run_until_complete(
rasa.core.test(stories, _agent, out_directory=output, **kwargs)
)
else:
logger.warning(
"Not able to test. Make sure both models, core and "
"nlu, are available."
)
else:
from rasa.core.test import compare, plot_curve
compare(model, stories, output)
story_n_path = os.path.join(model, "num_stories.json")
number_of_stories = core_utils.read_json_file(story_n_path)
plot_curve(output, number_of_stories)
def test_compare(models: List[Text], stories: Text, output: Text):
from rasa.core.test import compare, plot_curve
import rasa.core.utils as core_utils
model_directory = copy_models_to_compare(models)
loop = asyncio.get_event_loop()
loop.run_until_complete(compare(model_directory, stories, output))
story_n_path = os.path.join(model_directory, "num_stories.json")
number_of_stories = core_utils.read_json_file(story_n_path)
plot_curve(output, number_of_stories)
def test_core(model: Text,
stories: Text,
endpoints: Text = None,
output: Text = DEFAULT_RESULTS_PATH,
model_path: Text = None,
**kwargs: Dict):
import rasa.core.test
import rasa.core.utils as core_utils
from rasa_nlu import utils as nlu_utils
from rasa.model import get_model
from rasa.core.interpreter import NaturalLanguageInterpreter
from rasa.core.agent import Agent
_endpoints = core_utils.AvailableEndpoints.read_endpoints(endpoints)
if output:
nlu_utils.create_dir(output)
if os.path.isfile(model):
model_path = get_model(model)
if model_path:
# Single model: Normal evaluation
loop = asyncio.get_event_loop()
model_path = get_model(model)
core_path, nlu_path = get_model_subdirectories(model_path)
_interpreter = NaturalLanguageInterpreter.create(
nlu_path, _endpoints.nlu)
_agent = Agent.load(core_path, interpreter=_interpreter)
kwargs = minimal_kwargs(kwargs, rasa.core.test)
loop.run_until_complete(
rasa.core.test(stories, _agent, out_directory=output, **kwargs))
else:
from rasa.core.test import compare, plot_curve
compare(model, stories, output)
story_n_path = os.path.join(model, 'num_stories.json')
number_of_stories = core_utils.read_json_file(story_n_path)
plot_curve(output, number_of_stories)
async def test_interactive_domain_persistence(mock_endpoint, tmpdir):
# Test method interactive._write_domain_to_file
tracker_dump = "data/test_trackers/tracker_moodbot.json"
tracker_json = utils.read_json_file(tracker_dump)
events = tracker_json.get("events", [])
domain_path = tmpdir.join("interactive_domain_save.yml").strpath
url = "{}/domain".format(mock_endpoint.url)
with aioresponses() as mocked:
mocked.get(url, payload={})
await interactive._write_domain_to_file(domain_path, events,
mock_endpoint)
saved_domain = rasa.utils.io.read_yaml_file(domain_path)
for default_action in default_actions():
assert default_action.name() not in saved_domain["actions"]
