def test_read_story_file_with_cycles(tmpdir, default_domain):
graph = extract_story_graph_from_file(
"data/test_stories/stories_with_cycle.md", default_domain)
assert len(graph.story_steps) == 5
graph_without_cycles = graph.with_cycles_removed()
assert graph.cyclic_edge_ids != set()
assert graph_without_cycles.cyclic_edge_ids == set()
assert len(graph.story_steps) == len(graph_without_cycles.story_steps) == 5
assert len(graph_without_cycles.story_end_checkpoints) == 2
def test_persist_and_read_test_story(tmpdir, default_domain):
graph = extract_story_graph_from_file("data/test_stories/stories.md",
default_domain)
out_path = tmpdir.join("persisted_story.md")
Story(graph.story_steps).dump_to_file(out_path.strpath)
recovered_trackers = extract_trackers_from_file(out_path.strpath,
default_domain,
BinaryFeaturizer())
existing_trackers = extract_trackers_from_file(
"data/test_stories/stories.md", default_domain, BinaryFeaturizer())
existing_stories = {t.export_stories() for t in existing_trackers}
for t in recovered_trackers:
story_str = t.export_stories()
assert story_str in existing_stories
existing_stories.discard(story_str)
def test_visualize_training_data_graph(tmpdir, default_domain):
graph = extract_story_graph_from_file(
"data/test_stories/stories_with_cycle.md", default_domain)
graph = graph.with_cycles_removed()
out_path = tmpdir.join("graph.png").strpath
# this will be the plotted networkx graph
G = graph.visualize(out_path)
assert os.path.exists(out_path)
# we can't check the exact topology - but this should be enough to ensure
# the visualisation created a sane graph
assert set(G.nodes()) == set(range(-1, 14))
assert len(G.edges()) == 16
def collect_story_predictions(story_file,
policy_model_path,
nlu_model_path,
max_stories=None,
shuffle_stories=True):
"""Test the stories from a file, running them through the stored model."""
def actions_since_last_utterance(tracker):
actions = []
for e in reversed(tracker.events):
if isinstance(e, UserUttered):
break
elif isinstance(e, ActionExecuted):
actions.append(e.action_name)
actions.reverse()
return actions
if nlu_model_path is not None:
interpreter = RasaNLUInterpreter(model_directory=nlu_model_path)
else:
interpreter = RegexInterpreter()
agent = Agent.load(policy_model_path, interpreter=interpreter)
story_graph = extract_story_graph_from_file(story_file, agent.domain,
interpreter)
preds = []
actual = []
max_history = agent.policy_ensemble.policies[0].max_history
g = TrainingsDataGenerator(story_graph,
agent.domain,
agent.featurizer,
max_history=max_history,
use_story_concatenation=False,
tracker_limit=100)
data = g.generate()
completed_trackers = data.metadata["trackers"]
logger.info("Evaluating {} stories\nProgress:".format(
len(completed_trackers)))
for tracker in tqdm(completed_trackers):
sender_id = "default-" + uuid.uuid4().hex
events = list(tracker.events)
actions_between_utterances = []
last_prediction = []
for i, event in enumerate(events[1:]):
if isinstance(event, UserUttered):
p, a = _min_list_distance(last_prediction,
actions_between_utterances)
preds.extend(p)
actual.extend(a)
actions_between_utterances = []
agent.handle_message(event.text, sender_id=sender_id)
tracker = agent.tracker_store.retrieve(sender_id)
last_prediction = actions_since_last_utterance(tracker)
elif isinstance(event, ActionExecuted):
actions_between_utterances.append(event.action_name)
if last_prediction:
preds.extend(last_prediction)
preds_padding = len(actions_between_utterances) - \
len(last_prediction)
preds.extend(["None"] * preds_padding)
actual.extend(actions_between_utterances)
actual_padding = len(last_prediction) - \
len(actions_between_utterances)
actual.extend(["None"] * actual_padding)
return actual, preds