def test_read_story_file_with_cycles(tmpdir, default_domain): graph = extract_story_graph_from_file( "data/test_stories/stories_with_cycle.md", default_domain) assert len(graph.story_steps) == 5 graph_without_cycles = graph.with_cycles_removed() assert graph.cyclic_edge_ids != set() assert graph_without_cycles.cyclic_edge_ids == set() assert len(graph.story_steps) == len(graph_without_cycles.story_steps) == 5 assert len(graph_without_cycles.story_end_checkpoints) == 2
def test_persist_and_read_test_story(tmpdir, default_domain): graph = extract_story_graph_from_file("data/test_stories/stories.md", default_domain) out_path = tmpdir.join("persisted_story.md") Story(graph.story_steps).dump_to_file(out_path.strpath) recovered_trackers = extract_trackers_from_file(out_path.strpath, default_domain, BinaryFeaturizer()) existing_trackers = extract_trackers_from_file( "data/test_stories/stories.md", default_domain, BinaryFeaturizer()) existing_stories = {t.export_stories() for t in existing_trackers} for t in recovered_trackers: story_str = t.export_stories() assert story_str in existing_stories existing_stories.discard(story_str)
def test_visualize_training_data_graph(tmpdir, default_domain): graph = extract_story_graph_from_file( "data/test_stories/stories_with_cycle.md", default_domain) graph = graph.with_cycles_removed() out_path = tmpdir.join("graph.png").strpath # this will be the plotted networkx graph G = graph.visualize(out_path) assert os.path.exists(out_path) # we can't check the exact topology - but this should be enough to ensure # the visualisation created a sane graph assert set(G.nodes()) == set(range(-1, 14)) assert len(G.edges()) == 16
def collect_story_predictions(story_file, policy_model_path, nlu_model_path, max_stories=None, shuffle_stories=True): """Test the stories from a file, running them through the stored model.""" def actions_since_last_utterance(tracker): actions = [] for e in reversed(tracker.events): if isinstance(e, UserUttered): break elif isinstance(e, ActionExecuted): actions.append(e.action_name) actions.reverse() return actions if nlu_model_path is not None: interpreter = RasaNLUInterpreter(model_directory=nlu_model_path) else: interpreter = RegexInterpreter() agent = Agent.load(policy_model_path, interpreter=interpreter) story_graph = extract_story_graph_from_file(story_file, agent.domain, interpreter) preds = [] actual = [] max_history = agent.policy_ensemble.policies[0].max_history g = TrainingsDataGenerator(story_graph, agent.domain, agent.featurizer, max_history=max_history, use_story_concatenation=False, tracker_limit=100) data = g.generate() completed_trackers = data.metadata["trackers"] logger.info("Evaluating {} stories\nProgress:".format( len(completed_trackers))) for tracker in tqdm(completed_trackers): sender_id = "default-" + uuid.uuid4().hex events = list(tracker.events) actions_between_utterances = [] last_prediction = [] for i, event in enumerate(events[1:]): if isinstance(event, UserUttered): p, a = _min_list_distance(last_prediction, actions_between_utterances) preds.extend(p) actual.extend(a) actions_between_utterances = [] agent.handle_message(event.text, sender_id=sender_id) tracker = agent.tracker_store.retrieve(sender_id) last_prediction = actions_since_last_utterance(tracker) elif isinstance(event, ActionExecuted): actions_between_utterances.append(event.action_name) if last_prediction: preds.extend(last_prediction) preds_padding = len(actions_between_utterances) - \ len(last_prediction) preds.extend(["None"] * preds_padding) actual.extend(actions_between_utterances) actual_padding = len(last_prediction) - \ len(actions_between_utterances) actual.extend(["None"] * actual_padding) return actual, preds