def _predict_tracker_actions(tracker, agent, fail_on_prediction_errors=False, use_e2e=False): processor = agent.create_processor() tracker_eval_store = EvaluationStore() events = list(tracker.events) partial_tracker = DialogueStateTracker.from_events(tracker.sender_id, events[:1], agent.domain.slots) for event in events[1:]: if isinstance(event, ActionExecuted): action_executed_result = \ _collect_action_executed_predictions( processor, partial_tracker, event, fail_on_prediction_errors ) tracker_eval_store.merge_store(action_executed_result) elif use_e2e and isinstance(event, UserUttered): user_uttered_result = \ _collect_user_uttered_predictions( event, partial_tracker, fail_on_prediction_errors) tracker_eval_store.merge_store(user_uttered_result) else: partial_tracker.update(event) return tracker_eval_store, partial_tracker
def _is_in_training_data(tracker, agent, fail_on_prediction_errors=False): processor = agent.create_processor() events = list(tracker.events) partial_tracker = DialogueStateTracker.from_events(tracker.sender_id, events[:1], agent.domain.slots) in_training_data = True test_in_training_data = True for event in events[1:]: if isinstance(event, ActionExecuted): _, policy = \ _collect_action_executed_predictions( processor, partial_tracker, event, fail_on_prediction_errors ) if (test_in_training_data and policy is not None and SimplePolicyEnsemble.is_not_memo_policy(policy)): in_training_data = False test_in_training_data = False return in_training_data
def _predict_tracker_actions(tracker, agent, fail_on_prediction_errors=False): processor = agent.create_processor() golds = [] predictions = [] events = list(tracker.events) partial_tracker = DialogueStateTracker.from_events(tracker.sender_id, events[:1], agent.domain.slots) for event in events[1:]: if isinstance(event, ActionExecuted): action, _, _ = processor.predict_next_action(partial_tracker) predicted = action.name() gold = event.action_name predictions.append(predicted) golds.append(gold) if predicted != gold: partial_tracker.update(WronglyPredictedAction(gold, predicted)) if fail_on_prediction_errors: raise ValueError( "Model predicted a wrong action. Failed Story: " "\n\n{}".format(partial_tracker.export_stories())) else: partial_tracker.update(event) else: partial_tracker.update(event) return golds, predictions, partial_tracker
def _predict_tracker_actions(tracker, agent: 'Agent', fail_on_prediction_errors=False, use_e2e=False): from rasa_core.trackers import DialogueStateTracker processor = agent.create_processor() tracker_eval_store = EvaluationStore() events = list(tracker.events) partial_tracker = DialogueStateTracker.from_events(tracker.sender_id, events[:1], agent.domain.slots) tracker_actions = [] for event in events[1:]: if isinstance(event, ActionExecuted): action_executed_result, policy, confidence = \ _collect_action_executed_predictions( processor, partial_tracker, event, fail_on_prediction_errors ) tracker_eval_store.merge_store(action_executed_result) tracker_actions.append({ "action": action_executed_result.action_targets[0], "predicted": action_executed_result.action_predictions[0], "policy": policy, "confidence": confidence }) elif use_e2e and isinstance(event, UserUttered): user_uttered_result = \ _collect_user_uttered_predictions( event, partial_tracker, fail_on_prediction_errors) tracker_eval_store.merge_store(user_uttered_result) else: partial_tracker.update(event) return tracker_eval_store, partial_tracker, tracker_actions
def test_policy_priority(): domain = Domain.load("data/test_domains/default.yml") tracker = DialogueStateTracker.from_events("test", [UserUttered("hi")], []) priority_1 = ConstantPolicy(priority=1, predict_index=0) priority_2 = ConstantPolicy(priority=2, predict_index=1) policy_ensemble_0 = SimplePolicyEnsemble([priority_1, priority_2]) policy_ensemble_1 = SimplePolicyEnsemble([priority_2, priority_1]) priority_2_result = priority_2.predict_action_probabilities( tracker, domain) i = 1 # index of priority_2 in ensemble_0 result, best_policy = policy_ensemble_0.probabilities_using_best_policy( tracker, domain) assert best_policy == 'policy_{}_{}'.format(i, type(priority_2).__name__) assert (result.tolist() == priority_2_result) i = 0 # index of priority_2 in ensemble_1 result, best_policy = policy_ensemble_1.probabilities_using_best_policy( tracker, domain) assert best_policy == 'policy_{}_{}'.format(i, type(priority_2).__name__) assert (result.tolist() == priority_2_result)
def get_tracker(events: List[Event]) -> DialogueStateTracker: return DialogueStateTracker.from_events("sender", events, [], 20)