def _predict_tracker_actions(tracker,
agent,
fail_on_prediction_errors=False,
use_e2e=False):
processor = agent.create_processor()
tracker_eval_store = EvaluationStore()
events = list(tracker.events)
partial_tracker = DialogueStateTracker.from_events(tracker.sender_id,
events[:1],
agent.domain.slots)
for event in events[1:]:
if isinstance(event, ActionExecuted):
action_executed_result = \
_collect_action_executed_predictions(
processor, partial_tracker, event,
fail_on_prediction_errors
)
tracker_eval_store.merge_store(action_executed_result)
elif use_e2e and isinstance(event, UserUttered):
user_uttered_result = \
_collect_user_uttered_predictions(
event, partial_tracker, fail_on_prediction_errors)
tracker_eval_store.merge_store(user_uttered_result)
else:
partial_tracker.update(event)
return tracker_eval_store, partial_tracker
def _is_in_training_data(tracker, agent, fail_on_prediction_errors=False):
processor = agent.create_processor()
events = list(tracker.events)
partial_tracker = DialogueStateTracker.from_events(tracker.sender_id,
events[:1],
agent.domain.slots)
in_training_data = True
test_in_training_data = True
for event in events[1:]:
if isinstance(event, ActionExecuted):
_, policy = \
_collect_action_executed_predictions(
processor, partial_tracker, event,
fail_on_prediction_errors
)
if (test_in_training_data and
policy is not None and
SimplePolicyEnsemble.is_not_memo_policy(policy)):
in_training_data = False
test_in_training_data = False
return in_training_data
def _predict_tracker_actions(tracker, agent, fail_on_prediction_errors=False):
processor = agent.create_processor()
golds = []
predictions = []
events = list(tracker.events)
partial_tracker = DialogueStateTracker.from_events(tracker.sender_id,
events[:1],
agent.domain.slots)
for event in events[1:]:
if isinstance(event, ActionExecuted):
action, _, _ = processor.predict_next_action(partial_tracker)
predicted = action.name()
gold = event.action_name
predictions.append(predicted)
golds.append(gold)
if predicted != gold:
partial_tracker.update(WronglyPredictedAction(gold, predicted))
if fail_on_prediction_errors:
raise ValueError(
"Model predicted a wrong action. Failed Story: "
"\n\n{}".format(partial_tracker.export_stories()))
else:
partial_tracker.update(event)
else:
partial_tracker.update(event)
return golds, predictions, partial_tracker
def _predict_tracker_actions(tracker,
agent: 'Agent',
fail_on_prediction_errors=False,
use_e2e=False):
from rasa_core.trackers import DialogueStateTracker
processor = agent.create_processor()
tracker_eval_store = EvaluationStore()
events = list(tracker.events)
partial_tracker = DialogueStateTracker.from_events(tracker.sender_id,
events[:1],
agent.domain.slots)
tracker_actions = []
for event in events[1:]:
if isinstance(event, ActionExecuted):
action_executed_result, policy, confidence = \
_collect_action_executed_predictions(
processor, partial_tracker, event,
fail_on_prediction_errors
)
tracker_eval_store.merge_store(action_executed_result)
tracker_actions.append({
"action":
action_executed_result.action_targets[0],
"predicted":
action_executed_result.action_predictions[0],
"policy":
policy,
"confidence":
confidence
})
elif use_e2e and isinstance(event, UserUttered):
user_uttered_result = \
_collect_user_uttered_predictions(
event, partial_tracker, fail_on_prediction_errors)
tracker_eval_store.merge_store(user_uttered_result)
else:
partial_tracker.update(event)
return tracker_eval_store, partial_tracker, tracker_actions
def test_policy_priority():
domain = Domain.load("data/test_domains/default.yml")
tracker = DialogueStateTracker.from_events("test", [UserUttered("hi")], [])
priority_1 = ConstantPolicy(priority=1, predict_index=0)
priority_2 = ConstantPolicy(priority=2, predict_index=1)
policy_ensemble_0 = SimplePolicyEnsemble([priority_1, priority_2])
policy_ensemble_1 = SimplePolicyEnsemble([priority_2, priority_1])
priority_2_result = priority_2.predict_action_probabilities(
tracker, domain)
i = 1 # index of priority_2 in ensemble_0
result, best_policy = policy_ensemble_0.probabilities_using_best_policy(
tracker, domain)
assert best_policy == 'policy_{}_{}'.format(i, type(priority_2).__name__)
assert (result.tolist() == priority_2_result)
i = 0 # index of priority_2 in ensemble_1
result, best_policy = policy_ensemble_1.probabilities_using_best_policy(
tracker, domain)
assert best_policy == 'policy_{}_{}'.format(i, type(priority_2).__name__)
assert (result.tolist() == priority_2_result)
def get_tracker(events: List[Event]) -> DialogueStateTracker:
return DialogueStateTracker.from_events("sender", events, [], 20)
