def prepare_token_serialisation(
tracker_store: TrackerStore,
response_selector_interpreter: Interpreter,
sender_id: Text,
):
text = "Good morning"
tokenizer = WhitespaceTokenizer()
tokens = tokenizer.tokenize(Message(data={"text": text}), "text")
indices = [[t.start, t.end] for t in tokens]
tracker = tracker_store.get_or_create_tracker(sender_id=sender_id)
parse_data = response_selector_interpreter.parse(text)
event = UserUttered(
"Good morning",
parse_data.get("intent"),
parse_data.get("entities", []),
parse_data,
)
tracker.update(event)
tracker_store.save(tracker)
retrieved_tracker = tracker_store.retrieve(sender_id=sender_id)
event = retrieved_tracker.get_last_event_for(event_type=UserUttered)
event_tokens = event.as_dict().get("parse_data").get("text_tokens")
assert event_tokens == indices
async def test_interpreter_parses_text_tokens(
response_selector_interpreter: Interpreter,
):
text = "Hello there"
tokenizer = WhitespaceTokenizer()
tokens = tokenizer.tokenize(Message(data={"text": text}), "text")
indices = [(t.start, t.end) for t in tokens]
parsed_data = response_selector_interpreter.parse(text)
assert "text_tokens" in parsed_data.keys()
parsed_tokens = parsed_data.get("text_tokens")
assert parsed_tokens == indices
def get_eval_data(
interpreter: Interpreter, test_data: TrainingData
) -> Tuple[
List[IntentEvaluationResult],
List[EntityEvaluationResult],
]: # pragma: no cover
"""Runs the model for the test set and extracts targets and predictions.
Returns intent results (intent targets and predictions, the original
messages and the confidences of the predictions), as well as entity
results(entity_targets, entity_predictions, and tokens)."""
logger.info("Running model for predictions:")
intent_results, entity_results = [], []
intent_labels = [e.get("intent") for e in test_data.intent_examples]
should_eval_intents = (
is_intent_classifier_present(interpreter) and len(set(intent_labels)) >= 1
)
should_eval_entities = is_entity_extractor_present(interpreter)
for example in tqdm(test_data.training_examples):
result = interpreter.parse(example.text, only_output_properties=False)
if should_eval_intents:
intent_prediction = result.get("intent", {}) or {}
intent_results.append(
IntentEvaluationResult(
example.get("intent", ""),
intent_prediction.get("name"),
result.get("text", {}),
intent_prediction.get("confidence"),
)
)
if should_eval_entities:
entity_results.append(
EntityEvaluationResult(
example.get("entities", []),
result.get("entities", []),
result.get("tokens", []),
result.get("text", ""),
)
)
return intent_results, entity_results
def get_eval_data(
interpreter: Interpreter, test_data: TrainingData
) -> Tuple[List[IntentEvaluationResult],
List[ResponseSelectionEvaluationResult],
List[EntityEvaluationResult], ]: # pragma: no cover
"""Runs the model for the test set and extracts targets and predictions.
Returns intent results (intent targets and predictions, the original
messages and the confidences of the predictions), as well as entity
results(entity_targets, entity_predictions, and tokens)."""
logger.info("Running model for predictions:")
intent_results, entity_results, response_selection_results = [], [], []
response_labels = [
e.get("response") for e in test_data.intent_examples
if e.get("response") is not None
]
intent_labels = [e.get("intent") for e in test_data.intent_examples]
should_eval_intents = (is_intent_classifier_present(interpreter)
and len(set(intent_labels)) >= 2)
should_eval_response_selection = (is_response_selector_present(interpreter)
and len(set(response_labels)) >= 2)
available_response_selector_types = get_available_response_selector_types(
interpreter)
should_eval_entities = is_entity_extractor_present(interpreter)
for example in tqdm(test_data.training_examples):
result = interpreter.parse(example.text, only_output_properties=False)
if should_eval_intents:
intent_prediction = result.get("intent", {}) or {}
intent_results.append(
IntentEvaluationResult(
example.get("intent", ""),
intent_prediction.get("name"),
result.get("text", {}),
intent_prediction.get("confidence"),
))
if should_eval_response_selection:
# including all examples here. Empty response examples are filtered at the time of metric calculation
intent_target = example.get("intent", "")
selector_properties = result.get(MESSAGE_SELECTOR_PROPERTY_NAME,
{})
if intent_target in available_response_selector_types:
response_prediction_key = intent_target
else:
response_prediction_key = DEFAULT_OPEN_UTTERANCE_TYPE
response_prediction = selector_properties.get(
response_prediction_key, {}).get(OPEN_UTTERANCE_PREDICTION_KEY,
{})
response_target = example.get("response", "")
response_selection_results.append(
ResponseSelectionEvaluationResult(
intent_target,
response_target,
response_prediction.get("name"),
result.get("text", {}),
response_prediction.get("confidence"),
))
if should_eval_entities:
entity_results.append(
EntityEvaluationResult(
example.get("entities", []),
result.get("entities", []),
result.get("tokens", []),
result.get("text", ""),
))
return intent_results, response_selection_results, entity_results
