def prepare_token_serialisation( tracker_store: TrackerStore, response_selector_interpreter: Interpreter, sender_id: Text, ): text = "Good morning" tokenizer = WhitespaceTokenizer() tokens = tokenizer.tokenize(Message(data={"text": text}), "text") indices = [[t.start, t.end] for t in tokens] tracker = tracker_store.get_or_create_tracker(sender_id=sender_id) parse_data = response_selector_interpreter.parse(text) event = UserUttered( "Good morning", parse_data.get("intent"), parse_data.get("entities", []), parse_data, ) tracker.update(event) tracker_store.save(tracker) retrieved_tracker = tracker_store.retrieve(sender_id=sender_id) event = retrieved_tracker.get_last_event_for(event_type=UserUttered) event_tokens = event.as_dict().get("parse_data").get("text_tokens") assert event_tokens == indices
async def test_interpreter_parses_text_tokens( response_selector_interpreter: Interpreter, ): text = "Hello there" tokenizer = WhitespaceTokenizer() tokens = tokenizer.tokenize(Message(data={"text": text}), "text") indices = [(t.start, t.end) for t in tokens] parsed_data = response_selector_interpreter.parse(text) assert "text_tokens" in parsed_data.keys() parsed_tokens = parsed_data.get("text_tokens") assert parsed_tokens == indices
def get_eval_data( interpreter: Interpreter, test_data: TrainingData ) -> Tuple[ List[IntentEvaluationResult], List[EntityEvaluationResult], ]: # pragma: no cover """Runs the model for the test set and extracts targets and predictions. Returns intent results (intent targets and predictions, the original messages and the confidences of the predictions), as well as entity results(entity_targets, entity_predictions, and tokens).""" logger.info("Running model for predictions:") intent_results, entity_results = [], [] intent_labels = [e.get("intent") for e in test_data.intent_examples] should_eval_intents = ( is_intent_classifier_present(interpreter) and len(set(intent_labels)) >= 1 ) should_eval_entities = is_entity_extractor_present(interpreter) for example in tqdm(test_data.training_examples): result = interpreter.parse(example.text, only_output_properties=False) if should_eval_intents: intent_prediction = result.get("intent", {}) or {} intent_results.append( IntentEvaluationResult( example.get("intent", ""), intent_prediction.get("name"), result.get("text", {}), intent_prediction.get("confidence"), ) ) if should_eval_entities: entity_results.append( EntityEvaluationResult( example.get("entities", []), result.get("entities", []), result.get("tokens", []), result.get("text", ""), ) ) return intent_results, entity_results
def get_eval_data( interpreter: Interpreter, test_data: TrainingData ) -> Tuple[List[IntentEvaluationResult], List[ResponseSelectionEvaluationResult], List[EntityEvaluationResult], ]: # pragma: no cover """Runs the model for the test set and extracts targets and predictions. Returns intent results (intent targets and predictions, the original messages and the confidences of the predictions), as well as entity results(entity_targets, entity_predictions, and tokens).""" logger.info("Running model for predictions:") intent_results, entity_results, response_selection_results = [], [], [] response_labels = [ e.get("response") for e in test_data.intent_examples if e.get("response") is not None ] intent_labels = [e.get("intent") for e in test_data.intent_examples] should_eval_intents = (is_intent_classifier_present(interpreter) and len(set(intent_labels)) >= 2) should_eval_response_selection = (is_response_selector_present(interpreter) and len(set(response_labels)) >= 2) available_response_selector_types = get_available_response_selector_types( interpreter) should_eval_entities = is_entity_extractor_present(interpreter) for example in tqdm(test_data.training_examples): result = interpreter.parse(example.text, only_output_properties=False) if should_eval_intents: intent_prediction = result.get("intent", {}) or {} intent_results.append( IntentEvaluationResult( example.get("intent", ""), intent_prediction.get("name"), result.get("text", {}), intent_prediction.get("confidence"), )) if should_eval_response_selection: # including all examples here. Empty response examples are filtered at the time of metric calculation intent_target = example.get("intent", "") selector_properties = result.get(MESSAGE_SELECTOR_PROPERTY_NAME, {}) if intent_target in available_response_selector_types: response_prediction_key = intent_target else: response_prediction_key = DEFAULT_OPEN_UTTERANCE_TYPE response_prediction = selector_properties.get( response_prediction_key, {}).get(OPEN_UTTERANCE_PREDICTION_KEY, {}) response_target = example.get("response", "") response_selection_results.append( ResponseSelectionEvaluationResult( intent_target, response_target, response_prediction.get("name"), result.get("text", {}), response_prediction.get("confidence"), )) if should_eval_entities: entity_results.append( EntityEvaluationResult( example.get("entities", []), result.get("entities", []), result.get("tokens", []), result.get("text", ""), )) return intent_results, response_selection_results, entity_results