def compute_train_test_nlu_metrics(train_dataset, test_dataset, training_engine_class, inference_engine_class, include_slot_metrics=True, slot_matching_lambda=None): """Compute pure NLU metrics on `test_dataset` after having trained on `train_dataset` Args train_dataset (dict or str): Dataset or path to dataset used for training test_dataset (dict or str): Dataset or path to dataset used for testing training_engine_class: Python class to use for training inference_engine_class: Python class to use for inference include_slot_metrics (bool, true): If false, the slots metrics and the slots parsing errors will not be reported. slot_matching_lambda (lambda, optional): lambda expected_slot, actual_slot -> bool, if defined, this function will be use to match slots when computing metrics, otherwise exact match will be used. `expected_slot` corresponds to the slot as defined in the dataset, and `actual_slot` corresponds to the slot as returned by the NLU Returns dict: Metrics results containing the following data - "metrics": the computed metrics - "parsing_errors": the list of parsing errors """ engine_class = build_nlu_engine_class(training_engine_class, inference_engine_class) return compute_train_test_metrics(train_dataset, test_dataset, engine_class, include_slot_metrics, slot_matching_lambda)
def test_end_to_end_train_test_metrics(self): # Given dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "beverage_dataset.json") with io.open(dataset_path, encoding="utf8") as f: dataset = json.load(f) # When/Then try: engine_class = build_nlu_engine_class(MockTrainingEngine, MockInferenceEngine) compute_train_test_metrics(train_dataset=dataset, test_dataset=dataset, engine_class=engine_class) except Exception as e: self.fail(e.args[0])
def compute_cross_val_nlu_metrics(dataset, training_engine_class, inference_engine_class, nb_folds=5, train_size_ratio=1.0, drop_entities=False, include_slot_metrics=True, slot_matching_lambda=None, progression_handler=None): """Compute pure NLU metrics on the dataset using cross validation Args: dataset (dict or str): Dataset or path to dataset training_engine_class: Python class to use for training inference_engine_class: Python class to use for inference nb_folds (int, optional): Number of folds to use for cross validation train_size_ratio (float, optional): Ratio of intent utterances to use for training drop_entities (bool, false): Specify whether not all entity values should be removed from training data include_slot_metrics (bool, true): If false, the slots metrics and the slots parsing errors will not be reported. slot_matching_lambda (lambda, optional): lambda expected_slot, actual_slot -> bool, if defined, this function will be use to match slots when computing metrics, otherwise exact match will be used. `expected_slot` corresponds to the slot as defined in the dataset, and `actual_slot` corresponds to the slot as returned by the NLU progression_handler (lambda, optional): handler called at each progression (%) step Returns dict: Metrics results containing the following data - "metrics": the computed metrics - "parsing_errors": the list of parsing errors """ engine_class = build_nlu_engine_class(training_engine_class, inference_engine_class) return compute_cross_val_metrics(dataset, engine_class, nb_folds, train_size_ratio, drop_entities, include_slot_metrics, slot_matching_lambda, progression_handler)