def test_should_raise_when_non_zero_exit(logger, beverage_dataset): # When/Then with pytest.raises(SystemExit): compute_cross_val_metrics( dataset=beverage_dataset, engine_class=MockEngineSegfault, nb_folds=4, num_workers=4, )
def test_should_raise_when_non_zero_exit(self): # Given dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "beverage_dataset.json") with io.open(dataset_path, encoding="utf8") as f: dataset = json.load(f) # When/Then with self.assertRaises(SystemExit): compute_cross_val_metrics( dataset=dataset, engine_class=MockEngineSegfault, nb_folds=4, num_workers=4)
def test_end_to_end_cross_val_metrics(self): # Given dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "beverage_dataset.json") with io.open(dataset_path, encoding="utf8") as f: dataset = json.load(f) # When/Then try: engine_class = build_nlu_engine_class(MockTrainingEngine, MockInferenceEngine) compute_cross_val_metrics(dataset=dataset, engine_class=engine_class, nb_folds=5) except Exception as e: self.fail(e.args[0])
def test_compute_cross_val_metrics_without_slot_metrics(self): # Given dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "beverage_dataset.json") with io.open(dataset_path, encoding="utf8") as f: dataset = json.load(f) # When/Then try: res = compute_cross_val_metrics( dataset=dataset, engine_class=MockEngine, nb_folds=2, include_slot_metrics=False) except Exception as e: self.fail(e.args[0]) expected_metrics = { "null": { "intent": { "true_positive": 0, "false_positive": 11, "false_negative": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0 }, "intent_utterances": 0, "exact_parsings": 0 }, "MakeCoffee": { "intent": { "true_positive": 0, "false_positive": 0, "false_negative": 7, "precision": 0.0, "recall": 0.0, "f1": 0.0 }, "intent_utterances": 7, "exact_parsings": 0 }, "MakeTea": { "intent": { "true_positive": 0, "false_positive": 0, "false_negative": 4, "precision": 0.0, "recall": 0.0, "f1": 0.0 }, "intent_utterances": 4, "exact_parsings": 0 } } self.assertDictEqual(expected_metrics, res["metrics"])
def test_compute_cross_val_metrics_with_intents_filter(self): # Given dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "keyword_matching_dataset.json") with io.open(dataset_path, encoding="utf8") as f: dataset = json.load(f) # When/Then res = compute_cross_val_metrics( dataset=dataset, engine_class=KeyWordMatchingEngine, nb_folds=2, intents_filter=["intent2", "intent3"], include_slot_metrics=False, seed=42) expected_metrics = { "null": { "intent": { "true_positive": 0, "false_positive": 2, "false_negative": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0 }, "exact_parsings": 0, "intent_utterances": 0 }, "intent2": { "intent": { "true_positive": 3, "false_positive": 0, "false_negative": 1, "precision": 1.0, "recall": 3. / 4., "f1": 0.8571428571428571 }, "exact_parsings": 3, "intent_utterances": 4 }, "intent3": { "intent": { "true_positive": 2, "false_positive": 0, "false_negative": 1, "precision": 1.0, "recall": 2. / 3., "f1": 0.8 }, "exact_parsings": 2, "intent_utterances": 3 }, } self.assertDictEqual(expected_metrics, res["metrics"])
def test_compute_cross_val_metrics_without_slot_metrics( logger, beverage_dataset): # When/Then try: res = compute_cross_val_metrics( dataset=beverage_dataset, engine_class=MockEngine, nb_folds=2, include_slot_metrics=False, ) except Exception as e: raise AssertionError(e.args[0]) expected_metrics = { "null": { "intent": { "true_positive": 0, "false_positive": 11, "false_negative": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, }, "intent_utterances": 0, "exact_parsings": 0, }, "MakeCoffee": { "intent": { "true_positive": 0, "false_positive": 0, "false_negative": 7, "precision": 0.0, "recall": 0.0, "f1": 0.0, }, "intent_utterances": 7, "exact_parsings": 0, }, "MakeTea": { "intent": { "true_positive": 0, "false_positive": 0, "false_negative": 4, "precision": 0.0, "recall": 0.0, "f1": 0.0, }, "intent_utterances": 4, "exact_parsings": 0, }, } assert expected_metrics, res["metrics"]
def test_compute_cross_val_metrics_with_intents_filter( logger, keyword_matching_dataset): # When/Then res = compute_cross_val_metrics( dataset=keyword_matching_dataset, engine_class=KeyWordMatchingEngine, nb_folds=2, intents_filter=["intent2", "intent3"], include_slot_metrics=False, seed=42, ) expected_metrics = { "null": { "intent": { "true_positive": 0, "false_positive": 2, "false_negative": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, }, "exact_parsings": 0, "intent_utterances": 0, }, "intent2": { "intent": { "true_positive": 3, "false_positive": 0, "false_negative": 1, "precision": 1.0, "recall": 3.0 / 4.0, "f1": 0.8571428571428571, }, "exact_parsings": 3, "intent_utterances": 4, }, "intent3": { "intent": { "true_positive": 2, "false_positive": 0, "false_negative": 1, "precision": 1.0, "recall": 2.0 / 3.0, "f1": 0.8, }, "exact_parsings": 2, "intent_utterances": 3, }, } assert expected_metrics, res["metrics"]
def test_cross_val_metrics_should_skip_when_not_enough_data(self): # Given dataset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources", "beverage_dataset.json") # When result = compute_cross_val_metrics(dataset=dataset_path, engine_class=MockEngine, nb_folds=11) # Then expected_result = {METRICS: None, PARSING_ERRORS: []} self.assertDictEqual(expected_result, result)
def test_cross_val_metrics_should_skip_when_not_enough_data( logger, beverage_dataset_path): # When result = compute_cross_val_metrics(dataset=beverage_dataset_path, engine_class=MockEngine, nb_folds=11) # Then expected_result = { AVERAGE_METRICS: None, CONFUSION_MATRIX: None, METRICS: None, PARSING_ERRORS: [], } assert expected_result, result
def test_compute_cross_val_metrics_with_multiple_workers( logger, beverage_dataset): # When/Then expected_metrics = { "null": { "intent": { "true_positive": 0, "false_positive": 11, "false_negative": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, }, "exact_parsings": 0, "slots": {}, "intent_utterances": 0, }, "MakeCoffee": { "intent": { "true_positive": 0, "false_positive": 0, "false_negative": 7, "precision": 0.0, "recall": 0.0, "f1": 0.0, }, "exact_parsings": 0, "slots": { "number_of_cups": { "true_positive": 0, "false_positive": 0, "false_negative": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, } }, "intent_utterances": 7, }, "MakeTea": { "intent": { "true_positive": 0, "false_positive": 0, "false_negative": 4, "precision": 0.0, "recall": 0.0, "f1": 0.0, }, "exact_parsings": 0, "slots": { "number_of_cups": { "true_positive": 0, "false_positive": 0, "false_negative": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, }, "beverage_temperature": { "true_positive": 0, "false_positive": 0, "false_negative": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, }, }, "intent_utterances": 4, }, } try: res = compute_cross_val_metrics(dataset=beverage_dataset, engine_class=MockEngine, nb_folds=2, num_workers=4) except Exception as e: raise AssertionError(e.args[0]) assert expected_metrics, res["metrics"]