def cross_validation(data_path, config_path): # same as rasa_nlu's evaluate interface data = training_data.load_data(data_path) data = drop_intents_below_freq(data, cutoff=5) # same as rasa_nlu's evaluate interface default_folds = 10 # same as rasa_nlu's evaluate interface nlu_config = config.load(config_path) intent_results, entity_results = run_cv_evaluation(data, default_folds, nlu_config) template_result = { 'intent': { 'train': {}, 'test': {} }, 'entity': { 'train': {}, 'test': {} }, } set_template_result(intent_results.train, template_result['intent']['train']) set_template_result(intent_results.test, template_result['intent']['test']) set_template_result(entity_results.train.values()[0], template_result['entity']['train']) set_template_result(entity_results.test.values()[0], template_result['entity']['test']) return template_result
def test_run_cv_evaluation(): td = training_data.load_data('data/examples/rasa/demo-rasa.json') nlu_config = RasaNLUConfig("sample_configs/config_spacy.json") n_folds = 3 results = run_cv_evaluation(td, n_folds, nlu_config) assert len(results.train["Accuracy"]) == n_folds assert len(results.train["Precision"]) == n_folds assert len(results.train["F1-score"]) == n_folds assert len(results.test["Accuracy"]) == n_folds assert len(results.test["Precision"]) == n_folds assert len(results.test["F1-score"]) == n_folds
def test_run_cv_evaluation(): import numpy as np td = load_data('data/examples/rasa/demo-rasa.json') n_folds = 3 nlu_config = RasaNLUConfig("sample_configs/config_defaults.json") np.seed(2018) results = run_cv_evaluation(td, n_folds, nlu_config) rel_tol = 1e-09 abs_tol = 0.01 acc = np.mean(results["accuracy"]) exp_acc = 0.65 # expected result np.testing.assert_approx_equal(acc, exp_acc, significant=5)
def test_run_cv_evaluation(): import numpy as np td = training_data.load_data('data/examples/rasa/demo-rasa.json') n_folds = 3 nlu_config = RasaNLUConfig("sample_configs/config_defaults.json") np.seed(2018) results = run_cv_evaluation(td, n_folds, nlu_config) rel_tol=1e-09 abs_tol=0.01 acc = np.mean(results["accuracy"]) exp_acc = 0.65 # expected result np.testing.assert_approx_equal(acc, exp_acc, significant=5)
def test_run_cv_evaluation(): td = training_data.load_data('data/examples/rasa/demo-rasa.json') nlu_config = config.load("sample_configs/config_spacy.yml") n_folds = 2 results, entity_results = run_cv_evaluation(td, n_folds, nlu_config) assert len(results.train["Accuracy"]) == n_folds assert len(results.train["Precision"]) == n_folds assert len(results.train["F1-score"]) == n_folds assert len(results.test["Accuracy"]) == n_folds assert len(results.test["Precision"]) == n_folds assert len(results.test["F1-score"]) == n_folds assert len(entity_results.train['ner_crf']["Accuracy"]) == n_folds assert len(entity_results.train['ner_crf']["Precision"]) == n_folds assert len(entity_results.train['ner_crf']["F1-score"]) == n_folds assert len(entity_results.test['ner_crf']["Accuracy"]) == n_folds assert len(entity_results.test['ner_crf']["Precision"]) == n_folds assert len(entity_results.test['ner_crf']["F1-score"]) == n_folds
def test_run_cv_evaluation(): td = training_data.load_data('data/examples/rasa/demo-rasa.json') nlu_config = config.load("sample_configs/config_spacy.yml") n_folds = 2 results, entity_results = run_cv_evaluation(td, n_folds, nlu_config) assert len(results.train["Accuracy"]) == n_folds assert len(results.train["Precision"]) == n_folds assert len(results.train["F1-score"]) == n_folds assert len(results.test["Accuracy"]) == n_folds assert len(results.test["Precision"]) == n_folds assert len(results.test["F1-score"]) == n_folds assert len(entity_results.train['ner_crf']["Accuracy"]) == n_folds assert len(entity_results.train['ner_crf']["Precision"]) == n_folds assert len(entity_results.train['ner_crf']["F1-score"]) == n_folds assert len(entity_results.test['ner_crf']["Accuracy"]) == n_folds assert len(entity_results.test['ner_crf']["Precision"]) == n_folds assert len(entity_results.test['ner_crf']["F1-score"]) == n_folds
def CV_eval(td_file, config_file, Nfolds=10): # trains a model with crossvalidation using the training data and config td = load_data(td_file) configuration = config.load(config_file) evaluate.run_cv_evaluation(td, Nfolds, configuration)