def cross_validation(data_path, config_path):
# same as rasa_nlu's evaluate interface
data = training_data.load_data(data_path)
data = drop_intents_below_freq(data, cutoff=5)
# same as rasa_nlu's evaluate interface
default_folds = 10
# same as rasa_nlu's evaluate interface
nlu_config = config.load(config_path)
intent_results, entity_results = run_cv_evaluation(data, default_folds,
nlu_config)
template_result = {
'intent': {
'train': {},
'test': {}
},
'entity': {
'train': {},
'test': {}
},
}
set_template_result(intent_results.train,
template_result['intent']['train'])
set_template_result(intent_results.test, template_result['intent']['test'])
set_template_result(entity_results.train.values()[0],
template_result['entity']['train'])
set_template_result(entity_results.test.values()[0],
template_result['entity']['test'])
return template_result
def test_run_cv_evaluation():
td = training_data.load_data('data/examples/rasa/demo-rasa.json')
nlu_config = RasaNLUConfig("sample_configs/config_spacy.json")
n_folds = 3
results = run_cv_evaluation(td, n_folds, nlu_config)
assert len(results.train["Accuracy"]) == n_folds
assert len(results.train["Precision"]) == n_folds
assert len(results.train["F1-score"]) == n_folds
assert len(results.test["Accuracy"]) == n_folds
assert len(results.test["Precision"]) == n_folds
assert len(results.test["F1-score"]) == n_folds
def test_run_cv_evaluation():
import numpy as np
td = load_data('data/examples/rasa/demo-rasa.json')
n_folds = 3
nlu_config = RasaNLUConfig("sample_configs/config_defaults.json")
np.seed(2018)
results = run_cv_evaluation(td, n_folds, nlu_config)
rel_tol = 1e-09
abs_tol = 0.01
acc = np.mean(results["accuracy"])
exp_acc = 0.65 # expected result
np.testing.assert_approx_equal(acc, exp_acc, significant=5)
def test_run_cv_evaluation():
import numpy as np
td = training_data.load_data('data/examples/rasa/demo-rasa.json')
n_folds = 3
nlu_config = RasaNLUConfig("sample_configs/config_defaults.json")
np.seed(2018)
results = run_cv_evaluation(td, n_folds, nlu_config)
rel_tol=1e-09
abs_tol=0.01
acc = np.mean(results["accuracy"])
exp_acc = 0.65 # expected result
np.testing.assert_approx_equal(acc, exp_acc, significant=5)
def test_run_cv_evaluation():
td = training_data.load_data('data/examples/rasa/demo-rasa.json')
nlu_config = config.load("sample_configs/config_spacy.yml")
n_folds = 2
results, entity_results = run_cv_evaluation(td, n_folds, nlu_config)
assert len(results.train["Accuracy"]) == n_folds
assert len(results.train["Precision"]) == n_folds
assert len(results.train["F1-score"]) == n_folds
assert len(results.test["Accuracy"]) == n_folds
assert len(results.test["Precision"]) == n_folds
assert len(results.test["F1-score"]) == n_folds
assert len(entity_results.train['ner_crf']["Accuracy"]) == n_folds
assert len(entity_results.train['ner_crf']["Precision"]) == n_folds
assert len(entity_results.train['ner_crf']["F1-score"]) == n_folds
assert len(entity_results.test['ner_crf']["Accuracy"]) == n_folds
assert len(entity_results.test['ner_crf']["Precision"]) == n_folds
assert len(entity_results.test['ner_crf']["F1-score"]) == n_folds
def test_run_cv_evaluation():
td = training_data.load_data('data/examples/rasa/demo-rasa.json')
nlu_config = config.load("sample_configs/config_spacy.yml")
n_folds = 2
results, entity_results = run_cv_evaluation(td, n_folds, nlu_config)
assert len(results.train["Accuracy"]) == n_folds
assert len(results.train["Precision"]) == n_folds
assert len(results.train["F1-score"]) == n_folds
assert len(results.test["Accuracy"]) == n_folds
assert len(results.test["Precision"]) == n_folds
assert len(results.test["F1-score"]) == n_folds
assert len(entity_results.train['ner_crf']["Accuracy"]) == n_folds
assert len(entity_results.train['ner_crf']["Precision"]) == n_folds
assert len(entity_results.train['ner_crf']["F1-score"]) == n_folds
assert len(entity_results.test['ner_crf']["Accuracy"]) == n_folds
assert len(entity_results.test['ner_crf']["Precision"]) == n_folds
assert len(entity_results.test['ner_crf']["F1-score"]) == n_folds
def CV_eval(td_file, config_file, Nfolds=10):
# trains a model with crossvalidation using the training data and config
td = load_data(td_file)
configuration = config.load(config_file)
evaluate.run_cv_evaluation(td, Nfolds, configuration)