def test_run_cv_evaluation():
td = training_data.load_data('data/examples/rasa/demo-rasa.json')
nlu_config = config.load(
"sample_configs/config_pretrained_embeddings_spacy.yml")
n_folds = 2
results, entity_results = cross_validate(td, n_folds, nlu_config)
assert len(results.train["Accuracy"]) == n_folds
assert len(results.train["Precision"]) == n_folds
assert len(results.train["F1-score"]) == n_folds
assert len(results.test["Accuracy"]) == n_folds
assert len(results.test["Precision"]) == n_folds
assert len(results.test["F1-score"]) == n_folds
assert len(entity_results.train[
'CRFEntityExtractor']["Accuracy"]) == n_folds
assert len(entity_results.train[
'CRFEntityExtractor']["Precision"]) == n_folds
assert len(entity_results.train[
'CRFEntityExtractor']["F1-score"]) == n_folds
assert len(entity_results.test[
'CRFEntityExtractor']["Accuracy"]) == n_folds
assert len(entity_results.test[
'CRFEntityExtractor']["Precision"]) == n_folds
assert len(entity_results.test[
'CRFEntityExtractor']["F1-score"]) == n_folds
def test_run_cv_evaluation():
td = training_data.load_data('data/examples/rasa/demo-rasa.json')
nlu_config = config.load(
"sample_configs/config_pretrained_embeddings_spacy.yml")
n_folds = 2
results, entity_results = cross_validate(td, n_folds, nlu_config)
assert len(results.train["Accuracy"]) == n_folds
assert len(results.train["Precision"]) == n_folds
assert len(results.train["F1-score"]) == n_folds
assert len(results.test["Accuracy"]) == n_folds
assert len(results.test["Precision"]) == n_folds
assert len(results.test["F1-score"]) == n_folds
assert len(entity_results.train[
'CRFEntityExtractor']["Accuracy"]) == n_folds
assert len(entity_results.train[
'CRFEntityExtractor']["Precision"]) == n_folds
assert len(entity_results.train[
'CRFEntityExtractor']["F1-score"]) == n_folds
assert len(entity_results.test[
'CRFEntityExtractor']["Accuracy"]) == n_folds
assert len(entity_results.test[
'CRFEntityExtractor']["Precision"]) == n_folds
assert len(entity_results.test[
'CRFEntityExtractor']["F1-score"]) == n_folds
def test_nlu_with_cross_validation(config: Text, nlu: Text, folds: int = 3):
import rasa_nlu.config
import rasa_nlu.test as nlu_test
nlu_config = rasa_nlu.config.load(config)
data = rasa_nlu.training_data.load_data(nlu)
data = nlu_test.drop_intents_below_freq(data, cutoff=5)
results, entity_results = nlu_test.cross_validate(data, int(folds),
nlu_config)
logger.info("CV evaluation (n={})".format(folds))
if any(results):
logger.info("Intent evaluation results")
nlu_test.return_results(results.train, "train")
nlu_test.return_results(results.test, "test")
if any(entity_results):
logger.info("Entity evaluation results")
nlu_test.return_entity_results(entity_results.train, "train")
nlu_test.return_entity_results(entity_results.test, "test")