def perform_nlu_cross_validation(config: Text, nlu: Text,
kwargs: Optional[Dict[Text, Any]]):
import rasa.nlu.config
from rasa.nlu.test import (
drop_intents_below_freq,
cross_validate,
return_results,
return_entity_results,
)
kwargs = kwargs or {}
folds = int(kwargs.get("folds", 3))
nlu_config = rasa.nlu.config.load(config)
data = rasa.nlu.training_data.load_data(nlu)
data = drop_intents_below_freq(data, cutoff=folds)
kwargs = minimal_kwargs(kwargs, cross_validate)
results, entity_results = cross_validate(data, folds, nlu_config, **kwargs)
logger.info("CV evaluation (n={})".format(folds))
if any(results):
logger.info("Intent evaluation results")
return_results(results.train, "train")
return_results(results.test, "test")
if any(entity_results):
logger.info("Entity evaluation results")
return_entity_results(entity_results.train, "train")
return_entity_results(entity_results.test, "test")
def test_nlu_with_cross_validation(config: Text, nlu: Text, folds: int = 3):
import rasa.nlu.config
import rasa.nlu.test as nlu_test
nlu_config = rasa.nlu.config.load(config)
data = rasa.nlu.training_data.load_data(nlu)
data = nlu_test.drop_intents_below_freq(data, cutoff=5)
results, entity_results = nlu_test.cross_validate(data, int(folds), nlu_config)
logger.info("CV evaluation (n={})".format(folds))
if any(results):
logger.info("Intent evaluation results")
nlu_test.return_results(results.train, "train")
nlu_test.return_results(results.test, "test")
if any(entity_results):
logger.info("Entity evaluation results")
nlu_test.return_entity_results(entity_results.train, "train")
nlu_test.return_entity_results(entity_results.test, "test")
def perform_nlu_cross_validation(
config: Text,
nlu: Text,
output: Text,
additional_arguments: Optional[Dict[Text, Any]],
):
import rasa.nlu.config
from rasa.nlu.test import (
drop_intents_below_freq,
cross_validate,
return_results,
return_entity_results,
)
additional_arguments = additional_arguments or {}
folds = int(additional_arguments.get("folds", 3))
nlu_config = rasa.nlu.config.load(config)
data = rasa.nlu.training_data.load_data(nlu)
data = drop_intents_below_freq(data, cutoff=folds)
kwargs = utils.minimal_kwargs(additional_arguments, cross_validate)
results, entity_results, response_selection_results = cross_validate(
data, folds, nlu_config, output, **kwargs)
logger.info(f"CV evaluation (n={folds})")
if any(results):
logger.info("Intent evaluation results")
return_results(results.train, "train")
return_results(results.test, "test")
if any(entity_results):
logger.info("Entity evaluation results")
return_entity_results(entity_results.train, "train")
return_entity_results(entity_results.test, "test")
if any(response_selection_results):
logger.info("Response Selection evaluation results")
return_results(response_selection_results.train, "train")
return_results(response_selection_results.test, "test")
