def perform_nlu_cross_validation(
config: Text,
data: TrainingData,
output: Text,
additional_arguments: Optional[Dict[Text, Any]],
) -> None:
"""Runs cross-validation on test data.
Args:
config: The model configuration.
data: The data which is used for the cross-validation.
output: Output directory for the cross-validation results.
additional_arguments: Additional arguments which are passed to the
cross-validation, like number of `disable_plotting`.
"""
import rasa.nlu.config
from rasa.nlu.test import (
drop_intents_below_freq,
cross_validate,
log_results,
log_entity_results,
)
additional_arguments = additional_arguments or {}
folds = int(additional_arguments.get("folds", 3))
nlu_config = rasa.nlu.config.load(config)
data = drop_intents_below_freq(data, cutoff=folds)
kwargs = rasa.shared.utils.common.minimal_kwargs(
additional_arguments, cross_validate
)
results, entity_results, response_selection_results = cross_validate(
data, folds, nlu_config, output, **kwargs
)
logger.info(f"CV evaluation (n={folds})")
if any(results):
logger.info("Intent evaluation results")
log_results(results.train, "train")
log_results(results.test, "test")
if any(entity_results):
logger.info("Entity evaluation results")
log_entity_results(entity_results.train, "train")
log_entity_results(entity_results.test, "test")
if any(response_selection_results):
logger.info("Response Selection evaluation results")
log_results(response_selection_results.train, "train")
log_results(response_selection_results.test, "test")
def perform_nlu_cross_validation(
config: Text,
nlu: Text,
output: Text,
additional_arguments: Optional[Dict[Text, Any]],
):
import rasa.nlu.config
from rasa.nlu.test import (
drop_intents_below_freq,
cross_validate,
log_results,
log_entity_results,
)
additional_arguments = additional_arguments or {}
folds = int(additional_arguments.get("folds", 3))
nlu_config = rasa.nlu.config.load(config)
data = rasa.shared.nlu.training_data.loading.load_data(nlu)
data = drop_intents_below_freq(data, cutoff=folds)
kwargs = rasa.shared.utils.common.minimal_kwargs(additional_arguments,
cross_validate)
results, entity_results, response_selection_results = cross_validate(
data, folds, nlu_config, output, **kwargs)
logger.info(f"CV evaluation (n={folds})")
if any(results):
logger.info("Intent evaluation results")
log_results(results.train, "train")
log_results(results.test, "test")
if any(entity_results):
logger.info("Entity evaluation results")
log_entity_results(entity_results.train, "train")
log_entity_results(entity_results.test, "test")
if any(response_selection_results):
logger.info("Response Selection evaluation results")
log_results(response_selection_results.train, "train")
log_results(response_selection_results.test, "test")
