def perform_nlu_cross_validation( config: Text, data: TrainingData, output: Text, additional_arguments: Optional[Dict[Text, Any]], ) -> None: """Runs cross-validation on test data. Args: config: The model configuration. data: The data which is used for the cross-validation. output: Output directory for the cross-validation results. additional_arguments: Additional arguments which are passed to the cross-validation, like number of `disable_plotting`. """ import rasa.nlu.config from rasa.nlu.test import ( drop_intents_below_freq, cross_validate, log_results, log_entity_results, ) additional_arguments = additional_arguments or {} folds = int(additional_arguments.get("folds", 3)) nlu_config = rasa.nlu.config.load(config) data = drop_intents_below_freq(data, cutoff=folds) kwargs = rasa.shared.utils.common.minimal_kwargs( additional_arguments, cross_validate ) results, entity_results, response_selection_results = cross_validate( data, folds, nlu_config, output, **kwargs ) logger.info(f"CV evaluation (n={folds})") if any(results): logger.info("Intent evaluation results") log_results(results.train, "train") log_results(results.test, "test") if any(entity_results): logger.info("Entity evaluation results") log_entity_results(entity_results.train, "train") log_entity_results(entity_results.test, "test") if any(response_selection_results): logger.info("Response Selection evaluation results") log_results(response_selection_results.train, "train") log_results(response_selection_results.test, "test")
def perform_nlu_cross_validation( config: Text, nlu: Text, output: Text, additional_arguments: Optional[Dict[Text, Any]], ): import rasa.nlu.config from rasa.nlu.test import ( drop_intents_below_freq, cross_validate, log_results, log_entity_results, ) additional_arguments = additional_arguments or {} folds = int(additional_arguments.get("folds", 3)) nlu_config = rasa.nlu.config.load(config) data = rasa.shared.nlu.training_data.loading.load_data(nlu) data = drop_intents_below_freq(data, cutoff=folds) kwargs = rasa.shared.utils.common.minimal_kwargs(additional_arguments, cross_validate) results, entity_results, response_selection_results = cross_validate( data, folds, nlu_config, output, **kwargs) logger.info(f"CV evaluation (n={folds})") if any(results): logger.info("Intent evaluation results") log_results(results.train, "train") log_results(results.test, "test") if any(entity_results): logger.info("Entity evaluation results") log_entity_results(entity_results.train, "train") log_entity_results(entity_results.test, "test") if any(response_selection_results): logger.info("Response Selection evaluation results") log_results(response_selection_results.train, "train") log_results(response_selection_results.test, "test")