def test_nlu(
model: Optional[Text],
nlu_data: Optional[Text],
output_directory: Text = DEFAULT_RESULTS_PATH,
kwargs: Optional[Dict] = None,
):
from rasa.nlu.test import run_evaluation
from rasa.model import get_model
try:
unpacked_model = get_model(model)
except ModelNotFound:
print_error(
"Could not find any model. Use 'rasa train nlu' to train a "
"Rasa model and provide it via the '--model' argument.")
return
io_utils.create_directory(output_directory)
nlu_model = os.path.join(unpacked_model, "nlu")
if os.path.exists(nlu_model):
kwargs = utils.minimal_kwargs(kwargs, run_evaluation,
["data_path", "model"])
run_evaluation(nlu_data,
nlu_model,
output_directory=output_directory,
**kwargs)
else:
print_error(
"Could not find any model. Use 'rasa train nlu' to train a "
"Rasa model and provide it via the '--model' argument.")
def run_core_test(args: argparse.Namespace) -> None:
"""Run core tests."""
from rasa import data
from rasa.test import test_core_models_in_directory, test_core, test_core_models
endpoints = cli_utils.get_validated_path(args.endpoints, "endpoints",
DEFAULT_ENDPOINTS_PATH, True)
stories = cli_utils.get_validated_path(args.stories, "stories",
DEFAULT_DATA_PATH)
stories = data.get_core_directory(stories)
output = args.out or DEFAULT_RESULTS_PATH
io_utils.create_directory(output)
if isinstance(args.model, list) and len(args.model) == 1:
args.model = args.model[0]
if isinstance(args.model, str):
model_path = cli_utils.get_validated_path(args.model, "model",
DEFAULT_MODELS_PATH)
if args.evaluate_model_directory:
test_core_models_in_directory(args.model, stories, output)
else:
test_core(
model=model_path,
stories=stories,
endpoints=endpoints,
output=output,
additional_arguments=vars(args),
)
else:
test_core_models(args.model, stories, output)
def run_evaluation(
data_path: Text,
model_path: Text,
out_directory: Optional[Text] = None,
report: Optional[Text] = None,
successes: Optional[Text] = None,
errors: Optional[Text] = "errors.json",
confmat: Optional[Text] = None,
histogram: Optional[Text] = None,
component_builder: Optional[ComponentBuilder] = None,
) -> Dict: # pragma: no cover
"""
Evaluate intent classification and entity extraction.
:param data_path: path to the test data
:param model_path: path to the model
:param out_directory: path to folder where all output will be stored
:param report: path to folder where reports are stored
:param successes: path to file that will contain success cases
:param errors: path to file that will contain error cases
:param confmat: path to file that will show the confusion matrix
:param histogram: path fo file that will show a histogram
:param component_builder: component builder
:return: dictionary containing evaluation results
"""
# get the metadata config from the package data
interpreter = Interpreter.load(model_path, component_builder)
interpreter.pipeline = remove_pretrained_extractors(interpreter.pipeline)
test_data = training_data.load_data(data_path,
interpreter.model_metadata.language)
result = {
"intent_evaluation": None,
"entity_evaluation": None,
} # type: Dict[Text, Optional[Dict]]
if report:
if out_directory:
report = os.path.join(out_directory, report)
io_utils.create_directory(report)
intent_results, entity_results = get_eval_data(interpreter, test_data)
if intent_results:
logger.info("Intent evaluation results:")
result["intent_evaluation"] = evaluate_intents(intent_results, report,
successes, errors,
confmat, histogram,
out_directory)
if entity_results:
logger.info("Entity evaluation results:")
extractors = get_entity_extractors(interpreter)
result["entity_evaluation"] = evaluate_entities(
entity_results, extractors, report, out_directory)
return result
def test_core(
model: Optional[Text] = None,
stories: Optional[Text] = None,
endpoints: Optional[Text] = None,
output: Text = DEFAULT_RESULTS_PATH,
kwargs: Optional[Dict] = None,
):
import rasa.core.test
import rasa.core.utils as core_utils
import rasa.model
from rasa.core.interpreter import RegexInterpreter, NaturalLanguageInterpreter
from rasa.core.agent import Agent
_endpoints = core_utils.AvailableEndpoints.read_endpoints(endpoints)
if kwargs is None:
kwargs = {}
if output:
io_utils.create_directory(output)
try:
unpacked_model = rasa.model.get_model(model)
except ModelNotFound:
print_error(
"Unable to test: could not find a model. Use 'rasa train' to train a "
"Rasa model and provide it via the '--model' argument."
)
return
core_path, nlu_path = rasa.model.get_model_subdirectories(unpacked_model)
if not core_path:
print_error(
"Unable to test: could not find a Core model. Use 'rasa train' to train a "
"Rasa model and provide it via the '--model' argument."
)
use_e2e = kwargs["e2e"] if "e2e" in kwargs else False
_interpreter = RegexInterpreter()
if use_e2e:
if nlu_path:
_interpreter = NaturalLanguageInterpreter.create(_endpoints.nlu or nlu_path)
else:
print_warning(
"No NLU model found. Using default 'RegexInterpreter' for end-to-end "
"evaluation."
)
_agent = Agent.load(unpacked_model, interpreter=_interpreter)
kwargs = utils.minimal_kwargs(kwargs, rasa.core.test, ["stories", "agent"])
loop = asyncio.get_event_loop()
loop.run_until_complete(
rasa.core.test(stories, _agent, out_directory=output, **kwargs)
)
def test_nlu(args: argparse.Namespace) -> None:
from rasa import data
from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu
nlu_data = cli_utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH)
nlu_data = data.get_nlu_directory(nlu_data)
output = args.out or DEFAULT_RESULTS_PATH
io_utils.create_directory(output)
if args.config is not None and len(args.config) == 1:
args.config = os.path.abspath(args.config[0])
if os.path.isdir(args.config):
config_dir = args.config
config_files = os.listdir(config_dir)
args.config = [
os.path.join(config_dir, os.path.abspath(config))
for config in config_files
]
if isinstance(args.config, list):
logger.info(
"Multiple configuration files specified, running nlu comparison mode."
)
config_files = []
for file in args.config:
try:
validation_utils.validate_yaml_schema(
io_utils.read_file(file),
CONFIG_SCHEMA_FILE,
show_validation_errors=False,
)
config_files.append(file)
except validation_utils.InvalidYamlFileError:
logger.debug(
"Ignoring file '{}' as it is not a valid config file.".
format(file))
continue
compare_nlu_models(
configs=config_files,
nlu=nlu_data,
output=output,
runs=args.runs,
exclusion_percentages=args.percentages,
)
elif args.cross_validation:
logger.info("Test model using cross validation.")
config = cli_utils.get_validated_path(args.config, "config",
DEFAULT_CONFIG_PATH)
perform_nlu_cross_validation(config, nlu_data, output, vars(args))
else:
model_path = cli_utils.get_validated_path(args.model, "model",
DEFAULT_MODELS_PATH)
test_nlu(model_path, nlu_data, output, vars(args))
def _add_ssh_credentials(self, ssh_key: Text,
repository_id: int) -> Dict[Text, Text]:
base_path = self._directory_for_git_clones / SSH_FILES_DIRECTORY
io_utils.create_directory(base_path)
path_to_key = base_path / f"{repository_id}.key"
path_to_executable = self._ssh_executable_path(repository_id)
return self._add_ssh_credentials_to_disk(ssh_key, path_to_key,
path_to_executable)
def _create_directory_for_git_clones(
directory_for_git_clones: Union[Text, Path]) -> Path:
try:
io_utils.create_directory(directory_for_git_clones)
except OSError as e:
# This should only happen when running tests locally
logger.error(e)
import tempfile
directory_for_git_clones = tempfile.mkdtemp()
return Path(directory_for_git_clones)
def test_core(
model: Optional[Text] = None,
stories: Optional[Text] = None,
output: Text = DEFAULT_RESULTS_PATH,
additional_arguments: Optional[Dict] = None,
) -> None:
import rasa.model
from rasa.core.interpreter import RegexInterpreter
from rasa.core.agent import Agent
if additional_arguments is None:
additional_arguments = {}
if output:
io_utils.create_directory(output)
try:
unpacked_model = rasa.model.get_model(model)
except ModelNotFound:
cli_utils.print_error(
"Unable to test: could not find a model. Use 'rasa train' to train a "
"Rasa model and provide it via the '--model' argument."
)
return
_agent = Agent.load(unpacked_model)
if _agent.policy_ensemble is None:
cli_utils.print_error(
"Unable to test: could not find a Core model. Use 'rasa train' to train a "
"Rasa model and provide it via the '--model' argument."
)
if isinstance(_agent.interpreter, RegexInterpreter):
cli_utils.print_warning(
"No NLU model found. Using default 'RegexInterpreter' for end-to-end "
"evaluation. If you added actual user messages to your test stories "
"this will likely lead to the tests failing. In that case, you need "
"to train a NLU model first, e.g. using `rasa train`."
)
from rasa.core.test import test
kwargs = utils.minimal_kwargs(additional_arguments, test, ["stories", "agent"])
_test_core(stories, _agent, output, **kwargs)
def save_repository(
self, repository_information: Dict[Text, Union[Text, int]]
) -> Dict[Text, Union[Text, int]]:
"""Saves the credentials for a repository.
Args:
repository_information: The information for the repository as `Dict`.
Returns:
The saved repository information including database ID.
"""
repository_credentials = self._get_repository_from_dict(
repository_information)
if not self.has_required_remote_permission(
repository_credentials.repository_url,
repository_credentials.ssh_key):
raise ValueError(
"Given repository credentials don't provide write "
"permissions to the repository. Please make sure the ssh "
"key is correct and the administrator of the remote "
"repository gave you the required permissions.")
self.add(repository_credentials)
self.flush() # to assign database id
self._add_ssh_credentials(repository_credentials.ssh_key,
repository_credentials.id)
io_utils.create_directory(
self.repository_path(repository_credentials.id))
self.set_repository(repository_credentials.id)
was_created_via_ui = bool(
repository_information.get(REPOSITORY_USE_GENERATED_KEYS_KEY))
telemetry.track_repository_creation(
repository_credentials.target_branch, was_created_via_ui)
return repository_credentials.as_dict()
def test_create_directory_if_already_exists(tmp_path: Path):
# This should not throw an exception
io_utils.create_directory(str(tmp_path))
assert True
def test_create_directory_if_new(tmp_path: Path):
directory = str(tmp_path / "a" / "b")
io_utils.create_directory(directory)
assert os.path.exists(directory)
def cross_validate(
data: TrainingData,
n_folds: int,
nlu_config: Union[RasaNLUModelConfig, Text],
output: Optional[Text] = None,
successes: bool = False,
errors: bool = False,
confmat: Optional[Text] = None,
histogram: Optional[Text] = None,
) -> Tuple[CVEvaluationResult, CVEvaluationResult]:
"""Stratified cross validation on data.
Args:
data: Training Data
n_folds: integer, number of cv folds
nlu_config: nlu config file
report: path to folder where reports are stored
successes: if true successful predictions are written to a file
errors: if true incorrect predictions are written to a file
confmat: path to file that will show the confusion matrix
histogram: path fo file that will show a histogram
Returns:
dictionary with key, list structure, where each entry in list
corresponds to the relevant result for one fold
"""
from collections import defaultdict
if isinstance(nlu_config, str):
nlu_config = config.load(nlu_config)
if output:
io_utils.create_directory(output)
trainer = Trainer(nlu_config)
trainer.pipeline = remove_pretrained_extractors(trainer.pipeline)
intent_train_metrics = defaultdict(list) # type: IntentMetrics
intent_test_metrics = defaultdict(list) # type: IntentMetrics
entity_train_metrics = defaultdict(
lambda: defaultdict(list)) # type: EntityMetrics
entity_test_metrics = defaultdict(
lambda: defaultdict(list)) # type: EntityMetrics
intent_test_results = [] # type: List[IntentEvaluationResult]
entity_test_results = [] # type: List[EntityEvaluationResult]
intent_classifier_present = False
extractors = set() # type: Set[Text]
for train, test in generate_folds(n_folds, data):
interpreter = trainer.train(train)
# calculate train accuracy
combine_result(intent_train_metrics, entity_train_metrics, interpreter,
train)
# calculate test accuracy
combine_result(
intent_test_metrics,
entity_test_metrics,
interpreter,
test,
intent_test_results,
entity_test_results,
)
if not extractors:
extractors = get_entity_extractors(interpreter)
if is_intent_classifier_present(interpreter):
intent_classifier_present = True
if intent_classifier_present:
logger.info("Accumulated test folds intent evaluation results:")
evaluate_intents(intent_test_results, output, successes, errors,
confmat, histogram)
if extractors:
logger.info("Accumulated test folds entity evaluation results:")
evaluate_entities(entity_test_results, extractors, output, successes,
errors)
return (
CVEvaluationResult(dict(intent_train_metrics),
dict(intent_test_metrics)),
CVEvaluationResult(dict(entity_train_metrics),
dict(entity_test_metrics)),
)
def run_evaluation(
data_path: Text,
model_path: Text,
output_directory: Optional[Text] = None,
successes: bool = False,
errors: bool = False,
confmat: Optional[Text] = None,
histogram: Optional[Text] = None,
component_builder: Optional[ComponentBuilder] = None,
) -> Dict: # pragma: no cover
"""
Evaluate intent classification, response selection and entity extraction.
:param data_path: path to the test data
:param model_path: path to the model
:param output_directory: path to folder where all output will be stored
:param successes: if true successful predictions are written to a file
:param errors: if true incorrect predictions are written to a file
:param confmat: path to file that will show the confusion matrix
:param histogram: path fo file that will show a histogram
:param component_builder: component builder
:return: dictionary containing evaluation results
"""
if isinstance(model, Interpreter):
interpreter = model
# get the correct interpreter based on incremental flag
elif incremental is False:
interpreter = Interpreter.load(model, component_builder)
else:
interpreter = IncrementalInterpreter.load(model, component_builder)
test_data = training_data.load_data(data_path,
interpreter.model_metadata.language)
# get the metadata config from the package data
#interpreter = Interpreter.load(model_path, component_builder)
interpreter.pipeline = remove_pretrained_extractors(interpreter.pipeline)
test_data = training_data.load_data(data_path, interpreter.model_metadata.language)
result = {
"intent_evaluation": None,
"entity_evaluation": None,
"response_selection_evaluation": None,
} # type: Dict[Text, Optional[Dict]]
if output_directory:
io_utils.create_directory(output_directory)
intent_results, response_selection_results, entity_results, = get_eval_data(
interpreter, test_data
)
if intent_results:
logger.info("Intent evaluation results:")
result["intent_evaluation"] = evaluate_intents(
intent_results, output_directory, successes, errors, confmat, histogram
)
if response_selection_results:
logger.info("Response selection evaluation results:")
result["response_selection_evaluation"] = evaluate_response_selections(
response_selection_results, output_directory
)
if entity_results:
logger.info("Entity evaluation results:")
extractors = get_entity_extractors(interpreter)
result["entity_evaluation"] = evaluate_entities(
entity_results, extractors, output_directory, successes, errors
)
return result
def test_creation_of_existing_dir(tmpdir):
# makes sure there is no exception
assert io_utils.create_directory(tmpdir.strpath) is None
