def convert_and_write(cls, source_path: Path, output_path: Path) -> None:
"""Converts the given training data file and saves it to the output directory.
Args:
source_path: Path to the training data file.
output_path: Path to the output directory.
"""
from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
KEY_ACTIVE_LOOP, )
output_core_path = cls.generate_path_for_converted_training_data_file(
source_path, output_path)
reader = MarkdownStoryReader(unfold_or_utterances=False)
writer = YAMLStoryWriter()
loop = asyncio.get_event_loop()
steps = loop.run_until_complete(reader.read_from_file(source_path))
if YAMLStoryWriter.stories_contain_loops(steps):
print_warning(
f"Training data file '{source_path}' contains forms. "
f"Any 'form' events will be converted to '{KEY_ACTIVE_LOOP}' events. "
f"Please note that in order for these stories to work you still "
f"need the 'FormPolicy' to be active. However the 'FormPolicy' is "
f"deprecated, please consider switching to the new 'RulePolicy', "
f"for which you can find the documentation here: {DOCS_URL_RULES}."
)
writer.dump(output_core_path, steps)
print_success(
f"Converted Core file: '{source_path}' >> '{output_core_path}'.")
def _get_sanitized_model_directory(model_directory: Text) -> Text:
"""Adjusts the `--model` argument of `rasa test core` when called with
`--evaluate-model-directory`.
By default rasa uses the latest model for the `--model` parameter. However, for
`--evaluate-model-directory` we need a directory. This function checks if the
passed parameter is a model or an individual model file.
Args:
model_directory: The model_directory argument that was given to
`test_core_models_in_directory`.
Returns: The adjusted model_directory that should be used in
`test_core_models_in_directory`.
"""
import rasa.model
p = Path(model_directory)
if p.is_file():
if model_directory != rasa.model.get_latest_model():
cli_utils.print_warning(
"You passed a file as '--model'. Will use the directory containing "
"this file instead.")
model_directory = str(p.parent)
return model_directory
def _convert_to_yaml(args: argparse.Namespace, is_nlu: bool) -> None:
output = Path(args.out)
if not os.path.exists(output):
print_error_and_exit(
f"The output path '{output}' doesn't exist. Please make sure to specify "
f"an existing directory and try again."
)
training_data = Path(args.data)
if not os.path.exists(training_data):
print_error_and_exit(
f"The training data path {training_data} doesn't exist "
f"and will be skipped."
)
num_of_files_converted = 0
for file in os.listdir(training_data):
source_path = training_data / file
output_path = output / f"{source_path.stem}{CONVERTED_FILE_SUFFIX}"
if MarkdownReader.is_markdown_nlu_file(source_path):
if not is_nlu:
continue
_write_nlu_yaml(source_path, output_path, source_path)
num_of_files_converted += 1
elif not is_nlu and MarkdownStoryReader.is_markdown_story_file(source_path):
_write_core_yaml(source_path, output_path, source_path)
num_of_files_converted += 1
else:
print_warning(f"Skipped file: '{source_path}'.")
print_info(f"Converted {num_of_files_converted} file(s), saved in '{output}'.")
def _validate_rasa_x_start(args: argparse.Namespace, project_path: Text):
if not is_rasa_x_installed():
cli_utils.print_error_and_exit(
"Rasa X is not installed. The `rasa x` "
"command requires an installation of Rasa X. "
"Instructions on how to install Rasa X can be found here: "
"https://rasa.com/docs/rasa-x/installation-and-setup/.")
if args.port == args.rasa_x_port:
cli_utils.print_error_and_exit(
"The port for Rasa X '{}' and the port of the Rasa server '{}' are the "
"same. We need two different ports, one to run Rasa X (e.g. delivering the "
"UI) and another one to run a normal Rasa server.\nPlease specify two "
"different ports using the arguments '--port' and '--rasa-x-port'."
.format(args.rasa_x_port, args.port))
if not is_rasa_project_setup(project_path):
cli_utils.print_error_and_exit(
"This directory is not a valid Rasa project. Use 'rasa init' "
"to create a new Rasa project or switch to a valid Rasa project "
"directory (see http://rasa.com/docs/rasa/user-guide/"
"rasa-tutorial/#create-a-new-project).")
_validate_domain(os.path.join(project_path, DEFAULT_DOMAIN_PATH))
if args.data and not os.path.exists(args.data):
cli_utils.print_warning(
"The provided data path ('{}') does not exists. Rasa X will start "
"without any training data.".format(args.data))
def _convert_to_yaml(args: argparse.Namespace,
converter: TrainingDataConverter) -> None:
output = Path(args.out)
if not os.path.exists(output):
print_error_and_exit(
f"The output path '{output}' doesn't exist. Please make sure to specify "
f"an existing directory and try again.")
training_data = Path(args.data)
if not os.path.exists(training_data):
print_error_and_exit(
f"The training data path {training_data} doesn't exist "
f"and will be skipped.")
num_of_files_converted = 0
if os.path.isfile(training_data):
if _convert_file_to_yaml(training_data, output, converter):
num_of_files_converted += 1
elif os.path.isdir(training_data):
for root, _, files in os.walk(training_data, followlinks=True):
for f in sorted(files):
source_path = Path(os.path.join(root, f))
if _convert_file_to_yaml(source_path, output, converter):
num_of_files_converted += 1
if num_of_files_converted:
print_info(
f"Converted {num_of_files_converted} file(s), saved in '{output}'."
)
else:
print_warning(
f"Didn't convert any files under '{training_data}' path. "
"Did you specify the correct file/directory?")
def test_core(
model: Optional[Text] = None,
stories: Optional[Text] = None,
endpoints: Optional[Text] = None,
output: Text = DEFAULT_RESULTS_PATH,
kwargs: Optional[Dict] = None,
):
import rasa.core.test
import rasa.core.utils as core_utils
import rasa.model
from rasa.core.interpreter import RegexInterpreter, NaturalLanguageInterpreter
from rasa.core.agent import Agent
_endpoints = core_utils.AvailableEndpoints.read_endpoints(endpoints)
if kwargs is None:
kwargs = {}
if output:
io_utils.create_directory(output)
try:
unpacked_model = rasa.model.get_model(model)
except ModelNotFound:
print_error(
"Unable to test: could not find a model. Use 'rasa train' to train a "
"Rasa model and provide it via the '--model' argument."
)
return
core_path, nlu_path = rasa.model.get_model_subdirectories(unpacked_model)
if not core_path:
print_error(
"Unable to test: could not find a Core model. Use 'rasa train' to train a "
"Rasa model and provide it via the '--model' argument."
)
use_e2e = kwargs["e2e"] if "e2e" in kwargs else False
_interpreter = RegexInterpreter()
if use_e2e:
if nlu_path:
_interpreter = NaturalLanguageInterpreter.create(_endpoints.nlu or nlu_path)
else:
print_warning(
"No NLU model found. Using default 'RegexInterpreter' for end-to-end "
"evaluation."
)
_agent = Agent.load(unpacked_model, interpreter=_interpreter)
kwargs = utils.minimal_kwargs(kwargs, rasa.core.test, ["stories", "agent"])
loop = asyncio.get_event_loop()
loop.run_until_complete(
rasa.core.test(stories, _agent, out_directory=output, **kwargs)
)
def test_core(
model: Optional[Text] = None,
stories: Optional[Text] = None,
endpoints: Optional[Text] = None,
output: Text = DEFAULT_RESULTS_PATH,
kwargs: Optional[Dict] = None,
):
import rasa.core.test
import rasa.core.utils as core_utils
from rasa.nlu import utils as nlu_utils
from rasa.model import get_model
from rasa.core.interpreter import NaturalLanguageInterpreter
from rasa.core.agent import Agent
_endpoints = core_utils.AvailableEndpoints.read_endpoints(endpoints)
if kwargs is None:
kwargs = {}
if output:
nlu_utils.create_dir(output)
unpacked_model = get_model(model)
if unpacked_model is None:
print_error(
"Unable to test: could not find a model. Use 'rasa train' to train a "
"Rasa model."
)
return
core_path, nlu_path = get_model_subdirectories(unpacked_model)
if not os.path.exists(core_path):
print_error(
"Unable to test: could not find a Core model. Use 'rasa train' to "
"train a model."
)
use_e2e = kwargs["e2e"] if "e2e" in kwargs else False
_interpreter = RegexInterpreter()
if use_e2e:
if os.path.exists(nlu_path):
_interpreter = NaturalLanguageInterpreter.create(nlu_path, _endpoints.nlu)
else:
print_warning(
"No NLU model found. Using default 'RegexInterpreter' for end-to-end "
"evaluation."
)
_agent = Agent.load(unpacked_model, interpreter=_interpreter)
kwargs = minimal_kwargs(kwargs, rasa.core.test, ["stories", "agent"])
loop = asyncio.get_event_loop()
loop.run_until_complete(
rasa.core.test(stories, _agent, out_directory=output, **kwargs)
)
async def handle_domain_if_not_exists(file_importer: TrainingDataImporter,
output_path, fixed_model_name):
nlu_model_only = await _train_nlu_with_validated_data(
file_importer, output=output_path, fixed_model_name=fixed_model_name)
print_warning(
"Core training was skipped because no valid domain file was found. Only an nlu-model was created."
"Please specify a valid domain using '--domain' argument or check if the provided domain file exists."
)
return nlu_model_only
def rasa_x(args: argparse.Namespace):
from rasa.cli.utils import print_success, print_error, signal_handler
from rasa.core.utils import AvailableEndpoints
signal.signal(signal.SIGINT, signal_handler)
_configure_logging(args)
if args.production:
print_success("Starting Rasa X in production mode... 🚀")
args.endpoints = get_validated_path(args.endpoints, "endpoints",
DEFAULT_ENDPOINTS_PATH, True)
endpoints = AvailableEndpoints.read_endpoints(args.endpoints)
_rasa_service(args, endpoints)
else:
if not is_rasa_x_installed():
print_error("Rasa X is not installed. The `rasa x` "
"command requires an installation of Rasa X.")
sys.exit(1)
project_path = "."
if not is_rasa_project_setup(project_path):
print_error(
"This directory is not a valid Rasa project. Use 'rasa init' "
"to create a new Rasa project or switch to a valid Rasa project "
"directory.")
sys.exit(1)
_validate_domain(os.path.join(project_path, DEFAULT_DOMAIN_PATH))
if args.data and not os.path.exists(args.data):
print_warning(
"The provided data path ('{}') does not exists. Rasa X will start "
"without any training data.".format(args.data))
# noinspection PyUnresolvedReferences
from rasax.community import local
local.check_license_and_metrics(args)
rasa_x_token = generate_rasa_x_token()
process = start_rasa_for_local_rasa_x(args, rasa_x_token=rasa_x_token)
try:
local.main(args, project_path, args.data, token=rasa_x_token)
except Exception:
print(traceback.format_exc())
print_error(
"Sorry, something went wrong (see error above). Make sure to start "
"Rasa X with valid data and valid domain and config files. Please, "
"also check any warnings that popped up.\nIf you need help fixing "
"the issue visit our forum: https://forum.rasa.com/.")
finally:
process.terminate()
def run(
model: Text,
endpoints: Text,
connector: Text = None,
credentials: Text = None,
**kwargs: Dict
):
"""Runs a Rasa model.
Args:
model: Path to model archive.
endpoints: Path to endpoints file.
connector: Connector which should be use (overwrites `credentials`
field).
credentials: Path to channel credentials file.
**kwargs: Additional arguments which are passed to
`rasa.core.run.serve_application`.
"""
import rasa.core.run
import rasa.nlu.run
from rasa.core.utils import AvailableEndpoints
model_path = get_model(model)
if not model_path:
print_error(
"No model found. Train a model before running the "
"server using `rasa train`."
)
return
_endpoints = AvailableEndpoints.read_endpoints(endpoints)
if not connector and not credentials:
connector = "rest"
print_warning(
"No chat connector configured, falling back to the "
"REST input channel. To connect your bot to another channel, "
"read the docs here: {}/user-guide/"
"messaging-and-voice-channels".format(DOCS_BASE_URL)
)
kwargs = minimal_kwargs(kwargs, rasa.core.run.serve_application)
rasa.core.run.serve_application(
model,
channel=connector,
credentials=credentials,
endpoints=_endpoints,
**kwargs
)
shutil.rmtree(model_path)
def handle_domain_if_not_exists(config, nlu_data_directory, output_path,
fixed_model_name):
nlu_model_only = _train_nlu_with_validated_data(
config=config,
nlu_data_directory=nlu_data_directory,
output=output_path,
fixed_model_name=fixed_model_name,
)
print_warning(
"Core training was skipped because no valid domain file was found. Only an nlu-model was created."
"Please specify a valid domain using '--domain' argument or check if the provided domain file exists."
)
return nlu_model_only
def perform_interactive_learning(args, zipped_model):
from rasa.core.train import do_interactive_learning
if zipped_model:
model_path = model.unpack_model(zipped_model)
args.core, args.nlu = model.get_model_subdirectories(model_path)
stories_directory = data.get_core_directory(args.data)
do_interactive_learning(args, stories_directory)
shutil.rmtree(model_path)
else:
print_warning("No initial zipped trained model found.")
def _get_valid_config(config: Text, mandatory_keys: List[Text]) -> Text:
config_path = get_validated_path(config, "config", FALLBACK_CONFIG_PATH)
missing_keys = missing_config_keys(config_path, mandatory_keys)
if missing_keys:
print_warning(
"Configuration file '{}' is missing mandatory parameters: "
"{}. Filling missing parameters from fallback configuration file: '{}'."
"".format(config, ", ".join(missing_keys), FALLBACK_CONFIG_PATH))
_enrich_config(config_path, missing_keys, FALLBACK_CONFIG_PATH)
return config_path
def rasa_x(args: argparse.Namespace):
from rasa.cli.utils import print_success, print_error, signal_handler
from rasa.core.utils import AvailableEndpoints
signal.signal(signal.SIGINT, signal_handler)
_configure_logging(args)
if args.production:
print_success("Starting Rasa X in production mode... 🚀")
args.endpoints = get_validated_path(
args.endpoints, "endpoints", DEFAULT_ENDPOINTS_PATH, True
)
endpoints = AvailableEndpoints.read_endpoints(args.endpoints)
_rasa_service(args, endpoints)
else:
if not is_rasa_x_installed():
print_error(
"Rasa X is not installed. The `rasa x` "
"command requires an installation of Rasa X."
)
sys.exit(1)
project_path = "."
if not is_rasa_project_setup(project_path):
print_error(
"This directory is not a valid Rasa project. Use 'rasa init' "
"to create a new Rasa project or switch to a valid Rasa project "
"directory."
)
sys.exit(1)
if args.data and not os.path.exists(args.data):
print_warning(
"The provided data path ('{}') does not exists. Rasa X will start "
"without any training data.".format(args.data)
)
# noinspection PyUnresolvedReferences
from rasax.community import local
local.check_license_and_metrics(args)
rasa_x_token = generate_rasa_x_token()
process = start_rasa_for_local_rasa_x(args, rasa_x_token=rasa_x_token)
try:
local.main(args, project_path, args.data, token=rasa_x_token)
finally:
process.terminate()
def _validate_all_requested_ids_exist(
self, conversation_ids_in_tracker_store: Set[Text]) -> None:
"""Warn user if `self.requested_conversation_ids` contains IDs not found in
`conversation_ids_in_tracker_store`
Args:
conversation_ids_in_tracker_store: Set of conversation IDs contained in
the tracker store.
"""
missing_ids_in_tracker_store = (set(self.requested_conversation_ids) -
conversation_ids_in_tracker_store)
if missing_ids_in_tracker_store:
cli_utils.print_warning(
f"Could not find the following requested "
f"conversation IDs in connected tracker store: "
f"{', '.join(sorted(missing_ids_in_tracker_store))}")
def test_core(
model: Optional[Text] = None,
stories: Optional[Text] = None,
output: Text = DEFAULT_RESULTS_PATH,
additional_arguments: Optional[Dict] = None,
) -> None:
import rasa.model
from rasa.core.interpreter import RegexInterpreter
from rasa.core.agent import Agent
if additional_arguments is None:
additional_arguments = {}
if output:
io_utils.create_directory(output)
try:
unpacked_model = rasa.model.get_model(model)
except ModelNotFound:
cli_utils.print_error(
"Unable to test: could not find a model. Use 'rasa train' to train a "
"Rasa model and provide it via the '--model' argument."
)
return
_agent = Agent.load(unpacked_model)
if _agent.policy_ensemble is None:
cli_utils.print_error(
"Unable to test: could not find a Core model. Use 'rasa train' to train a "
"Rasa model and provide it via the '--model' argument."
)
if isinstance(_agent.interpreter, RegexInterpreter):
cli_utils.print_warning(
"No NLU model found. Using default 'RegexInterpreter' for end-to-end "
"evaluation. If you added actual user messages to your test stories "
"this will likely lead to the tests failing. In that case, you need "
"to train a NLU model first, e.g. using `rasa train`."
)
from rasa.core.test import test
kwargs = utils.minimal_kwargs(additional_arguments, test, ["stories", "agent"])
_test_core(stories, _agent, output, **kwargs)
def run(
model: Text,
endpoints: Text,
connector: Text = None,
credentials: Text = None,
**kwargs: Dict,
):
"""Runs a Rasa model.
Args:
model: Path to model archive.
endpoints: Path to endpoints file.
connector: Connector which should be use (overwrites `credentials`
field).
credentials: Path to channel credentials file.
**kwargs: Additional arguments which are passed to
`rasa.core.run.serve_application`.
"""
import rasa.core.run
import rasa.nlu.run
from rasa.core.utils import AvailableEndpoints
import rasa.utils.common as utils
_endpoints = AvailableEndpoints.read_endpoints(endpoints)
if not connector and not credentials:
connector = "rest"
print_warning(
"No chat connector configured, falling back to the "
"REST input channel. To connect your bot to another channel, "
"read the docs here: {}/user-guide/"
"messaging-and-voice-channels".format(DOCS_BASE_URL))
kwargs = utils.minimal_kwargs(kwargs, run_core.serve_application)
# 重写core run serve_application 方法
run_core.serve_application(
model,
channel=connector,
credentials=credentials,
endpoints=_endpoints,
**kwargs,
)
def warn_missing_templates(action_names: List[Text],
templates: Dict[Text, Any]) -> None:
"""Warn user of utterance names which have no specified template."""
utterances = [
act for act in action_names if act.startswith(action.UTTER_PREFIX)
]
missing_templates = [
t for t in utterances if t not in templates.keys()
]
if missing_templates:
message = ""
for template in missing_templates:
message += ("\nUtterance '{}' is listed as an "
"action in the domain file, but there is "
"no matching utterance template. Please "
"check your domain.").format(template)
print_warning(message)
def _convert_file_to_yaml(source_file: Path, target_dir: Path,
converter: TrainingDataConverter) -> bool:
"""Converts a single training data file to `YAML` format.
Args:
source_file: Training data file to be converted.
target_dir: Target directory for the converted file.
converter: Converter to be used.
Returns:
`True` if file was converted, `False` otherwise.
"""
if not is_valid_filetype(source_file):
return False
if converter.filter(source_file):
converter.convert_and_write(source_file, target_dir)
return True
print_warning(f"Skipped file: '{source_file}'.")
return False
async def _discover_models(self,
project_id: Text = config.project_name) -> None:
minimum_version = await self._retry_fetching_minimum_compatible_version(
)
available_model_names = []
for path in glob.glob(os.path.join(self.model_directory, "*.tar.gz")):
metadata = self.get_model_metadata(path)
model_version = self.get_model_version(metadata)
is_compatible = self.is_model_compatible(
minimum_version, model_version=model_version)
if not is_compatible:
rasa_cli_utils.print_warning(
"Version of model {} version is not compatible. "
"The model was trained with Rasa version {} "
"but the current Rasa requires a minimum version of {}. "
"Please retrain your model with a more recent Rasa "
"version."
"".format(os.path.basename(path), model_version,
minimum_version))
filename: Text = os.path.basename(path)
model_name = filename.split(".tar.gz")[0]
available_model_names.append(model_name)
existing_model = self._get_model_by_name(project_id, model_name)
if not existing_model:
_ = await self.add_model(project_id, model_name, path)
logger.debug(f"Imported model '{model_name}'.")
elif not is_compatible:
# Delete all tags from existing incompatible model
existing_model.tags = []
logger.info(
f"Deleting all tags from model '{existing_model.name}' since its "
f"current version {existing_model.version} does not suffice the "
f"minimum compatible version {minimum_version}.")
self.delete_not_existing_models_from_db(project_id,
available_model_names)
def _write_core_yaml(training_data_path: Path, output_path: Path,
source_path: Path) -> None:
from rasa.core.training.story_reader.yaml_story_reader import KEY_ACTIVE_LOOP
reader = MarkdownStoryReader()
writer = YAMLStoryWriter()
loop = asyncio.get_event_loop()
steps = loop.run_until_complete(reader.read_from_file(training_data_path))
if YAMLStoryWriter.stories_contain_loops(steps):
print_warning(
f"Training data file '{source_path}' contains forms. "
f"Any 'form' events will be converted to '{KEY_ACTIVE_LOOP}' events. "
f"Please note that in order for these stories to work you still "
f"need the 'FormPolicy' to be active. However the 'FormPolicy' is "
f"deprecated, please consider switching to the new 'RulePolicy', "
f"for which you can find the documentation here: {DOCS_URL_RULES}."
)
writer.dump(output_path, steps)
print_success(f"Converted Core file: '{source_path}' >> '{output_path}'.")
async def train_async(
domain: Text,
config: Text,
training_files: Union[Text, List[Text]],
output: Text = DEFAULT_MODELS_PATH,
force_training: bool = False,
kwargs: Optional[Dict] = None,
) -> Optional[Text]:
"""Trains a Rasa model (Core and NLU).
Args:
domain: Path to the domain file.
config: Path to the config for Core and NLU.
training_files: Paths to the training data for Core and NLU.
output: Output path.
force_training: If `True` retrain model even if data has not changed.
kwargs: Additional training parameters.
Returns:
Path of the trained model archive.
"""
config = get_valid_config(config, CONFIG_MANDATORY_KEYS)
train_path = tempfile.mkdtemp()
old_model = model.get_latest_model(output)
retrain_core = True
retrain_nlu = True
story_directory, nlu_data_directory = data.get_core_nlu_directories(training_files)
new_fingerprint = model.model_fingerprint(
config, domain, nlu_data_directory, story_directory
)
dialogue_data_not_present = not os.listdir(story_directory)
nlu_data_not_present = not os.listdir(nlu_data_directory)
if dialogue_data_not_present and nlu_data_not_present:
print_error(
"No training data given. Please provide dialogue and NLU data in "
"order to train a Rasa model."
)
return
if dialogue_data_not_present:
print_warning(
"No dialogue data present. Just a Rasa NLU model will be trained."
)
return train_nlu(config, nlu_data_directory, output, None)
if nlu_data_not_present:
print_warning("No NLU data present. Just a Rasa Core model will be trained.")
return await train_core_async(
domain, config, story_directory, output, None, kwargs
)
if not force_training and old_model:
unpacked = model.unpack_model(old_model)
old_core, old_nlu = model.get_model_subdirectories(unpacked)
last_fingerprint = model.fingerprint_from_path(unpacked)
if not model.core_fingerprint_changed(last_fingerprint, new_fingerprint):
target_path = os.path.join(train_path, "core")
retrain_core = not model.merge_model(old_core, target_path)
if not model.nlu_fingerprint_changed(last_fingerprint, new_fingerprint):
target_path = os.path.join(train_path, "nlu")
retrain_nlu = not model.merge_model(old_nlu, target_path)
if force_training or retrain_core:
await train_core_async(
domain, config, story_directory, output, train_path, kwargs
)
else:
print (
"Dialogue data / configuration did not change. "
"No need to retrain dialogue model."
)
if force_training or retrain_nlu:
train_nlu(config, nlu_data_directory, output, train_path)
else:
print ("NLU data / configuration did not change. No need to retrain NLU model.")
if retrain_core or retrain_nlu:
output = create_output_path(output)
model.create_package_rasa(train_path, output, new_fingerprint)
print_success("Your bot is trained and ready to take for a spin!")
return output
else:
print_success(
"Nothing changed. You can use the old model stored at '{}'"
"".format(os.path.abspath(old_model))
)
return old_model
async def _train_async_internal(
domain: Union[Domain, Text],
config: Text,
train_path: Text,
nlu_data_directory: Text,
story_directory: Text,
output_path: Text,
force_training: bool,
fixed_model_name: Optional[Text],
kwargs: Optional[Dict],
) -> Optional[Text]:
"""Trains a Rasa model (Core and NLU). Use only from `train_async`.
Args:
domain: Path to the domain file.
config: Path to the config for Core and NLU.
train_path: Directory in which to train the model.
nlu_data_directory: Path to NLU training files.
story_directory: Path to Core training files.
output_path: Output path.
force_training: If `True` retrain model even if data has not changed.
fixed_model_name: Name of model to be stored.
kwargs: Additional training parameters.
Returns:
Path of the trained model archive.
"""
new_fingerprint = model.model_fingerprint(config, domain,
nlu_data_directory,
story_directory)
dialogue_data_not_present = not os.listdir(story_directory)
nlu_data_not_present = not os.listdir(nlu_data_directory)
if dialogue_data_not_present and nlu_data_not_present:
print_error(
"No training data given. Please provide stories and NLU data in "
"order to train a Rasa model using the '--data' argument.")
return
if dialogue_data_not_present:
print_warning(
"No dialogue data present. Just a Rasa NLU model will be trained.")
return _train_nlu_with_validated_data(
config=config,
nlu_data_directory=nlu_data_directory,
output=output_path,
fixed_model_name=fixed_model_name,
)
if nlu_data_not_present:
print_warning(
"No NLU data present. Just a Rasa Core model will be trained.")
return await _train_core_with_validated_data(
domain=domain,
config=config,
story_directory=story_directory,
output=output_path,
fixed_model_name=fixed_model_name,
kwargs=kwargs,
)
old_model = model.get_latest_model(output_path)
retrain_core, retrain_nlu = should_retrain(new_fingerprint, old_model,
train_path)
if force_training or retrain_core or retrain_nlu:
await _do_training(
domain=domain,
config=config,
output_path=output_path,
train_path=train_path,
nlu_data_directory=nlu_data_directory,
story_directory=story_directory,
force_training=force_training,
retrain_core=retrain_core,
retrain_nlu=retrain_nlu,
fixed_model_name=fixed_model_name,
kwargs=kwargs,
)
return _package_model(
new_fingerprint=new_fingerprint,
output_path=output_path,
train_path=train_path,
fixed_model_name=fixed_model_name,
)
print_success("Nothing changed. You can use the old model stored at '{}'."
"".format(os.path.abspath(old_model)))
return old_model
async def train_async(
domain: Optional,
config: Text,
training_files: Optional[Union[Text, List[Text]]],
output_path: Text = DEFAULT_MODELS_PATH,
force_training: bool = False,
fixed_model_name: Optional[Text] = None,
uncompress: bool = False,
kwargs: Optional[Dict] = None,
) -> Optional[Text]:
"""Trains a Rasa model (Core and NLU).
Args:
domain: Path to the domain file.
config: Path to the config for Core and NLU.
training_files: Paths to the training data for Core and NLU.
output_path: Output path.
force_training: If `True` retrain model even if data has not changed.
fixed_model_name: Name of model to be stored.
uncompress: If `True` the model will not be compressed.
kwargs: Additional training parameters.
Returns:
Path of the trained model archive.
"""
config = get_valid_config(config, CONFIG_MANDATORY_KEYS)
train_path = tempfile.mkdtemp()
old_model = model.get_latest_model(output_path)
retrain_core = True
retrain_nlu = True
skill_imports = SkillSelector.load(config)
try:
domain = Domain.load(domain, skill_imports)
except InvalidDomain as e:
print_error(e)
return None
story_directory, nlu_data_directory = data.get_core_nlu_directories(
training_files, skill_imports)
new_fingerprint = model.model_fingerprint(config, domain,
nlu_data_directory,
story_directory)
dialogue_data_not_present = not os.listdir(story_directory)
nlu_data_not_present = not os.listdir(nlu_data_directory)
if dialogue_data_not_present and nlu_data_not_present:
print_error(
"No training data given. Please provide dialogue and NLU data in "
"order to train a Rasa model.")
return
if dialogue_data_not_present:
print_warning(
"No dialogue data present. Just a Rasa NLU model will be trained.")
return _train_nlu_with_validated_data(
config=config,
nlu_data_directory=nlu_data_directory,
output=output_path,
fixed_model_name=fixed_model_name,
uncompress=uncompress,
)
if nlu_data_not_present:
print_warning(
"No NLU data present. Just a Rasa Core model will be trained.")
return await _train_core_with_validated_data(
domain=domain,
config=config,
story_directory=story_directory,
output=output_path,
fixed_model_name=fixed_model_name,
uncompress=uncompress,
kwargs=kwargs,
)
if not force_training and old_model:
unpacked = model.unpack_model(old_model)
old_core, old_nlu = model.get_model_subdirectories(unpacked)
last_fingerprint = model.fingerprint_from_path(unpacked)
if not model.core_fingerprint_changed(last_fingerprint,
new_fingerprint):
target_path = os.path.join(train_path, "core")
retrain_core = not model.merge_model(old_core, target_path)
if not model.nlu_fingerprint_changed(last_fingerprint,
new_fingerprint):
target_path = os.path.join(train_path, "nlu")
retrain_nlu = not model.merge_model(old_nlu, target_path)
if force_training or retrain_core:
await _train_core_with_validated_data(
domain=domain,
config=config,
story_directory=story_directory,
output=output_path,
train_path=train_path,
fixed_model_name=fixed_model_name,
uncompress=uncompress,
kwargs=kwargs,
)
else:
print("Dialogue data / configuration did not change. "
"No need to retrain dialogue model.")
if force_training or retrain_nlu:
_train_nlu_with_validated_data(
config=config,
nlu_data_directory=nlu_data_directory,
output=output_path,
train_path=train_path,
fixed_model_name=fixed_model_name,
uncompress=uncompress,
)
else:
print(
"NLU data / configuration did not change. No need to retrain NLU model."
)
if retrain_core or retrain_nlu:
output_path = create_output_path(output_path,
fixed_name=fixed_model_name)
model.create_package_rasa(train_path, output_path, new_fingerprint)
if uncompress:
output_path = decompress(output_path)
print_success("Your Rasa model is trained and saved at '{}'.".format(
output_path))
return output_path
else:
print_success(
"Nothing changed. You can use the old model stored at '{}'"
"".format(os.path.abspath(old_model)))
return old_model
async def _train_async_internal(
file_importer: TrainingDataImporter,
train_path: Text,
output_path: Text,
force_training: bool,
fixed_model_name: Optional[Text],
persist_nlu_training_data: bool,
kwargs: Optional[Dict],
) -> Optional[Text]:
"""Trains a Rasa model (Core and NLU). Use only from `train_async`.
Args:
file_importer: `TrainingDataImporter` which supplies the training data.
train_path: Directory in which to train the model.
output_path: Output path.
force_training: If `True` retrain model even if data has not changed.
persist_nlu_training_data: `True` if the NLU training data should be persisted
with the model.
fixed_model_name: Name of model to be stored.
kwargs: Additional training parameters.
Returns:
Path of the trained model archive.
"""
stories = await file_importer.get_stories()
nlu_data = await file_importer.get_nlu_data()
if stories.is_empty() and nlu_data.is_empty():
print_error(
"No training data given. Please provide stories and NLU data in "
"order to train a Rasa model using the '--data' argument.")
return
if stories.is_empty():
print_warning(
"No stories present. Just a Rasa NLU model will be trained.")
return await _train_nlu_with_validated_data(
file_importer,
output=output_path,
fixed_model_name=fixed_model_name,
persist_nlu_training_data=persist_nlu_training_data,
)
if nlu_data.is_empty():
print_warning(
"No NLU data present. Just a Rasa Core model will be trained.")
return await _train_core_with_validated_data(
file_importer,
output=output_path,
fixed_model_name=fixed_model_name,
kwargs=kwargs,
)
new_fingerprint = await model.model_fingerprint(file_importer)
old_model = model.get_latest_model(output_path)
fingerprint_comparison = FingerprintComparisonResult(
force_training=force_training)
if not force_training:
fingerprint_comparison = model.should_retrain(new_fingerprint,
old_model, train_path)
if fingerprint_comparison.is_training_required():
await _do_training(
file_importer,
output_path=output_path,
train_path=train_path,
fingerprint_comparison_result=fingerprint_comparison,
fixed_model_name=fixed_model_name,
persist_nlu_training_data=persist_nlu_training_data,
kwargs=kwargs,
)
return model.package_model(
fingerprint=new_fingerprint,
output_directory=output_path,
train_path=train_path,
fixed_model_name=fixed_model_name,
)
print_success("Nothing changed. You can use the old model stored at '{}'."
"".format(os.path.abspath(old_model)))
return old_model
async def train_async(
domain: Union[Domain, Text],
config: Text,
training_files: Optional[Union[Text, List[Text]]],
output_path: Text = DEFAULT_MODELS_PATH,
force_training: bool = False,
fixed_model_name: Optional[Text] = None,
uncompress: bool = False,
kwargs: Optional[Dict] = None,
) -> Optional[Text]:
"""Trains a Rasa model (Core and NLU).
Args:
domain: Path to the domain file.
config: Path to the config for Core and NLU.
training_files: Paths to the training data for Core and NLU.
output_path: Output path.
force_training: If `True` retrain model even if data has not changed.
fixed_model_name: Name of model to be stored.
uncompress: If `True` the model will not be compressed.
kwargs: Additional training parameters.
Returns:
Path of the trained model archive.
"""
config = _get_valid_config(config, CONFIG_MANDATORY_KEYS)
train_path = tempfile.mkdtemp()
skill_imports = SkillSelector.load(config)
try:
domain = Domain.load(domain, skill_imports)
except InvalidDomain as e:
print_error(
"Could not load domain due to: '{}'. To specify a valid domain path use "
"the '--domain' argument.".format(e))
return None
story_directory, nlu_data_directory = data.get_core_nlu_directories(
training_files, skill_imports)
new_fingerprint = model.model_fingerprint(config, domain,
nlu_data_directory,
story_directory)
dialogue_data_not_present = not os.listdir(story_directory)
nlu_data_not_present = not os.listdir(nlu_data_directory)
if dialogue_data_not_present and nlu_data_not_present:
print_error(
"No training data given. Please provide stories and NLU data in "
"order to train a Rasa model using the '--data' argument.")
return
if dialogue_data_not_present:
print_warning(
"No dialogue data present. Just a Rasa NLU model will be trained.")
return _train_nlu_with_validated_data(
config=config,
nlu_data_directory=nlu_data_directory,
output=output_path,
fixed_model_name=fixed_model_name,
uncompress=uncompress,
)
if nlu_data_not_present:
print_warning(
"No NLU data present. Just a Rasa Core model will be trained.")
return await _train_core_with_validated_data(
domain=domain,
config=config,
story_directory=story_directory,
output=output_path,
fixed_model_name=fixed_model_name,
uncompress=uncompress,
kwargs=kwargs,
)
old_model = model.get_latest_model(output_path)
retrain_core, retrain_nlu = should_retrain(new_fingerprint, old_model,
train_path)
if force_training or retrain_core or retrain_nlu:
await _do_training(
domain=domain,
config=config,
output_path=output_path,
train_path=train_path,
nlu_data_directory=nlu_data_directory,
story_directory=story_directory,
force_training=force_training,
retrain_core=retrain_core,
retrain_nlu=retrain_nlu,
fixed_model_name=fixed_model_name,
uncompress=uncompress,
kwargs=kwargs,
)
return _package_model(
new_fingerprint=new_fingerprint,
output_path=output_path,
train_path=train_path,
fixed_model_name=fixed_model_name,
uncompress=uncompress,
)
print_success("Nothing changed. You can use the old model stored at '{}'."
"".format(os.path.abspath(old_model)))
return old_model
async def _train_async_internal(
file_importer: TrainingDataImporter,
train_path: Text,
output_path: Text,
force_training: bool,
fixed_model_name: Optional[Text],
kwargs: Optional[Dict],
) -> Optional[Text]:
"""Trains a Rasa model (Core and NLU). Use only from `train_async`.
Args:
domain: Path to the domain file.
config: Path to the config for Core and NLU.
train_path: Directory in which to train the model.
nlu_data_directory: Path to NLU training files.
story_directory: Path to Core training files.
output_path: Output path.
force_training: If `True` retrain model even if data has not changed.
fixed_model_name: Name of model to be stored.
kwargs: Additional training parameters.
Returns:
Path of the trained model archive.
"""
new_fingerprint = await model.model_fingerprint(file_importer)
stories = await file_importer.get_stories()
nlu_data = await file_importer.get_nlu_data()
if stories.is_empty() and nlu_data.is_empty():
print_error(
"No training data given. Please provide stories and NLU data in "
"order to train a Rasa model using the '--data' argument.")
return
if stories.is_empty():
print_warning(
"No stories present. Just a Rasa NLU model will be trained.")
return await _train_nlu_with_validated_data(
file_importer,
output=output_path,
fixed_model_name=fixed_model_name)
if nlu_data.is_empty():
print_warning(
"No NLU data present. Just a Rasa Core model will be trained.")
return await _train_core_with_validated_data(
file_importer,
output=output_path,
fixed_model_name=fixed_model_name,
kwargs=kwargs,
)
old_model = model.get_latest_model(output_path)
retrain_core, retrain_nlu = model.should_retrain(new_fingerprint,
old_model, train_path)
if force_training or retrain_core or retrain_nlu:
await _do_training(
file_importer,
output_path=output_path,
train_path=train_path,
force_training=force_training,
retrain_core=retrain_core,
retrain_nlu=retrain_nlu,
fixed_model_name=fixed_model_name,
kwargs=kwargs,
)
return model.package_model(
fingerprint=new_fingerprint,
output_directory=output_path,
train_path=train_path,
fixed_model_name=fixed_model_name,
)
print_success("Nothing changed. You can use the old model stored at '{}'."
"".format(os.path.abspath(old_model)))
return old_model
def _read_data(paths: List[Text]) -> Generator[Tuple[Text, Text], None, None]:
for filename in paths:
try:
yield read_file(filename), filename
except ValueError:
rasa_cli_utils.print_warning(f"Cannot read file {filename}")