def convert_and_write(cls, source_path: Path, output_path: Path) -> None: """Converts the given training data file and saves it to the output directory. Args: source_path: Path to the training data file. output_path: Path to the output directory. """ from rasa.shared.core.training_data.story_reader.yaml_story_reader import ( KEY_ACTIVE_LOOP, ) output_core_path = cls.generate_path_for_converted_training_data_file( source_path, output_path) reader = MarkdownStoryReader(unfold_or_utterances=False) writer = YAMLStoryWriter() loop = asyncio.get_event_loop() steps = loop.run_until_complete(reader.read_from_file(source_path)) if YAMLStoryWriter.stories_contain_loops(steps): print_warning( f"Training data file '{source_path}' contains forms. " f"Any 'form' events will be converted to '{KEY_ACTIVE_LOOP}' events. " f"Please note that in order for these stories to work you still " f"need the 'FormPolicy' to be active. However the 'FormPolicy' is " f"deprecated, please consider switching to the new 'RulePolicy', " f"for which you can find the documentation here: {DOCS_URL_RULES}." ) writer.dump(output_core_path, steps) print_success( f"Converted Core file: '{source_path}' >> '{output_core_path}'.")
def _get_sanitized_model_directory(model_directory: Text) -> Text: """Adjusts the `--model` argument of `rasa test core` when called with `--evaluate-model-directory`. By default rasa uses the latest model for the `--model` parameter. However, for `--evaluate-model-directory` we need a directory. This function checks if the passed parameter is a model or an individual model file. Args: model_directory: The model_directory argument that was given to `test_core_models_in_directory`. Returns: The adjusted model_directory that should be used in `test_core_models_in_directory`. """ import rasa.model p = Path(model_directory) if p.is_file(): if model_directory != rasa.model.get_latest_model(): cli_utils.print_warning( "You passed a file as '--model'. Will use the directory containing " "this file instead.") model_directory = str(p.parent) return model_directory
def _convert_to_yaml(args: argparse.Namespace, is_nlu: bool) -> None: output = Path(args.out) if not os.path.exists(output): print_error_and_exit( f"The output path '{output}' doesn't exist. Please make sure to specify " f"an existing directory and try again." ) training_data = Path(args.data) if not os.path.exists(training_data): print_error_and_exit( f"The training data path {training_data} doesn't exist " f"and will be skipped." ) num_of_files_converted = 0 for file in os.listdir(training_data): source_path = training_data / file output_path = output / f"{source_path.stem}{CONVERTED_FILE_SUFFIX}" if MarkdownReader.is_markdown_nlu_file(source_path): if not is_nlu: continue _write_nlu_yaml(source_path, output_path, source_path) num_of_files_converted += 1 elif not is_nlu and MarkdownStoryReader.is_markdown_story_file(source_path): _write_core_yaml(source_path, output_path, source_path) num_of_files_converted += 1 else: print_warning(f"Skipped file: '{source_path}'.") print_info(f"Converted {num_of_files_converted} file(s), saved in '{output}'.")
def _validate_rasa_x_start(args: argparse.Namespace, project_path: Text): if not is_rasa_x_installed(): cli_utils.print_error_and_exit( "Rasa X is not installed. The `rasa x` " "command requires an installation of Rasa X. " "Instructions on how to install Rasa X can be found here: " "https://rasa.com/docs/rasa-x/installation-and-setup/.") if args.port == args.rasa_x_port: cli_utils.print_error_and_exit( "The port for Rasa X '{}' and the port of the Rasa server '{}' are the " "same. We need two different ports, one to run Rasa X (e.g. delivering the " "UI) and another one to run a normal Rasa server.\nPlease specify two " "different ports using the arguments '--port' and '--rasa-x-port'." .format(args.rasa_x_port, args.port)) if not is_rasa_project_setup(project_path): cli_utils.print_error_and_exit( "This directory is not a valid Rasa project. Use 'rasa init' " "to create a new Rasa project or switch to a valid Rasa project " "directory (see http://rasa.com/docs/rasa/user-guide/" "rasa-tutorial/#create-a-new-project).") _validate_domain(os.path.join(project_path, DEFAULT_DOMAIN_PATH)) if args.data and not os.path.exists(args.data): cli_utils.print_warning( "The provided data path ('{}') does not exists. Rasa X will start " "without any training data.".format(args.data))
def _convert_to_yaml(args: argparse.Namespace, converter: TrainingDataConverter) -> None: output = Path(args.out) if not os.path.exists(output): print_error_and_exit( f"The output path '{output}' doesn't exist. Please make sure to specify " f"an existing directory and try again.") training_data = Path(args.data) if not os.path.exists(training_data): print_error_and_exit( f"The training data path {training_data} doesn't exist " f"and will be skipped.") num_of_files_converted = 0 if os.path.isfile(training_data): if _convert_file_to_yaml(training_data, output, converter): num_of_files_converted += 1 elif os.path.isdir(training_data): for root, _, files in os.walk(training_data, followlinks=True): for f in sorted(files): source_path = Path(os.path.join(root, f)) if _convert_file_to_yaml(source_path, output, converter): num_of_files_converted += 1 if num_of_files_converted: print_info( f"Converted {num_of_files_converted} file(s), saved in '{output}'." ) else: print_warning( f"Didn't convert any files under '{training_data}' path. " "Did you specify the correct file/directory?")
def test_core( model: Optional[Text] = None, stories: Optional[Text] = None, endpoints: Optional[Text] = None, output: Text = DEFAULT_RESULTS_PATH, kwargs: Optional[Dict] = None, ): import rasa.core.test import rasa.core.utils as core_utils import rasa.model from rasa.core.interpreter import RegexInterpreter, NaturalLanguageInterpreter from rasa.core.agent import Agent _endpoints = core_utils.AvailableEndpoints.read_endpoints(endpoints) if kwargs is None: kwargs = {} if output: io_utils.create_directory(output) try: unpacked_model = rasa.model.get_model(model) except ModelNotFound: print_error( "Unable to test: could not find a model. Use 'rasa train' to train a " "Rasa model and provide it via the '--model' argument." ) return core_path, nlu_path = rasa.model.get_model_subdirectories(unpacked_model) if not core_path: print_error( "Unable to test: could not find a Core model. Use 'rasa train' to train a " "Rasa model and provide it via the '--model' argument." ) use_e2e = kwargs["e2e"] if "e2e" in kwargs else False _interpreter = RegexInterpreter() if use_e2e: if nlu_path: _interpreter = NaturalLanguageInterpreter.create(_endpoints.nlu or nlu_path) else: print_warning( "No NLU model found. Using default 'RegexInterpreter' for end-to-end " "evaluation." ) _agent = Agent.load(unpacked_model, interpreter=_interpreter) kwargs = utils.minimal_kwargs(kwargs, rasa.core.test, ["stories", "agent"]) loop = asyncio.get_event_loop() loop.run_until_complete( rasa.core.test(stories, _agent, out_directory=output, **kwargs) )
def test_core( model: Optional[Text] = None, stories: Optional[Text] = None, endpoints: Optional[Text] = None, output: Text = DEFAULT_RESULTS_PATH, kwargs: Optional[Dict] = None, ): import rasa.core.test import rasa.core.utils as core_utils from rasa.nlu import utils as nlu_utils from rasa.model import get_model from rasa.core.interpreter import NaturalLanguageInterpreter from rasa.core.agent import Agent _endpoints = core_utils.AvailableEndpoints.read_endpoints(endpoints) if kwargs is None: kwargs = {} if output: nlu_utils.create_dir(output) unpacked_model = get_model(model) if unpacked_model is None: print_error( "Unable to test: could not find a model. Use 'rasa train' to train a " "Rasa model." ) return core_path, nlu_path = get_model_subdirectories(unpacked_model) if not os.path.exists(core_path): print_error( "Unable to test: could not find a Core model. Use 'rasa train' to " "train a model." ) use_e2e = kwargs["e2e"] if "e2e" in kwargs else False _interpreter = RegexInterpreter() if use_e2e: if os.path.exists(nlu_path): _interpreter = NaturalLanguageInterpreter.create(nlu_path, _endpoints.nlu) else: print_warning( "No NLU model found. Using default 'RegexInterpreter' for end-to-end " "evaluation." ) _agent = Agent.load(unpacked_model, interpreter=_interpreter) kwargs = minimal_kwargs(kwargs, rasa.core.test, ["stories", "agent"]) loop = asyncio.get_event_loop() loop.run_until_complete( rasa.core.test(stories, _agent, out_directory=output, **kwargs) )
async def handle_domain_if_not_exists(file_importer: TrainingDataImporter, output_path, fixed_model_name): nlu_model_only = await _train_nlu_with_validated_data( file_importer, output=output_path, fixed_model_name=fixed_model_name) print_warning( "Core training was skipped because no valid domain file was found. Only an nlu-model was created." "Please specify a valid domain using '--domain' argument or check if the provided domain file exists." ) return nlu_model_only
def rasa_x(args: argparse.Namespace): from rasa.cli.utils import print_success, print_error, signal_handler from rasa.core.utils import AvailableEndpoints signal.signal(signal.SIGINT, signal_handler) _configure_logging(args) if args.production: print_success("Starting Rasa X in production mode... 🚀") args.endpoints = get_validated_path(args.endpoints, "endpoints", DEFAULT_ENDPOINTS_PATH, True) endpoints = AvailableEndpoints.read_endpoints(args.endpoints) _rasa_service(args, endpoints) else: if not is_rasa_x_installed(): print_error("Rasa X is not installed. The `rasa x` " "command requires an installation of Rasa X.") sys.exit(1) project_path = "." if not is_rasa_project_setup(project_path): print_error( "This directory is not a valid Rasa project. Use 'rasa init' " "to create a new Rasa project or switch to a valid Rasa project " "directory.") sys.exit(1) _validate_domain(os.path.join(project_path, DEFAULT_DOMAIN_PATH)) if args.data and not os.path.exists(args.data): print_warning( "The provided data path ('{}') does not exists. Rasa X will start " "without any training data.".format(args.data)) # noinspection PyUnresolvedReferences from rasax.community import local local.check_license_and_metrics(args) rasa_x_token = generate_rasa_x_token() process = start_rasa_for_local_rasa_x(args, rasa_x_token=rasa_x_token) try: local.main(args, project_path, args.data, token=rasa_x_token) except Exception: print(traceback.format_exc()) print_error( "Sorry, something went wrong (see error above). Make sure to start " "Rasa X with valid data and valid domain and config files. Please, " "also check any warnings that popped up.\nIf you need help fixing " "the issue visit our forum: https://forum.rasa.com/.") finally: process.terminate()
def run( model: Text, endpoints: Text, connector: Text = None, credentials: Text = None, **kwargs: Dict ): """Runs a Rasa model. Args: model: Path to model archive. endpoints: Path to endpoints file. connector: Connector which should be use (overwrites `credentials` field). credentials: Path to channel credentials file. **kwargs: Additional arguments which are passed to `rasa.core.run.serve_application`. """ import rasa.core.run import rasa.nlu.run from rasa.core.utils import AvailableEndpoints model_path = get_model(model) if not model_path: print_error( "No model found. Train a model before running the " "server using `rasa train`." ) return _endpoints = AvailableEndpoints.read_endpoints(endpoints) if not connector and not credentials: connector = "rest" print_warning( "No chat connector configured, falling back to the " "REST input channel. To connect your bot to another channel, " "read the docs here: {}/user-guide/" "messaging-and-voice-channels".format(DOCS_BASE_URL) ) kwargs = minimal_kwargs(kwargs, rasa.core.run.serve_application) rasa.core.run.serve_application( model, channel=connector, credentials=credentials, endpoints=_endpoints, **kwargs ) shutil.rmtree(model_path)
def handle_domain_if_not_exists(config, nlu_data_directory, output_path, fixed_model_name): nlu_model_only = _train_nlu_with_validated_data( config=config, nlu_data_directory=nlu_data_directory, output=output_path, fixed_model_name=fixed_model_name, ) print_warning( "Core training was skipped because no valid domain file was found. Only an nlu-model was created." "Please specify a valid domain using '--domain' argument or check if the provided domain file exists." ) return nlu_model_only
def perform_interactive_learning(args, zipped_model): from rasa.core.train import do_interactive_learning if zipped_model: model_path = model.unpack_model(zipped_model) args.core, args.nlu = model.get_model_subdirectories(model_path) stories_directory = data.get_core_directory(args.data) do_interactive_learning(args, stories_directory) shutil.rmtree(model_path) else: print_warning("No initial zipped trained model found.")
def _get_valid_config(config: Text, mandatory_keys: List[Text]) -> Text: config_path = get_validated_path(config, "config", FALLBACK_CONFIG_PATH) missing_keys = missing_config_keys(config_path, mandatory_keys) if missing_keys: print_warning( "Configuration file '{}' is missing mandatory parameters: " "{}. Filling missing parameters from fallback configuration file: '{}'." "".format(config, ", ".join(missing_keys), FALLBACK_CONFIG_PATH)) _enrich_config(config_path, missing_keys, FALLBACK_CONFIG_PATH) return config_path
def rasa_x(args: argparse.Namespace): from rasa.cli.utils import print_success, print_error, signal_handler from rasa.core.utils import AvailableEndpoints signal.signal(signal.SIGINT, signal_handler) _configure_logging(args) if args.production: print_success("Starting Rasa X in production mode... 🚀") args.endpoints = get_validated_path( args.endpoints, "endpoints", DEFAULT_ENDPOINTS_PATH, True ) endpoints = AvailableEndpoints.read_endpoints(args.endpoints) _rasa_service(args, endpoints) else: if not is_rasa_x_installed(): print_error( "Rasa X is not installed. The `rasa x` " "command requires an installation of Rasa X." ) sys.exit(1) project_path = "." if not is_rasa_project_setup(project_path): print_error( "This directory is not a valid Rasa project. Use 'rasa init' " "to create a new Rasa project or switch to a valid Rasa project " "directory." ) sys.exit(1) if args.data and not os.path.exists(args.data): print_warning( "The provided data path ('{}') does not exists. Rasa X will start " "without any training data.".format(args.data) ) # noinspection PyUnresolvedReferences from rasax.community import local local.check_license_and_metrics(args) rasa_x_token = generate_rasa_x_token() process = start_rasa_for_local_rasa_x(args, rasa_x_token=rasa_x_token) try: local.main(args, project_path, args.data, token=rasa_x_token) finally: process.terminate()
def _validate_all_requested_ids_exist( self, conversation_ids_in_tracker_store: Set[Text]) -> None: """Warn user if `self.requested_conversation_ids` contains IDs not found in `conversation_ids_in_tracker_store` Args: conversation_ids_in_tracker_store: Set of conversation IDs contained in the tracker store. """ missing_ids_in_tracker_store = (set(self.requested_conversation_ids) - conversation_ids_in_tracker_store) if missing_ids_in_tracker_store: cli_utils.print_warning( f"Could not find the following requested " f"conversation IDs in connected tracker store: " f"{', '.join(sorted(missing_ids_in_tracker_store))}")
def test_core( model: Optional[Text] = None, stories: Optional[Text] = None, output: Text = DEFAULT_RESULTS_PATH, additional_arguments: Optional[Dict] = None, ) -> None: import rasa.model from rasa.core.interpreter import RegexInterpreter from rasa.core.agent import Agent if additional_arguments is None: additional_arguments = {} if output: io_utils.create_directory(output) try: unpacked_model = rasa.model.get_model(model) except ModelNotFound: cli_utils.print_error( "Unable to test: could not find a model. Use 'rasa train' to train a " "Rasa model and provide it via the '--model' argument." ) return _agent = Agent.load(unpacked_model) if _agent.policy_ensemble is None: cli_utils.print_error( "Unable to test: could not find a Core model. Use 'rasa train' to train a " "Rasa model and provide it via the '--model' argument." ) if isinstance(_agent.interpreter, RegexInterpreter): cli_utils.print_warning( "No NLU model found. Using default 'RegexInterpreter' for end-to-end " "evaluation. If you added actual user messages to your test stories " "this will likely lead to the tests failing. In that case, you need " "to train a NLU model first, e.g. using `rasa train`." ) from rasa.core.test import test kwargs = utils.minimal_kwargs(additional_arguments, test, ["stories", "agent"]) _test_core(stories, _agent, output, **kwargs)
def run( model: Text, endpoints: Text, connector: Text = None, credentials: Text = None, **kwargs: Dict, ): """Runs a Rasa model. Args: model: Path to model archive. endpoints: Path to endpoints file. connector: Connector which should be use (overwrites `credentials` field). credentials: Path to channel credentials file. **kwargs: Additional arguments which are passed to `rasa.core.run.serve_application`. """ import rasa.core.run import rasa.nlu.run from rasa.core.utils import AvailableEndpoints import rasa.utils.common as utils _endpoints = AvailableEndpoints.read_endpoints(endpoints) if not connector and not credentials: connector = "rest" print_warning( "No chat connector configured, falling back to the " "REST input channel. To connect your bot to another channel, " "read the docs here: {}/user-guide/" "messaging-and-voice-channels".format(DOCS_BASE_URL)) kwargs = utils.minimal_kwargs(kwargs, run_core.serve_application) # 重写core run serve_application 方法 run_core.serve_application( model, channel=connector, credentials=credentials, endpoints=_endpoints, **kwargs, )
def warn_missing_templates(action_names: List[Text], templates: Dict[Text, Any]) -> None: """Warn user of utterance names which have no specified template.""" utterances = [ act for act in action_names if act.startswith(action.UTTER_PREFIX) ] missing_templates = [ t for t in utterances if t not in templates.keys() ] if missing_templates: message = "" for template in missing_templates: message += ("\nUtterance '{}' is listed as an " "action in the domain file, but there is " "no matching utterance template. Please " "check your domain.").format(template) print_warning(message)
def _convert_file_to_yaml(source_file: Path, target_dir: Path, converter: TrainingDataConverter) -> bool: """Converts a single training data file to `YAML` format. Args: source_file: Training data file to be converted. target_dir: Target directory for the converted file. converter: Converter to be used. Returns: `True` if file was converted, `False` otherwise. """ if not is_valid_filetype(source_file): return False if converter.filter(source_file): converter.convert_and_write(source_file, target_dir) return True print_warning(f"Skipped file: '{source_file}'.") return False
async def _discover_models(self, project_id: Text = config.project_name) -> None: minimum_version = await self._retry_fetching_minimum_compatible_version( ) available_model_names = [] for path in glob.glob(os.path.join(self.model_directory, "*.tar.gz")): metadata = self.get_model_metadata(path) model_version = self.get_model_version(metadata) is_compatible = self.is_model_compatible( minimum_version, model_version=model_version) if not is_compatible: rasa_cli_utils.print_warning( "Version of model {} version is not compatible. " "The model was trained with Rasa version {} " "but the current Rasa requires a minimum version of {}. " "Please retrain your model with a more recent Rasa " "version." "".format(os.path.basename(path), model_version, minimum_version)) filename: Text = os.path.basename(path) model_name = filename.split(".tar.gz")[0] available_model_names.append(model_name) existing_model = self._get_model_by_name(project_id, model_name) if not existing_model: _ = await self.add_model(project_id, model_name, path) logger.debug(f"Imported model '{model_name}'.") elif not is_compatible: # Delete all tags from existing incompatible model existing_model.tags = [] logger.info( f"Deleting all tags from model '{existing_model.name}' since its " f"current version {existing_model.version} does not suffice the " f"minimum compatible version {minimum_version}.") self.delete_not_existing_models_from_db(project_id, available_model_names)
def _write_core_yaml(training_data_path: Path, output_path: Path, source_path: Path) -> None: from rasa.core.training.story_reader.yaml_story_reader import KEY_ACTIVE_LOOP reader = MarkdownStoryReader() writer = YAMLStoryWriter() loop = asyncio.get_event_loop() steps = loop.run_until_complete(reader.read_from_file(training_data_path)) if YAMLStoryWriter.stories_contain_loops(steps): print_warning( f"Training data file '{source_path}' contains forms. " f"Any 'form' events will be converted to '{KEY_ACTIVE_LOOP}' events. " f"Please note that in order for these stories to work you still " f"need the 'FormPolicy' to be active. However the 'FormPolicy' is " f"deprecated, please consider switching to the new 'RulePolicy', " f"for which you can find the documentation here: {DOCS_URL_RULES}." ) writer.dump(output_path, steps) print_success(f"Converted Core file: '{source_path}' >> '{output_path}'.")
async def train_async( domain: Text, config: Text, training_files: Union[Text, List[Text]], output: Text = DEFAULT_MODELS_PATH, force_training: bool = False, kwargs: Optional[Dict] = None, ) -> Optional[Text]: """Trains a Rasa model (Core and NLU). Args: domain: Path to the domain file. config: Path to the config for Core and NLU. training_files: Paths to the training data for Core and NLU. output: Output path. force_training: If `True` retrain model even if data has not changed. kwargs: Additional training parameters. Returns: Path of the trained model archive. """ config = get_valid_config(config, CONFIG_MANDATORY_KEYS) train_path = tempfile.mkdtemp() old_model = model.get_latest_model(output) retrain_core = True retrain_nlu = True story_directory, nlu_data_directory = data.get_core_nlu_directories(training_files) new_fingerprint = model.model_fingerprint( config, domain, nlu_data_directory, story_directory ) dialogue_data_not_present = not os.listdir(story_directory) nlu_data_not_present = not os.listdir(nlu_data_directory) if dialogue_data_not_present and nlu_data_not_present: print_error( "No training data given. Please provide dialogue and NLU data in " "order to train a Rasa model." ) return if dialogue_data_not_present: print_warning( "No dialogue data present. Just a Rasa NLU model will be trained." ) return train_nlu(config, nlu_data_directory, output, None) if nlu_data_not_present: print_warning("No NLU data present. Just a Rasa Core model will be trained.") return await train_core_async( domain, config, story_directory, output, None, kwargs ) if not force_training and old_model: unpacked = model.unpack_model(old_model) old_core, old_nlu = model.get_model_subdirectories(unpacked) last_fingerprint = model.fingerprint_from_path(unpacked) if not model.core_fingerprint_changed(last_fingerprint, new_fingerprint): target_path = os.path.join(train_path, "core") retrain_core = not model.merge_model(old_core, target_path) if not model.nlu_fingerprint_changed(last_fingerprint, new_fingerprint): target_path = os.path.join(train_path, "nlu") retrain_nlu = not model.merge_model(old_nlu, target_path) if force_training or retrain_core: await train_core_async( domain, config, story_directory, output, train_path, kwargs ) else: print ( "Dialogue data / configuration did not change. " "No need to retrain dialogue model." ) if force_training or retrain_nlu: train_nlu(config, nlu_data_directory, output, train_path) else: print ("NLU data / configuration did not change. No need to retrain NLU model.") if retrain_core or retrain_nlu: output = create_output_path(output) model.create_package_rasa(train_path, output, new_fingerprint) print_success("Your bot is trained and ready to take for a spin!") return output else: print_success( "Nothing changed. You can use the old model stored at '{}'" "".format(os.path.abspath(old_model)) ) return old_model
async def _train_async_internal( domain: Union[Domain, Text], config: Text, train_path: Text, nlu_data_directory: Text, story_directory: Text, output_path: Text, force_training: bool, fixed_model_name: Optional[Text], kwargs: Optional[Dict], ) -> Optional[Text]: """Trains a Rasa model (Core and NLU). Use only from `train_async`. Args: domain: Path to the domain file. config: Path to the config for Core and NLU. train_path: Directory in which to train the model. nlu_data_directory: Path to NLU training files. story_directory: Path to Core training files. output_path: Output path. force_training: If `True` retrain model even if data has not changed. fixed_model_name: Name of model to be stored. kwargs: Additional training parameters. Returns: Path of the trained model archive. """ new_fingerprint = model.model_fingerprint(config, domain, nlu_data_directory, story_directory) dialogue_data_not_present = not os.listdir(story_directory) nlu_data_not_present = not os.listdir(nlu_data_directory) if dialogue_data_not_present and nlu_data_not_present: print_error( "No training data given. Please provide stories and NLU data in " "order to train a Rasa model using the '--data' argument.") return if dialogue_data_not_present: print_warning( "No dialogue data present. Just a Rasa NLU model will be trained.") return _train_nlu_with_validated_data( config=config, nlu_data_directory=nlu_data_directory, output=output_path, fixed_model_name=fixed_model_name, ) if nlu_data_not_present: print_warning( "No NLU data present. Just a Rasa Core model will be trained.") return await _train_core_with_validated_data( domain=domain, config=config, story_directory=story_directory, output=output_path, fixed_model_name=fixed_model_name, kwargs=kwargs, ) old_model = model.get_latest_model(output_path) retrain_core, retrain_nlu = should_retrain(new_fingerprint, old_model, train_path) if force_training or retrain_core or retrain_nlu: await _do_training( domain=domain, config=config, output_path=output_path, train_path=train_path, nlu_data_directory=nlu_data_directory, story_directory=story_directory, force_training=force_training, retrain_core=retrain_core, retrain_nlu=retrain_nlu, fixed_model_name=fixed_model_name, kwargs=kwargs, ) return _package_model( new_fingerprint=new_fingerprint, output_path=output_path, train_path=train_path, fixed_model_name=fixed_model_name, ) print_success("Nothing changed. You can use the old model stored at '{}'." "".format(os.path.abspath(old_model))) return old_model
async def train_async( domain: Optional, config: Text, training_files: Optional[Union[Text, List[Text]]], output_path: Text = DEFAULT_MODELS_PATH, force_training: bool = False, fixed_model_name: Optional[Text] = None, uncompress: bool = False, kwargs: Optional[Dict] = None, ) -> Optional[Text]: """Trains a Rasa model (Core and NLU). Args: domain: Path to the domain file. config: Path to the config for Core and NLU. training_files: Paths to the training data for Core and NLU. output_path: Output path. force_training: If `True` retrain model even if data has not changed. fixed_model_name: Name of model to be stored. uncompress: If `True` the model will not be compressed. kwargs: Additional training parameters. Returns: Path of the trained model archive. """ config = get_valid_config(config, CONFIG_MANDATORY_KEYS) train_path = tempfile.mkdtemp() old_model = model.get_latest_model(output_path) retrain_core = True retrain_nlu = True skill_imports = SkillSelector.load(config) try: domain = Domain.load(domain, skill_imports) except InvalidDomain as e: print_error(e) return None story_directory, nlu_data_directory = data.get_core_nlu_directories( training_files, skill_imports) new_fingerprint = model.model_fingerprint(config, domain, nlu_data_directory, story_directory) dialogue_data_not_present = not os.listdir(story_directory) nlu_data_not_present = not os.listdir(nlu_data_directory) if dialogue_data_not_present and nlu_data_not_present: print_error( "No training data given. Please provide dialogue and NLU data in " "order to train a Rasa model.") return if dialogue_data_not_present: print_warning( "No dialogue data present. Just a Rasa NLU model will be trained.") return _train_nlu_with_validated_data( config=config, nlu_data_directory=nlu_data_directory, output=output_path, fixed_model_name=fixed_model_name, uncompress=uncompress, ) if nlu_data_not_present: print_warning( "No NLU data present. Just a Rasa Core model will be trained.") return await _train_core_with_validated_data( domain=domain, config=config, story_directory=story_directory, output=output_path, fixed_model_name=fixed_model_name, uncompress=uncompress, kwargs=kwargs, ) if not force_training and old_model: unpacked = model.unpack_model(old_model) old_core, old_nlu = model.get_model_subdirectories(unpacked) last_fingerprint = model.fingerprint_from_path(unpacked) if not model.core_fingerprint_changed(last_fingerprint, new_fingerprint): target_path = os.path.join(train_path, "core") retrain_core = not model.merge_model(old_core, target_path) if not model.nlu_fingerprint_changed(last_fingerprint, new_fingerprint): target_path = os.path.join(train_path, "nlu") retrain_nlu = not model.merge_model(old_nlu, target_path) if force_training or retrain_core: await _train_core_with_validated_data( domain=domain, config=config, story_directory=story_directory, output=output_path, train_path=train_path, fixed_model_name=fixed_model_name, uncompress=uncompress, kwargs=kwargs, ) else: print("Dialogue data / configuration did not change. " "No need to retrain dialogue model.") if force_training or retrain_nlu: _train_nlu_with_validated_data( config=config, nlu_data_directory=nlu_data_directory, output=output_path, train_path=train_path, fixed_model_name=fixed_model_name, uncompress=uncompress, ) else: print( "NLU data / configuration did not change. No need to retrain NLU model." ) if retrain_core or retrain_nlu: output_path = create_output_path(output_path, fixed_name=fixed_model_name) model.create_package_rasa(train_path, output_path, new_fingerprint) if uncompress: output_path = decompress(output_path) print_success("Your Rasa model is trained and saved at '{}'.".format( output_path)) return output_path else: print_success( "Nothing changed. You can use the old model stored at '{}'" "".format(os.path.abspath(old_model))) return old_model
async def _train_async_internal( file_importer: TrainingDataImporter, train_path: Text, output_path: Text, force_training: bool, fixed_model_name: Optional[Text], persist_nlu_training_data: bool, kwargs: Optional[Dict], ) -> Optional[Text]: """Trains a Rasa model (Core and NLU). Use only from `train_async`. Args: file_importer: `TrainingDataImporter` which supplies the training data. train_path: Directory in which to train the model. output_path: Output path. force_training: If `True` retrain model even if data has not changed. persist_nlu_training_data: `True` if the NLU training data should be persisted with the model. fixed_model_name: Name of model to be stored. kwargs: Additional training parameters. Returns: Path of the trained model archive. """ stories = await file_importer.get_stories() nlu_data = await file_importer.get_nlu_data() if stories.is_empty() and nlu_data.is_empty(): print_error( "No training data given. Please provide stories and NLU data in " "order to train a Rasa model using the '--data' argument.") return if stories.is_empty(): print_warning( "No stories present. Just a Rasa NLU model will be trained.") return await _train_nlu_with_validated_data( file_importer, output=output_path, fixed_model_name=fixed_model_name, persist_nlu_training_data=persist_nlu_training_data, ) if nlu_data.is_empty(): print_warning( "No NLU data present. Just a Rasa Core model will be trained.") return await _train_core_with_validated_data( file_importer, output=output_path, fixed_model_name=fixed_model_name, kwargs=kwargs, ) new_fingerprint = await model.model_fingerprint(file_importer) old_model = model.get_latest_model(output_path) fingerprint_comparison = FingerprintComparisonResult( force_training=force_training) if not force_training: fingerprint_comparison = model.should_retrain(new_fingerprint, old_model, train_path) if fingerprint_comparison.is_training_required(): await _do_training( file_importer, output_path=output_path, train_path=train_path, fingerprint_comparison_result=fingerprint_comparison, fixed_model_name=fixed_model_name, persist_nlu_training_data=persist_nlu_training_data, kwargs=kwargs, ) return model.package_model( fingerprint=new_fingerprint, output_directory=output_path, train_path=train_path, fixed_model_name=fixed_model_name, ) print_success("Nothing changed. You can use the old model stored at '{}'." "".format(os.path.abspath(old_model))) return old_model
async def train_async( domain: Union[Domain, Text], config: Text, training_files: Optional[Union[Text, List[Text]]], output_path: Text = DEFAULT_MODELS_PATH, force_training: bool = False, fixed_model_name: Optional[Text] = None, uncompress: bool = False, kwargs: Optional[Dict] = None, ) -> Optional[Text]: """Trains a Rasa model (Core and NLU). Args: domain: Path to the domain file. config: Path to the config for Core and NLU. training_files: Paths to the training data for Core and NLU. output_path: Output path. force_training: If `True` retrain model even if data has not changed. fixed_model_name: Name of model to be stored. uncompress: If `True` the model will not be compressed. kwargs: Additional training parameters. Returns: Path of the trained model archive. """ config = _get_valid_config(config, CONFIG_MANDATORY_KEYS) train_path = tempfile.mkdtemp() skill_imports = SkillSelector.load(config) try: domain = Domain.load(domain, skill_imports) except InvalidDomain as e: print_error( "Could not load domain due to: '{}'. To specify a valid domain path use " "the '--domain' argument.".format(e)) return None story_directory, nlu_data_directory = data.get_core_nlu_directories( training_files, skill_imports) new_fingerprint = model.model_fingerprint(config, domain, nlu_data_directory, story_directory) dialogue_data_not_present = not os.listdir(story_directory) nlu_data_not_present = not os.listdir(nlu_data_directory) if dialogue_data_not_present and nlu_data_not_present: print_error( "No training data given. Please provide stories and NLU data in " "order to train a Rasa model using the '--data' argument.") return if dialogue_data_not_present: print_warning( "No dialogue data present. Just a Rasa NLU model will be trained.") return _train_nlu_with_validated_data( config=config, nlu_data_directory=nlu_data_directory, output=output_path, fixed_model_name=fixed_model_name, uncompress=uncompress, ) if nlu_data_not_present: print_warning( "No NLU data present. Just a Rasa Core model will be trained.") return await _train_core_with_validated_data( domain=domain, config=config, story_directory=story_directory, output=output_path, fixed_model_name=fixed_model_name, uncompress=uncompress, kwargs=kwargs, ) old_model = model.get_latest_model(output_path) retrain_core, retrain_nlu = should_retrain(new_fingerprint, old_model, train_path) if force_training or retrain_core or retrain_nlu: await _do_training( domain=domain, config=config, output_path=output_path, train_path=train_path, nlu_data_directory=nlu_data_directory, story_directory=story_directory, force_training=force_training, retrain_core=retrain_core, retrain_nlu=retrain_nlu, fixed_model_name=fixed_model_name, uncompress=uncompress, kwargs=kwargs, ) return _package_model( new_fingerprint=new_fingerprint, output_path=output_path, train_path=train_path, fixed_model_name=fixed_model_name, uncompress=uncompress, ) print_success("Nothing changed. You can use the old model stored at '{}'." "".format(os.path.abspath(old_model))) return old_model
async def _train_async_internal( file_importer: TrainingDataImporter, train_path: Text, output_path: Text, force_training: bool, fixed_model_name: Optional[Text], kwargs: Optional[Dict], ) -> Optional[Text]: """Trains a Rasa model (Core and NLU). Use only from `train_async`. Args: domain: Path to the domain file. config: Path to the config for Core and NLU. train_path: Directory in which to train the model. nlu_data_directory: Path to NLU training files. story_directory: Path to Core training files. output_path: Output path. force_training: If `True` retrain model even if data has not changed. fixed_model_name: Name of model to be stored. kwargs: Additional training parameters. Returns: Path of the trained model archive. """ new_fingerprint = await model.model_fingerprint(file_importer) stories = await file_importer.get_stories() nlu_data = await file_importer.get_nlu_data() if stories.is_empty() and nlu_data.is_empty(): print_error( "No training data given. Please provide stories and NLU data in " "order to train a Rasa model using the '--data' argument.") return if stories.is_empty(): print_warning( "No stories present. Just a Rasa NLU model will be trained.") return await _train_nlu_with_validated_data( file_importer, output=output_path, fixed_model_name=fixed_model_name) if nlu_data.is_empty(): print_warning( "No NLU data present. Just a Rasa Core model will be trained.") return await _train_core_with_validated_data( file_importer, output=output_path, fixed_model_name=fixed_model_name, kwargs=kwargs, ) old_model = model.get_latest_model(output_path) retrain_core, retrain_nlu = model.should_retrain(new_fingerprint, old_model, train_path) if force_training or retrain_core or retrain_nlu: await _do_training( file_importer, output_path=output_path, train_path=train_path, force_training=force_training, retrain_core=retrain_core, retrain_nlu=retrain_nlu, fixed_model_name=fixed_model_name, kwargs=kwargs, ) return model.package_model( fingerprint=new_fingerprint, output_directory=output_path, train_path=train_path, fixed_model_name=fixed_model_name, ) print_success("Nothing changed. You can use the old model stored at '{}'." "".format(os.path.abspath(old_model))) return old_model
def _read_data(paths: List[Text]) -> Generator[Tuple[Text, Text], None, None]: for filename in paths: try: yield read_file(filename), filename except ValueError: rasa_cli_utils.print_warning(f"Cannot read file {filename}")