async def convert_and_write(cls, source_path: Path, output_path: Path) -> None: """Converts the given training data file and saves it to the output directory. Args: source_path: Path to the training data file. output_path: Path to the output directory. """ from rasa.shared.core.training_data.story_reader.yaml_story_reader import ( KEY_ACTIVE_LOOP, ) output_core_path = cls.generate_path_for_converted_training_data_file( source_path, output_path) reader = MarkdownStoryReader(unfold_or_utterances=False) writer = YAMLStoryWriter() steps = await reader.read_from_file(source_path) if YAMLStoryWriter.stories_contain_loops(steps): print_warning( f"Training data file '{source_path}' contains forms. " f"Any 'form' events will be converted to '{KEY_ACTIVE_LOOP}' events. " f"Please note that in order for these stories to work you still " f"need the 'FormPolicy' to be active. However the 'FormPolicy' is " f"deprecated, please consider switching to the new 'RulePolicy', " f"for which you can find the documentation here: {DOCS_URL_RULES}." ) writer.dump(output_core_path, steps) print_success( f"Converted Core file: '{source_path}' >> '{output_core_path}'.")
def _convert_to_yaml(args: argparse.Namespace, converter: TrainingDataConverter) -> None: output = Path(args.out) if not os.path.exists(output): print_error_and_exit( f"The output path '{output}' doesn't exist. Please make sure to specify " f"an existing directory and try again.") training_data = Path(args.data) if not os.path.exists(training_data): print_error_and_exit( f"The training data path {training_data} doesn't exist " f"and will be skipped.") num_of_files_converted = 0 if os.path.isfile(training_data): if _convert_file_to_yaml(training_data, output, converter): num_of_files_converted += 1 elif os.path.isdir(training_data): for root, _, files in os.walk(training_data, followlinks=True): for f in sorted(files): source_path = Path(os.path.join(root, f)) if _convert_file_to_yaml(source_path, output, converter): num_of_files_converted += 1 if num_of_files_converted: print_info( f"Converted {num_of_files_converted} file(s), saved in '{output}'." ) else: print_warning( f"Didn't convert any files under '{training_data}' path. " "Did you specify the correct file/directory?")
async def handle_domain_if_not_exists(file_importer: TrainingDataImporter, output_path, fixed_model_name): nlu_model_only = await _train_nlu_with_validated_data( file_importer, output=output_path, fixed_model_name=fixed_model_name) print_warning( "Core training was skipped because no valid domain file was found. Only an nlu-model was created." "Please specify a valid domain using '--domain' argument or check if the provided domain file exists." ) return nlu_model_only
def run( model: "Text", endpoints: "Text", connector: "Text" = None, credentials: "Text" = None, **kwargs: "Dict[Text, Any]", ) -> None: """Runs a Rasa model. Args: model: Path to model archive. endpoints: Path to endpoints file. connector: Connector which should be use (overwrites `credentials` field). credentials: Path to channel credentials file. **kwargs: Additional arguments which are passed to `rasa.core.run.serve_application`. """ import rasa.core.run from rasa.core.utils import AvailableEndpoints from rasa.shared.utils.cli import print_warning import rasa.shared.utils.common from rasa.shared.constants import DOCS_BASE_URL _endpoints = AvailableEndpoints.read_endpoints(endpoints) if not connector and not credentials: connector = "rest" print_warning( f"No chat connector configured, falling back to the " f"REST input channel. To connect your bot to another channel, " f"read the docs here: {DOCS_BASE_URL}/messaging-and-voice-channels" ) kwargs = rasa.shared.utils.common.minimal_kwargs( kwargs, rasa.core.run.serve_application ) rasa.core.run.serve_application( model, channel=connector, credentials=credentials, endpoints=_endpoints, **kwargs, )
def run( model: Text, endpoints: Text, connector: Text = None, credentials: Text = None, **kwargs: Dict, ): """Runs a Rasa model. Args: model: Path to model archive. endpoints: Path to endpoints file. connector: Connector which should be use (overwrites `credentials` field). credentials: Path to channel credentials file. **kwargs: Additional arguments which are passed to `rasa.core.run.serve_application`. """ import rasa.core.run import rasa.nlu.run from rasa.core.utils import AvailableEndpoints import rasa.utils.common as utils _endpoints = AvailableEndpoints.read_endpoints(endpoints) if not connector and not credentials: connector = "rest" print_warning( "No chat connector configured, falling back to the " "REST input channel. To connect your bot to another channel, " "read the docs here: {}/user-guide/" "messaging-and-voice-channels".format(DOCS_BASE_URL)) kwargs = rasa.shared.utils.common.minimal_kwargs( kwargs, rasa.core.run.serve_application) rasa.core.run.serve_application( model, channel=connector, credentials=credentials, endpoints=_endpoints, **kwargs, )
def _convert_file_to_yaml(source_file: Path, target_dir: Path, converter: TrainingDataConverter) -> bool: """Converts a single training data file to `YAML` format. Args: source_file: Training data file to be converted. target_dir: Target directory for the converted file. converter: Converter to be used. Returns: `True` if file was converted, `False` otherwise. """ if not is_valid_filetype(source_file): return False if converter.filter(source_file): converter.convert_and_write(source_file, target_dir) return True print_warning(f"Skipped file: '{source_file}'.") return False
async def _train_async_internal( file_importer: TrainingDataImporter, train_path: Text, output_path: Text, dry_run: bool, force_training: bool, fixed_model_name: Optional[Text], persist_nlu_training_data: bool, core_additional_arguments: Optional[Dict] = None, nlu_additional_arguments: Optional[Dict] = None, model_to_finetune: Optional[Text] = None, finetuning_epoch_fraction: float = 1.0, ) -> TrainingResult: """Trains a Rasa model (Core and NLU). Use only from `train_async`. Args: file_importer: `TrainingDataImporter` which supplies the training data. train_path: Directory in which to train the model. output_path: Output path. dry_run: If `True` then no training will be done, and the information about whether the training needs to be done will be printed. force_training: If `True` retrain model even if data has not changed. fixed_model_name: Name of model to be stored. persist_nlu_training_data: `True` if the NLU training data should be persisted with the model. core_additional_arguments: Additional training parameters for core training. nlu_additional_arguments: Additional training parameters forwarded to training method of each NLU component. model_to_finetune: Optional path to a model which should be finetuned or a directory in case the latest trained model should be used. finetuning_epoch_fraction: The fraction currently specified training epochs in the model configuration which should be used for finetuning. Returns: An instance of `TrainingResult`. """ stories, nlu_data = await asyncio.gather(file_importer.get_stories(), file_importer.get_nlu_data()) new_fingerprint = await model.model_fingerprint(file_importer) old_model = model.get_latest_model(output_path) fingerprint_comparison = model.should_retrain( new_fingerprint, old_model, train_path, force_training=force_training) if dry_run: code, texts = dry_run_result(fingerprint_comparison) for text in texts: print_warning(text) if code > 0 else print_success(text) return TrainingResult(code=code) if nlu_data.has_e2e_examples(): rasa.shared.utils.common.mark_as_experimental_feature( "end-to-end training") if stories.is_empty() and nlu_data.contains_no_pure_nlu_data(): rasa.shared.utils.cli.print_error( "No training data given. Please provide stories and NLU data in " "order to train a Rasa model using the '--data' argument.") return TrainingResult() if stories.is_empty(): rasa.shared.utils.cli.print_warning( "No stories present. Just a Rasa NLU model will be trained.") trained_model = await _train_nlu_with_validated_data( file_importer, output=output_path, fixed_model_name=fixed_model_name, persist_nlu_training_data=persist_nlu_training_data, additional_arguments=nlu_additional_arguments, model_to_finetune=model_to_finetune, finetuning_epoch_fraction=finetuning_epoch_fraction, ) return TrainingResult(model=trained_model) # We will train nlu if there are any nlu example, including from e2e stories. if nlu_data.contains_no_pure_nlu_data( ) and not nlu_data.has_e2e_examples(): rasa.shared.utils.cli.print_warning( "No NLU data present. Just a Rasa Core model will be trained.") trained_model = await _train_core_with_validated_data( file_importer, output=output_path, fixed_model_name=fixed_model_name, additional_arguments=core_additional_arguments, model_to_finetune=model_to_finetune, finetuning_epoch_fraction=finetuning_epoch_fraction, ) return TrainingResult(model=trained_model) new_fingerprint = await model.model_fingerprint(file_importer) old_model = model.get_latest_model(output_path) if not force_training: fingerprint_comparison = model.should_retrain( new_fingerprint, old_model, train_path, has_e2e_examples=nlu_data.has_e2e_examples(), ) else: fingerprint_comparison = FingerprintComparisonResult( force_training=True) if fingerprint_comparison.is_training_required(): async with telemetry.track_model_training( file_importer, model_type="rasa", ): await _do_training( file_importer, output_path=output_path, train_path=train_path, fingerprint_comparison_result=fingerprint_comparison, fixed_model_name=fixed_model_name, persist_nlu_training_data=persist_nlu_training_data, core_additional_arguments=core_additional_arguments, nlu_additional_arguments=nlu_additional_arguments, old_model_zip_path=old_model, model_to_finetune=model_to_finetune, finetuning_epoch_fraction=finetuning_epoch_fraction, ) trained_model = model.package_model( fingerprint=new_fingerprint, output_directory=output_path, train_path=train_path, fixed_model_name=fixed_model_name, ) return TrainingResult(model=trained_model) rasa.shared.utils.cli.print_success( "Nothing changed. You can use the old model stored at '{}'." "".format(os.path.abspath(old_model))) return TrainingResult(model=old_model)
async def _train_async_internal( file_importer: TrainingDataImporter, train_path: Text, output_path: Text, force_training: bool, fixed_model_name: Optional[Text], persist_nlu_training_data: bool, core_additional_arguments: Optional[Dict] = None, nlu_additional_arguments: Optional[Dict] = None, ) -> Optional[Text]: """Trains a Rasa model (Core and NLU). Use only from `train_async`. Args: file_importer: `TrainingDataImporter` which supplies the training data. train_path: Directory in which to train the model. output_path: Output path. force_training: If `True` retrain model even if data has not changed. fixed_model_name: Name of model to be stored. persist_nlu_training_data: `True` if the NLU training data should be persisted with the model. core_additional_arguments: Additional training parameters for core training. nlu_additional_arguments: Additional training parameters forwarded to training method of each NLU component. Returns: Path of the trained model archive. """ stories, nlu_data = await asyncio.gather(file_importer.get_stories(), file_importer.get_nlu_data()) if stories.is_empty() and nlu_data.is_empty(): print_error( "No training data given. Please provide stories and NLU data in " "order to train a Rasa model using the '--data' argument.") return if stories.is_empty(): print_warning( "No stories present. Just a Rasa NLU model will be trained.") return await _train_nlu_with_validated_data( file_importer, output=output_path, fixed_model_name=fixed_model_name, persist_nlu_training_data=persist_nlu_training_data, additional_arguments=nlu_additional_arguments, ) if nlu_data.is_empty(): print_warning( "No NLU data present. Just a Rasa Core model will be trained.") return await _train_core_with_validated_data( file_importer, output=output_path, fixed_model_name=fixed_model_name, additional_arguments=core_additional_arguments, ) new_fingerprint = await model.model_fingerprint(file_importer) old_model = model.get_latest_model(output_path) fingerprint_comparison = FingerprintComparisonResult( force_training=force_training) if not force_training: fingerprint_comparison = model.should_retrain(new_fingerprint, old_model, train_path) if fingerprint_comparison.is_training_required(): await _do_training( file_importer, output_path=output_path, train_path=train_path, fingerprint_comparison_result=fingerprint_comparison, fixed_model_name=fixed_model_name, persist_nlu_training_data=persist_nlu_training_data, core_additional_arguments=core_additional_arguments, nlu_additional_arguments=nlu_additional_arguments, old_model_zip_path=old_model, ) return model.package_model( fingerprint=new_fingerprint, output_directory=output_path, train_path=train_path, fixed_model_name=fixed_model_name, ) print_success("Nothing changed. You can use the old model stored at '{}'." "".format(os.path.abspath(old_model))) return old_model