async def convert_and_write(cls, source_path: Path,
output_path: Path) -> None:
"""Converts the given training data file and saves it to the output directory.
Args:
source_path: Path to the training data file.
output_path: Path to the output directory.
"""
from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
KEY_ACTIVE_LOOP, )
output_core_path = cls.generate_path_for_converted_training_data_file(
source_path, output_path)
reader = MarkdownStoryReader(unfold_or_utterances=False)
writer = YAMLStoryWriter()
steps = await reader.read_from_file(source_path)
if YAMLStoryWriter.stories_contain_loops(steps):
print_warning(
f"Training data file '{source_path}' contains forms. "
f"Any 'form' events will be converted to '{KEY_ACTIVE_LOOP}' events. "
f"Please note that in order for these stories to work you still "
f"need the 'FormPolicy' to be active. However the 'FormPolicy' is "
f"deprecated, please consider switching to the new 'RulePolicy', "
f"for which you can find the documentation here: {DOCS_URL_RULES}."
)
writer.dump(output_core_path, steps)
print_success(
f"Converted Core file: '{source_path}' >> '{output_core_path}'.")
def _convert_to_yaml(args: argparse.Namespace,
converter: TrainingDataConverter) -> None:
output = Path(args.out)
if not os.path.exists(output):
print_error_and_exit(
f"The output path '{output}' doesn't exist. Please make sure to specify "
f"an existing directory and try again.")
training_data = Path(args.data)
if not os.path.exists(training_data):
print_error_and_exit(
f"The training data path {training_data} doesn't exist "
f"and will be skipped.")
num_of_files_converted = 0
if os.path.isfile(training_data):
if _convert_file_to_yaml(training_data, output, converter):
num_of_files_converted += 1
elif os.path.isdir(training_data):
for root, _, files in os.walk(training_data, followlinks=True):
for f in sorted(files):
source_path = Path(os.path.join(root, f))
if _convert_file_to_yaml(source_path, output, converter):
num_of_files_converted += 1
if num_of_files_converted:
print_info(
f"Converted {num_of_files_converted} file(s), saved in '{output}'."
)
else:
print_warning(
f"Didn't convert any files under '{training_data}' path. "
"Did you specify the correct file/directory?")
async def handle_domain_if_not_exists(file_importer: TrainingDataImporter,
output_path, fixed_model_name):
nlu_model_only = await _train_nlu_with_validated_data(
file_importer, output=output_path, fixed_model_name=fixed_model_name)
print_warning(
"Core training was skipped because no valid domain file was found. Only an nlu-model was created."
"Please specify a valid domain using '--domain' argument or check if the provided domain file exists."
)
return nlu_model_only
def run(
model: "Text",
endpoints: "Text",
connector: "Text" = None,
credentials: "Text" = None,
**kwargs: "Dict[Text, Any]",
) -> None:
"""Runs a Rasa model.
Args:
model: Path to model archive.
endpoints: Path to endpoints file.
connector: Connector which should be use (overwrites `credentials`
field).
credentials: Path to channel credentials file.
**kwargs: Additional arguments which are passed to
`rasa.core.run.serve_application`.
"""
import rasa.core.run
from rasa.core.utils import AvailableEndpoints
from rasa.shared.utils.cli import print_warning
import rasa.shared.utils.common
from rasa.shared.constants import DOCS_BASE_URL
_endpoints = AvailableEndpoints.read_endpoints(endpoints)
if not connector and not credentials:
connector = "rest"
print_warning(
f"No chat connector configured, falling back to the "
f"REST input channel. To connect your bot to another channel, "
f"read the docs here: {DOCS_BASE_URL}/messaging-and-voice-channels"
)
kwargs = rasa.shared.utils.common.minimal_kwargs(
kwargs, rasa.core.run.serve_application
)
rasa.core.run.serve_application(
model,
channel=connector,
credentials=credentials,
endpoints=_endpoints,
**kwargs,
)
def run(
model: Text,
endpoints: Text,
connector: Text = None,
credentials: Text = None,
**kwargs: Dict,
):
"""Runs a Rasa model.
Args:
model: Path to model archive.
endpoints: Path to endpoints file.
connector: Connector which should be use (overwrites `credentials`
field).
credentials: Path to channel credentials file.
**kwargs: Additional arguments which are passed to
`rasa.core.run.serve_application`.
"""
import rasa.core.run
import rasa.nlu.run
from rasa.core.utils import AvailableEndpoints
import rasa.utils.common as utils
_endpoints = AvailableEndpoints.read_endpoints(endpoints)
if not connector and not credentials:
connector = "rest"
print_warning(
"No chat connector configured, falling back to the "
"REST input channel. To connect your bot to another channel, "
"read the docs here: {}/user-guide/"
"messaging-and-voice-channels".format(DOCS_BASE_URL))
kwargs = rasa.shared.utils.common.minimal_kwargs(
kwargs, rasa.core.run.serve_application)
rasa.core.run.serve_application(
model,
channel=connector,
credentials=credentials,
endpoints=_endpoints,
**kwargs,
)
def _convert_file_to_yaml(source_file: Path, target_dir: Path,
converter: TrainingDataConverter) -> bool:
"""Converts a single training data file to `YAML` format.
Args:
source_file: Training data file to be converted.
target_dir: Target directory for the converted file.
converter: Converter to be used.
Returns:
`True` if file was converted, `False` otherwise.
"""
if not is_valid_filetype(source_file):
return False
if converter.filter(source_file):
converter.convert_and_write(source_file, target_dir)
return True
print_warning(f"Skipped file: '{source_file}'.")
return False
async def _train_async_internal(
file_importer: TrainingDataImporter,
train_path: Text,
output_path: Text,
dry_run: bool,
force_training: bool,
fixed_model_name: Optional[Text],
persist_nlu_training_data: bool,
core_additional_arguments: Optional[Dict] = None,
nlu_additional_arguments: Optional[Dict] = None,
model_to_finetune: Optional[Text] = None,
finetuning_epoch_fraction: float = 1.0,
) -> TrainingResult:
"""Trains a Rasa model (Core and NLU). Use only from `train_async`.
Args:
file_importer: `TrainingDataImporter` which supplies the training data.
train_path: Directory in which to train the model.
output_path: Output path.
dry_run: If `True` then no training will be done, and the information about
whether the training needs to be done will be printed.
force_training: If `True` retrain model even if data has not changed.
fixed_model_name: Name of model to be stored.
persist_nlu_training_data: `True` if the NLU training data should be persisted
with the model.
core_additional_arguments: Additional training parameters for core training.
nlu_additional_arguments: Additional training parameters forwarded to training
method of each NLU component.
model_to_finetune: Optional path to a model which should be finetuned or
a directory in case the latest trained model should be used.
finetuning_epoch_fraction: The fraction currently specified training epochs
in the model configuration which should be used for finetuning.
Returns:
An instance of `TrainingResult`.
"""
stories, nlu_data = await asyncio.gather(file_importer.get_stories(),
file_importer.get_nlu_data())
new_fingerprint = await model.model_fingerprint(file_importer)
old_model = model.get_latest_model(output_path)
fingerprint_comparison = model.should_retrain(
new_fingerprint, old_model, train_path, force_training=force_training)
if dry_run:
code, texts = dry_run_result(fingerprint_comparison)
for text in texts:
print_warning(text) if code > 0 else print_success(text)
return TrainingResult(code=code)
if nlu_data.has_e2e_examples():
rasa.shared.utils.common.mark_as_experimental_feature(
"end-to-end training")
if stories.is_empty() and nlu_data.contains_no_pure_nlu_data():
rasa.shared.utils.cli.print_error(
"No training data given. Please provide stories and NLU data in "
"order to train a Rasa model using the '--data' argument.")
return TrainingResult()
if stories.is_empty():
rasa.shared.utils.cli.print_warning(
"No stories present. Just a Rasa NLU model will be trained.")
trained_model = await _train_nlu_with_validated_data(
file_importer,
output=output_path,
fixed_model_name=fixed_model_name,
persist_nlu_training_data=persist_nlu_training_data,
additional_arguments=nlu_additional_arguments,
model_to_finetune=model_to_finetune,
finetuning_epoch_fraction=finetuning_epoch_fraction,
)
return TrainingResult(model=trained_model)
# We will train nlu if there are any nlu example, including from e2e stories.
if nlu_data.contains_no_pure_nlu_data(
) and not nlu_data.has_e2e_examples():
rasa.shared.utils.cli.print_warning(
"No NLU data present. Just a Rasa Core model will be trained.")
trained_model = await _train_core_with_validated_data(
file_importer,
output=output_path,
fixed_model_name=fixed_model_name,
additional_arguments=core_additional_arguments,
model_to_finetune=model_to_finetune,
finetuning_epoch_fraction=finetuning_epoch_fraction,
)
return TrainingResult(model=trained_model)
new_fingerprint = await model.model_fingerprint(file_importer)
old_model = model.get_latest_model(output_path)
if not force_training:
fingerprint_comparison = model.should_retrain(
new_fingerprint,
old_model,
train_path,
has_e2e_examples=nlu_data.has_e2e_examples(),
)
else:
fingerprint_comparison = FingerprintComparisonResult(
force_training=True)
if fingerprint_comparison.is_training_required():
async with telemetry.track_model_training(
file_importer,
model_type="rasa",
):
await _do_training(
file_importer,
output_path=output_path,
train_path=train_path,
fingerprint_comparison_result=fingerprint_comparison,
fixed_model_name=fixed_model_name,
persist_nlu_training_data=persist_nlu_training_data,
core_additional_arguments=core_additional_arguments,
nlu_additional_arguments=nlu_additional_arguments,
old_model_zip_path=old_model,
model_to_finetune=model_to_finetune,
finetuning_epoch_fraction=finetuning_epoch_fraction,
)
trained_model = model.package_model(
fingerprint=new_fingerprint,
output_directory=output_path,
train_path=train_path,
fixed_model_name=fixed_model_name,
)
return TrainingResult(model=trained_model)
rasa.shared.utils.cli.print_success(
"Nothing changed. You can use the old model stored at '{}'."
"".format(os.path.abspath(old_model)))
return TrainingResult(model=old_model)
async def _train_async_internal(
file_importer: TrainingDataImporter,
train_path: Text,
output_path: Text,
force_training: bool,
fixed_model_name: Optional[Text],
persist_nlu_training_data: bool,
core_additional_arguments: Optional[Dict] = None,
nlu_additional_arguments: Optional[Dict] = None,
) -> Optional[Text]:
"""Trains a Rasa model (Core and NLU). Use only from `train_async`.
Args:
file_importer: `TrainingDataImporter` which supplies the training data.
train_path: Directory in which to train the model.
output_path: Output path.
force_training: If `True` retrain model even if data has not changed.
fixed_model_name: Name of model to be stored.
persist_nlu_training_data: `True` if the NLU training data should be persisted
with the model.
core_additional_arguments: Additional training parameters for core training.
nlu_additional_arguments: Additional training parameters forwarded to training
method of each NLU component.
Returns:
Path of the trained model archive.
"""
stories, nlu_data = await asyncio.gather(file_importer.get_stories(),
file_importer.get_nlu_data())
if stories.is_empty() and nlu_data.is_empty():
print_error(
"No training data given. Please provide stories and NLU data in "
"order to train a Rasa model using the '--data' argument.")
return
if stories.is_empty():
print_warning(
"No stories present. Just a Rasa NLU model will be trained.")
return await _train_nlu_with_validated_data(
file_importer,
output=output_path,
fixed_model_name=fixed_model_name,
persist_nlu_training_data=persist_nlu_training_data,
additional_arguments=nlu_additional_arguments,
)
if nlu_data.is_empty():
print_warning(
"No NLU data present. Just a Rasa Core model will be trained.")
return await _train_core_with_validated_data(
file_importer,
output=output_path,
fixed_model_name=fixed_model_name,
additional_arguments=core_additional_arguments,
)
new_fingerprint = await model.model_fingerprint(file_importer)
old_model = model.get_latest_model(output_path)
fingerprint_comparison = FingerprintComparisonResult(
force_training=force_training)
if not force_training:
fingerprint_comparison = model.should_retrain(new_fingerprint,
old_model, train_path)
if fingerprint_comparison.is_training_required():
await _do_training(
file_importer,
output_path=output_path,
train_path=train_path,
fingerprint_comparison_result=fingerprint_comparison,
fixed_model_name=fixed_model_name,
persist_nlu_training_data=persist_nlu_training_data,
core_additional_arguments=core_additional_arguments,
nlu_additional_arguments=nlu_additional_arguments,
old_model_zip_path=old_model,
)
return model.package_model(
fingerprint=new_fingerprint,
output_directory=output_path,
train_path=train_path,
fixed_model_name=fixed_model_name,
)
print_success("Nothing changed. You can use the old model stored at '{}'."
"".format(os.path.abspath(old_model)))
return old_model
