async def convert_and_write(cls, source_path: Path,
output_path: Path) -> None:
"""Converts the given training data file and saves it to the output directory.
Args:
source_path: Path to the training data file.
output_path: Path to the output directory.
"""
from rasa.shared.core.training_data.story_reader.yaml_story_reader import (
KEY_ACTIVE_LOOP, )
output_core_path = cls.generate_path_for_converted_training_data_file(
source_path, output_path)
reader = MarkdownStoryReader(unfold_or_utterances=False)
writer = YAMLStoryWriter()
steps = await reader.read_from_file(source_path)
if YAMLStoryWriter.stories_contain_loops(steps):
print_warning(
f"Training data file '{source_path}' contains forms. "
f"Any 'form' events will be converted to '{KEY_ACTIVE_LOOP}' events. "
f"Please note that in order for these stories to work you still "
f"need the 'FormPolicy' to be active. However the 'FormPolicy' is "
f"deprecated, please consider switching to the new 'RulePolicy', "
f"for which you can find the documentation here: {DOCS_URL_RULES}."
)
writer.dump(output_core_path, steps)
print_success(
f"Converted Core file: '{source_path}' >> '{output_core_path}'.")
async def convert_and_write(cls, source_path: Path, output_path: Path) -> None:
"""Converts the given training data file and saves it to the output directory.
Args:
source_path: Path to the training data file.
output_path: Path to the output directory.
"""
reader = NLGMarkdownReader()
writer = RasaYAMLWriter()
output_nlg_path = cls.generate_path_for_converted_training_data_file(
source_path, output_path
)
training_data = reader.read(source_path)
converted_responses = {}
for response_name, examples in training_data.responses.items():
new_response_name = cls._normalize_response_name(response_name)
converted_responses[new_response_name] = examples
converted_training_data = TrainingData(responses=converted_responses)
writer.dump(output_nlg_path, converted_training_data)
print_success(f"Converted NLG file: '{source_path}' >> '{output_nlg_path}'.")
def package_model(
fingerprint: Fingerprint,
output_directory: Text,
train_path: Text,
fixed_model_name: Optional[Text] = None,
model_prefix: Text = "",
) -> Text:
"""
Compress a trained model.
Args:
fingerprint: fingerprint of the model
output_directory: path to the directory in which the model should be stored
train_path: path to uncompressed model
fixed_model_name: name of the compressed model file
model_prefix: prefix of the compressed model file
Returns: path to 'tar.gz' model file
"""
output_directory = create_output_path(
output_directory, prefix=model_prefix, fixed_name=fixed_model_name
)
create_package_rasa(train_path, output_directory, fingerprint)
print_success(
"Your Rasa model is trained and saved at '{}'.".format(
os.path.abspath(output_directory)
)
)
return output_directory
def run_in_production(args: argparse.Namespace) -> None:
from rasa.shared.utils.cli import print_success
print_success("Starting Rasa X in production mode... 🚀")
credentials_path, endpoints_path = _get_credentials_and_endpoints_paths(args)
endpoints = AvailableEndpoints.read_endpoints(endpoints_path)
_rasa_service(args, endpoints, None, credentials_path)
def _display_bot_response(response: Dict):
from IPython.display import Image, display
for response_type, value in response.items():
if response_type == "text":
print_success(value)
if response_type == "image":
image = Image(url=value)
display(image)
def _ask_create_path(path: Text) -> None:
import questionary
should_create = questionary.confirm(
f"Path '{path}' does not exist 🧐. Create path?").ask()
if should_create:
os.makedirs(path)
else:
print_success("Ok. You can continue setting up by running "
"'rasa init' 🙋🏽♀️")
sys.exit(0)
def run(args: argparse.Namespace) -> None:
import questionary
print_success("Welcome to Rasa! 🤖\n")
if args.no_prompt:
print(
f"To get started quickly, an "
f"initial project will be created.\n"
f"If you need some help, check out "
f"the documentation at {DOCS_BASE_URL}.\n"
)
else:
print(
f"To get started quickly, an "
f"initial project will be created.\n"
f"If you need some help, check out "
f"the documentation at {DOCS_BASE_URL}.\n"
f"Now let's start! 👇🏽\n"
)
if args.init_dir is not None:
path = args.init_dir
else:
path = (
questionary.text(
"Please enter a path where the project will be "
"created [default: current directory]",
)
.skip_if(args.no_prompt, default="")
.ask()
)
# set the default directory. we can't use the `default` property
# in questionary as we want to avoid showing the "." in the prompt as the
# initial value. users tend to overlook it and it leads to invalid
# paths like: ".C:\mydir".
# Can't use `if not path` either, as `None` will be handled differently (abort)
if path == "":
path = "."
if args.no_prompt and not os.path.isdir(path):
print_error_and_exit(f"Project init path '{path}' not found.")
if path and not os.path.isdir(path):
_ask_create_path(path)
if path is None or not os.path.isdir(path):
print_cancel()
if not args.no_prompt and len(os.listdir(path)) > 0:
_ask_overwrite(path)
telemetry.track_project_init(path)
init_project(args, path)
def _ask_create_path(path: Text) -> None:
import questionary
should_create = questionary.confirm(
f"Path '{path}' does not exist 🧐. Create path?").ask()
if should_create:
try:
os.makedirs(path)
except (PermissionError, OSError, FileExistsError) as e:
print_error_and_exit(f"Failed to create project path at '{path}'. "
f"Error: {e}")
else:
print_success("Ok, will exit for now. You can continue setting up by "
"running 'rasa init' again 🙋🏽♀️")
sys.exit(0)
def run_cmdline(model_path: Text) -> None:
"""Loops over CLI input, passing each message to a loaded NLU model."""
agent = Agent.load(model_path)
print_success(
"NLU model loaded. Type a message and press enter to parse it.")
while True:
print_success("Next message:")
try:
message = input().strip()
except (EOFError, KeyboardInterrupt):
print_info("Wrapping up command line chat...")
break
result = asyncio.run(agent.parse_message(message))
print(json_to_string(result))
def convert_and_write(cls, source_path: Path, output_path: Path) -> None:
"""Converts the given training data file and saves it to the output directory.
Args:
source_path: Path to the training data file.
output_path: Path to the output directory.
"""
output_nlu_path = cls.generate_path_for_converted_training_data_file(
source_path, output_path)
yaml_training_data = MarkdownReader().read(source_path)
RasaYAMLWriter().dump(output_nlu_path, yaml_training_data)
for lookup_table in yaml_training_data.lookup_tables:
cls._write_nlu_lookup_table_yaml(lookup_table, output_path)
print_success(
f"Converted NLU file: '{source_path}' >> '{output_nlu_path}'.")
def run_cmdline(
model_path: Text,
component_builder: Optional["ComponentBuilder"] = None) -> None:
interpreter = Interpreter.load(model_path, component_builder)
regex_interpreter = RegexInterpreter()
print_success(
"NLU model loaded. Type a message and press enter to parse it.")
while True:
print_success("Next message:")
message = input().strip()
if message.startswith(INTENT_MESSAGE_PREFIX):
result = rasa.utils.common.run_in_loop(
regex_interpreter.parse(message))
else:
result = interpreter.parse(message)
print(json_to_string(result))
def convert_and_write(cls, source_path: Path, output_path: Path) -> None:
"""Converts the given training data file and saves it to the output directory.
Args:
source_path: Path to the training data file.
output_path: Path to the output directory.
"""
reader = NLGMarkdownReader()
writer = RasaYAMLWriter()
output_nlg_path = cls.generate_path_for_converted_training_data_file(
source_path, output_path)
yaml_training_data = reader.read(source_path)
writer.dump(output_nlg_path, yaml_training_data)
print_success(
f"Converted NLG file: '{source_path}' >> '{output_nlg_path}'.")
def print_run_or_instructions(args: argparse.Namespace) -> None:
from rasa.core import constants
import questionary
should_run = (
questionary.confirm(
"Do you want to speak to the trained assistant on the command line? 🤖"
)
.skip_if(args.no_prompt, default=False)
.ask()
)
if should_run:
# provide defaults for command line arguments
attributes = [
"endpoints",
"credentials",
"cors",
"auth_token",
"jwt_secret",
"jwt_method",
"enable_api",
"remote_storage",
]
for a in attributes:
setattr(args, a, None)
args.port = constants.DEFAULT_SERVER_PORT
shell(args)
else:
if args.no_prompt:
print(
"If you want to speak to the assistant, "
"run 'rasa shell' at any time inside "
"the project directory."
)
else:
print_success(
"Ok 👍🏼. "
"If you want to speak to the assistant, "
"run 'rasa shell' at any time inside "
"the project directory."
)
def run(args: argparse.Namespace) -> None:
import questionary
print_success("Welcome to Rasa! 🤖\n")
if args.no_prompt:
print(f"To get started quickly, an "
f"initial project will be created.\n"
f"If you need some help, check out "
f"the documentation at {DOCS_BASE_URL}.\n")
else:
print(f"To get started quickly, an "
f"initial project will be created.\n"
f"If you need some help, check out "
f"the documentation at {DOCS_BASE_URL}.\n"
f"Now let's start! 👇🏽\n")
if args.init_dir is not None:
path = args.init_dir
else:
path = (questionary.text(
"Please enter a path where the project will be "
"created [default: current directory]",
default=".",
).skip_if(args.no_prompt, default=".").ask())
if args.no_prompt and not os.path.isdir(path):
print_error_and_exit(f"Project init path '{path}' not found.")
if path and not os.path.isdir(path):
_ask_create_path(path)
if path is None or not os.path.isdir(path):
print_cancel()
if not args.no_prompt and len(os.listdir(path)) > 0:
_ask_overwrite(path)
telemetry.track_project_init(path)
init_project(args, path)
def print_train_or_instructions(args: argparse.Namespace, path: Text) -> None:
import questionary
print_success("Finished creating project structure.")
should_train = (questionary.confirm(
"Do you want to train an initial model? 💪🏽").skip_if(
args.no_prompt, default=True).ask())
if should_train:
print_success("Training an initial model...")
config = os.path.join(path, DEFAULT_CONFIG_PATH)
training_files = os.path.join(path, DEFAULT_DATA_PATH)
domain = os.path.join(path, DEFAULT_DOMAIN_PATH)
output = os.path.join(path, create_output_path())
training_result = rasa.train(domain, config, training_files, output)
args.model = training_result.model
print_run_or_instructions(args)
else:
print_success(
"No problem 👍🏼. You can also train a model later by going "
"to the project directory and running 'rasa train'.")
def print_train_or_instructions(args: argparse.Namespace) -> None:
"""Train a model if the user wants to."""
import questionary
import rasa
print_success("Finished creating project structure.")
should_train = (questionary.confirm(
"Do you want to train an initial model? 💪🏽").skip_if(
args.no_prompt, default=True).ask())
if should_train:
print_success("Training an initial model...")
training_result = rasa.train(
DEFAULT_DOMAIN_PATH,
DEFAULT_CONFIG_PATH,
DEFAULT_DATA_PATH,
DEFAULT_MODELS_PATH,
)
args.model = training_result.model
print_run_or_instructions(args)
else:
print_success(
"No problem 👍🏼. You can also train a model later by going "
"to the project directory and running 'rasa train'.")
def run_cmdline(
model_path: Text,
component_builder: Optional["ComponentBuilder"] = None) -> None:
interpreter = Interpreter.load(model_path, component_builder)
regex_interpreter = RegexInterpreter()
print_success(
"NLU model loaded. Type a message and press enter to parse it.")
while True:
print_success("Next message:")
try:
message = input().strip()
except (EOFError, KeyboardInterrupt):
print_info("Wrapping up command line chat...")
break
if message.startswith(INTENT_MESSAGE_PREFIX):
result = rasa.utils.common.run_in_loop(
regex_interpreter.parse(message))
else:
result = interpreter.parse(message)
print(json_to_string(result))
async def _train_async_internal(
file_importer: TrainingDataImporter,
train_path: Text,
output_path: Text,
force_training: bool,
fixed_model_name: Optional[Text],
persist_nlu_training_data: bool,
core_additional_arguments: Optional[Dict] = None,
nlu_additional_arguments: Optional[Dict] = None,
) -> Optional[Text]:
"""Trains a Rasa model (Core and NLU). Use only from `train_async`.
Args:
file_importer: `TrainingDataImporter` which supplies the training data.
train_path: Directory in which to train the model.
output_path: Output path.
force_training: If `True` retrain model even if data has not changed.
fixed_model_name: Name of model to be stored.
persist_nlu_training_data: `True` if the NLU training data should be persisted
with the model.
core_additional_arguments: Additional training parameters for core training.
nlu_additional_arguments: Additional training parameters forwarded to training
method of each NLU component.
Returns:
Path of the trained model archive.
"""
stories, nlu_data = await asyncio.gather(
file_importer.get_stories(), file_importer.get_nlu_data()
)
# if stories.is_empty() and nlu_data.can_train_nlu_model():
# print_error(
# "No training data given. Please provide stories and NLU data in "
# "order to train a Rasa model using the '--data' argument."
# )
# return
# if stories.is_empty():
# print_warning("No stories present. Just a Rasa NLU model will be trained.")
# return await _train_nlu_with_validated_data(
# file_importer,
# output=output_path,
# fixed_model_name=fixed_model_name,
# persist_nlu_training_data=persist_nlu_training_data,
# additional_arguments=nlu_additional_arguments,
# )
# if nlu_data.can_train_nlu_model():
# print_warning("No NLU data present. Just a Rasa Core model will be trained.")
# return await _train_core_with_validated_data(
# file_importer,
# output=output_path,
# fixed_model_name=fixed_model_name,
# additional_arguments=core_additional_arguments,
# )
new_fingerprint = await model.model_fingerprint(file_importer)
old_model = model.get_latest_model(output_path)
if not force_training:
fingerprint_comparison = model.should_retrain(
new_fingerprint, old_model, train_path
)
else:
fingerprint_comparison = FingerprintComparisonResult(force_training=True)
# bf mod >
if fingerprint_comparison.nlu == True: # replace True with list of all langs
fingerprint_comparison.nlu = list(new_fingerprint.get("nlu-config", {}).keys())
domain = await file_importer.get_domain()
core_untrainable = domain.is_empty() or stories.is_empty()
nlu_untrainable = [l for l, d in nlu_data.items() if d.is_empty()]
fingerprint_comparison.core = fingerprint_comparison.core and not core_untrainable
fingerprint_comparison.nlu = [
l for l in fingerprint_comparison.nlu if l not in nlu_untrainable
]
if core_untrainable:
print_color(
"Skipping Core training since domain or stories are empty.",
color=rasa.shared.utils.io.bcolors.OKBLUE,
)
for lang in nlu_untrainable:
print_color(
"No NLU data found for language <{}>, skipping training...".format(lang),
color=rasa.shared.utils.io.bcolors.OKBLUE,
)
# </ bf mod
if fingerprint_comparison.is_training_required():
async with telemetry.track_model_training(file_importer, model_type="rasa"):
await _do_training(
file_importer,
output_path=output_path,
train_path=train_path,
fingerprint_comparison_result=fingerprint_comparison,
fixed_model_name=fixed_model_name,
persist_nlu_training_data=persist_nlu_training_data,
core_additional_arguments=core_additional_arguments,
nlu_additional_arguments=nlu_additional_arguments,
old_model_zip_path=old_model,
)
return model.package_model(
fingerprint=new_fingerprint,
output_directory=output_path,
train_path=train_path,
fixed_model_name=fixed_model_name,
)
print_success(
"Nothing changed. You can use the old model stored at '{}'."
"".format(os.path.abspath(old_model))
)
return old_model
async def _train_async_internal(
file_importer: TrainingDataImporter,
train_path: Text,
output_path: Text,
force_training: bool,
fixed_model_name: Optional[Text],
persist_nlu_training_data: bool,
core_additional_arguments: Optional[Dict] = None,
nlu_additional_arguments: Optional[Dict] = None,
) -> Optional[Text]:
"""Trains a Rasa model (Core and NLU). Use only from `train_async`.
Args:
file_importer: `TrainingDataImporter` which supplies the training data.
train_path: Directory in which to train the model.
output_path: Output path.
force_training: If `True` retrain model even if data has not changed.
fixed_model_name: Name of model to be stored.
persist_nlu_training_data: `True` if the NLU training data should be persisted
with the model.
core_additional_arguments: Additional training parameters for core training.
nlu_additional_arguments: Additional training parameters forwarded to training
method of each NLU component.
Returns:
Path of the trained model archive.
"""
stories, nlu_data = await asyncio.gather(file_importer.get_stories(),
file_importer.get_nlu_data())
if stories.is_empty() and nlu_data.is_empty():
print_error(
"No training data given. Please provide stories and NLU data in "
"order to train a Rasa model using the '--data' argument.")
return
if stories.is_empty():
print_warning(
"No stories present. Just a Rasa NLU model will be trained.")
return await _train_nlu_with_validated_data(
file_importer,
output=output_path,
fixed_model_name=fixed_model_name,
persist_nlu_training_data=persist_nlu_training_data,
additional_arguments=nlu_additional_arguments,
)
if nlu_data.is_empty():
print_warning(
"No NLU data present. Just a Rasa Core model will be trained.")
return await _train_core_with_validated_data(
file_importer,
output=output_path,
fixed_model_name=fixed_model_name,
additional_arguments=core_additional_arguments,
)
new_fingerprint = await model.model_fingerprint(file_importer)
old_model = model.get_latest_model(output_path)
fingerprint_comparison = FingerprintComparisonResult(
force_training=force_training)
if not force_training:
fingerprint_comparison = model.should_retrain(new_fingerprint,
old_model, train_path)
if fingerprint_comparison.is_training_required():
await _do_training(
file_importer,
output_path=output_path,
train_path=train_path,
fingerprint_comparison_result=fingerprint_comparison,
fixed_model_name=fixed_model_name,
persist_nlu_training_data=persist_nlu_training_data,
core_additional_arguments=core_additional_arguments,
nlu_additional_arguments=nlu_additional_arguments,
old_model_zip_path=old_model,
)
return model.package_model(
fingerprint=new_fingerprint,
output_directory=output_path,
train_path=train_path,
fixed_model_name=fixed_model_name,
)
print_success("Nothing changed. You can use the old model stored at '{}'."
"".format(os.path.abspath(old_model)))
return old_model
def print_cancel() -> None:
print_success(
"Ok. You can continue setting up by running 'rasa init' 🙋🏽♀️")
sys.exit(0)
async def _train_async_internal(
file_importer: TrainingDataImporter,
train_path: Text,
output_path: Text,
dry_run: bool,
force_training: bool,
fixed_model_name: Optional[Text],
persist_nlu_training_data: bool,
core_additional_arguments: Optional[Dict] = None,
nlu_additional_arguments: Optional[Dict] = None,
model_to_finetune: Optional[Text] = None,
finetuning_epoch_fraction: float = 1.0,
) -> TrainingResult:
"""Trains a Rasa model (Core and NLU). Use only from `train_async`.
Args:
file_importer: `TrainingDataImporter` which supplies the training data.
train_path: Directory in which to train the model.
output_path: Output path.
dry_run: If `True` then no training will be done, and the information about
whether the training needs to be done will be printed.
force_training: If `True` retrain model even if data has not changed.
fixed_model_name: Name of model to be stored.
persist_nlu_training_data: `True` if the NLU training data should be persisted
with the model.
core_additional_arguments: Additional training parameters for core training.
nlu_additional_arguments: Additional training parameters forwarded to training
method of each NLU component.
model_to_finetune: Optional path to a model which should be finetuned or
a directory in case the latest trained model should be used.
finetuning_epoch_fraction: The fraction currently specified training epochs
in the model configuration which should be used for finetuning.
Returns:
An instance of `TrainingResult`.
"""
stories, nlu_data = await asyncio.gather(file_importer.get_stories(),
file_importer.get_nlu_data())
new_fingerprint = await model.model_fingerprint(file_importer)
old_model = model.get_latest_model(output_path)
fingerprint_comparison = model.should_retrain(
new_fingerprint, old_model, train_path, force_training=force_training)
if dry_run:
code, texts = dry_run_result(fingerprint_comparison)
for text in texts:
print_warning(text) if code > 0 else print_success(text)
return TrainingResult(code=code)
if nlu_data.has_e2e_examples():
rasa.shared.utils.common.mark_as_experimental_feature(
"end-to-end training")
if stories.is_empty() and nlu_data.contains_no_pure_nlu_data():
rasa.shared.utils.cli.print_error(
"No training data given. Please provide stories and NLU data in "
"order to train a Rasa model using the '--data' argument.")
return TrainingResult()
if stories.is_empty():
rasa.shared.utils.cli.print_warning(
"No stories present. Just a Rasa NLU model will be trained.")
trained_model = await _train_nlu_with_validated_data(
file_importer,
output=output_path,
fixed_model_name=fixed_model_name,
persist_nlu_training_data=persist_nlu_training_data,
additional_arguments=nlu_additional_arguments,
model_to_finetune=model_to_finetune,
finetuning_epoch_fraction=finetuning_epoch_fraction,
)
return TrainingResult(model=trained_model)
# We will train nlu if there are any nlu example, including from e2e stories.
if nlu_data.contains_no_pure_nlu_data(
) and not nlu_data.has_e2e_examples():
rasa.shared.utils.cli.print_warning(
"No NLU data present. Just a Rasa Core model will be trained.")
trained_model = await _train_core_with_validated_data(
file_importer,
output=output_path,
fixed_model_name=fixed_model_name,
additional_arguments=core_additional_arguments,
model_to_finetune=model_to_finetune,
finetuning_epoch_fraction=finetuning_epoch_fraction,
)
return TrainingResult(model=trained_model)
new_fingerprint = await model.model_fingerprint(file_importer)
old_model = model.get_latest_model(output_path)
if not force_training:
fingerprint_comparison = model.should_retrain(
new_fingerprint,
old_model,
train_path,
has_e2e_examples=nlu_data.has_e2e_examples(),
)
else:
fingerprint_comparison = FingerprintComparisonResult(
force_training=True)
if fingerprint_comparison.is_training_required():
async with telemetry.track_model_training(
file_importer,
model_type="rasa",
):
await _do_training(
file_importer,
output_path=output_path,
train_path=train_path,
fingerprint_comparison_result=fingerprint_comparison,
fixed_model_name=fixed_model_name,
persist_nlu_training_data=persist_nlu_training_data,
core_additional_arguments=core_additional_arguments,
nlu_additional_arguments=nlu_additional_arguments,
old_model_zip_path=old_model,
model_to_finetune=model_to_finetune,
finetuning_epoch_fraction=finetuning_epoch_fraction,
)
trained_model = model.package_model(
fingerprint=new_fingerprint,
output_directory=output_path,
train_path=train_path,
fixed_model_name=fixed_model_name,
)
return TrainingResult(model=trained_model)
rasa.shared.utils.cli.print_success(
"Nothing changed. You can use the old model stored at '{}'."
"".format(os.path.abspath(old_model)))
return TrainingResult(model=old_model)
