def get_database_session(is_local: bool = False,
db_url: Union[Text, URL, None] = None) -> Session:
"""Create a new database session.
Please use `session_scope` wherever possible. Using this function requires you to
manage the session state (committing, error handling, closing) yourself.
Args:
is_local: `True` if a local SQLite database should be used.
db_url: Database URL to use to create the session.
Returns:
A new database session.
"""
if not db_url:
db_url = get_db_url(is_local)
session_maker = create_session_maker(db_url)
session = session_maker()
try:
rasa.core.tracker_store.ensure_schema_exists(session)
except ValueError as e:
rasa_cli_utils.print_error_and_exit(
f"Requested PostgreSQL schema '{e}' was not found in the database. To "
f"continue, please create the schema by running 'CREATE DATABASE {e};' "
f"or unset the '{POSTGRESQL_SCHEMA}' environment variable in order to use "
f"the default schema. Exiting application.")
return session
def _convert_to_yaml(args: argparse.Namespace,
converter: TrainingDataConverter) -> None:
output = Path(args.out)
if not os.path.exists(output):
print_error_and_exit(
f"The output path '{output}' doesn't exist. Please make sure to specify "
f"an existing directory and try again.")
training_data = Path(args.data)
if not os.path.exists(training_data):
print_error_and_exit(
f"The training data path {training_data} doesn't exist "
f"and will be skipped.")
num_of_files_converted = 0
if os.path.isfile(training_data):
if _convert_file_to_yaml(training_data, output, converter):
num_of_files_converted += 1
elif os.path.isdir(training_data):
for root, _, files in os.walk(training_data, followlinks=True):
for f in sorted(files):
source_path = Path(os.path.join(root, f))
if _convert_file_to_yaml(source_path, output, converter):
num_of_files_converted += 1
if num_of_files_converted:
print_info(
f"Converted {num_of_files_converted} file(s), saved in '{output}'."
)
else:
print_warning(
f"Didn't convert any files under '{training_data}' path. "
"Did you specify the correct file/directory?")
def inject_domain(
domain_path: Text,
domain_service: "DomainService",
project_id: Text = config.project_name,
username: Text = config.default_username,
) -> Dict[Text, Any]:
"""Load Rasa Core domain at `path` and save it to database.
Quits the application if domain cannot be loaded.
"""
if not os.path.exists(domain_path):
rasa_cli_utils.print_error_and_exit(
f"domain.yml could not be found at '{os.path.abspath(domain_path)}'. "
f"Rasa X requires a domain in the project root directory.")
try:
domain_service.validate_and_store_domain_yaml(
domain_yaml=read_file(domain_path),
project_id=project_id,
path=domain_path,
store_responses=True,
username=username,
should_dump_domain=False,
)
except InvalidDomain as e:
rasa_cli_utils.print_error_and_exit(
f"Could not inject domain. Details:\n{e}")
return domain_service.get_or_create_domain(project_id)
def _convert_core_data(args: argparse.Namespace) -> None:
if args.format == "yaml":
_convert_to_yaml(args, False)
else:
print_error_and_exit(
"Could not recognize output format. Supported output formats: "
"'yaml'. Specify the desired output format with '--format'.")
def inject_config(
config_path: Text,
settings_service: "SettingsService") -> Optional[Dict[Text, Any]]:
"""Load a configuration file from `path` and save it to the database.
Quits the application if config cannot be loaded.
"""
if not os.path.exists(config_path):
rasa_cli_utils.print_error_and_exit(
f"Failed to inject Rasa configuration. The file "
f"'{os.path.abspath(config_path)}' does not exist.")
_config = read_yaml_file(config_path)
if not _config:
rasa_cli_utils.print_error_and_exit(
f"Failed to inject Rasa configuration:\n"
f"Reading of yaml '{os.path.abspath(config_path)}' file failed. Most "
f"likely the file was not found or uses the wrong syntax.")
settings_service.save_config(config.team_name,
"default",
_config,
config_path,
should_dump=False)
logger.debug("Loaded local configuration from '{}' into database".format(
os.path.abspath(config_path)))
return _config
def _convert_to_yaml(args: argparse.Namespace, is_nlu: bool) -> None:
output = Path(args.out)
if not os.path.exists(output):
print_error_and_exit(
f"The output path '{output}' doesn't exist. Please make sure to specify "
f"an existing directory and try again."
)
training_data = Path(args.data)
if not os.path.exists(training_data):
print_error_and_exit(
f"The training data path {training_data} doesn't exist "
f"and will be skipped."
)
num_of_files_converted = 0
for file in os.listdir(training_data):
source_path = training_data / file
output_path = output / f"{source_path.stem}{CONVERTED_FILE_SUFFIX}"
if MarkdownReader.is_markdown_nlu_file(source_path):
if not is_nlu:
continue
_write_nlu_yaml(source_path, output_path, source_path)
num_of_files_converted += 1
elif not is_nlu and MarkdownStoryReader.is_markdown_story_file(source_path):
_write_core_yaml(source_path, output_path, source_path)
num_of_files_converted += 1
else:
print_warning(f"Skipped file: '{source_path}'.")
print_info(f"Converted {num_of_files_converted} file(s), saved in '{output}'.")
def _convert_nlu_data(args: argparse.Namespace) -> None:
if args.format in ["json", "md"]:
convert_training_data(args.data, args.out, args.format, args.language)
elif args.format == "yaml":
_convert_to_yaml(args, True)
else:
print_error_and_exit(
"Could not recognize output format. Supported output formats: 'json', "
"'md', 'yaml'. Specify the desired output format with '--format'.")
def _validate_domain(domain_path: Text):
from rasa.core.domain import Domain, InvalidDomain
try:
Domain.load(domain_path)
except InvalidDomain as e:
cli_utils.print_error_and_exit(
"The provided domain file could not be loaded. "
"Error: {}".format(e))
def _convert_nlg_data(args: argparse.Namespace) -> None:
from rasa.nlu.training_data.converters.nlg_markdown_to_yaml_converter import (
NLGMarkdownToYamlConverter, )
if args.format == "yaml":
_convert_to_yaml(args, NLGMarkdownToYamlConverter())
else:
print_error_and_exit(
"Could not recognize output format. Supported output formats: "
"'yaml'. Specify the desired output format with '--format'.")
def _convert_nlu_data(args: argparse.Namespace) -> None:
from rasa.nlu.training_data.converters.nlu_markdown_to_yaml_converter import (
NLUMarkdownToYamlConverter, )
if args.format in ["json", "md"]:
convert_training_data(args.data, args.out, args.format, args.language)
elif args.format == "yaml":
_convert_to_yaml(args, NLUMarkdownToYamlConverter())
else:
print_error_and_exit(
"Could not recognize output format. Supported output formats: 'json', "
"'md', 'yaml'. Specify the desired output format with '--format'.")
def accept_terms_or_quit(args: argparse.Namespace) -> None:
"""Prompt the user to accept the Rasa terms."""
import webbrowser
import questionary
from rasax.community.constants import RASA_TERMS_URL
show_prompt = not hasattr(args, "no_prompt") or not args.no_prompt
if not show_prompt:
print(
f"By adding the '--no_prompt' parameter you agreed to the Rasa "
f"X license agreement ({RASA_TERMS_URL})"
)
return
rasa_cli_utils.print_success(
"Before you can use Rasa X, you have to agree to its "
"license agreement (you will only have to do this "
"once)."
)
should_open_in_browser = questionary.confirm(
"Would you like to view the license agreement in your web browser?"
).ask()
if should_open_in_browser:
webbrowser.open(RASA_TERMS_URL)
accepted_terms = questionary.confirm(
"\nRasa X License Agreement\n"
"===========================\n\n"
"Do you agree to the Rasa X license agreement ({})?\n"
"By typing 'y', you agree to the terms. "
"If you are using this software for a company, by confirming, "
"you acknowledge you have the authority to do so.\n"
"If you do not agree, type 'n' to stop Rasa X."
"".format(RASA_TERMS_URL),
default=False,
qmark="",
).ask()
if accepted_terms:
rasa_utils.write_global_config_value(CONFIG_FILE_TERMS_KEY, True)
else:
rasa_cli_utils.print_error_and_exit(
"Sorry, without accepting the terms, you cannot use Rasa X. "
"You can of course still use the (Apache 2 licensed) Rasa framework: "
"https://github.com/RasaHQ/rasa",
exit_code=0,
)
def session_scope(self):
"""Provide a transactional scope around a series of operations."""
session = self.sessionmaker()
try:
ensure_schema_exists(session)
yield session
except ValueError as e:
rasa_cli_utils.print_error_and_exit(
f"Requested PostgreSQL schema '{e}' was not found in the database. To "
f"continue, please create the schema by running 'CREATE DATABASE {e};' "
f"or unset the '{POSTGRESQL_SCHEMA}' environment variable in order to "
f"use the default schema. Exiting application.")
finally:
session.close()
def perform_interactive_learning(args: argparse.Namespace, zipped_model: Text,
file_importer: TrainingDataImporter) -> None:
from rasa.core.train import do_interactive_learning
args.model = zipped_model
with model.unpack_model(zipped_model) as model_path:
args.core, args.nlu = model.get_model_subdirectories(model_path)
if args.core is None:
utils.print_error_and_exit(
"Can not run interactive learning on an NLU-only model.")
args.endpoints = utils.get_validated_path(args.endpoints, "endpoints",
DEFAULT_ENDPOINTS_PATH, True)
do_interactive_learning(args, file_importer)
def run(args: argparse.Namespace) -> None:
import questionary
print_success("Welcome to Rasa! 🤖\n")
if args.no_prompt:
print(
"To get started quickly, an "
"initial project will be created.\n"
"If you need some help, check out "
"the documentation at {}.\n".format(DOCS_BASE_URL)
)
else:
print(
"To get started quickly, an "
"initial project will be created.\n"
"If you need some help, check out "
"the documentation at {}.\n"
"Now let's start! 👇🏽\n".format(DOCS_BASE_URL)
)
if args.init_dir is not None:
path = args.init_dir
else:
path = (
questionary.text(
"Please enter a path where the project will be "
"created [default: current directory]",
default=".",
)
.skip_if(args.no_prompt, default=".")
.ask()
)
if args.no_prompt and not os.path.isdir(path):
print_error_and_exit(f"Project init path '{path}' not found.")
if path and not os.path.isdir(path):
_ask_create_path(path)
if path is None or not os.path.isdir(path):
print_cancel()
if not args.no_prompt and len(os.listdir(path)) > 0:
_ask_overwrite(path)
init_project(args, path)
def _assert_max_timestamp_is_greater_than_min_timestamp(
args: argparse.Namespace, ) -> None:
"""Inspect CLI timestamp parameters.
Prints an error and exits if a maximum timestamp is provided that is smaller
than the provided minimum timestamp.
Args:
args: Command-line arguments to process.
"""
min_timestamp = args.minimum_timestamp
max_timestamp = args.maximum_timestamp
if (min_timestamp is not None and max_timestamp is not None
and max_timestamp < min_timestamp):
cli_utils.print_error_and_exit(
f"Maximum timestamp '{max_timestamp}' is smaller than minimum "
f"timestamp '{min_timestamp}'. Exiting.")
async def _pull_runtime_config_from_server(
config_endpoint: Optional[Text],
attempts: int = 60,
wait_time_between_pulls: Union[int, float] = 5,
keys: Iterable[Text] = ("endpoints", "credentials"),
) -> Optional[List[Text]]:
"""Pull runtime config from `config_endpoint`.
Returns a list of paths to yaml dumps, each containing the contents of one of
`keys`.
"""
while attempts:
try:
async with aiohttp.ClientSession() as session:
async with session.get(config_endpoint) as resp:
if resp.status == 200:
rjs = await resp.json()
try:
return [
io_utils.create_temporary_file(rjs[k]) for k in keys
]
except KeyError as e:
cli_utils.print_error_and_exit(
"Failed to find key '{}' in runtime config. "
"Exiting.".format(e)
)
else:
logger.debug(
"Failed to get a proper response from remote "
"server. Status Code: {}. Response: '{}'"
"".format(resp.status, await resp.text())
)
except aiohttp.ClientError as e:
logger.debug(f"Failed to connect to server. Retrying. {e}")
await asyncio.sleep(wait_time_between_pulls)
attempts -= 1
cli_utils.print_error_and_exit(
"Could not fetch runtime config from server at '{}'. "
"Exiting.".format(config_endpoint)
)
def export_trackers(args: argparse.Namespace) -> None:
"""Export events for a connected tracker store using an event broker.
Args:
args: Command-line arguments to process.
"""
_assert_max_timestamp_is_greater_than_min_timestamp(args)
endpoints = rasa_core_utils.read_endpoints_from_path(args.endpoints)
tracker_store = _get_tracker_store(endpoints)
event_broker = _get_event_broker(endpoints)
_prepare_event_broker(event_broker)
requested_conversation_ids = _get_requested_conversation_ids(args.conversation_ids)
from rasa.core.exporter import Exporter
exporter = Exporter(
tracker_store,
event_broker,
args.endpoints,
requested_conversation_ids,
args.minimum_timestamp,
args.maximum_timestamp,
)
try:
published_events = exporter.publish_events()
cli_utils.print_success(
f"Done! Successfully published {published_events} events 🎉"
)
except PublishingError as e:
command = _get_continuation_command(exporter, e.timestamp)
cli_utils.print_error_and_exit(
f"Encountered error while publishing event with timestamp '{e}'. To "
f"continue where I left off, run the following command:"
f"\n\n\t{command}\n\nExiting."
)
except RasaException as e:
cli_utils.print_error_and_exit(str(e))
def _validate_rasa_x_start(args: argparse.Namespace, project_path: Text):
if not is_rasa_x_installed():
cli_utils.print_error_and_exit(
"Rasa X is not installed. The `rasa x` "
"command requires an installation of Rasa X. "
"Instructions on how to install Rasa X can be found here: "
"https://rasa.com/docs/rasa-x/installation-and-setup/.")
if args.port == args.rasa_x_port:
cli_utils.print_error_and_exit(
"The port for Rasa X '{}' and the port of the Rasa server '{}' are the "
"same. We need two different ports, one to run Rasa X (e.g. delivering the "
"UI) and another one to run a normal Rasa server.\nPlease specify two "
"different ports using the arguments '--port' and '--rasa-x-port'."
.format(args.rasa_x_port, args.port))
if not is_rasa_project_setup(project_path):
cli_utils.print_error_and_exit(
"This directory is not a valid Rasa project. Use 'rasa init' "
"to create a new Rasa project or switch to a valid Rasa project "
"directory (see http://rasa.com/docs/rasa/user-guide/"
"rasa-tutorial/#create-a-new-project).")
_validate_domain(os.path.join(project_path, DEFAULT_DOMAIN_PATH))
if args.data and not os.path.exists(args.data):
cli_utils.print_warning(
"The provided data path ('{}') does not exists. Rasa X will start "
"without any training data.".format(args.data))
def interactive(args: argparse.Namespace) -> None:
_set_not_required_args(args)
file_importer = TrainingDataImporter.load_from_config(
args.config, args.domain, args.data
)
if args.model is None:
loop = asyncio.get_event_loop()
story_graph = loop.run_until_complete(file_importer.get_stories())
if not story_graph or story_graph.is_empty():
utils.print_error_and_exit(
"Could not run interactive learning without either core data or a model containing core data."
)
zipped_model = train.train_core(args) if args.core_only else train.train(args)
if not zipped_model:
utils.print_error_and_exit(
"Could not train an initial model. Either pass paths "
"to the relevant training files (`--data`, `--config`, `--domain`), "
"or use 'rasa train' to train a model."
)
else:
zipped_model = get_provided_model(args.model)
if not (zipped_model and os.path.exists(zipped_model)):
utils.print_error_and_exit(
f"Interactive learning process cannot be started as no initial model was "
f"found at path '{args.model}'. Use 'rasa train' to train a model."
)
if not args.skip_visualization:
logger.info(f"Loading visualization data from {args.data}.")
perform_interactive_learning(args, zipped_model, file_importer)
def _get_event_broker(endpoints: "AvailableEndpoints") -> Optional["EventBroker"]:
"""Get `EventBroker` from `endpoints`.
Prints an error and exits if no event broker could be loaded.
Args:
endpoints: `AvailableEndpoints` to initialize the event broker from.
Returns:
Initialized event broker.
"""
if not endpoints.event_broker:
cli_utils.print_error_and_exit(
f"Could not find an `event_broker` section in the supplied "
f"endpoints file. Instructions on how to configure an event broker "
f"can be found here: {DOCS_URL_EVENT_BROKERS}. Exiting."
)
from rasa.core.brokers.broker import EventBroker
return EventBroker.create(endpoints.event_broker)
def _get_tracker_store(endpoints: "AvailableEndpoints") -> "TrackerStore":
"""Get `TrackerStore` from `endpoints`.
Prints an error and exits if no tracker store could be loaded.
Args:
endpoints: `AvailableEndpoints` to initialize the tracker store from.
Returns:
Initialized tracker store.
"""
if not endpoints.tracker_store:
cli_utils.print_error_and_exit(
f"Could not find a `tracker_store` section in the supplied "
f"endpoints file. Instructions on how to configure a tracker store "
f"can be found here: {DOCS_URL_TRACKER_STORES}. "
f"Exiting. ")
from rasa.core.tracker_store import TrackerStore
return TrackerStore.create(endpoints.tracker_store)
def _prepare_event_broker(event_broker: "EventBroker") -> None:
"""Sets `should_keep_unpublished_messages` flag to `False` if
`self.event_broker` is a `PikaEventBroker`.
If publishing of events fails, the `PikaEventBroker` instance should not keep a
list of unpublished messages, so we can retry publishing them. This is because
the instance is launched as part of this short-lived export script, meaning the
object is destroyed before it might be published.
In addition, wait until the event broker reports a `ready` state.
"""
from rasa.core.brokers.pika import PikaEventBroker
if isinstance(event_broker, PikaEventBroker):
event_broker.should_keep_unpublished_messages = False
event_broker.raise_on_failure = True
if not event_broker.is_ready():
cli_utils.print_error_and_exit(
f"Event broker of type '{type(event_broker)}' is not ready. Exiting."
)
def _fetch_and_verify_jwt_keys_from_file(
private_key_path: Text = config.jwt_private_key_path,
public_key_path: Text = config.jwt_public_key_path,
) -> Tuple[bytes, bytes]:
"""Load the public and private JWT key files and verify them."""
try:
private_key = file_as_bytes(private_key_path)
public_key = file_as_bytes(public_key_path)
_verify_keys(private_key, public_key)
return private_key, public_key
except FileNotFoundError as e:
error_message = f"Could not find key file. Error: '{e}'"
except ValueError as e:
error_message = (
"Failed to load key data. Make sure the key "
"files are enclosed with the "
"'-----BEGIN PRIVATE KEY-----' etc. tags. Error: '{}'".format(e))
except InvalidSignatureError as e:
error_message = f"Failed to verify key signature. Error: '{e}'"
except Exception as e:
error_message = f"Encountered error trying to verify JWT keys: '{e}'"
rasa_cli_utils.print_error_and_exit(error_message)
