def get_database_session(is_local: bool = False, db_url: Union[Text, URL, None] = None) -> Session: """Create a new database session. Please use `session_scope` wherever possible. Using this function requires you to manage the session state (committing, error handling, closing) yourself. Args: is_local: `True` if a local SQLite database should be used. db_url: Database URL to use to create the session. Returns: A new database session. """ if not db_url: db_url = get_db_url(is_local) session_maker = create_session_maker(db_url) session = session_maker() try: rasa.core.tracker_store.ensure_schema_exists(session) except ValueError as e: rasa_cli_utils.print_error_and_exit( f"Requested PostgreSQL schema '{e}' was not found in the database. To " f"continue, please create the schema by running 'CREATE DATABASE {e};' " f"or unset the '{POSTGRESQL_SCHEMA}' environment variable in order to use " f"the default schema. Exiting application.") return session
def _convert_to_yaml(args: argparse.Namespace, converter: TrainingDataConverter) -> None: output = Path(args.out) if not os.path.exists(output): print_error_and_exit( f"The output path '{output}' doesn't exist. Please make sure to specify " f"an existing directory and try again.") training_data = Path(args.data) if not os.path.exists(training_data): print_error_and_exit( f"The training data path {training_data} doesn't exist " f"and will be skipped.") num_of_files_converted = 0 if os.path.isfile(training_data): if _convert_file_to_yaml(training_data, output, converter): num_of_files_converted += 1 elif os.path.isdir(training_data): for root, _, files in os.walk(training_data, followlinks=True): for f in sorted(files): source_path = Path(os.path.join(root, f)) if _convert_file_to_yaml(source_path, output, converter): num_of_files_converted += 1 if num_of_files_converted: print_info( f"Converted {num_of_files_converted} file(s), saved in '{output}'." ) else: print_warning( f"Didn't convert any files under '{training_data}' path. " "Did you specify the correct file/directory?")
def inject_domain( domain_path: Text, domain_service: "DomainService", project_id: Text = config.project_name, username: Text = config.default_username, ) -> Dict[Text, Any]: """Load Rasa Core domain at `path` and save it to database. Quits the application if domain cannot be loaded. """ if not os.path.exists(domain_path): rasa_cli_utils.print_error_and_exit( f"domain.yml could not be found at '{os.path.abspath(domain_path)}'. " f"Rasa X requires a domain in the project root directory.") try: domain_service.validate_and_store_domain_yaml( domain_yaml=read_file(domain_path), project_id=project_id, path=domain_path, store_responses=True, username=username, should_dump_domain=False, ) except InvalidDomain as e: rasa_cli_utils.print_error_and_exit( f"Could not inject domain. Details:\n{e}") return domain_service.get_or_create_domain(project_id)
def _convert_core_data(args: argparse.Namespace) -> None: if args.format == "yaml": _convert_to_yaml(args, False) else: print_error_and_exit( "Could not recognize output format. Supported output formats: " "'yaml'. Specify the desired output format with '--format'.")
def inject_config( config_path: Text, settings_service: "SettingsService") -> Optional[Dict[Text, Any]]: """Load a configuration file from `path` and save it to the database. Quits the application if config cannot be loaded. """ if not os.path.exists(config_path): rasa_cli_utils.print_error_and_exit( f"Failed to inject Rasa configuration. The file " f"'{os.path.abspath(config_path)}' does not exist.") _config = read_yaml_file(config_path) if not _config: rasa_cli_utils.print_error_and_exit( f"Failed to inject Rasa configuration:\n" f"Reading of yaml '{os.path.abspath(config_path)}' file failed. Most " f"likely the file was not found or uses the wrong syntax.") settings_service.save_config(config.team_name, "default", _config, config_path, should_dump=False) logger.debug("Loaded local configuration from '{}' into database".format( os.path.abspath(config_path))) return _config
def _convert_to_yaml(args: argparse.Namespace, is_nlu: bool) -> None: output = Path(args.out) if not os.path.exists(output): print_error_and_exit( f"The output path '{output}' doesn't exist. Please make sure to specify " f"an existing directory and try again." ) training_data = Path(args.data) if not os.path.exists(training_data): print_error_and_exit( f"The training data path {training_data} doesn't exist " f"and will be skipped." ) num_of_files_converted = 0 for file in os.listdir(training_data): source_path = training_data / file output_path = output / f"{source_path.stem}{CONVERTED_FILE_SUFFIX}" if MarkdownReader.is_markdown_nlu_file(source_path): if not is_nlu: continue _write_nlu_yaml(source_path, output_path, source_path) num_of_files_converted += 1 elif not is_nlu and MarkdownStoryReader.is_markdown_story_file(source_path): _write_core_yaml(source_path, output_path, source_path) num_of_files_converted += 1 else: print_warning(f"Skipped file: '{source_path}'.") print_info(f"Converted {num_of_files_converted} file(s), saved in '{output}'.")
def _convert_nlu_data(args: argparse.Namespace) -> None: if args.format in ["json", "md"]: convert_training_data(args.data, args.out, args.format, args.language) elif args.format == "yaml": _convert_to_yaml(args, True) else: print_error_and_exit( "Could not recognize output format. Supported output formats: 'json', " "'md', 'yaml'. Specify the desired output format with '--format'.")
def _validate_domain(domain_path: Text): from rasa.core.domain import Domain, InvalidDomain try: Domain.load(domain_path) except InvalidDomain as e: cli_utils.print_error_and_exit( "The provided domain file could not be loaded. " "Error: {}".format(e))
def _convert_nlg_data(args: argparse.Namespace) -> None: from rasa.nlu.training_data.converters.nlg_markdown_to_yaml_converter import ( NLGMarkdownToYamlConverter, ) if args.format == "yaml": _convert_to_yaml(args, NLGMarkdownToYamlConverter()) else: print_error_and_exit( "Could not recognize output format. Supported output formats: " "'yaml'. Specify the desired output format with '--format'.")
def _convert_nlu_data(args: argparse.Namespace) -> None: from rasa.nlu.training_data.converters.nlu_markdown_to_yaml_converter import ( NLUMarkdownToYamlConverter, ) if args.format in ["json", "md"]: convert_training_data(args.data, args.out, args.format, args.language) elif args.format == "yaml": _convert_to_yaml(args, NLUMarkdownToYamlConverter()) else: print_error_and_exit( "Could not recognize output format. Supported output formats: 'json', " "'md', 'yaml'. Specify the desired output format with '--format'.")
def accept_terms_or_quit(args: argparse.Namespace) -> None: """Prompt the user to accept the Rasa terms.""" import webbrowser import questionary from rasax.community.constants import RASA_TERMS_URL show_prompt = not hasattr(args, "no_prompt") or not args.no_prompt if not show_prompt: print( f"By adding the '--no_prompt' parameter you agreed to the Rasa " f"X license agreement ({RASA_TERMS_URL})" ) return rasa_cli_utils.print_success( "Before you can use Rasa X, you have to agree to its " "license agreement (you will only have to do this " "once)." ) should_open_in_browser = questionary.confirm( "Would you like to view the license agreement in your web browser?" ).ask() if should_open_in_browser: webbrowser.open(RASA_TERMS_URL) accepted_terms = questionary.confirm( "\nRasa X License Agreement\n" "===========================\n\n" "Do you agree to the Rasa X license agreement ({})?\n" "By typing 'y', you agree to the terms. " "If you are using this software for a company, by confirming, " "you acknowledge you have the authority to do so.\n" "If you do not agree, type 'n' to stop Rasa X." "".format(RASA_TERMS_URL), default=False, qmark="", ).ask() if accepted_terms: rasa_utils.write_global_config_value(CONFIG_FILE_TERMS_KEY, True) else: rasa_cli_utils.print_error_and_exit( "Sorry, without accepting the terms, you cannot use Rasa X. " "You can of course still use the (Apache 2 licensed) Rasa framework: " "https://github.com/RasaHQ/rasa", exit_code=0, )
def session_scope(self): """Provide a transactional scope around a series of operations.""" session = self.sessionmaker() try: ensure_schema_exists(session) yield session except ValueError as e: rasa_cli_utils.print_error_and_exit( f"Requested PostgreSQL schema '{e}' was not found in the database. To " f"continue, please create the schema by running 'CREATE DATABASE {e};' " f"or unset the '{POSTGRESQL_SCHEMA}' environment variable in order to " f"use the default schema. Exiting application.") finally: session.close()
def perform_interactive_learning(args: argparse.Namespace, zipped_model: Text, file_importer: TrainingDataImporter) -> None: from rasa.core.train import do_interactive_learning args.model = zipped_model with model.unpack_model(zipped_model) as model_path: args.core, args.nlu = model.get_model_subdirectories(model_path) if args.core is None: utils.print_error_and_exit( "Can not run interactive learning on an NLU-only model.") args.endpoints = utils.get_validated_path(args.endpoints, "endpoints", DEFAULT_ENDPOINTS_PATH, True) do_interactive_learning(args, file_importer)
def run(args: argparse.Namespace) -> None: import questionary print_success("Welcome to Rasa! 🤖\n") if args.no_prompt: print( "To get started quickly, an " "initial project will be created.\n" "If you need some help, check out " "the documentation at {}.\n".format(DOCS_BASE_URL) ) else: print( "To get started quickly, an " "initial project will be created.\n" "If you need some help, check out " "the documentation at {}.\n" "Now let's start! 👇🏽\n".format(DOCS_BASE_URL) ) if args.init_dir is not None: path = args.init_dir else: path = ( questionary.text( "Please enter a path where the project will be " "created [default: current directory]", default=".", ) .skip_if(args.no_prompt, default=".") .ask() ) if args.no_prompt and not os.path.isdir(path): print_error_and_exit(f"Project init path '{path}' not found.") if path and not os.path.isdir(path): _ask_create_path(path) if path is None or not os.path.isdir(path): print_cancel() if not args.no_prompt and len(os.listdir(path)) > 0: _ask_overwrite(path) init_project(args, path)
def _assert_max_timestamp_is_greater_than_min_timestamp( args: argparse.Namespace, ) -> None: """Inspect CLI timestamp parameters. Prints an error and exits if a maximum timestamp is provided that is smaller than the provided minimum timestamp. Args: args: Command-line arguments to process. """ min_timestamp = args.minimum_timestamp max_timestamp = args.maximum_timestamp if (min_timestamp is not None and max_timestamp is not None and max_timestamp < min_timestamp): cli_utils.print_error_and_exit( f"Maximum timestamp '{max_timestamp}' is smaller than minimum " f"timestamp '{min_timestamp}'. Exiting.")
async def _pull_runtime_config_from_server( config_endpoint: Optional[Text], attempts: int = 60, wait_time_between_pulls: Union[int, float] = 5, keys: Iterable[Text] = ("endpoints", "credentials"), ) -> Optional[List[Text]]: """Pull runtime config from `config_endpoint`. Returns a list of paths to yaml dumps, each containing the contents of one of `keys`. """ while attempts: try: async with aiohttp.ClientSession() as session: async with session.get(config_endpoint) as resp: if resp.status == 200: rjs = await resp.json() try: return [ io_utils.create_temporary_file(rjs[k]) for k in keys ] except KeyError as e: cli_utils.print_error_and_exit( "Failed to find key '{}' in runtime config. " "Exiting.".format(e) ) else: logger.debug( "Failed to get a proper response from remote " "server. Status Code: {}. Response: '{}'" "".format(resp.status, await resp.text()) ) except aiohttp.ClientError as e: logger.debug(f"Failed to connect to server. Retrying. {e}") await asyncio.sleep(wait_time_between_pulls) attempts -= 1 cli_utils.print_error_and_exit( "Could not fetch runtime config from server at '{}'. " "Exiting.".format(config_endpoint) )
def export_trackers(args: argparse.Namespace) -> None: """Export events for a connected tracker store using an event broker. Args: args: Command-line arguments to process. """ _assert_max_timestamp_is_greater_than_min_timestamp(args) endpoints = rasa_core_utils.read_endpoints_from_path(args.endpoints) tracker_store = _get_tracker_store(endpoints) event_broker = _get_event_broker(endpoints) _prepare_event_broker(event_broker) requested_conversation_ids = _get_requested_conversation_ids(args.conversation_ids) from rasa.core.exporter import Exporter exporter = Exporter( tracker_store, event_broker, args.endpoints, requested_conversation_ids, args.minimum_timestamp, args.maximum_timestamp, ) try: published_events = exporter.publish_events() cli_utils.print_success( f"Done! Successfully published {published_events} events 🎉" ) except PublishingError as e: command = _get_continuation_command(exporter, e.timestamp) cli_utils.print_error_and_exit( f"Encountered error while publishing event with timestamp '{e}'. To " f"continue where I left off, run the following command:" f"\n\n\t{command}\n\nExiting." ) except RasaException as e: cli_utils.print_error_and_exit(str(e))
def _validate_rasa_x_start(args: argparse.Namespace, project_path: Text): if not is_rasa_x_installed(): cli_utils.print_error_and_exit( "Rasa X is not installed. The `rasa x` " "command requires an installation of Rasa X. " "Instructions on how to install Rasa X can be found here: " "https://rasa.com/docs/rasa-x/installation-and-setup/.") if args.port == args.rasa_x_port: cli_utils.print_error_and_exit( "The port for Rasa X '{}' and the port of the Rasa server '{}' are the " "same. We need two different ports, one to run Rasa X (e.g. delivering the " "UI) and another one to run a normal Rasa server.\nPlease specify two " "different ports using the arguments '--port' and '--rasa-x-port'." .format(args.rasa_x_port, args.port)) if not is_rasa_project_setup(project_path): cli_utils.print_error_and_exit( "This directory is not a valid Rasa project. Use 'rasa init' " "to create a new Rasa project or switch to a valid Rasa project " "directory (see http://rasa.com/docs/rasa/user-guide/" "rasa-tutorial/#create-a-new-project).") _validate_domain(os.path.join(project_path, DEFAULT_DOMAIN_PATH)) if args.data and not os.path.exists(args.data): cli_utils.print_warning( "The provided data path ('{}') does not exists. Rasa X will start " "without any training data.".format(args.data))
def interactive(args: argparse.Namespace) -> None: _set_not_required_args(args) file_importer = TrainingDataImporter.load_from_config( args.config, args.domain, args.data ) if args.model is None: loop = asyncio.get_event_loop() story_graph = loop.run_until_complete(file_importer.get_stories()) if not story_graph or story_graph.is_empty(): utils.print_error_and_exit( "Could not run interactive learning without either core data or a model containing core data." ) zipped_model = train.train_core(args) if args.core_only else train.train(args) if not zipped_model: utils.print_error_and_exit( "Could not train an initial model. Either pass paths " "to the relevant training files (`--data`, `--config`, `--domain`), " "or use 'rasa train' to train a model." ) else: zipped_model = get_provided_model(args.model) if not (zipped_model and os.path.exists(zipped_model)): utils.print_error_and_exit( f"Interactive learning process cannot be started as no initial model was " f"found at path '{args.model}'. Use 'rasa train' to train a model." ) if not args.skip_visualization: logger.info(f"Loading visualization data from {args.data}.") perform_interactive_learning(args, zipped_model, file_importer)
def _get_event_broker(endpoints: "AvailableEndpoints") -> Optional["EventBroker"]: """Get `EventBroker` from `endpoints`. Prints an error and exits if no event broker could be loaded. Args: endpoints: `AvailableEndpoints` to initialize the event broker from. Returns: Initialized event broker. """ if not endpoints.event_broker: cli_utils.print_error_and_exit( f"Could not find an `event_broker` section in the supplied " f"endpoints file. Instructions on how to configure an event broker " f"can be found here: {DOCS_URL_EVENT_BROKERS}. Exiting." ) from rasa.core.brokers.broker import EventBroker return EventBroker.create(endpoints.event_broker)
def _get_tracker_store(endpoints: "AvailableEndpoints") -> "TrackerStore": """Get `TrackerStore` from `endpoints`. Prints an error and exits if no tracker store could be loaded. Args: endpoints: `AvailableEndpoints` to initialize the tracker store from. Returns: Initialized tracker store. """ if not endpoints.tracker_store: cli_utils.print_error_and_exit( f"Could not find a `tracker_store` section in the supplied " f"endpoints file. Instructions on how to configure a tracker store " f"can be found here: {DOCS_URL_TRACKER_STORES}. " f"Exiting. ") from rasa.core.tracker_store import TrackerStore return TrackerStore.create(endpoints.tracker_store)
def _prepare_event_broker(event_broker: "EventBroker") -> None: """Sets `should_keep_unpublished_messages` flag to `False` if `self.event_broker` is a `PikaEventBroker`. If publishing of events fails, the `PikaEventBroker` instance should not keep a list of unpublished messages, so we can retry publishing them. This is because the instance is launched as part of this short-lived export script, meaning the object is destroyed before it might be published. In addition, wait until the event broker reports a `ready` state. """ from rasa.core.brokers.pika import PikaEventBroker if isinstance(event_broker, PikaEventBroker): event_broker.should_keep_unpublished_messages = False event_broker.raise_on_failure = True if not event_broker.is_ready(): cli_utils.print_error_and_exit( f"Event broker of type '{type(event_broker)}' is not ready. Exiting." )
def _fetch_and_verify_jwt_keys_from_file( private_key_path: Text = config.jwt_private_key_path, public_key_path: Text = config.jwt_public_key_path, ) -> Tuple[bytes, bytes]: """Load the public and private JWT key files and verify them.""" try: private_key = file_as_bytes(private_key_path) public_key = file_as_bytes(public_key_path) _verify_keys(private_key, public_key) return private_key, public_key except FileNotFoundError as e: error_message = f"Could not find key file. Error: '{e}'" except ValueError as e: error_message = ( "Failed to load key data. Make sure the key " "files are enclosed with the " "'-----BEGIN PRIVATE KEY-----' etc. tags. Error: '{}'".format(e)) except InvalidSignatureError as e: error_message = f"Failed to verify key signature. Error: '{e}'" except Exception as e: error_message = f"Encountered error trying to verify JWT keys: '{e}'" rasa_cli_utils.print_error_and_exit(error_message)