def test_dump_config( tmp_path: Path, input_file: Text, expected_file: Text, capsys: CaptureFixture, autoconfig_keys: Set[Text], ): config_file = str(tmp_path / "config.yml") shutil.copyfile(str(CONFIG_FOLDER / input_file), config_file) autoconfig.get_configuration(config_file) actual = io_utils.read_file(config_file) expected = io_utils.read_file(str(CONFIG_FOLDER / expected_file)) assert actual == expected captured = capsys.readouterr() assert "does not exist or is empty" not in captured.out for k in CONFIG_AUTOCONFIGURABLE_KEYS: if k in autoconfig_keys: assert k in captured.out else: assert k not in captured.out
def guess_format(filename: Text) -> Text: """Applies heuristics to guess the data format of a file. Args: filename: file whose type should be guessed Returns: Guessed file format. """ from rasa.nlu.training_data.formats import RasaYAMLReader, markdown guess = UNK content = "" try: content = io_utils.read_file(filename) js = json.loads(content) except ValueError: if any(marker in content for marker in markdown.MARKDOWN_SECTION_MARKERS): guess = MARKDOWN elif _is_nlg_story_format(content): guess = MARKDOWN_NLG elif RasaYAMLReader.is_yaml_nlu_file(filename): guess = RASA_YAML else: for fformat, format_heuristic in _json_format_heuristics.items(): if format_heuristic(js, filename): guess = fformat break logger.debug(f"Training data format of '{filename}' is '{guess}'.") return guess
async def read_from_file(self, filename: Union[Text, Path]) -> List[StoryStep]: """Read stories or rules from file. Args: filename: Path to the story/rule file. Returns: `StoryStep`s read from `filename`. """ self.source_name = filename try: file_content = io_utils.read_file(filename, io_utils.DEFAULT_ENCODING) validate_yaml_schema(file_content, CORE_SCHEMA_FILE) yaml_content = io_utils.read_yaml(file_content) except (ValueError, ParserError) as e: common_utils.raise_warning( f"Failed to read YAML from '{filename}', it will be skipped. Error: {e}" ) return [] except InvalidYamlFileError as e: raise ValueError from e return self.read_from_parsed_yaml(yaml_content)
def guess_format(filename: Text) -> Text: """Applies heuristics to guess the data format of a file. Args: filename: file whose type should be guessed Returns: Guessed file format. """ guess = UNK content = "" try: content = io_utils.read_file(filename) js = json.loads(content) except ValueError: if any([marker in content for marker in _markdown_section_markers]): guess = MARKDOWN elif _is_nlg_story_format(content): guess = MARKDOWN_NLG else: for fformat, format_heuristic in _json_format_heuristics.items(): if format_heuristic(js, filename): guess = fformat break logger.debug("Training data format of '{}' is '{}'.".format( filename, guess)) return guess
def inject_domain( domain_path: Text, domain_service: "DomainService", project_id: Text = config.project_name, username: Text = config.default_username, ) -> Dict[Text, Any]: """Load Rasa Core domain at `path` and save it to database. Quits the application if domain cannot be loaded. """ if not os.path.exists(domain_path): rasa_cli_utils.print_error_and_exit( f"domain.yml could not be found at '{os.path.abspath(domain_path)}'. " f"Rasa X requires a domain in the project root directory.") try: domain_service.validate_and_store_domain_yaml( domain_yaml=read_file(domain_path), project_id=project_id, path=domain_path, store_responses=True, username=username, should_dump_domain=False, ) except InvalidDomain as e: rasa_cli_utils.print_error_and_exit( f"Could not inject domain. Details:\n{e}") return domain_service.get_or_create_domain(project_id)
def test_nlu(args: argparse.Namespace) -> None: from rasa import data from rasa.test import compare_nlu_models, perform_nlu_cross_validation, test_nlu nlu_data = cli_utils.get_validated_path(args.nlu, "nlu", DEFAULT_DATA_PATH) nlu_data = data.get_nlu_directory(nlu_data) output = args.out or DEFAULT_RESULTS_PATH io_utils.create_directory(output) if args.config is not None and len(args.config) == 1: args.config = os.path.abspath(args.config[0]) if os.path.isdir(args.config): config_dir = args.config config_files = os.listdir(config_dir) args.config = [ os.path.join(config_dir, os.path.abspath(config)) for config in config_files ] if isinstance(args.config, list): logger.info( "Multiple configuration files specified, running nlu comparison mode." ) config_files = [] for file in args.config: try: validation_utils.validate_yaml_schema( io_utils.read_file(file), CONFIG_SCHEMA_FILE, show_validation_errors=False, ) config_files.append(file) except validation_utils.InvalidYamlFileError: logger.debug( "Ignoring file '{}' as it is not a valid config file.". format(file)) continue compare_nlu_models( configs=config_files, nlu=nlu_data, output=output, runs=args.runs, exclusion_percentages=args.percentages, ) elif args.cross_validation: logger.info("Test model using cross validation.") config = cli_utils.get_validated_path(args.config, "config", DEFAULT_CONFIG_PATH) perform_nlu_cross_validation(config, nlu_data, output, vars(args)) else: model_path = cli_utils.get_validated_path(args.model, "model", DEFAULT_MODELS_PATH) test_nlu(model_path, nlu_data, output, vars(args))
def test_write_utf_8_yaml_file(tmp_path: Path): """This test makes sure that dumping a yaml doesn't result in Uxxxx sequences but rather directly dumps the unicode character.""" file_path = str(tmp_path / "test.yml") data = {"data": "amazing 🌈"} io_utils.write_yaml_file(data, file_path) assert io_utils.read_file(file_path) == "data: amazing 🌈\n"
def json_unpickle(file_name: Text) -> Any: """Unpickle an object from file using json.""" import jsonpickle.ext.numpy as jsonpickle_numpy import jsonpickle jsonpickle_numpy.register_handlers() file_content = io_utils.read_file(file_name) return jsonpickle.loads(file_content)
def get_private_ssh_key() -> Text: """Get the private key which Rasa X can use to authenticate with a Git server. Returns: A private SSH key which can be used together with the public key which the user has to store at their Git Server. """ if not Path(GIT_GENERATED_PRIVATE_KEY_PATH).exists(): raise ValueError("Please request the public key first.") return io_utils.read_file(GIT_GENERATED_PRIVATE_KEY_PATH)
def get_public_ssh_key() -> Text: """Get a public SSH key which can be used to setup Integrated Version Control. Returns: A public SSH key which has to be provided to the Git server so that Rasa X can authenticate with its private key. """ if not Path(GIT_GENERATED_PRIVATE_KEY_PATH).exists(): GitSSHKeyProvider.generate_new_ssh_keys() return io_utils.read_file(GIT_GENERATED_PUBLIC_KEY_PATH)
def test_get_configuration_for_different_training_types( tmp_path: Path, input_file: Text, expected_file: Text, expected_file_windows: Text, training_type: autoconfig.TrainingType, ): config_file = str(tmp_path / "config.yml") shutil.copyfile(str(CONFIG_FOLDER / input_file), config_file) autoconfig.get_configuration(config_file, training_type) actual = io_utils.read_file(config_file) if sys.platform == "win32": expected = io_utils.read_file( str(CONFIG_FOLDER / expected_file_windows)) else: expected = io_utils.read_file(str(CONFIG_FOLDER / expected_file)) assert actual == expected
async def test_pull_runtime_config_from_server(): config_url = "http://example.com/api/config?token=token" credentials = "rasa: http://example.com:5002/api" endpoint_config = """ event_broker: url: http://example.com/event_broker username: some_username password: PASSWORD queue: broker_queue """ with aioresponses() as mocked: mocked.get( config_url, payload={ "credentials": credentials, "endpoints": endpoint_config }, ) endpoints_path, credentials_path = await x._pull_runtime_config_from_server( config_url, 1, 0) assert io_utils.read_file(endpoints_path) == endpoint_config assert io_utils.read_file(credentials_path) == credentials
def load(path: Text) -> Optional["TrackerFeaturizer"]: """Load the featurizer from file. Args: path: The path to load the tracker featurizer from. Returns: The loaded tracker featurizer. """ featurizer_file = os.path.join(path, "featurizer.json") if os.path.isfile(featurizer_file): return jsonpickle.decode(io_utils.read_file(featurizer_file)) logger.error(f"Couldn't load featurizer for policy. " f"File '{featurizer_file}' doesn't exist.") return None
def persist_graph(graph: "networkx.Graph", output_file: Text) -> None: """Plots the graph and persists it into a html file.""" import networkx as nx import rasa.utils.io as io_utils expg = nx.nx_pydot.to_pydot(graph) template = io_utils.read_file(visualization_html_path()) # Insert graph into template template = template.replace("// { is-client }", "isClient = true", 1) graph_as_text = expg.to_string() # escape backslashes graph_as_text = graph_as_text.replace("\\", "\\\\") template = template.replace("// { graph-content }", f"graph = `{graph_as_text}`", 1) io_utils.write_text_file(template, output_file)
def get_tests_from_file(filename: Optional[Text] = None) -> List[Text]: """Returns an list of tests from a `filename`. Args: filename: Path to a test file. """ if not filename: filename = utils.get_project_directory() / DEFAULT_FILENAME try: content = io_utils.read_file(filename) return _split_tests(content) except ValueError as e: rasa_utils.raise_warning( f"Unable to get tests from {filename}:\n{e} " f"Please, make sure you have end-to-end tests added to your assistant. " f"See https://rasa.com/docs/rasa/user-guide/testing-your-assistant/ " f"for more information.", UserWarning, ) return []
def extract_partial_endpoint_config( endpoint_config_path: Text, key: Text ) -> Optional["EndpointConfig"]: """Extracts partial endpoint config at `key`. Args: endpoint_config_path: Path to endpoint config file to read. key: Endpoint config key (section) to extract. Returns: Endpoint config initialised only from `key`. """ from rasa.utils import endpoints # read endpoint config file and create dictionary containing only one # key-value pair content = rasa_io_utils.read_file(endpoint_config_path) endpoint_dict = {key: load_yaml(content)[key]} # dump this sub-dictionary to a temporary file and load endpoint config from it temp_path = dump_as_yaml_to_temporary_file(endpoint_dict) return endpoints.read_endpoint_config(temp_path, key)
def test_validate_yaml_schema(file, schema): # should raise no exception validation_utils.validate_yaml_schema(io_utils.read_file(file), schema)
def read(self, filename: Text, **kwargs: Any) -> "TrainingData": """Reads TrainingData from a file.""" self.filename = filename return self.reads(io_utils.read_file(filename), **kwargs)
def test_read_file_with_not_existing_path(): with pytest.raises(ValueError): io_utils.read_file("some path")
def is_markdown_nlu_file(filename: Union[Text, Path]) -> bool: content = io_utils.read_file(filename) return any(marker in content for marker in MARKDOWN_SECTION_MARKERS)
def test_validate_yaml_schema_raise_exception(file, schema): with pytest.raises(validation_utils.InvalidYamlFileError): validation_utils.validate_yaml_schema(io_utils.read_file(file), schema)
def _read_data(paths: List[Text]) -> Generator[Tuple[Text, Text], None, None]: for filename in paths: try: yield read_file(filename), filename except ValueError: rasa_cli_utils.print_warning(f"Cannot read file {filename}")