async def save_from_path(self,
path: Text,
bot: Text,
overwrite: bool = True,
user="******"):
try:
story_files, nlu_files = get_core_nlu_files(
os.path.join(path, DEFAULT_DATA_PATH))
nlu = utils.training_data_from_paths(nlu_files, "en")
domain = Domain.from_file(os.path.join(path, DEFAULT_DOMAIN_PATH))
domain.check_missing_templates()
story_steps = await StoryFileReader.read_from_files(
story_files, domain)
config = read_config_file(os.path.join(path, DEFAULT_CONFIG_PATH))
self.save_domain(domain, bot, user)
self.save_stories(story_steps, bot, user)
self.save_nlu(nlu, bot, user)
self.save_config(config, bot, user)
except InvalidDomain as e:
logging.info(e)
raise AppException("""Failed to validate yaml file.
Please make sure the file is initial and all mandatory parameters are specified"""
)
except Exception as e:
logging.info(e)
raise AppException(e)
def __init__(
self,
config_file: Text,
domain_path: Optional[Text] = None,
training_data_paths: Optional[Union[List[Text], Text]] = None,
project_directory: Optional[Text] = None,
):
self.config = io_utils.read_config_file(config_file)
if domain_path:
self._domain_paths = [domain_path]
else:
self._domain_paths = []
self._story_paths = []
self._nlu_paths = []
self._imports = []
self._additional_paths = training_data_paths or []
self._project_directory = project_directory or os.path.dirname(
config_file)
self._init_from_dict(self.config, self._project_directory)
extra_story_files, extra_nlu_files = data.get_core_nlu_files(
training_data_paths)
self._story_paths += list(extra_story_files)
self._nlu_paths += list(extra_nlu_files)
logger.debug("Selected projects: {}".format("".join(
[f"\n-{i}" for i in self._imports])))
rasa.utils.common.mark_as_experimental_feature(
feature_name="MultiProjectImporter")
def __init__(
self,
config_file: Text,
domain_path: Optional[Text] = None,
training_data_paths: Optional[Union[List[Text], Text]] = None,
project_directory: Optional[Text] = None,
):
self.config = io_utils.read_config_file(config_file)
if domain_path:
self._domain_paths = [domain_path]
else:
self._domain_paths = []
self._story_paths = []
self._nlu_paths = []
self._imports = set()
self._additional_paths = training_data_paths or []
self._project_directory = project_directory or os.path.dirname(
config_file)
self._init_from_dict(self.config, self._project_directory)
extra_story_files, extra_nlu_files = data.get_core_nlu_files(
training_data_paths)
self._story_paths += list(extra_story_files)
self._nlu_paths += list(extra_nlu_files)
logger.debug("Selected skills: {}".format("".join(
["\n-{}".format(i) for i in self._imports])))
def test_get_core_nlu_files(project):
data_dir = os.path.join(project, "data")
core_files, nlu_files = data.get_core_nlu_files([data_dir])
assert len(nlu_files) == 1
assert list(nlu_files)[0].endswith("nlu.md")
assert len(core_files) == 1
assert list(core_files)[0].endswith("stories.md")
def test_get_core_nlu_files(project):
data_dir = os.path.join(project, "data")
core_files, nlu_files = data.get_core_nlu_files([data_dir])
assert len(nlu_files) == 1
assert list(nlu_files)[0].endswith("nlu.yml")
assert len(core_files) == 2
assert any(file.endswith("stories.yml") for file in core_files)
assert any(file.endswith("rules.yml") for file in core_files)
def check_training_data(args):
training_files = [
get_validated_path(f, "data", DEFAULT_DATA_PATH, none_is_valid=True)
for f in args.data
]
story_files, nlu_files = data.get_core_nlu_files(training_files)
if not story_files or not nlu_files:
print_error(
"Cannot train initial Rasa model. Please provide NLU and Core data "
"using the '--data' argument.")
exit(1)
def _get_train_files_cmd():
"""Get the raw train data by fetching the train file given in the
command line arguments to the train script. When training the NLU model
explicitly, the training data will be in the "nlu" argument, otherwise
it will be in the "data" argument.
"""
cmdline_args = create_argument_parser().parse_args()
try:
files = list_files(cmdline_args.nlu)
except AttributeError:
files = list(get_core_nlu_files(cmdline_args.data)[1])
return [file for file in files if _guess_format(file) == RASA_NLU]
def __init__(
self,
config_file: Optional[Text] = None,
domain_path: Optional[Text] = None,
training_data_paths: Optional[Union[List[Text], Text]] = None,
):
self._domain_path = domain_path
self._story_files, self._nlu_files = data.get_core_nlu_files(
training_data_paths
)
self.config = autoconfig.get_configuration(config_file)
def __init__(
self,
config_file: Optional[Text] = None,
domain_path: Optional[Text] = None,
training_data_paths: Optional[Union[List[Text], Text]] = None,
):
if config_file and os.path.exists(config_file):
self.config = io_utils.read_config_file(config_file)
else:
self.config = {}
self._domain_path = domain_path
self._story_files, self._nlu_files = data.get_core_nlu_files(
training_data_paths)
def save_from_path(self, path: Text, bot: Text, user="******"):
try:
story_files, nlu_files = get_core_nlu_files(
os.path.join(path, DEFAULT_DATA_PATH))
nlu = utils.training_data_from_paths(nlu_files, "en")
domain = Domain.from_file(os.path.join(path, DEFAULT_DOMAIN_PATH))
loop = asyncio.new_event_loop()
story_steps = loop.run_until_complete(
StoryFileReader.read_from_files(story_files, domain))
self.save_domain(domain, bot, user)
self.save_stories(story_steps, bot, user)
self.save_nlu(nlu, bot, user)
self.__save_config(
read_config_file(os.path.join(path, DEFAULT_CONFIG_PATH)), bot,
user)
except InvalidDomain as e:
logging.info(e)
raise AppException("""Failed to validate yaml file.
Please make sure the file is initial and all mandatory parameters are specified"""
)
except Exception as e:
logging.info(e)
raise AppException(e)
def __init__(
self,
config_file: Optional[Union[List[Text], Text]] = None,
domain_path: Optional[Text] = None,
training_data_paths: Optional[Union[List[Text], Text]] = None,
):
self._domain_path = domain_path
self._story_files, self._nlu_files = data.get_core_nlu_files(
training_data_paths)
self.core_config = {}
self.nlu_config = {}
if config_file:
if not isinstance(config_file, list): config_file = [config_file]
for file in config_file:
if not os.path.exists(file): continue
config = io_utils.read_config_file(file)
lang = config["language"]
self.core_config = {"policies": config["policies"]}
self.nlu_config[lang] = {
"pipeline": config["pipeline"],
"language": lang
}
def test_find_nlu_files_with_different_formats(test_input, expected):
examples_dir = "data/examples"
data_dir = os.path.join(examples_dir, test_input)
core_files, nlu_files = data.get_core_nlu_files([data_dir])
assert nlu_files == expected
def test_get_story_file_with_yaml():
examples_dir = "data/test_yaml_stories"
core_files, nlu_files = data.get_core_nlu_files([examples_dir])
assert core_files
