async def test_rasa_packaging( trained_rasa_model: Text, project: Text, use_fingerprint: bool, tmp_path: Path ): unpacked_model_path = get_model(trained_rasa_model) os.remove(os.path.join(unpacked_model_path, FINGERPRINT_FILE_PATH)) if use_fingerprint: fingerprint = await model_fingerprint(_project_files(project)) else: fingerprint = None output_path = str(tmp_path / "test.tar.gz") create_package_rasa(unpacked_model_path, output_path, fingerprint) unpacked = get_model(output_path) assert ( os.path.exists(os.path.join(unpacked, FINGERPRINT_FILE_PATH)) == use_fingerprint ) assert os.path.exists(os.path.join(unpacked, DEFAULT_CORE_SUBDIRECTORY_NAME)) assert os.path.exists(os.path.join(unpacked, "nlu-en")) # bf assert os.path.exists(os.path.join(unpacked, "nlu-fr")) # bf assert not os.path.exists(unpacked_model_path)
def train_nlu(config: Text, nlu_data: Text, output: Text, train_path: Optional[Text]) -> Optional["Interpreter"]: """Trains a NLU model. Args: config: Path to the config file for NLU. nlu_data: Path to the NLU training data. output: Output path. train_path: If `None` the model will be trained in a temporary directory, otherwise in the provided directory. Returns: If `train_path` is given it returns the path to the model archive, otherwise the path to the directory with the trained model files. """ import rasa_nlu _train_path = train_path or tempfile.mkdtemp() _, nlu_model, _ = rasa_nlu.train(config, nlu_data, _train_path, project="", fixed_model_name="nlu") if not train_path: nlu_data = data.get_nlu_directory(nlu_data) output_path = create_output_path(output, prefix="nlu-") new_fingerprint = model.model_fingerprint(config, nlu_data=nlu_data) model.create_package_rasa(_train_path, output_path, new_fingerprint) print_success("Your Rasa NLU model is trained and saved at '{}'." "".format(output_path)) return nlu_model
def trained_nlu_model(request): cfg = RasaNLUModelConfig({"pipeline": "keyword"}) trainer = Trainer(cfg) td = training_data.load_data(DEFAULT_DATA_PATH) trainer.train(td) model_path = trainer.persist(NLU_MODEL_PATH) nlu_data = data.get_nlu_directory(DEFAULT_DATA_PATH) output_path = os.path.join(NLU_MODEL_PATH, NLU_MODEL_NAME) new_fingerprint = model.model_fingerprint(NLU_DEFAULT_CONFIG_PATH, nlu_data=nlu_data) model.create_package_rasa(model_path, output_path, new_fingerprint) def fin(): if os.path.exists(NLU_MODEL_PATH): shutil.rmtree(NLU_MODEL_PATH) if os.path.exists(output_path): shutil.rmtree(output_path) request.addfinalizer(fin) return output_path
async def train(request): # if set will not generate a model name but use the passed one model_name = request.args.get("model", None) try: model_config, data_dict = extract_data_and_config(request) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse( 500, "ServerError", "An unexpected error occurred.", details={"error": str(e)}, ) data_file = dump_to_data_file(data_dict) config_file = dump_to_data_file(model_config, "_config") try: path_to_model = await data_router.start_train_process( data_file, RasaNLUModelConfig(model_config), model_name) # store trained model as tar.gz file output_path = create_model_path(model_name, path_to_model) nlu_data = data.get_nlu_directory(data_file) new_fingerprint = model.model_fingerprint(config_file, nlu_data=nlu_data) model.create_package_rasa(path_to_model, output_path, new_fingerprint) logger.info("Rasa NLU model trained and persisted to '{}'.".format( output_path)) await data_router.load_model(output_path) return await response.file(output_path) except MaxWorkerProcessError as e: raise ErrorResponse( 403, "NoFreeProcess", "No process available for training.", details={"error": str(e)}, ) except InvalidModelError as e: raise ErrorResponse( 404, "ModelNotFound", "Model '{}' not found.".format(model_name), details={"error": str(e)}, ) except TrainingException as e: logger.debug(traceback.format_exc()) raise ErrorResponse( 500, "ServerError", "An unexpected error occurred.", details={"error": str(e)}, )
def train_nlu(config: Text, nlu_data: Text, output: Text, train_path: Optional[Text]) -> Optional[Text]: """Trains a NLU model. Args: config: Path to the config file for NLU. nlu_data: Path to the NLU training data. output: Output path. train_path: If `None` the model will be trained in a temporary directory, otherwise in the provided directory. Returns: If `train_path` is given it returns the path to the model archive, otherwise the path to the directory with the trained model files. """ import rasa.nlu.train config = get_valid_config(config, CONFIG_MANDATORY_KEYS_NLU) if not train_path: # training NLU only hence the training files still have to be selected skill_imports = SkillSelector.load(config) nlu_data_directory = data.get_nlu_directory(nlu_data, skill_imports) else: nlu_data_directory = nlu_data if not os.listdir(nlu_data_directory): print_error( "No NLU data given. Please provide NLU data in order to train " "a Rasa NLU model.") return _train_path = train_path or tempfile.mkdtemp() print_color("Start training NLU model ...", color=bcolors.OKBLUE) _, nlu_model, _ = rasa.nlu.train(config, nlu_data_directory, _train_path, fixed_model_name="nlu") print_color("Done.", color=bcolors.OKBLUE) if not train_path: output_path = create_output_path(output, prefix="nlu-") new_fingerprint = model.model_fingerprint(config, nlu_data=nlu_data_directory) model.create_package_rasa(_train_path, output_path, new_fingerprint) print_success( "Your Rasa NLU model is trained and saved at '{}'.".format( output_path)) return output_path return _train_path
def set_fingerprint(trained_rasa_model: Text, fingerprint: Fingerprint, tmp_path: Path) -> Text: unpacked_model_path = get_model(trained_rasa_model) os.remove(os.path.join(unpacked_model_path, FINGERPRINT_FILE_PATH)) output_path = str(tmp_path / "test.tar.gz") create_package_rasa(unpacked_model_path, output_path, fingerprint) return output_path
def set_fingerprint(trained_rasa_model: Text, fingerprint: Fingerprint) -> Text: unpacked_model_path = get_model(trained_rasa_model) os.remove(os.path.join(unpacked_model_path, FINGERPRINT_FILE_PATH)) tempdir = tempfile.mkdtemp() output_path = os.path.join(tempdir, "test.tar.gz") create_package_rasa(unpacked_model_path, output_path, fingerprint) return output_path
async def _train_core_with_validated_data( domain: Domain, config: Text, story_directory: Text, output: Text, train_path: Optional[Text] = None, fixed_model_name: Optional[Text] = None, uncompress: bool = False, kwargs: Optional[Dict] = None, ) -> Optional[Text]: """Train Core with validated training and config data.""" import rasa.core.train if not os.listdir(story_directory): print_error( "No dialogue data given. Please provide dialogue data in order to " "train a Rasa Core model.") return _train_path = train_path or tempfile.mkdtemp() # normal (not compare) training print_color("Start training dialogue model ...", color=bcolors.OKBLUE) await rasa.core.train( domain_file=domain, stories_file=story_directory, output_path=os.path.join(_train_path, "core"), policy_config=config, kwargs=kwargs, ) print_color("Done.", color=bcolors.OKBLUE) if not train_path: # Only Core was trained. output_path = create_output_path(output, prefix="core-", fixed_name=fixed_model_name) new_fingerprint = model.model_fingerprint(config, domain, stories=story_directory) model.create_package_rasa(_train_path, output_path, new_fingerprint) if uncompress: output_path = decompress(output_path) print_success( "Your Rasa Core model is trained and saved at '{}'.".format( output_path)) return output_path return _train_path
async def train_core_async( domain: Text, config: Text, stories: Text, output: Text, train_path: Optional[Text] = None, kwargs: Optional[Dict] = None, ) -> Optional[Text]: """Trains a Core model. Args: domain: Path to the domain file. config: Path to the config file for Core. stories: Path to the Core training data. output: Output path. train_path: If `None` the model will be trained in a temporary directory, otherwise in the provided directory. kwargs: Additional training parameters. Returns: If `train_path` is given it returns the path to the model archive, otherwise the path to the directory with the trained model files. """ import rasa.core.train config = get_valid_config(config, CONFIG_MANDATORY_KEYS_CORE) _train_path = train_path or tempfile.mkdtemp() # normal (not compare) training core_model = await rasa.core.train( domain_file=domain, stories_file=data.get_core_directory(stories), output_path=os.path.join(_train_path, "core"), policy_config=config, kwargs=kwargs, ) if not train_path: # Only Core was trained. stories = data.get_core_directory(stories) output_path = create_output_path(output, prefix="core-") new_fingerprint = model.model_fingerprint(config, domain, stories=stories) model.create_package_rasa(_train_path, output_path, new_fingerprint) print_success( "Your Rasa Core model is trained and saved at '{}'.".format( output_path)) return core_model
def _async_train(self, config, nlu_data, model_name): training_start = timer() with self.lock: self.training_status[model_name] = { "status": "TRAINING", } data = self.data_reader.read_from_json({'rasa_nlu_data': nlu_data}) with self.interpreter_cache.lock: trainer = Trainer(RasaNLUModelConfig(config), self.interpreter_cache.component_builder) interpreter = trainer.train(data) tempdir = tempfile.mkdtemp() trainer.persist(tempdir, None, "nlu") _model_package = create_package_rasa(tempdir, os.path.join("models", model_name)) self.interpreter_cache.store(model_name, interpreter) with self.lock: training_end = timer() self.training_status[model_name] = { "status": "READY", "training_time": f"{training_end - training_start:.2f}" }
def _package_model( new_fingerprint: Fingerprint, output_path: Text, train_path: Text, fixed_model_name: Optional[Text] = None, model_prefix: Text = "", ): output_path = create_output_path(output_path, prefix=model_prefix, fixed_name=fixed_model_name) model.create_package_rasa(train_path, output_path, new_fingerprint) print_success("Your Rasa model is trained and saved at '{}'.".format( os.path.abspath(output_path))) return output_path
def _train_nlu_with_validated_data( config: Text, nlu_data_directory: Text, output: Text, train_path: Optional[Text] = None, fixed_model_name: Optional[Text] = None, uncompress: bool = False, ) -> Optional[Text]: """Train NLU with validated training and config data.""" import rasa.nlu.train if not os.listdir(nlu_data_directory): print_error( "No NLU data given. Please provide NLU data in order to train " "a Rasa NLU model.") return _train_path = train_path or tempfile.mkdtemp() print_color("Start training NLU model ...", color=bcolors.OKBLUE) _, nlu_model, _ = rasa.nlu.train(config, nlu_data_directory, _train_path, fixed_model_name="nlu") print_color("Done.", color=bcolors.OKBLUE) if not train_path: # Only NLU was trained output_path = create_output_path(output, prefix="nlu-", fixed_name=fixed_model_name) new_fingerprint = model.model_fingerprint(config, nlu_data=nlu_data_directory) model.create_package_rasa(_train_path, output_path, new_fingerprint) if uncompress: output_path = decompress(output_path) print_success( "Your Rasa NLU model is trained and saved at '{}'.".format( output_path)) return output_path return _train_path
def test_rasa_packaging(trained_model, project, use_fingerprint): unpacked_model_path = get_model(trained_model) os.remove(os.path.join(unpacked_model_path, FINGERPRINT_FILE_PATH)) if use_fingerprint: fingerprint = model_fingerprint(**_project_files(project)) else: fingerprint = None tempdir = tempfile.mkdtemp() output_path = os.path.join(tempdir, "test.tar.gz") create_package_rasa(unpacked_model_path, output_path, fingerprint) unpacked = get_model(output_path) assert (os.path.exists(os.path.join( unpacked, FINGERPRINT_FILE_PATH)) == use_fingerprint) assert os.path.exists(os.path.join(unpacked, "core")) assert os.path.exists(os.path.join(unpacked, "nlu")) assert not os.path.exists(unpacked_model_path)
async def train_core_async(domain: Text, config: Text, stories: Text, output: Text, train_path: Optional[Text]) -> Optional[Text]: """Trains a Core model. Args: domain: Path to the domain file. config: Path to the config file for Core. stories: Path to the Core training data. output: Output path. train_path: If `None` the model will be trained in a temporary directory, otherwise in the provided directory. Returns: If `train_path` is given it returns the path to the model archive, otherwise the path to the directory with the trained model files. """ import rasa_core.train # normal (not compare) training core_model = await rasa_core.train(domain_file=domain, stories_file=stories, output_path=os.path.join( train_path, "core"), policy_config=config) if not train_path: # Only Core was trained. stories = data.get_core_directory(stories) output_path = create_output_path(output, prefix="core-") new_fingerprint = model.model_fingerprint(config, domain, stories=stories) model.create_package_rasa(train_path, output_path, new_fingerprint) print_success("Your Rasa Core model is trained and saved at '{}'." "".format(output_path)) return core_model
async def train_async(domain: Text, config: Text, training_files: Union[Text, List[Text]], output: Text = DEFAULT_MODELS_PATH, force_training: bool = False) -> Optional[Text]: """Trains a Rasa model (Core and NLU). Args: domain: Path to the domain file. config: Path to the config for Core and NLU. training_files: Paths to the training data for Core and NLU. output: Output path. force_training: If `True` retrain model even if data has not changed. Returns: Path of the trained model archive. """ train_path = tempfile.mkdtemp() old_model = model.get_latest_model(output) retrain_core = True retrain_nlu = True story_directory, nlu_data_directory = data.get_core_nlu_directories( training_files) new_fingerprint = model.model_fingerprint(config, domain, nlu_data_directory, story_directory) if not force_training and old_model: unpacked = model.unpack_model(old_model) old_core, old_nlu = model.get_model_subdirectories(unpacked) last_fingerprint = model.fingerprint_from_path(unpacked) if not model.core_fingerprint_changed(last_fingerprint, new_fingerprint): target_path = os.path.join(train_path, "core") retrain_core = not model.merge_model(old_core, target_path) if not model.nlu_fingerprint_changed(last_fingerprint, new_fingerprint): target_path = os.path.join(train_path, "nlu") retrain_nlu = not model.merge_model(old_nlu, target_path) if force_training or retrain_core: await train_core_async(domain, config, story_directory, output, train_path) else: print("Dialogue data / configuration did not change. " "No need to retrain dialogue model.") if force_training or retrain_nlu: train_nlu(config, nlu_data_directory, output, train_path) else: print("NLU data / configuration did not change. " "No need to retrain NLU model.") if retrain_core or retrain_nlu: output = create_output_path(output) model.create_package_rasa(train_path, output, new_fingerprint) print("Train path: '{}'.".format(train_path)) print_success("Your bot is trained and ready to take for a spin!") return output else: print("Nothing changed. You can use the old model stored at {}" "".format(os.path.abspath(old_model))) return old_model
async def train_async( domain: Text, config: Text, training_files: Union[Text, List[Text]], output: Text = DEFAULT_MODELS_PATH, force_training: bool = False, kwargs: Optional[Dict] = None, ) -> Optional[Text]: """Trains a Rasa model (Core and NLU). Args: domain: Path to the domain file. config: Path to the config for Core and NLU. training_files: Paths to the training data for Core and NLU. output: Output path. force_training: If `True` retrain model even if data has not changed. kwargs: Additional training parameters. Returns: Path of the trained model archive. """ config = get_valid_config(config, CONFIG_MANDATORY_KEYS) train_path = tempfile.mkdtemp() old_model = model.get_latest_model(output) retrain_core = True retrain_nlu = True story_directory, nlu_data_directory = data.get_core_nlu_directories(training_files) new_fingerprint = model.model_fingerprint( config, domain, nlu_data_directory, story_directory ) dialogue_data_not_present = not os.listdir(story_directory) nlu_data_not_present = not os.listdir(nlu_data_directory) if dialogue_data_not_present and nlu_data_not_present: print_error( "No training data given. Please provide dialogue and NLU data in " "order to train a Rasa model." ) return if dialogue_data_not_present: print_warning( "No dialogue data present. Just a Rasa NLU model will be trained." ) return train_nlu(config, nlu_data_directory, output, None) if nlu_data_not_present: print_warning("No NLU data present. Just a Rasa Core model will be trained.") return await train_core_async( domain, config, story_directory, output, None, kwargs ) if not force_training and old_model: unpacked = model.unpack_model(old_model) old_core, old_nlu = model.get_model_subdirectories(unpacked) last_fingerprint = model.fingerprint_from_path(unpacked) if not model.core_fingerprint_changed(last_fingerprint, new_fingerprint): target_path = os.path.join(train_path, "core") retrain_core = not model.merge_model(old_core, target_path) if not model.nlu_fingerprint_changed(last_fingerprint, new_fingerprint): target_path = os.path.join(train_path, "nlu") retrain_nlu = not model.merge_model(old_nlu, target_path) if force_training or retrain_core: await train_core_async( domain, config, story_directory, output, train_path, kwargs ) else: print ( "Dialogue data / configuration did not change. " "No need to retrain dialogue model." ) if force_training or retrain_nlu: train_nlu(config, nlu_data_directory, output, train_path) else: print ("NLU data / configuration did not change. No need to retrain NLU model.") if retrain_core or retrain_nlu: output = create_output_path(output) model.create_package_rasa(train_path, output, new_fingerprint) print_success("Your bot is trained and ready to take for a spin!") return output else: print_success( "Nothing changed. You can use the old model stored at '{}'" "".format(os.path.abspath(old_model)) ) return old_model
async def train_core_async( domain: Union[Domain, Text], config: Text, stories: Text, output: Text, train_path: Optional[Text] = None, kwargs: Optional[Dict] = None, ) -> Optional[Text]: """Trains a Core model. Args: domain: Path to the domain file. config: Path to the config file for Core. stories: Path to the Core training data. output: Output path. train_path: If `None` the model will be trained in a temporary directory, otherwise in the provided directory. kwargs: Additional training parameters. Returns: If `train_path` is given it returns the path to the model archive, otherwise the path to the directory with the trained model files. """ import rasa.core.train config = get_valid_config(config, CONFIG_MANDATORY_KEYS_CORE) _train_path = train_path or tempfile.mkdtemp() if isinstance(Domain, str) or not train_path: skill_imports = SkillSelector.load(config) domain = Domain.load(domain, skill_imports) story_directory = data.get_core_directory(stories, skill_imports) else: story_directory = stories if not os.listdir(story_directory): print_error( "No dialogue data given. Please provide dialogue data in order to " "train a Rasa Core model.") return # normal (not compare) training print_color("Start training dialogue model ...", color=bcolors.OKBLUE) await rasa.core.train( domain_file=domain, stories_file=story_directory, output_path=os.path.join(_train_path, "core"), policy_config=config, kwargs=kwargs, ) print_color("Done.", color=bcolors.OKBLUE) if not train_path: # Only Core was trained. output_path = create_output_path(output, prefix="core-") new_fingerprint = model.model_fingerprint(config, domain, stories=story_directory) model.create_package_rasa(_train_path, output_path, new_fingerprint) print_success( "Your Rasa Core model is trained and saved at '{}'.".format( output_path)) return output_path return _train_path
async def train_async( domain: Optional, config: Text, training_files: Optional[Union[Text, List[Text]]], output_path: Text = DEFAULT_MODELS_PATH, force_training: bool = False, fixed_model_name: Optional[Text] = None, uncompress: bool = False, kwargs: Optional[Dict] = None, ) -> Optional[Text]: """Trains a Rasa model (Core and NLU). Args: domain: Path to the domain file. config: Path to the config for Core and NLU. training_files: Paths to the training data for Core and NLU. output_path: Output path. force_training: If `True` retrain model even if data has not changed. fixed_model_name: Name of model to be stored. uncompress: If `True` the model will not be compressed. kwargs: Additional training parameters. Returns: Path of the trained model archive. """ config = get_valid_config(config, CONFIG_MANDATORY_KEYS) train_path = tempfile.mkdtemp() old_model = model.get_latest_model(output_path) retrain_core = True retrain_nlu = True skill_imports = SkillSelector.load(config) try: domain = Domain.load(domain, skill_imports) except InvalidDomain as e: print_error(e) return None story_directory, nlu_data_directory = data.get_core_nlu_directories( training_files, skill_imports) new_fingerprint = model.model_fingerprint(config, domain, nlu_data_directory, story_directory) dialogue_data_not_present = not os.listdir(story_directory) nlu_data_not_present = not os.listdir(nlu_data_directory) if dialogue_data_not_present and nlu_data_not_present: print_error( "No training data given. Please provide dialogue and NLU data in " "order to train a Rasa model.") return if dialogue_data_not_present: print_warning( "No dialogue data present. Just a Rasa NLU model will be trained.") return _train_nlu_with_validated_data( config=config, nlu_data_directory=nlu_data_directory, output=output_path, fixed_model_name=fixed_model_name, uncompress=uncompress, ) if nlu_data_not_present: print_warning( "No NLU data present. Just a Rasa Core model will be trained.") return await _train_core_with_validated_data( domain=domain, config=config, story_directory=story_directory, output=output_path, fixed_model_name=fixed_model_name, uncompress=uncompress, kwargs=kwargs, ) if not force_training and old_model: unpacked = model.unpack_model(old_model) old_core, old_nlu = model.get_model_subdirectories(unpacked) last_fingerprint = model.fingerprint_from_path(unpacked) if not model.core_fingerprint_changed(last_fingerprint, new_fingerprint): target_path = os.path.join(train_path, "core") retrain_core = not model.merge_model(old_core, target_path) if not model.nlu_fingerprint_changed(last_fingerprint, new_fingerprint): target_path = os.path.join(train_path, "nlu") retrain_nlu = not model.merge_model(old_nlu, target_path) if force_training or retrain_core: await _train_core_with_validated_data( domain=domain, config=config, story_directory=story_directory, output=output_path, train_path=train_path, fixed_model_name=fixed_model_name, uncompress=uncompress, kwargs=kwargs, ) else: print("Dialogue data / configuration did not change. " "No need to retrain dialogue model.") if force_training or retrain_nlu: _train_nlu_with_validated_data( config=config, nlu_data_directory=nlu_data_directory, output=output_path, train_path=train_path, fixed_model_name=fixed_model_name, uncompress=uncompress, ) else: print( "NLU data / configuration did not change. No need to retrain NLU model." ) if retrain_core or retrain_nlu: output_path = create_output_path(output_path, fixed_name=fixed_model_name) model.create_package_rasa(train_path, output_path, new_fingerprint) if uncompress: output_path = decompress(output_path) print_success("Your Rasa model is trained and saved at '{}'.".format( output_path)) return output_path else: print_success( "Nothing changed. You can use the old model stored at '{}'" "".format(os.path.abspath(old_model))) return old_model