def persist_specification(self, model_path: Text) -> None: """Persists the domain specification to storage.""" domain_spec_path = os.path.join(model_path, "domain.json") utils.create_dir_for_file(domain_spec_path) metadata = {"states": self.input_states} utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
async def compare_models(models: List[Text], stories_file: Text, output: Text) -> None: """Evaluates provided trained models on a test set.""" from rasa.core import utils number_correct = defaultdict(list) for model in models: number_of_correct_stories = await _evaluate_core_model(model, stories_file) number_correct[os.path.basename(model)].append(number_of_correct_stories) utils.dump_obj_as_json_to_file(os.path.join(output, RESULTS_FILE), number_correct)
def persist_fingerprint(output_path: Text, fingerprint: Fingerprint): """Persists a model fingerprint. Args: output_path: Directory in which the fingerprint should be saved. fingerprint: The fingerprint to be persisted. """ from rasa.core.utils import dump_obj_as_json_to_file path = os.path.join(output_path, FINGERPRINT_FILE_PATH) dump_obj_as_json_to_file(path, fingerprint)
def persist(self, path: Text) -> None: """Persists the policy to storage.""" config_file = os.path.join(path, 'fallback_policy.json') meta = { "priority": self.priority, "nlu_threshold": self.nlu_threshold, "core_threshold": self.core_threshold, "fallback_action_name": self.fallback_action_name } utils.create_dir_for_file(config_file) utils.dump_obj_as_json_to_file(config_file, meta)
def persist(self, path: Text) -> None: self.featurizer.persist(path) memorized_file = os.path.join(path, 'memorized_turns.json') data = { "priority": self.priority, "max_history": self.max_history, "lookup": self.lookup } utils.create_dir_for_file(memorized_file) utils.dump_obj_as_json_to_file(memorized_file, data)
def persist(self, path: Text) -> None: """Persists the policy to storage.""" config_file = os.path.join(path, "two_stage_fallback_policy.json") meta = { "priority": self.priority, "nlu_threshold": self.nlu_threshold, "core_threshold": self.core_threshold, "fallback_core_action_name": self.fallback_action_name, "fallback_nlu_action_name": self.fallback_nlu_action_name, "deny_suggestion_intent_name": self.deny_suggestion_intent_name, } rasa.utils.io.create_directory_for_file(config_file) utils.dump_obj_as_json_to_file(config_file, meta)
async def test_dump_and_restore_as_json(default_agent, tmpdir_factory): trackers = await default_agent.load_data(DEFAULT_STORIES_FILE) for tracker in trackers: out_path = tmpdir_factory.mktemp("tracker").join("dumped_tracker.json") dumped = tracker.current_state(EventVerbosity.AFTER_RESTART) utils.dump_obj_as_json_to_file(out_path.strpath, dumped) restored_tracker = restore.load_tracker_from_json( out_path.strpath, default_agent.domain) assert restored_tracker == tracker
def persist(self, path: Text) -> None: """Persists the policy to storage.""" config_file = os.path.join(path, "custom_policy.json") meta = { "priority": self.priority, "nlu_threshold": self.nlu_threshold, "ambiguity_threshold": self.ambiguity_threshold, "core_threshold": self.core_threshold, "fallback_action_name": self.fallback_action_name, } rasa.utils.io.create_directory_for_file(config_file) utils.dump_obj_as_json_to_file(config_file, meta)
def persist(self, path: Text) -> None: if self.model: self.featurizer.persist(path) meta = {"priority": self.priority} meta_file = os.path.join(path, 'sklearn_policy.json') utils.dump_obj_as_json_to_file(meta_file, meta) filename = os.path.join(path, 'sklearn_model.pkl') with open(filename, 'wb') as f: pickle.dump(self._state, f) else: warnings.warn("Persist called without a trained model present. " "Nothing to persist then!")
def persist(self, path: Text) -> None: """Persists the policy to storage.""" config_file = os.path.join(path, "botfront_disambiguation_policy.json") meta = { "priority": self.priority, "disambiguation_trigger": self.disambiguation_trigger, "fallback_trigger": self.fallback_trigger, "deny_suggestions": self.deny_suggestions, "excluded_intents": self.excluded_intents, "n_suggestions": self.n_suggestions, "intent_mappings": self.intent_mappings, "disambiguation_title": self.disambiguation_title } rasa.utils.io.create_directory_for_file(config_file) utils.dump_obj_as_json_to_file(config_file, meta)
async def post_data_convert(request: Request): """Converts current domain in yaml or json format.""" validate_request_body( request, "You must provide training data in the request body in order to " "train your model.", ) rjs = request.json if 'data' not in rjs: raise ErrorResponse( 400, "BadRequest", "Must provide training data in 'data' property") if 'output_format' not in rjs or rjs["output_format"] not in [ "json", "md" ]: raise ErrorResponse( 400, "BadRequest", "'output_format' is required and must be either 'md' or 'json") if 'language' not in rjs: raise ErrorResponse(400, "BadRequest", "'language' is required") temp_dir = tempfile.mkdtemp() out_dir = tempfile.mkdtemp() nlu_data_path = os.path.join(temp_dir, "nlu_data") output_path = os.path.join(out_dir, "output") # botfront: several nlu files if type(rjs["data"] is dict): from rasa.core.utils import dump_obj_as_json_to_file dump_obj_as_json_to_file(nlu_data_path, rjs["data"]) else: dump_obj_as_str_to_file(nlu_data_path, rjs["data"]) # botfront end from rasa.nlu.convert import convert_training_data convert_training_data(nlu_data_path, output_path, rjs["output_format"], rjs["language"]) with open(output_path, encoding='utf-8') as f: data = f.read() if rjs["output_format"] == 'json': import json data = json.loads(data, encoding='utf-8') return response.json({"data": data})
def persist(self, path: Text) -> None: """Persists the policy to a storage.""" if self.session is None: warnings.warn("Method `persist(...)` was called " "without a trained model present. " "Nothing to persist then!") return self.featurizer.persist(path) meta = {"priority": self.priority} meta_file = os.path.join(path, "embedding_policy.json") utils.dump_obj_as_json_to_file(meta_file, meta) file_name = "tensorflow_embedding.ckpt" checkpoint = os.path.join(path, file_name) rasa.utils.io.create_directory_for_file(checkpoint) with self.graph.as_default(): train_utils.persist_tensor("user_placeholder", self.a_in, self.graph) train_utils.persist_tensor("bot_placeholder", self.b_in, self.graph) train_utils.persist_tensor("similarity_all", self.sim_all, self.graph) train_utils.persist_tensor("pred_confidence", self.pred_confidence, self.graph) train_utils.persist_tensor("similarity", self.sim, self.graph) train_utils.persist_tensor("dial_embed", self.dial_embed, self.graph) train_utils.persist_tensor("bot_embed", self.bot_embed, self.graph) train_utils.persist_tensor("all_bot_embed", self.all_bot_embed, self.graph) train_utils.persist_tensor("attention_weights", self.attention_weights, self.graph) saver = tf.train.Saver() saver.save(self.session, checkpoint) with open(os.path.join(path, file_name + ".tf_config.pkl"), "wb") as f: pickle.dump(self._tf_config, f)
async def do_compare_training(cmdline_args, stories, additional_arguments): from rasa.core import utils await train_comparison_models(stories, cmdline_args.domain, cmdline_args.out, cmdline_args.percentages, cmdline_args.config, cmdline_args.runs, cmdline_args.dump_stories, additional_arguments) no_stories = await get_no_of_stories(cmdline_args.stories, cmdline_args.domain) # store the list of the number of stories present at each exclusion # percentage story_range = [ no_stories - round((x / 100.0) * no_stories) for x in cmdline_args.percentages ] story_n_path = os.path.join(cmdline_args.out, 'num_stories.json') utils.dump_obj_as_json_to_file(story_n_path, story_range)
def _persist_metadata(self, path: Text, dump_flattened_stories: bool = False) -> None: """Persists the domain specification to storage.""" # make sure the directory we persist exists domain_spec_path = os.path.join(path, "metadata.json") training_data_path = os.path.join(path, "stories.md") rasa.utils.io.create_directory_for_file(domain_spec_path) policy_names = [ utils.module_path_from_instance(p) for p in self.policies ] training_events = self._training_events_from_trackers( self.training_trackers) action_fingerprints = self._create_action_fingerprints(training_events) metadata = { "action_fingerprints": action_fingerprints, "python": ".".join([str(s) for s in sys.version_info[:3]]), "max_histories": self._max_histories(), "ensemble_name": self.__module__ + "." + self.__class__.__name__, "policy_names": policy_names, "trained_at": self.date_trained, } self._add_package_version_info(metadata) utils.dump_obj_as_json_to_file(domain_spec_path, metadata) # if there are lots of stories, saving flattened stories takes a long # time, so this is turned off by default if dump_flattened_stories: training.persist_data(self.training_trackers, training_data_path)
async def train(request: Request): """Train a Rasa Model.""" from rasa.train import train_async validate_request_body( request, "You must provide training data in the request body in order to " "train your model.", ) rjs = request.json validate_request(rjs) # create a temporary directory to store config, domain and # training data temp_dir = tempfile.mkdtemp() filenames = { "config": "config.yml", "nlu": "nlu.md", "stories": "stories.md", "domain": "domain.yml" } for (key, filename) in filenames.items(): if key not in rjs: continue temp_path = os.path.join(temp_dir, filename) temp_data = rjs[key] if isinstance(temp_data, dict): dump_obj_as_json_to_file(temp_path, temp_data) else: dump_obj_as_str_to_file(temp_path, temp_data) if "domain" in rjs: domain_path = os.path.join(temp_dir, filenames['domain']) else: domain_path = DEFAULT_DOMAIN_PATH try: model_path = await train_async( domain=domain_path, config=os.path.join(temp_dir, filenames["config"]), training_files=temp_dir, output_path=rjs.get("out", DEFAULT_MODELS_PATH), force_training=rjs.get("force", False), ) filename = os.path.basename(model_path) if model_path else None return await response.file(model_path, filename=filename, headers={"filename": filename}) except InvalidDomain as e: raise ErrorResponse( 400, "InvalidDomainError", "Provided domain file is invalid. Error: {}".format(e), ) except Exception as e: logger.debug(traceback.format_exc()) raise ErrorResponse( 500, "TrainingError", "An unexpected error occurred during training. Error: {}". format(e), )