def _persist_metadata(self, path: Text, dump_flattened_stories: bool = False) -> None: """Persists the domain specification to storage.""" # make sure the directory we persist exists domain_spec_path = os.path.join(path, 'metadata.json') training_data_path = os.path.join(path, 'stories.md') utils.create_dir_for_file(domain_spec_path) policy_names = [utils.module_path_from_instance(p) for p in self.policies] training_events = self._training_events_from_trackers( self.training_trackers) action_fingerprints = self._create_action_fingerprints(training_events) metadata = { "action_fingerprints": action_fingerprints, "python": ".".join([str(s) for s in sys.version_info[:3]]), "max_histories": self._max_histories(), "ensemble_name": self.__module__ + "." + self.__class__.__name__, "policy_names": policy_names, "trained_at": self.date_trained } self._add_package_version_info(metadata) utils.dump_obj_as_json_to_file(domain_spec_path, metadata) # if there are lots of stories, saving flattened stories takes a long # time, so this is turned off by default if dump_flattened_stories: training.persist_data(self.training_trackers, training_data_path)
def _persist_configuration(self, config_file): model_config = { "arch": "keras_arch.json", "weights": "keras_weights.h5", "epochs": self.current_epoch} utils.dump_obj_as_json_to_file(config_file, model_config)
def run_comparison_evaluation(models: Text, stories_file: Text, output: Text) -> None: """Evaluates multiple trained models on a test set""" num_correct = defaultdict(list) for run in nlu_utils.list_subdirectories(models): num_correct_run = defaultdict(list) for model in sorted(nlu_utils.list_subdirectories(run)): logger.info("Evaluating model {}".format(model)) agent = Agent.load(model) completed_trackers = _generate_trackers(stories_file, agent) story_eval_store, no_of_stories = \ collect_story_predictions(completed_trackers, agent) failed_stories = story_eval_store.failed_stories policy_name = ''.join( [i for i in os.path.basename(model) if not i.isdigit()]) num_correct_run[policy_name].append(no_of_stories - len(failed_stories)) for k, v in num_correct_run.items(): num_correct[k].append(v) utils.dump_obj_as_json_to_file(os.path.join(output, 'results.json'), num_correct)
def _persist_metadata(self, path, dump_flattened_stories=False): # type: (Text) -> None """Persists the domain specification to storage.""" # make sure the directory we persist to exists domain_spec_path = os.path.join(path, 'policy_metadata.json') training_data_path = os.path.join(path, 'stories.md') utils.create_dir_for_file(domain_spec_path) policy_names = [utils.module_path_from_instance(p) for p in self.policies] training_events = self._training_events_from_trackers( self.training_trackers) action_fingerprints = self._create_action_fingerprints(training_events) metadata = { "action_fingerprints": action_fingerprints, "rasa_core": rasa_core.__version__, "max_histories": self._max_histories(), "ensemble_name": self.__module__ + "." + self.__class__.__name__, "policy_names": policy_names } utils.dump_obj_as_json_to_file(domain_spec_path, metadata) # if there are lots of stories, saving flattened stories takes a long # time, so this is turned off by default if dump_flattened_stories: training.persist_data(self.training_trackers, training_data_path)
def persist(self, path: Text) -> None: if self.model: self.featurizer.persist(path) meta = { "priority": self.priority, "model": "keras_model.h5", "epochs": self.current_epoch } meta_file = os.path.join(path, 'keras_policy.json') utils.dump_obj_as_json_to_file(meta_file, meta) model_file = os.path.join(path, meta['model']) # makes sure the model directory exists utils.create_dir_for_file(model_file) with self.graph.as_default(), self.session.as_default(): self.model.save(model_file, overwrite=True) tf_config_file = os.path.join(path, "keras_policy.tf_config.pkl") with open(tf_config_file, 'wb') as f: pickle.dump(self._tf_config, f) else: warnings.warn("Persist called without a trained model present. " "Nothing to persist then!")
def _persist_metadata(self, path, dump_flattened_stories=False): # type: (Text) -> None """Persists the domain specification to storage.""" # make sure the directory we persist to exists domain_spec_path = os.path.join(path, 'policy_metadata.json') training_data_path = os.path.join(path, 'stories.md') utils.create_dir_for_file(domain_spec_path) policy_names = [ utils.module_path_from_instance(p) for p in self.policies ] training_events = self._training_events_from_trackers( self.training_trackers) action_fingerprints = self._create_action_fingerprints(training_events) metadata = { "action_fingerprints": action_fingerprints, "rasa_core": rasa_core.__version__, "max_histories": self._max_histories(), "ensemble_name": self.__module__ + "." + self.__class__.__name__, "policy_names": policy_names } utils.dump_obj_as_json_to_file(domain_spec_path, metadata) # if there are lots of stories, saving flattened stories takes a long # time, so this is turned off by default if dump_flattened_stories: training.persist_data(self.training_trackers, training_data_path)
def persist_specification(self, model_path): # type: (Text, List[Text]) -> None """Persists the domain specification to storage.""" domain_spec_path = os.path.join(model_path, 'domain.json') utils.create_dir_for_file(domain_spec_path) metadata = {"features": self.input_features} utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
def persist_specification(self, model_path: Text) -> None: """Persists the domain specification to storage.""" domain_spec_path = os.path.join(model_path, 'domain.json') utils.create_dir_for_file(domain_spec_path) metadata = {"states": self.input_states} utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
def persist(self, path: Text) -> None: self.featurizer.persist(path) memorized_file = os.path.join(path, 'memorized_turns.json') data = {"max_history": self.max_history, "lookup": self.lookup} utils.create_dir_for_file(memorized_file) utils.dump_obj_as_json_to_file(memorized_file, data)
def persist(self, path): # type: (Text) -> None """Persists the policy to storage.""" config_file = os.path.join(path, 'confirmation_policy.json') meta = { "nlu_threshold": self.nlu_threshold, "confirmation_action_name": self.confirmation_action_name } utils.create_dir_for_file(config_file) utils.dump_obj_as_json_to_file(config_file, meta)
def persist(self, path: Text) -> None: """Persists the policy to storage.""" config_file = os.path.join(path, 'fallback_policy.json') meta = { "nlu_threshold": self.nlu_threshold, "core_threshold": self.core_threshold, "fallback_action_name": self.fallback_action_name } utils.create_dir_for_file(config_file) utils.dump_obj_as_json_to_file(config_file, meta)
def persist(self, path): # type: (Text) -> None """Persists the policy to storage.""" config_file = os.path.join(path, 'fallback_policy.json') meta = { "nlu_threshold": self.nlu_threshold, "core_threshold": self.core_threshold, "fallback_action_name": self.fallback_action_name } utils.create_dir_for_file(config_file) utils.dump_obj_as_json_to_file(config_file, meta)
def persist_specification(self, model_path): # type: (Text) -> None """Persists the domain specification to storage.""" domain_spec_path = os.path.join(model_path, 'domain.json') utils.create_dir_for_file(domain_spec_path) metadata = { "states": self.input_states } utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
def persist_fingerprint(output_path: Text, fingerprint: Fingerprint): """Persists a model fingerprint. Args: output_path: Directory in which the fingerprint should be saved. fingerprint: The fingerprint to be persisted. """ from rasa_core.utils import dump_obj_as_json_to_file path = os.path.join(output_path, FINGERPRINT_FILE_PATH) dump_obj_as_json_to_file(path, fingerprint)
def persist(self, path): # type: (Text) -> None self.featurizer.persist(path) memorized_file = os.path.join(path, 'memorized_turns.json') data = { "max_history": self.max_history, "lookup": self.lookup } utils.create_dir_for_file(memorized_file) utils.dump_obj_as_json_to_file(memorized_file, data)
def persist(self, path: Text) -> None: """Persists the policy to storage.""" config_file = os.path.join(path, "bottis_policy.json") meta = { "priority": self.priority, "nlu_threshold": self.nlu_threshold, "core_threshold": self.core_threshold, "custom_response_action_name": self.custom_response_action_name, } utils.create_dir_for_file(config_file) utils.dump_obj_as_json_to_file(config_file, meta)
async def test_dump_and_restore_as_json(default_agent, tmpdir_factory): trackers = await default_agent.load_data(DEFAULT_STORIES_FILE) for tracker in trackers: out_path = tmpdir_factory.mktemp("tracker").join("dumped_tracker.json") dumped = tracker.current_state(EventVerbosity.AFTER_RESTART) utils.dump_obj_as_json_to_file(out_path.strpath, dumped) restored_tracker = restore.load_tracker_from_json( out_path.strpath, default_agent.domain) assert restored_tracker == tracker
def test_dump_and_restore_as_json(default_agent, tmpdir_factory): trackers = default_agent.load_data(DEFAULT_STORIES_FILE) for tracker in trackers: out_path = tmpdir_factory.mktemp("tracker").join("dumped_tracker.json") dumped = tracker.current_state(should_include_events=True) utils.dump_obj_as_json_to_file(out_path.strpath, dumped) restored_tracker = restore.load_tracker_from_json( out_path.strpath, default_agent.domain) assert restored_tracker == tracker
def persist(self, path: Text) -> None: """Persists the policy to storage.""" config_file = os.path.join(path, 'two_stage_fallback_policy.json') meta = { "priority": self.priority, "nlu_threshold": self.nlu_threshold, "core_threshold": self.core_threshold, "fallback_core_action_name": self.fallback_action_name, "fallback_nlu_action_name": self.fallback_nlu_action_name, "deny_suggestion_intent_name": self.deny_suggestion_intent_name, } utils.create_dir_for_file(config_file) utils.dump_obj_as_json_to_file(config_file, meta)
def test_dump_and_restore_as_json(default_agent, tmpdir_factory): trackers = default_agent.load_data(DEFAULT_STORIES_FILE) for tracker in trackers: out_path = tmpdir_factory.mktemp("tracker").join("dumped_tracker.json") dumped = tracker.current_state(should_include_events=True) utils.dump_obj_as_json_to_file(out_path.strpath, dumped) restored_tracker = restore.load_tracker_from_json(out_path.strpath, default_agent.domain) assert restored_tracker == tracker
def persist(self, path): # tupe: (Text) -> None if self.model: self.featurizer.persist(path) meta = {"model": "torch_model.h5", "epochs": self.current_epoch} config_file = os.path.join(path, "torch_policy.json") utils.dump_obj_as_json_to_file(config_file, meta) model_file = os.path.join(path, meta["model"]) utils.create_dir_for_file(model_file) torch.save(self.model, model_file) else: warnings.warn("Persist called without a trained model present. " "Nothing to persist.")
def persist(self, path: Text) -> None: if self.model: self.featurizer.persist(path) meta = {"priority": self.priority} meta_file = os.path.join(path, 'sklearn_policy.json') utils.dump_obj_as_json_to_file(meta_file, meta) filename = os.path.join(path, 'sklearn_model.pkl') with open(filename, 'wb') as f: pickle.dump(self._state, f) else: warnings.warn("Persist called without a trained model present. " "Nothing to persist then!")
def test_dump_and_restore_as_json(default_agent, tmpdir_factory): trackers = default_agent.load_data(DEFAULT_STORIES_FILE) for tracker in trackers: out_path = tmpdir_factory.mktemp("tracker").join("dumped_tracker.json") dumped = tracker.current_state(should_include_events=True) utils.dump_obj_as_json_to_file(out_path.strpath, dumped) tracker_json = json.loads(utils.read_file(out_path.strpath)) sender_id = tracker_json.get("sender_id", UserMessage.DEFAULT_SENDER_ID) restored_tracker = DialogueStateTracker.from_dict( sender_id, tracker_json.get("events", []), default_agent.domain) assert restored_tracker == tracker
def do_compare_training(cmdline_args, stories, additional_arguments): train_comparison_models(stories, cmdline_args.domain, cmdline_args.out, cmdline_args.percentages, cmdline_args.config, cmdline_args.runs, cmdline_args.dump_stories, additional_arguments) no_stories = get_no_of_stories(cmdline_args.stories, cmdline_args.domain) # store the list of the number of stories present at each exclusion # percentage story_range = [ no_stories - round((x / 100.0) * no_stories) for x in cmdline_args.percentages ] story_n_path = os.path.join(cmdline_args.out, 'num_stories.json') utils.dump_obj_as_json_to_file(story_n_path, story_range)
def test_dump_and_restore_as_json(default_agent, tmpdir): trackers = training.extract_trackers( DEFAULT_STORIES_FILE, default_agent.domain, default_agent.featurizer, default_agent.interpreter, default_agent.policy_ensemble.max_history()) out_path = tmpdir.join("dumped_tracker.json") for tracker in trackers: dumped = tracker.current_state(should_include_events=True) utils.dump_obj_as_json_to_file(out_path.strpath, dumped) tracker_json = json.loads(utils.read_file(out_path.strpath)) sender_id = tracker_json.get("sender_id", UserMessage.DEFAULT_SENDER_ID) restored_tracker = DialogueStateTracker.from_dict( sender_id, tracker_json.get("events", []), default_agent.domain) assert restored_tracker == tracker
def persist(self, path): # type: (Text) -> None if self.model: self.featurizer.persist(path) meta = {"model": "keras_model.h5", "epochs": self.current_epoch} config_file = os.path.join(path, 'keras_policy.json') utils.dump_obj_as_json_to_file(config_file, meta) model_file = os.path.join(path, meta['model']) # makes sure the model directory exists utils.create_dir_for_file(model_file) with self.graph.as_default(), self.session.as_default(): self.model.save(model_file, overwrite=True) else: warnings.warn("Persist called without a trained model present. " "Nothing to persist then!")
def _persist_metadata(self, path, max_history): # type: (Text, Optional[int]) -> None """Persists the domain specification to storage.""" # make sure the directory we persist to exists domain_spec_path = os.path.join(path, 'policy_metadata.json') utils.create_dir_for_file(domain_spec_path) policy_names = [utils.module_path_from_instance(p) for p in self.policies] training_events = self.training_metadata.get("events", {}) action_fingerprints = self._create_action_fingerprints(training_events) metadata = { "action_fingerprints": action_fingerprints, "rasa_core": rasa_core.__version__, "max_history": max_history, "ensemble_name": self.__module__ + "." + self.__class__.__name__, "policy_names": policy_names } utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
def persist(self, path): memorized_file = os.path.join(path, 'memorized_turns.json') data = {"lookup": self.lookup} utils.create_dir_for_file(memorized_file) utils.dump_obj_as_json_to_file(memorized_file, data)