def _persist_metadata(self,
path: Text,
dump_flattened_stories: bool = False) -> None:
"""Persists the domain specification to storage."""
# make sure the directory we persist exists
domain_spec_path = os.path.join(path, 'metadata.json')
training_data_path = os.path.join(path, 'stories.md')
utils.create_dir_for_file(domain_spec_path)
policy_names = [utils.module_path_from_instance(p)
for p in self.policies]
training_events = self._training_events_from_trackers(
self.training_trackers)
action_fingerprints = self._create_action_fingerprints(training_events)
metadata = {
"action_fingerprints": action_fingerprints,
"python": ".".join([str(s) for s in sys.version_info[:3]]),
"max_histories": self._max_histories(),
"ensemble_name": self.__module__ + "." + self.__class__.__name__,
"policy_names": policy_names,
"trained_at": self.date_trained
}
self._add_package_version_info(metadata)
utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
# if there are lots of stories, saving flattened stories takes a long
# time, so this is turned off by default
if dump_flattened_stories:
training.persist_data(self.training_trackers, training_data_path)
def _persist_configuration(self, config_file):
model_config = {
"arch": "keras_arch.json",
"weights": "keras_weights.h5",
"epochs": self.current_epoch}
utils.dump_obj_as_json_to_file(config_file, model_config)
def run_comparison_evaluation(models: Text, stories_file: Text,
output: Text) -> None:
"""Evaluates multiple trained models on a test set"""
num_correct = defaultdict(list)
for run in nlu_utils.list_subdirectories(models):
num_correct_run = defaultdict(list)
for model in sorted(nlu_utils.list_subdirectories(run)):
logger.info("Evaluating model {}".format(model))
agent = Agent.load(model)
completed_trackers = _generate_trackers(stories_file, agent)
story_eval_store, no_of_stories = \
collect_story_predictions(completed_trackers,
agent)
failed_stories = story_eval_store.failed_stories
policy_name = ''.join(
[i for i in os.path.basename(model) if not i.isdigit()])
num_correct_run[policy_name].append(no_of_stories -
len(failed_stories))
for k, v in num_correct_run.items():
num_correct[k].append(v)
utils.dump_obj_as_json_to_file(os.path.join(output, 'results.json'),
num_correct)
def _persist_metadata(self, path, dump_flattened_stories=False):
# type: (Text) -> None
"""Persists the domain specification to storage."""
# make sure the directory we persist to exists
domain_spec_path = os.path.join(path, 'policy_metadata.json')
training_data_path = os.path.join(path, 'stories.md')
utils.create_dir_for_file(domain_spec_path)
policy_names = [utils.module_path_from_instance(p)
for p in self.policies]
training_events = self._training_events_from_trackers(
self.training_trackers)
action_fingerprints = self._create_action_fingerprints(training_events)
metadata = {
"action_fingerprints": action_fingerprints,
"rasa_core": rasa_core.__version__,
"max_histories": self._max_histories(),
"ensemble_name": self.__module__ + "." + self.__class__.__name__,
"policy_names": policy_names
}
utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
# if there are lots of stories, saving flattened stories takes a long
# time, so this is turned off by default
if dump_flattened_stories:
training.persist_data(self.training_trackers, training_data_path)
def persist(self, path: Text) -> None:
if self.model:
self.featurizer.persist(path)
meta = {
"priority": self.priority,
"model": "keras_model.h5",
"epochs": self.current_epoch
}
meta_file = os.path.join(path, 'keras_policy.json')
utils.dump_obj_as_json_to_file(meta_file, meta)
model_file = os.path.join(path, meta['model'])
# makes sure the model directory exists
utils.create_dir_for_file(model_file)
with self.graph.as_default(), self.session.as_default():
self.model.save(model_file, overwrite=True)
tf_config_file = os.path.join(path, "keras_policy.tf_config.pkl")
with open(tf_config_file, 'wb') as f:
pickle.dump(self._tf_config, f)
else:
warnings.warn("Persist called without a trained model present. "
"Nothing to persist then!")
def _persist_metadata(self, path, dump_flattened_stories=False):
# type: (Text) -> None
"""Persists the domain specification to storage."""
# make sure the directory we persist to exists
domain_spec_path = os.path.join(path, 'policy_metadata.json')
training_data_path = os.path.join(path, 'stories.md')
utils.create_dir_for_file(domain_spec_path)
policy_names = [
utils.module_path_from_instance(p) for p in self.policies
]
training_events = self._training_events_from_trackers(
self.training_trackers)
action_fingerprints = self._create_action_fingerprints(training_events)
metadata = {
"action_fingerprints": action_fingerprints,
"rasa_core": rasa_core.__version__,
"max_histories": self._max_histories(),
"ensemble_name": self.__module__ + "." + self.__class__.__name__,
"policy_names": policy_names
}
utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
# if there are lots of stories, saving flattened stories takes a long
# time, so this is turned off by default
if dump_flattened_stories:
training.persist_data(self.training_trackers, training_data_path)
def persist_specification(self, model_path):
# type: (Text, List[Text]) -> None
"""Persists the domain specification to storage."""
domain_spec_path = os.path.join(model_path, 'domain.json')
utils.create_dir_for_file(domain_spec_path)
metadata = {"features": self.input_features}
utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
def persist_specification(self, model_path: Text) -> None:
"""Persists the domain specification to storage."""
domain_spec_path = os.path.join(model_path, 'domain.json')
utils.create_dir_for_file(domain_spec_path)
metadata = {"states": self.input_states}
utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
def persist(self, path: Text) -> None:
self.featurizer.persist(path)
memorized_file = os.path.join(path, 'memorized_turns.json')
data = {"max_history": self.max_history, "lookup": self.lookup}
utils.create_dir_for_file(memorized_file)
utils.dump_obj_as_json_to_file(memorized_file, data)
def persist(self, path):
# type: (Text) -> None
"""Persists the policy to storage."""
config_file = os.path.join(path, 'confirmation_policy.json')
meta = {
"nlu_threshold": self.nlu_threshold,
"confirmation_action_name": self.confirmation_action_name
}
utils.create_dir_for_file(config_file)
utils.dump_obj_as_json_to_file(config_file, meta)
def persist(self, path: Text) -> None:
"""Persists the policy to storage."""
config_file = os.path.join(path, 'fallback_policy.json')
meta = {
"nlu_threshold": self.nlu_threshold,
"core_threshold": self.core_threshold,
"fallback_action_name": self.fallback_action_name
}
utils.create_dir_for_file(config_file)
utils.dump_obj_as_json_to_file(config_file, meta)
def persist(self, path):
# type: (Text) -> None
"""Persists the policy to storage."""
config_file = os.path.join(path, 'fallback_policy.json')
meta = {
"nlu_threshold": self.nlu_threshold,
"core_threshold": self.core_threshold,
"fallback_action_name": self.fallback_action_name
}
utils.create_dir_for_file(config_file)
utils.dump_obj_as_json_to_file(config_file, meta)
def persist_specification(self, model_path):
# type: (Text) -> None
"""Persists the domain specification to storage."""
domain_spec_path = os.path.join(model_path, 'domain.json')
utils.create_dir_for_file(domain_spec_path)
metadata = {
"states": self.input_states
}
utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
def persist_fingerprint(output_path: Text, fingerprint: Fingerprint):
"""Persists a model fingerprint.
Args:
output_path: Directory in which the fingerprint should be saved.
fingerprint: The fingerprint to be persisted.
"""
from rasa_core.utils import dump_obj_as_json_to_file
path = os.path.join(output_path, FINGERPRINT_FILE_PATH)
dump_obj_as_json_to_file(path, fingerprint)
def persist(self, path):
# type: (Text) -> None
self.featurizer.persist(path)
memorized_file = os.path.join(path, 'memorized_turns.json')
data = {
"max_history": self.max_history,
"lookup": self.lookup
}
utils.create_dir_for_file(memorized_file)
utils.dump_obj_as_json_to_file(memorized_file, data)
def persist(self, path: Text) -> None:
"""Persists the policy to storage."""
config_file = os.path.join(path, "bottis_policy.json")
meta = {
"priority": self.priority,
"nlu_threshold": self.nlu_threshold,
"core_threshold": self.core_threshold,
"custom_response_action_name": self.custom_response_action_name,
}
utils.create_dir_for_file(config_file)
utils.dump_obj_as_json_to_file(config_file, meta)
async def test_dump_and_restore_as_json(default_agent, tmpdir_factory):
trackers = await default_agent.load_data(DEFAULT_STORIES_FILE)
for tracker in trackers:
out_path = tmpdir_factory.mktemp("tracker").join("dumped_tracker.json")
dumped = tracker.current_state(EventVerbosity.AFTER_RESTART)
utils.dump_obj_as_json_to_file(out_path.strpath, dumped)
restored_tracker = restore.load_tracker_from_json(
out_path.strpath, default_agent.domain)
assert restored_tracker == tracker
def test_dump_and_restore_as_json(default_agent, tmpdir_factory):
trackers = default_agent.load_data(DEFAULT_STORIES_FILE)
for tracker in trackers:
out_path = tmpdir_factory.mktemp("tracker").join("dumped_tracker.json")
dumped = tracker.current_state(should_include_events=True)
utils.dump_obj_as_json_to_file(out_path.strpath, dumped)
restored_tracker = restore.load_tracker_from_json(
out_path.strpath, default_agent.domain)
assert restored_tracker == tracker
def persist(self, path: Text) -> None:
"""Persists the policy to storage."""
config_file = os.path.join(path, 'two_stage_fallback_policy.json')
meta = {
"priority": self.priority,
"nlu_threshold": self.nlu_threshold,
"core_threshold": self.core_threshold,
"fallback_core_action_name": self.fallback_action_name,
"fallback_nlu_action_name": self.fallback_nlu_action_name,
"deny_suggestion_intent_name": self.deny_suggestion_intent_name,
}
utils.create_dir_for_file(config_file)
utils.dump_obj_as_json_to_file(config_file, meta)
def test_dump_and_restore_as_json(default_agent, tmpdir_factory):
trackers = default_agent.load_data(DEFAULT_STORIES_FILE)
for tracker in trackers:
out_path = tmpdir_factory.mktemp("tracker").join("dumped_tracker.json")
dumped = tracker.current_state(should_include_events=True)
utils.dump_obj_as_json_to_file(out_path.strpath, dumped)
restored_tracker = restore.load_tracker_from_json(out_path.strpath,
default_agent.domain)
assert restored_tracker == tracker
def persist(self, path):
# tupe: (Text) -> None
if self.model:
self.featurizer.persist(path)
meta = {"model": "torch_model.h5", "epochs": self.current_epoch}
config_file = os.path.join(path, "torch_policy.json")
utils.dump_obj_as_json_to_file(config_file, meta)
model_file = os.path.join(path, meta["model"])
utils.create_dir_for_file(model_file)
torch.save(self.model, model_file)
else:
warnings.warn("Persist called without a trained model present. "
"Nothing to persist.")
def persist(self, path: Text) -> None:
if self.model:
self.featurizer.persist(path)
meta = {"priority": self.priority}
meta_file = os.path.join(path, 'sklearn_policy.json')
utils.dump_obj_as_json_to_file(meta_file, meta)
filename = os.path.join(path, 'sklearn_model.pkl')
with open(filename, 'wb') as f:
pickle.dump(self._state, f)
else:
warnings.warn("Persist called without a trained model present. "
"Nothing to persist then!")
def test_dump_and_restore_as_json(default_agent, tmpdir_factory):
trackers = default_agent.load_data(DEFAULT_STORIES_FILE)
for tracker in trackers:
out_path = tmpdir_factory.mktemp("tracker").join("dumped_tracker.json")
dumped = tracker.current_state(should_include_events=True)
utils.dump_obj_as_json_to_file(out_path.strpath, dumped)
tracker_json = json.loads(utils.read_file(out_path.strpath))
sender_id = tracker_json.get("sender_id",
UserMessage.DEFAULT_SENDER_ID)
restored_tracker = DialogueStateTracker.from_dict(
sender_id, tracker_json.get("events", []), default_agent.domain)
assert restored_tracker == tracker
def do_compare_training(cmdline_args, stories, additional_arguments):
train_comparison_models(stories, cmdline_args.domain, cmdline_args.out,
cmdline_args.percentages, cmdline_args.config,
cmdline_args.runs, cmdline_args.dump_stories,
additional_arguments)
no_stories = get_no_of_stories(cmdline_args.stories, cmdline_args.domain)
# store the list of the number of stories present at each exclusion
# percentage
story_range = [
no_stories - round((x / 100.0) * no_stories)
for x in cmdline_args.percentages
]
story_n_path = os.path.join(cmdline_args.out, 'num_stories.json')
utils.dump_obj_as_json_to_file(story_n_path, story_range)
def test_dump_and_restore_as_json(default_agent, tmpdir):
trackers = training.extract_trackers(
DEFAULT_STORIES_FILE, default_agent.domain, default_agent.featurizer,
default_agent.interpreter, default_agent.policy_ensemble.max_history())
out_path = tmpdir.join("dumped_tracker.json")
for tracker in trackers:
dumped = tracker.current_state(should_include_events=True)
utils.dump_obj_as_json_to_file(out_path.strpath, dumped)
tracker_json = json.loads(utils.read_file(out_path.strpath))
sender_id = tracker_json.get("sender_id",
UserMessage.DEFAULT_SENDER_ID)
restored_tracker = DialogueStateTracker.from_dict(
sender_id, tracker_json.get("events", []), default_agent.domain)
assert restored_tracker == tracker
def persist(self, path):
# type: (Text) -> None
if self.model:
self.featurizer.persist(path)
meta = {"model": "keras_model.h5", "epochs": self.current_epoch}
config_file = os.path.join(path, 'keras_policy.json')
utils.dump_obj_as_json_to_file(config_file, meta)
model_file = os.path.join(path, meta['model'])
# makes sure the model directory exists
utils.create_dir_for_file(model_file)
with self.graph.as_default(), self.session.as_default():
self.model.save(model_file, overwrite=True)
else:
warnings.warn("Persist called without a trained model present. "
"Nothing to persist then!")
def persist(self, path):
# type: (Text) -> None
if self.model:
self.featurizer.persist(path)
meta = {"model": "keras_model.h5",
"epochs": self.current_epoch}
config_file = os.path.join(path, 'keras_policy.json')
utils.dump_obj_as_json_to_file(config_file, meta)
model_file = os.path.join(path, meta['model'])
# makes sure the model directory exists
utils.create_dir_for_file(model_file)
with self.graph.as_default(), self.session.as_default():
self.model.save(model_file, overwrite=True)
else:
warnings.warn("Persist called without a trained model present. "
"Nothing to persist then!")
def _persist_metadata(self, path, max_history):
# type: (Text, Optional[int]) -> None
"""Persists the domain specification to storage."""
# make sure the directory we persist to exists
domain_spec_path = os.path.join(path, 'policy_metadata.json')
utils.create_dir_for_file(domain_spec_path)
policy_names = [utils.module_path_from_instance(p)
for p in self.policies]
training_events = self.training_metadata.get("events", {})
action_fingerprints = self._create_action_fingerprints(training_events)
metadata = {
"action_fingerprints": action_fingerprints,
"rasa_core": rasa_core.__version__,
"max_history": max_history,
"ensemble_name": self.__module__ + "." + self.__class__.__name__,
"policy_names": policy_names
}
utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
def persist(self, path):
memorized_file = os.path.join(path, 'memorized_turns.json')
data = {"lookup": self.lookup}
utils.create_dir_for_file(memorized_file)
utils.dump_obj_as_json_to_file(memorized_file, data)
