def persist_specification(self, model_path: Text) -> None:
"""Persists the domain specification to storage."""
domain_spec_path = os.path.join(model_path, "domain.json")
utils.create_dir_for_file(domain_spec_path)
metadata = {"states": self.input_states}
utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
async def compare_models(models: List[Text], stories_file: Text, output: Text) -> None:
"""Evaluates provided trained models on a test set."""
from rasa.core import utils
number_correct = defaultdict(list)
for model in models:
number_of_correct_stories = await _evaluate_core_model(model, stories_file)
number_correct[os.path.basename(model)].append(number_of_correct_stories)
utils.dump_obj_as_json_to_file(os.path.join(output, RESULTS_FILE), number_correct)
def persist_fingerprint(output_path: Text, fingerprint: Fingerprint):
"""Persists a model fingerprint.
Args:
output_path: Directory in which the fingerprint should be saved.
fingerprint: The fingerprint to be persisted.
"""
from rasa.core.utils import dump_obj_as_json_to_file
path = os.path.join(output_path, FINGERPRINT_FILE_PATH)
dump_obj_as_json_to_file(path, fingerprint)
def persist(self, path: Text) -> None:
"""Persists the policy to storage."""
config_file = os.path.join(path, 'fallback_policy.json')
meta = {
"priority": self.priority,
"nlu_threshold": self.nlu_threshold,
"core_threshold": self.core_threshold,
"fallback_action_name": self.fallback_action_name
}
utils.create_dir_for_file(config_file)
utils.dump_obj_as_json_to_file(config_file, meta)
def persist(self, path: Text) -> None:
self.featurizer.persist(path)
memorized_file = os.path.join(path, 'memorized_turns.json')
data = {
"priority": self.priority,
"max_history": self.max_history,
"lookup": self.lookup
}
utils.create_dir_for_file(memorized_file)
utils.dump_obj_as_json_to_file(memorized_file, data)
def persist(self, path: Text) -> None:
"""Persists the policy to storage."""
config_file = os.path.join(path, "two_stage_fallback_policy.json")
meta = {
"priority": self.priority,
"nlu_threshold": self.nlu_threshold,
"core_threshold": self.core_threshold,
"fallback_core_action_name": self.fallback_action_name,
"fallback_nlu_action_name": self.fallback_nlu_action_name,
"deny_suggestion_intent_name": self.deny_suggestion_intent_name,
}
rasa.utils.io.create_directory_for_file(config_file)
utils.dump_obj_as_json_to_file(config_file, meta)
async def test_dump_and_restore_as_json(default_agent, tmpdir_factory):
trackers = await default_agent.load_data(DEFAULT_STORIES_FILE)
for tracker in trackers:
out_path = tmpdir_factory.mktemp("tracker").join("dumped_tracker.json")
dumped = tracker.current_state(EventVerbosity.AFTER_RESTART)
utils.dump_obj_as_json_to_file(out_path.strpath, dumped)
restored_tracker = restore.load_tracker_from_json(
out_path.strpath, default_agent.domain)
assert restored_tracker == tracker
def persist(self, path: Text) -> None:
"""Persists the policy to storage."""
config_file = os.path.join(path, "custom_policy.json")
meta = {
"priority": self.priority,
"nlu_threshold": self.nlu_threshold,
"ambiguity_threshold": self.ambiguity_threshold,
"core_threshold": self.core_threshold,
"fallback_action_name": self.fallback_action_name,
}
rasa.utils.io.create_directory_for_file(config_file)
utils.dump_obj_as_json_to_file(config_file, meta)
def persist(self, path: Text) -> None:
if self.model:
self.featurizer.persist(path)
meta = {"priority": self.priority}
meta_file = os.path.join(path, 'sklearn_policy.json')
utils.dump_obj_as_json_to_file(meta_file, meta)
filename = os.path.join(path, 'sklearn_model.pkl')
with open(filename, 'wb') as f:
pickle.dump(self._state, f)
else:
warnings.warn("Persist called without a trained model present. "
"Nothing to persist then!")
def persist(self, path: Text) -> None:
"""Persists the policy to storage."""
config_file = os.path.join(path, "botfront_disambiguation_policy.json")
meta = {
"priority": self.priority,
"disambiguation_trigger": self.disambiguation_trigger,
"fallback_trigger": self.fallback_trigger,
"deny_suggestions": self.deny_suggestions,
"excluded_intents": self.excluded_intents,
"n_suggestions": self.n_suggestions,
"intent_mappings": self.intent_mappings,
"disambiguation_title": self.disambiguation_title
}
rasa.utils.io.create_directory_for_file(config_file)
utils.dump_obj_as_json_to_file(config_file, meta)
async def post_data_convert(request: Request):
"""Converts current domain in yaml or json format."""
validate_request_body(
request,
"You must provide training data in the request body in order to "
"train your model.",
)
rjs = request.json
if 'data' not in rjs:
raise ErrorResponse(
400, "BadRequest",
"Must provide training data in 'data' property")
if 'output_format' not in rjs or rjs["output_format"] not in [
"json", "md"
]:
raise ErrorResponse(
400, "BadRequest",
"'output_format' is required and must be either 'md' or 'json")
if 'language' not in rjs:
raise ErrorResponse(400, "BadRequest", "'language' is required")
temp_dir = tempfile.mkdtemp()
out_dir = tempfile.mkdtemp()
nlu_data_path = os.path.join(temp_dir, "nlu_data")
output_path = os.path.join(out_dir, "output")
# botfront: several nlu files
if type(rjs["data"] is dict):
from rasa.core.utils import dump_obj_as_json_to_file
dump_obj_as_json_to_file(nlu_data_path, rjs["data"])
else:
dump_obj_as_str_to_file(nlu_data_path, rjs["data"])
# botfront end
from rasa.nlu.convert import convert_training_data
convert_training_data(nlu_data_path, output_path, rjs["output_format"],
rjs["language"])
with open(output_path, encoding='utf-8') as f:
data = f.read()
if rjs["output_format"] == 'json':
import json
data = json.loads(data, encoding='utf-8')
return response.json({"data": data})
def persist(self, path: Text) -> None:
"""Persists the policy to a storage."""
if self.session is None:
warnings.warn("Method `persist(...)` was called "
"without a trained model present. "
"Nothing to persist then!")
return
self.featurizer.persist(path)
meta = {"priority": self.priority}
meta_file = os.path.join(path, "embedding_policy.json")
utils.dump_obj_as_json_to_file(meta_file, meta)
file_name = "tensorflow_embedding.ckpt"
checkpoint = os.path.join(path, file_name)
rasa.utils.io.create_directory_for_file(checkpoint)
with self.graph.as_default():
train_utils.persist_tensor("user_placeholder", self.a_in,
self.graph)
train_utils.persist_tensor("bot_placeholder", self.b_in,
self.graph)
train_utils.persist_tensor("similarity_all", self.sim_all,
self.graph)
train_utils.persist_tensor("pred_confidence", self.pred_confidence,
self.graph)
train_utils.persist_tensor("similarity", self.sim, self.graph)
train_utils.persist_tensor("dial_embed", self.dial_embed,
self.graph)
train_utils.persist_tensor("bot_embed", self.bot_embed, self.graph)
train_utils.persist_tensor("all_bot_embed", self.all_bot_embed,
self.graph)
train_utils.persist_tensor("attention_weights",
self.attention_weights, self.graph)
saver = tf.train.Saver()
saver.save(self.session, checkpoint)
with open(os.path.join(path, file_name + ".tf_config.pkl"), "wb") as f:
pickle.dump(self._tf_config, f)
async def do_compare_training(cmdline_args, stories, additional_arguments):
from rasa.core import utils
await train_comparison_models(stories, cmdline_args.domain,
cmdline_args.out, cmdline_args.percentages,
cmdline_args.config, cmdline_args.runs,
cmdline_args.dump_stories,
additional_arguments)
no_stories = await get_no_of_stories(cmdline_args.stories,
cmdline_args.domain)
# store the list of the number of stories present at each exclusion
# percentage
story_range = [
no_stories - round((x / 100.0) * no_stories)
for x in cmdline_args.percentages
]
story_n_path = os.path.join(cmdline_args.out, 'num_stories.json')
utils.dump_obj_as_json_to_file(story_n_path, story_range)
def _persist_metadata(self,
path: Text,
dump_flattened_stories: bool = False) -> None:
"""Persists the domain specification to storage."""
# make sure the directory we persist exists
domain_spec_path = os.path.join(path, "metadata.json")
training_data_path = os.path.join(path, "stories.md")
rasa.utils.io.create_directory_for_file(domain_spec_path)
policy_names = [
utils.module_path_from_instance(p) for p in self.policies
]
training_events = self._training_events_from_trackers(
self.training_trackers)
action_fingerprints = self._create_action_fingerprints(training_events)
metadata = {
"action_fingerprints": action_fingerprints,
"python": ".".join([str(s) for s in sys.version_info[:3]]),
"max_histories": self._max_histories(),
"ensemble_name": self.__module__ + "." + self.__class__.__name__,
"policy_names": policy_names,
"trained_at": self.date_trained,
}
self._add_package_version_info(metadata)
utils.dump_obj_as_json_to_file(domain_spec_path, metadata)
# if there are lots of stories, saving flattened stories takes a long
# time, so this is turned off by default
if dump_flattened_stories:
training.persist_data(self.training_trackers, training_data_path)
async def train(request: Request):
"""Train a Rasa Model."""
from rasa.train import train_async
validate_request_body(
request,
"You must provide training data in the request body in order to "
"train your model.",
)
rjs = request.json
validate_request(rjs)
# create a temporary directory to store config, domain and
# training data
temp_dir = tempfile.mkdtemp()
filenames = {
"config": "config.yml",
"nlu": "nlu.md",
"stories": "stories.md",
"domain": "domain.yml"
}
for (key, filename) in filenames.items():
if key not in rjs: continue
temp_path = os.path.join(temp_dir, filename)
temp_data = rjs[key]
if isinstance(temp_data, dict):
dump_obj_as_json_to_file(temp_path, temp_data)
else:
dump_obj_as_str_to_file(temp_path, temp_data)
if "domain" in rjs:
domain_path = os.path.join(temp_dir, filenames['domain'])
else:
domain_path = DEFAULT_DOMAIN_PATH
try:
model_path = await train_async(
domain=domain_path,
config=os.path.join(temp_dir, filenames["config"]),
training_files=temp_dir,
output_path=rjs.get("out", DEFAULT_MODELS_PATH),
force_training=rjs.get("force", False),
)
filename = os.path.basename(model_path) if model_path else None
return await response.file(model_path,
filename=filename,
headers={"filename": filename})
except InvalidDomain as e:
raise ErrorResponse(
400,
"InvalidDomainError",
"Provided domain file is invalid. Error: {}".format(e),
)
except Exception as e:
logger.debug(traceback.format_exc())
raise ErrorResponse(
500,
"TrainingError",
"An unexpected error occurred during training. Error: {}".
format(e),
)
