def parse(self, data):
project = data.get("project") or RasaNLUConfig.DEFAULT_PROJECT_NAME
model = data.get("model")
if project not in self.project_store:
projects = os.listdir(self.config['path'])
if project not in projects:
raise InvalidProjectError(
"No project found with name '{}'.".format(project))
else:
try:
self.project_store[project] = Project(
self.config, self.component_builder, project)
except Exception as e:
raise InvalidProjectError(
"Unable to load project '{}'. Error: {}".format(
project, e))
response, used_model = self.project_store[project].parse(
data['text'], data.get('time', None), model)
if self.responses:
self.responses.info(user_input=response,
project=project,
model=used_model)
return self.format_response(response)
def parse(self, data):
project = data.get("project", RasaNLUModelConfig.DEFAULT_PROJECT_NAME)
model = data.get("model")
if project not in self.project_store:
projects = self._list_projects(self.project_dir)
cloud_provided_projects = self._list_projects_in_cloud()
projects.extend(cloud_provided_projects)
if project not in projects:
raise InvalidProjectError(
"No project found with name '{}'.".format(project))
else:
try:
self.project_store[project] = Project(
self.component_builder, project, self.project_dir,
self.remote_storage)
except Exception as e:
raise InvalidProjectError(
"Unable to load project '{}'. Error: {}".format(
project, e))
time = data.get('time')
response, used_model = self.project_store[project].parse(
data['text'], time, model)
if self.responses:
self.responses.info('',
user_input=response,
project=project,
model=used_model)
return self.format_response(response)
def unload_model(self, project, model):
# type: (Text, Text) -> Dict[Text]
"""Unload a model from server memory."""
if project is None:
raise InvalidProjectError("No project specified".format(project))
elif project not in self.project_store:
raise InvalidProjectError("Project {} could not "
"be found".format(project))
try:
unloaded_model = self.project_store[project].unload(model)
return unloaded_model
except KeyError:
raise InvalidProjectError("Failed to unload model {} "
"for project {}.".format(model, project))
def evaluate(self, data, project=None, model=None):
# type: (Text, Optional[Text], Optional[Text]) -> Dict[Text, Any]
"""Perform a model evaluation."""
project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME
model = model or None
file_name = utils.create_temporary_file(data, "_training_data")
if project not in self.project_store:
raise InvalidProjectError("Project {} could not "
"be found".format(project))
model_name = self.project_store[project]._dynamic_load_model(model)
self.project_store[project]._loader_lock.acquire()
try:
if not self.project_store[project]._models.get(model_name):
interpreter = self.project_store[project]. \
_interpreter_for_model(model_name)
self.project_store[project]._models[model_name] = interpreter
finally:
self.project_store[project]._loader_lock.release()
return run_evaluation(
data_path=file_name,
model=self.project_store[project]._models[model_name],
errors_filename=None)
def start_train_process(self,
data_file: Text,
project: Text,
train_config: RasaNLUModelConfig,
model_name: Optional[Text] = None
) -> Deferred:
"""Start a model training."""
if not project:
raise InvalidProjectError("Missing project name to train")
if self._training_processes <= self._current_training_processes:
raise MaxTrainingError
if project in self.project_store:
self.project_store[project].status = STATUS_TRAINING
elif project not in self.project_store:
self.project_store[project] = Project(
self.component_builder, project,
self.project_dir, self.remote_storage)
self.project_store[project].status = STATUS_TRAINING
def training_callback(model_path):
model_dir = os.path.basename(os.path.normpath(model_path))
self.project_store[project].update(model_dir)
self._current_training_processes -= 1
self.project_store[project].current_training_processes -= 1
if (self.project_store[project].status == STATUS_TRAINING and
self.project_store[project].current_training_processes ==
0):
self.project_store[project].status = STATUS_READY
return model_path
def training_errback(failure):
logger.warning(failure)
self._current_training_processes -= 1
self.project_store[project].current_training_processes -= 1
self.project_store[project].status = STATUS_FAILED
self.project_store[project].error_message = str(failure)
return failure
logger.debug("New training queued")
self._current_training_processes += 1
self.project_store[project].current_training_processes += 1
result = self.pool.submit(do_train_in_worker,
train_config,
data_file,
path=self.project_dir,
project=project,
fixed_model_name=model_name,
storage=self.remote_storage)
result = deferred_from_future(result)
result.addCallback(training_callback)
result.addErrback(training_errback)
return result
def start_train_process(self, data, config_values):
# type: (Text, Dict[Text, Any]) -> Deferred
"""Start a model training."""
if PY3:
f = tempfile.NamedTemporaryFile("w+",
suffix="_training_data",
delete=False,
encoding="utf-8")
f.write(data)
else:
f = tempfile.NamedTemporaryFile("w+",
suffix="_training_data",
delete=False)
f.write(data.encode("utf-8"))
f.close()
# TODO: fix config handling
_config = self.config.as_dict()
for key, val in config_values.items():
_config[key] = val
_config["data"] = f.name
train_config = RasaNLUConfig(cmdline_args=_config)
project = _config.get("project")
if not project:
raise InvalidProjectError("Missing project name to train")
elif project in self.project_store:
if self.project_store[project].status == 1:
raise AlreadyTrainingError
else:
self.project_store[project].status = 1
elif project not in self.project_store:
self.project_store[project] = Project(self.config,
self.component_builder,
project)
self.project_store[project].status = 1
def training_callback(model_path):
model_dir = os.path.basename(os.path.normpath(model_path))
self.project_store[project].update(model_dir)
return model_dir
def training_errback(failure):
target_project = self.project_store.get(
failure.value.failed_target_project)
if target_project:
target_project.status = 0
return failure
logger.debug("New training queued")
result = self.pool.submit(do_train_in_worker, train_config)
result = deferred_from_future(result)
result.addCallback(training_callback)
result.addErrback(training_errback)
return result
def unload_model(self, project, model):
# type: (Text, Text) -> Dict[Text]
"""Unload a model from server memory."""
if project is None:
raise InvalidProjectError("No project specified".format(project))
if not self.project_manager.project_exists(project):
raise InvalidProjectError("Project {} could not "
"be found".format(project))
project_instance = self.project_manager.load_project(project)
try:
unloaded_model = project_instance.unload(model)
return unloaded_model
except KeyError:
raise InvalidProjectError("Failed to unload model {} "
"for project {}.".format(model, project))
def _try_load_project(self, project):
# type: (str) -> None
# Try to load project from local storage, if failed
# an InvalidProjectError exception will raise
try:
self.project_store[project] = Project(self.component_builder,
project, self.project_dir,
self.remote_storage)
except Exception as e:
raise InvalidProjectError(
"Unable to load project '{}'. Error: {}".format(project, e))
def start_train_process(
self,
data_file, # type: Text
project, # type: Text
train_config, # type: RasaNLUModelConfig
model_name=None # type: Optional[Text]
):
# type: (...) -> Deferred
"""Start a model training."""
if not project:
raise InvalidProjectError("Missing project name to train")
if project in self.project_store:
if self.project_store[project].status == 1:
raise AlreadyTrainingError
else:
self.project_store[project].status = 1
elif project not in self.project_store:
self.project_store[project] = Project(self.component_builder,
project, self.project_dir,
self.remote_storage)
self.project_store[project].status = 1
def training_callback(model_path):
model_dir = os.path.basename(os.path.normpath(model_path))
self.project_store[project].update(model_dir)
return model_dir
def training_errback(failure):
logger.warn(failure)
target_project = self.project_store.get(
failure.value.failed_target_project)
if target_project:
target_project.status = 0
return failure
logger.debug("New training queued")
result = self.pool.submit(do_train_in_worker,
train_config,
data_file,
path=self.project_dir,
project=project,
fixed_model_name=model_name,
storage=self.remote_storage)
result = deferred_from_future(result)
result.addCallback(training_callback)
result.addErrback(training_errback)
return result
def evaluate(self, data, project=None, model=None):
# type: (Text, Optional[Text], Optional[Text]) -> Dict[Text, Any]
"""Perform a model evaluation."""
project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME
model = model or None
file_name = utils.create_temporary_file(data, "_training_data")
test_data = load_data(file_name)
if project not in self.project_store:
raise InvalidProjectError("Project {} could not "
"be found".format(project))
preds_json = self.parse_training_examples(test_data.intent_examples,
project,
model)
predictions = [
{"text": e.text,
"intent": e.data.get("intent"),
"predicted": p.get("intent", {}).get("name"),
"confidence": p.get("intent", {}).get("confidence")}
for e, p in zip(test_data.intent_examples, preds_json)
]
y_true = [e.data.get("intent") for e in test_data.intent_examples]
y_true = clean_intent_labels(y_true)
y_pred = [p.get("intent", {}).get("name") for p in preds_json]
y_pred = clean_intent_labels(y_pred)
report, precision, f1, accuracy = get_evaluation_metrics(y_true,
y_pred)
return {
"intent_evaluation": {
"report": report,
"predictions": predictions,
"precision": precision,
"f1_score": f1,
"accuracy": accuracy}
}
def load_project(self, project=None):
# type: (Union[None, str]) -> Project
# Try to load a project from memory first, if not exists
# try load from local file system or cloud by name,
# if project not exists or can not be loaded correctly,
# a InvalidProjectError exception will raise
project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME
if project not in self.project_store:
projects = self._list_projects(self.project_dir)
cloud_provided_projects = self._list_projects_in_cloud()
projects.extend(cloud_provided_projects)
if project not in projects:
raise InvalidProjectError(
"No project found with name '{}'.".format(project))
else:
self._try_load_project(project)
return self.project_store[project]
def start_train_process(
self,
data_file, # type: Text
project, # type: Text
train_config, # type: RasaNLUModelConfig
model_name=None # type: Optional[Text]
):
# type: (...) -> Deferred
"""Start a model training."""
if not project:
raise InvalidProjectError("Missing project name to train")
if self._training_processes <= self._current_training_processes:
raise MaxTrainingError
if project in self.project_store:
self.project_store[project].status = STATUS_TRAINING
elif project not in self.project_store:
self.project_store[project] = Project(self.component_builder,
project, self.project_dir,
self.remote_storage)
self.project_store[project].status = STATUS_TRAINING
def training_callback(model_path):
model_dir = os.path.basename(os.path.normpath(model_path))
self.project_store[project].update(model_dir)
self._current_training_processes -= 1
self.project_store[project].current_training_processes -= 1
if (self.project_store[project].status == STATUS_TRAINING
and self.project_store[project].current_training_processes
== 0):
self.project_store[project].status = STATUS_READY
return model_dir
def training_errback(failure):
logger.warning(failure)
self._current_training_processes -= 1
self.project_store[project].current_training_processes -= 1
self.project_store[project].status = STATUS_FAILED
self.project_store[project].error_message = str(failure)
return failure
logger.debug("New training queued")
self._current_training_processes += 1
self.project_store[project].current_training_processes += 1
# tensorflow training is not executed in a separate thread on python 2,
# as this may cause training to freeze
if six.PY2 and self._tf_in_pipeline(train_config):
try:
logger.warning("Training a pipeline with a tensorflow "
"component. This blocks the server during "
"training.")
model_path = do_train_in_worker(train_config,
data_file,
path=self.project_dir,
project=project,
fixed_model_name=model_name,
storage=self.remote_storage)
model_dir = os.path.basename(os.path.normpath(model_path))
training_callback(model_dir)
return model_dir
except TrainingException as e:
logger.warning(e)
target_project = self.project_store.get(
e.failed_target_project)
if target_project:
target_project.status = STATUS_READY
raise e
else:
result = self.pool.submit(do_train_in_worker,
train_config,
data_file,
path=self.project_dir,
project=project,
fixed_model_name=model_name,
storage=self.remote_storage)
result = deferred_from_future(result)
result.addCallback(training_callback)
result.addErrback(training_errback)
return result
