def parse(self, message, project_name):
from rasa_nlu.project import Project
self.logger.info('Parsing message with %s: %s' %
(project_name, message))
project = self._projects.get(project_name)
if project is None:
project = Project(project=project_name,
project_dir=self._project_dir)
self._projects[project_name] = project
# --------------------------
# RasaNLU intent JSON format
# --------------------------
# {
# "intent": {
# "name": "...",
# "confidence": 1.0
# },
# "entities": [
# {
# "value": "...",
# "entity": "..."
# }
# ],
# "text": "..."
# }
result = project.parse(message)
self.logger.debug(str(result))
return result
def _create_project_store(self, project_dir):
default_project = RasaNLUModelConfig.DEFAULT_PROJECT_NAME
projects = self._collect_projects(project_dir)
project_store = {}
if self.model_server is not None:
project_store[default_project] = load_from_server(
self.component_builder, default_project, self.project_dir,
self.remote_storage, self.model_server,
self.wait_time_between_pulls)
else:
for project in projects:
project_store[project] = Project(self.component_builder,
project, self.project_dir,
self.remote_storage)
if not project_store:
project_store[default_project] = Project(
project=default_project,
project_dir=self.project_dir,
remote_storage=self.remote_storage)
return project_store
def test_dynamic_load_model_with_model_is_none():
LATEST_MODEL_NAME = 'latest_model_name'
def mocked_init(*args, **kwargs):
return None
def mocked_search_for_models(self):
pass
def mocked_latest_project_model(self):
return LATEST_MODEL_NAME
with mock.patch.object(Project, "__init__", mocked_init):
with mock.patch.object(Project, "_search_for_models",
mocked_search_for_models):
with mock.patch.object(Project, "_latest_project_model",
mocked_latest_project_model):
project = Project()
project._models = ()
project.pull_models = None
result = project._dynamic_load_model(None)
assert result == LATEST_MODEL_NAME
def test_dynamic_load_model_with_model_is_none():
LATEST_MODEL_NAME = 'latest_model_name'
def mocked_init(*args, **kwargs):
return None
def mocked_search_for_models(self):
pass
def mocked_latest_project_model(self):
return LATEST_MODEL_NAME
with mock.patch.object(Project, "__init__", mocked_init):
with mock.patch.object(Project, "_search_for_models",
mocked_search_for_models):
with mock.patch.object(Project, "_latest_project_model",
mocked_latest_project_model):
project = Project()
project._models = ()
project.pull_models = None
result = project._dynamic_load_model(None)
assert result == LATEST_MODEL_NAME
def _create_project_store(self):
projects = self._collect_projects()
project_store = {}
for project in projects:
project_store[project] = Project(self.config,
self.component_builder, project)
if not project_store:
project_store[RasaNLUConfig.DEFAULT_PROJECT_NAME] = Project(
self.config)
return project_store
def read_model_metadata(model_dir, config):
if model_dir is None:
data = Project._default_model_metadata()
return Metadata(data, model_dir)
else:
if not os.path.isabs(model_dir):
model_dir = os.path.join(config['path'], model_dir)
# download model from S3 if needed
if not os.path.isdir(model_dir):
Project._load_model_from_cloud(model_dir, config)
return Metadata.load(model_dir)
def _create_project_store(self):
projects = []
if os.path.isdir(self.config['path']):
projects = os.listdir(self.config['path'])
project_store = {}
for project in projects:
project_store[project] = Project(self.config, self.component_builder, project)
if not project_store:
project_store[RasaNLUConfig.DEFAULT_PROJECT_NAME] = Project(self.config)
return project_store
def test_dynamic_load_model_with_exists_model():
MODEL_NAME = 'model_name'
def mocked_init(*args, **kwargs):
return None
with mock.patch.object(Project, "__init__", mocked_init):
project = Project()
project._models = (MODEL_NAME, )
result = project._dynamic_load_model(MODEL_NAME)
assert result == MODEL_NAME
def test_dynamic_load_model_with_exists_model():
MODEL_NAME = 'model_name'
def mocked_init(*args, **kwargs):
return None
with mock.patch.object(Project, "__init__", mocked_init):
project = Project()
project._models = (MODEL_NAME,)
project.pull_models = None
result = project._dynamic_load_model(MODEL_NAME)
assert result == MODEL_NAME
def start_train_process(self,
data_file: Text,
project: Text,
train_config: RasaNLUModelConfig,
model_name: Optional[Text] = None
) -> Deferred:
"""Start a model training."""
if not project:
raise InvalidProjectError("Missing project name to train")
if self._training_processes <= self._current_training_processes:
raise MaxTrainingError
if project in self.project_store:
self.project_store[project].status = STATUS_TRAINING
elif project not in self.project_store:
self.project_store[project] = Project(
self.component_builder, project,
self.project_dir, self.remote_storage)
self.project_store[project].status = STATUS_TRAINING
def training_callback(model_path):
model_dir = os.path.basename(os.path.normpath(model_path))
self.project_store[project].update(model_dir)
self._current_training_processes -= 1
self.project_store[project].current_training_processes -= 1
if (self.project_store[project].status == STATUS_TRAINING and
self.project_store[project].current_training_processes ==
0):
self.project_store[project].status = STATUS_READY
return model_path
def training_errback(failure):
logger.warning(failure)
self._current_training_processes -= 1
self.project_store[project].current_training_processes -= 1
self.project_store[project].status = STATUS_FAILED
self.project_store[project].error_message = str(failure)
return failure
logger.debug("New training queued")
self._current_training_processes += 1
self.project_store[project].current_training_processes += 1
result = self.pool.submit(do_train_in_worker,
train_config,
data_file,
path=self.project_dir,
project=project,
fixed_model_name=model_name,
storage=self.remote_storage)
result = deferred_from_future(result)
result.addCallback(training_callback)
result.addErrback(training_errback)
return result
def parse(self, data):
project = data.get("project", RasaNLUModelConfig.DEFAULT_PROJECT_NAME)
model = data.get("model")
if project not in self.project_store:
projects = self._list_projects(self.project_dir)
cloud_provided_projects = self._list_projects_in_cloud()
projects.extend(cloud_provided_projects)
if project not in projects:
raise InvalidProjectError(
"No project found with name '{}'.".format(project))
else:
try:
self.project_store[project] = Project(
self.component_builder, project, self.project_dir,
self.remote_storage)
except Exception as e:
raise InvalidProjectError(
"Unable to load project '{}'. Error: {}".format(
project, e))
time = data.get('time')
response, used_model = self.project_store[project].parse(
data['text'], time, model)
if self.responses:
self.responses.info('',
user_input=response,
project=project,
model=used_model)
return self.format_response(response)
def _create_project_store(self, project_dir):
projects = self._collect_projects(project_dir)
project_store = {}
for project in projects:
project_store[project] = Project(self.component_builder, project,
self.project_dir,
self.remote_storage)
if not project_store:
default_model = RasaNLUModelConfig.DEFAULT_PROJECT_NAME
project_store[default_model] = Project(
project_dir=self.project_dir,
remote_storage=self.remote_storage)
return project_store
def parse(self, data):
project = data.get("project") or RasaNLUConfig.DEFAULT_PROJECT_NAME
model = data.get("model")
if project not in self.project_store:
projects = os.listdir(self.config['path'])
if project not in projects:
raise InvalidProjectError(
"No project found with name '{}'.".format(project))
else:
try:
self.project_store[project] = Project(
self.config, self.component_builder, project)
except Exception as e:
raise InvalidProjectError(
"Unable to load project '{}'. Error: {}".format(
project, e))
response, used_model = self.project_store[project].parse(
data['text'], data.get('time', None), model)
if self.responses:
self.responses.info(user_input=response,
project=project,
model=used_model)
return self.format_response(response)
def start_train_process(self, data, config_values):
# type: (Text, Dict[Text, Any]) -> Deferred
"""Start a model training."""
if PY3:
f = tempfile.NamedTemporaryFile("w+",
suffix="_training_data",
delete=False,
encoding="utf-8")
f.write(data)
else:
f = tempfile.NamedTemporaryFile("w+",
suffix="_training_data",
delete=False)
f.write(data.encode("utf-8"))
f.close()
# TODO: fix config handling
_config = self.config.as_dict()
for key, val in config_values.items():
_config[key] = val
_config["data"] = f.name
train_config = RasaNLUConfig(cmdline_args=_config)
project = _config.get("project")
if not project:
raise InvalidProjectError("Missing project name to train")
elif project in self.project_store:
if self.project_store[project].status == 1:
raise AlreadyTrainingError
else:
self.project_store[project].status = 1
elif project not in self.project_store:
self.project_store[project] = Project(self.config,
self.component_builder,
project)
self.project_store[project].status = 1
def training_callback(model_path):
model_dir = os.path.basename(os.path.normpath(model_path))
self.project_store[project].update(model_dir)
return model_dir
def training_errback(failure):
target_project = self.project_store.get(
failure.value.failed_target_project)
if target_project:
target_project.status = 0
return failure
logger.debug("New training queued")
result = self.pool.submit(do_train_in_worker, train_config)
result = deferred_from_future(result)
result.addCallback(training_callback)
result.addErrback(training_errback)
return result
def test_dynamic_load_model_with_refresh_exists_model():
MODEL_NAME = 'model_name'
def mocked_init(*args, **kwargs):
return None
def mocked_search_for_models(self):
self._models = (MODEL_NAME, )
with mock.patch.object(Project, "__init__", mocked_init):
with mock.patch.object(Project, '_search_for_models', mocked_search_for_models):
project = Project()
project._models = ()
result = project._dynamic_load_model(MODEL_NAME)
assert result == MODEL_NAME
def _try_load_project(self, project):
# type: (str) -> None
# Try to load project from local storage, if failed
# an InvalidProjectError exception will raise
try:
self.project_store[project] = Project(self.component_builder,
project, self.project_dir,
self.remote_storage)
except Exception as e:
raise InvalidProjectError(
"Unable to load project '{}'. Error: {}".format(project, e))
def load_or_create_project(self, project):
try:
project_instance = self.load_project(project)
except InvalidProjectError:
# project not exists, create one
project_instance = Project(self.component_builder, project,
self.project_dir, self.remote_storage)
# add to cache
self.project_store[project] = project_instance
return project_instance
def test_dynamic_load_model_with_refresh_exists_model():
MODEL_NAME = 'model_name'
def mocked_init(*args, **kwargs):
return None
def mocked_search_for_models(self):
self._models = (MODEL_NAME,)
with mock.patch.object(Project, "__init__", mocked_init):
with mock.patch.object(Project, '_search_for_models',
mocked_search_for_models):
project = Project()
project._models = ()
project.pull_models = None
result = project._dynamic_load_model(MODEL_NAME)
assert result == MODEL_NAME
def start_train_process(
self,
data_file, # type: Text
project, # type: Text
train_config, # type: RasaNLUModelConfig
model_name=None # type: Optional[Text]
):
# type: (...) -> Deferred
"""Start a model training."""
if not project:
raise InvalidProjectError("Missing project name to train")
if project in self.project_store:
if self.project_store[project].status == 1:
raise AlreadyTrainingError
else:
self.project_store[project].status = 1
elif project not in self.project_store:
self.project_store[project] = Project(self.component_builder,
project, self.project_dir,
self.remote_storage)
self.project_store[project].status = 1
def training_callback(model_path):
model_dir = os.path.basename(os.path.normpath(model_path))
self.project_store[project].update(model_dir)
return model_dir
def training_errback(failure):
logger.warn(failure)
target_project = self.project_store.get(
failure.value.failed_target_project)
if target_project:
target_project.status = 0
return failure
logger.debug("New training queued")
result = self.pool.submit(do_train_in_worker,
train_config,
data_file,
path=self.project_dir,
project=project,
fixed_model_name=model_name,
storage=self.remote_storage)
result = deferred_from_future(result)
result.addCallback(training_callback)
result.addErrback(training_errback)
return result
def start_train_process(
self,
data_file, # type: Text
project, # type: Text
train_config, # type: RasaNLUModelConfig
model_name=None # type: Optional[Text]
):
# type: (...) -> Deferred
"""Start a model training."""
if not project:
raise InvalidProjectError("Missing project name to train")
if self._training_processes <= self._current_training_processes:
raise MaxTrainingError
if project in self.project_store:
self.project_store[project].status = STATUS_TRAINING
elif project not in self.project_store:
self.project_store[project] = Project(self.component_builder,
project, self.project_dir,
self.remote_storage)
self.project_store[project].status = STATUS_TRAINING
def training_callback(model_path):
model_dir = os.path.basename(os.path.normpath(model_path))
self.project_store[project].update(model_dir)
self._current_training_processes -= 1
self.project_store[project].current_training_processes -= 1
if (self.project_store[project].status == STATUS_TRAINING
and self.project_store[project].current_training_processes
== 0):
self.project_store[project].status = STATUS_READY
return model_dir
def training_errback(failure):
logger.warning(failure)
self._current_training_processes -= 1
self.project_store[project].current_training_processes -= 1
self.project_store[project].status = STATUS_FAILED
self.project_store[project].error_message = str(failure)
return failure
logger.debug("New training queued")
self._current_training_processes += 1
self.project_store[project].current_training_processes += 1
# tensorflow training is not executed in a separate thread on python 2,
# as this may cause training to freeze
if six.PY2 and self._tf_in_pipeline(train_config):
try:
logger.warning("Training a pipeline with a tensorflow "
"component. This blocks the server during "
"training.")
model_path = do_train_in_worker(train_config,
data_file,
path=self.project_dir,
project=project,
fixed_model_name=model_name,
storage=self.remote_storage)
model_dir = os.path.basename(os.path.normpath(model_path))
training_callback(model_dir)
return model_dir
except TrainingException as e:
logger.warning(e)
target_project = self.project_store.get(
e.failed_target_project)
if target_project:
target_project.status = STATUS_READY
raise e
else:
result = self.pool.submit(do_train_in_worker,
train_config,
data_file,
path=self.project_dir,
project=project,
fixed_model_name=model_name,
storage=self.remote_storage)
result = deferred_from_future(result)
result.addCallback(training_callback)
result.addErrback(training_errback)
return result
def get_intent(profile, text):
system = profile.intent.get('system', 'fuzzywuzzy')
if system == 'rasa':
rasa_config = profile.intent[system]
# Use rasaNLU
global intent_projects
project = intent_projects.get(profile.name, None)
if project is None:
import rasa_nlu
from rasa_nlu.project import Project
project_dir = profile.read_path(rasa_config['project_dir'])
project_name = rasa_config['project_name']
project = Project(project=project_name, project_dir=project_dir)
intent_projects[profile.name] = project
return project.parse(text)
elif system == 'remote':
remote_url = profile.intent[system]['url']
headers = {'Content-Type': 'text/plain'}
# Pass profile name through
params = {'profile': profile.name, 'nohass': True}
response = requests.post(remote_url,
headers=headers,
data=text,
params=params)
response.raise_for_status()
# Return intent directly
return response.json()
else:
fuzzy_config = profile.intent[system]
# Use fuzzywuzzy
global intent_examples
if not profile.name in intent_examples:
examples_path = profile.read_path(fuzzy_config['examples_json'])
with open(examples_path, 'r') as examples_file:
intent_examples[profile.name] = json.load(examples_file)
text, intent_name, slots = best_intent(intent_examples[profile.name],
text)
# Try to match RasaNLU format for future compatibility
intent = {
'text':
text,
'intent': {
'name': intent_name,
},
'entities': [{
'entity': name,
'value': values[0]
} for name, values in slots.items()]
}
return intent
def get_project(project_name, project_dir):
from rasa_nlu.project import Project
return Project(project=project_name, project_dir=project_dir)
