def parse(self, message, project_name): from rasa_nlu.project import Project self.logger.info('Parsing message with %s: %s' % (project_name, message)) project = self._projects.get(project_name) if project is None: project = Project(project=project_name, project_dir=self._project_dir) self._projects[project_name] = project # -------------------------- # RasaNLU intent JSON format # -------------------------- # { # "intent": { # "name": "...", # "confidence": 1.0 # }, # "entities": [ # { # "value": "...", # "entity": "..." # } # ], # "text": "..." # } result = project.parse(message) self.logger.debug(str(result)) return result
def _create_project_store(self, project_dir): default_project = RasaNLUModelConfig.DEFAULT_PROJECT_NAME projects = self._collect_projects(project_dir) project_store = {} if self.model_server is not None: project_store[default_project] = load_from_server( self.component_builder, default_project, self.project_dir, self.remote_storage, self.model_server, self.wait_time_between_pulls) else: for project in projects: project_store[project] = Project(self.component_builder, project, self.project_dir, self.remote_storage) if not project_store: project_store[default_project] = Project( project=default_project, project_dir=self.project_dir, remote_storage=self.remote_storage) return project_store
def test_dynamic_load_model_with_model_is_none(): LATEST_MODEL_NAME = 'latest_model_name' def mocked_init(*args, **kwargs): return None def mocked_search_for_models(self): pass def mocked_latest_project_model(self): return LATEST_MODEL_NAME with mock.patch.object(Project, "__init__", mocked_init): with mock.patch.object(Project, "_search_for_models", mocked_search_for_models): with mock.patch.object(Project, "_latest_project_model", mocked_latest_project_model): project = Project() project._models = () project.pull_models = None result = project._dynamic_load_model(None) assert result == LATEST_MODEL_NAME
def _create_project_store(self): projects = self._collect_projects() project_store = {} for project in projects: project_store[project] = Project(self.config, self.component_builder, project) if not project_store: project_store[RasaNLUConfig.DEFAULT_PROJECT_NAME] = Project( self.config) return project_store
def read_model_metadata(model_dir, config): if model_dir is None: data = Project._default_model_metadata() return Metadata(data, model_dir) else: if not os.path.isabs(model_dir): model_dir = os.path.join(config['path'], model_dir) # download model from S3 if needed if not os.path.isdir(model_dir): Project._load_model_from_cloud(model_dir, config) return Metadata.load(model_dir)
def _create_project_store(self): projects = [] if os.path.isdir(self.config['path']): projects = os.listdir(self.config['path']) project_store = {} for project in projects: project_store[project] = Project(self.config, self.component_builder, project) if not project_store: project_store[RasaNLUConfig.DEFAULT_PROJECT_NAME] = Project(self.config) return project_store
def test_dynamic_load_model_with_exists_model(): MODEL_NAME = 'model_name' def mocked_init(*args, **kwargs): return None with mock.patch.object(Project, "__init__", mocked_init): project = Project() project._models = (MODEL_NAME, ) result = project._dynamic_load_model(MODEL_NAME) assert result == MODEL_NAME
def test_dynamic_load_model_with_exists_model(): MODEL_NAME = 'model_name' def mocked_init(*args, **kwargs): return None with mock.patch.object(Project, "__init__", mocked_init): project = Project() project._models = (MODEL_NAME,) project.pull_models = None result = project._dynamic_load_model(MODEL_NAME) assert result == MODEL_NAME
def start_train_process(self, data_file: Text, project: Text, train_config: RasaNLUModelConfig, model_name: Optional[Text] = None ) -> Deferred: """Start a model training.""" if not project: raise InvalidProjectError("Missing project name to train") if self._training_processes <= self._current_training_processes: raise MaxTrainingError if project in self.project_store: self.project_store[project].status = STATUS_TRAINING elif project not in self.project_store: self.project_store[project] = Project( self.component_builder, project, self.project_dir, self.remote_storage) self.project_store[project].status = STATUS_TRAINING def training_callback(model_path): model_dir = os.path.basename(os.path.normpath(model_path)) self.project_store[project].update(model_dir) self._current_training_processes -= 1 self.project_store[project].current_training_processes -= 1 if (self.project_store[project].status == STATUS_TRAINING and self.project_store[project].current_training_processes == 0): self.project_store[project].status = STATUS_READY return model_path def training_errback(failure): logger.warning(failure) self._current_training_processes -= 1 self.project_store[project].current_training_processes -= 1 self.project_store[project].status = STATUS_FAILED self.project_store[project].error_message = str(failure) return failure logger.debug("New training queued") self._current_training_processes += 1 self.project_store[project].current_training_processes += 1 result = self.pool.submit(do_train_in_worker, train_config, data_file, path=self.project_dir, project=project, fixed_model_name=model_name, storage=self.remote_storage) result = deferred_from_future(result) result.addCallback(training_callback) result.addErrback(training_errback) return result
def parse(self, data): project = data.get("project", RasaNLUModelConfig.DEFAULT_PROJECT_NAME) model = data.get("model") if project not in self.project_store: projects = self._list_projects(self.project_dir) cloud_provided_projects = self._list_projects_in_cloud() projects.extend(cloud_provided_projects) if project not in projects: raise InvalidProjectError( "No project found with name '{}'.".format(project)) else: try: self.project_store[project] = Project( self.component_builder, project, self.project_dir, self.remote_storage) except Exception as e: raise InvalidProjectError( "Unable to load project '{}'. Error: {}".format( project, e)) time = data.get('time') response, used_model = self.project_store[project].parse( data['text'], time, model) if self.responses: self.responses.info('', user_input=response, project=project, model=used_model) return self.format_response(response)
def _create_project_store(self, project_dir): projects = self._collect_projects(project_dir) project_store = {} for project in projects: project_store[project] = Project(self.component_builder, project, self.project_dir, self.remote_storage) if not project_store: default_model = RasaNLUModelConfig.DEFAULT_PROJECT_NAME project_store[default_model] = Project( project_dir=self.project_dir, remote_storage=self.remote_storage) return project_store
def parse(self, data): project = data.get("project") or RasaNLUConfig.DEFAULT_PROJECT_NAME model = data.get("model") if project not in self.project_store: projects = os.listdir(self.config['path']) if project not in projects: raise InvalidProjectError( "No project found with name '{}'.".format(project)) else: try: self.project_store[project] = Project( self.config, self.component_builder, project) except Exception as e: raise InvalidProjectError( "Unable to load project '{}'. Error: {}".format( project, e)) response, used_model = self.project_store[project].parse( data['text'], data.get('time', None), model) if self.responses: self.responses.info(user_input=response, project=project, model=used_model) return self.format_response(response)
def start_train_process(self, data, config_values): # type: (Text, Dict[Text, Any]) -> Deferred """Start a model training.""" if PY3: f = tempfile.NamedTemporaryFile("w+", suffix="_training_data", delete=False, encoding="utf-8") f.write(data) else: f = tempfile.NamedTemporaryFile("w+", suffix="_training_data", delete=False) f.write(data.encode("utf-8")) f.close() # TODO: fix config handling _config = self.config.as_dict() for key, val in config_values.items(): _config[key] = val _config["data"] = f.name train_config = RasaNLUConfig(cmdline_args=_config) project = _config.get("project") if not project: raise InvalidProjectError("Missing project name to train") elif project in self.project_store: if self.project_store[project].status == 1: raise AlreadyTrainingError else: self.project_store[project].status = 1 elif project not in self.project_store: self.project_store[project] = Project(self.config, self.component_builder, project) self.project_store[project].status = 1 def training_callback(model_path): model_dir = os.path.basename(os.path.normpath(model_path)) self.project_store[project].update(model_dir) return model_dir def training_errback(failure): target_project = self.project_store.get( failure.value.failed_target_project) if target_project: target_project.status = 0 return failure logger.debug("New training queued") result = self.pool.submit(do_train_in_worker, train_config) result = deferred_from_future(result) result.addCallback(training_callback) result.addErrback(training_errback) return result
def test_dynamic_load_model_with_refresh_exists_model(): MODEL_NAME = 'model_name' def mocked_init(*args, **kwargs): return None def mocked_search_for_models(self): self._models = (MODEL_NAME, ) with mock.patch.object(Project, "__init__", mocked_init): with mock.patch.object(Project, '_search_for_models', mocked_search_for_models): project = Project() project._models = () result = project._dynamic_load_model(MODEL_NAME) assert result == MODEL_NAME
def _try_load_project(self, project): # type: (str) -> None # Try to load project from local storage, if failed # an InvalidProjectError exception will raise try: self.project_store[project] = Project(self.component_builder, project, self.project_dir, self.remote_storage) except Exception as e: raise InvalidProjectError( "Unable to load project '{}'. Error: {}".format(project, e))
def load_or_create_project(self, project): try: project_instance = self.load_project(project) except InvalidProjectError: # project not exists, create one project_instance = Project(self.component_builder, project, self.project_dir, self.remote_storage) # add to cache self.project_store[project] = project_instance return project_instance
def test_dynamic_load_model_with_refresh_exists_model(): MODEL_NAME = 'model_name' def mocked_init(*args, **kwargs): return None def mocked_search_for_models(self): self._models = (MODEL_NAME,) with mock.patch.object(Project, "__init__", mocked_init): with mock.patch.object(Project, '_search_for_models', mocked_search_for_models): project = Project() project._models = () project.pull_models = None result = project._dynamic_load_model(MODEL_NAME) assert result == MODEL_NAME
def start_train_process( self, data_file, # type: Text project, # type: Text train_config, # type: RasaNLUModelConfig model_name=None # type: Optional[Text] ): # type: (...) -> Deferred """Start a model training.""" if not project: raise InvalidProjectError("Missing project name to train") if project in self.project_store: if self.project_store[project].status == 1: raise AlreadyTrainingError else: self.project_store[project].status = 1 elif project not in self.project_store: self.project_store[project] = Project(self.component_builder, project, self.project_dir, self.remote_storage) self.project_store[project].status = 1 def training_callback(model_path): model_dir = os.path.basename(os.path.normpath(model_path)) self.project_store[project].update(model_dir) return model_dir def training_errback(failure): logger.warn(failure) target_project = self.project_store.get( failure.value.failed_target_project) if target_project: target_project.status = 0 return failure logger.debug("New training queued") result = self.pool.submit(do_train_in_worker, train_config, data_file, path=self.project_dir, project=project, fixed_model_name=model_name, storage=self.remote_storage) result = deferred_from_future(result) result.addCallback(training_callback) result.addErrback(training_errback) return result
def start_train_process( self, data_file, # type: Text project, # type: Text train_config, # type: RasaNLUModelConfig model_name=None # type: Optional[Text] ): # type: (...) -> Deferred """Start a model training.""" if not project: raise InvalidProjectError("Missing project name to train") if self._training_processes <= self._current_training_processes: raise MaxTrainingError if project in self.project_store: self.project_store[project].status = STATUS_TRAINING elif project not in self.project_store: self.project_store[project] = Project(self.component_builder, project, self.project_dir, self.remote_storage) self.project_store[project].status = STATUS_TRAINING def training_callback(model_path): model_dir = os.path.basename(os.path.normpath(model_path)) self.project_store[project].update(model_dir) self._current_training_processes -= 1 self.project_store[project].current_training_processes -= 1 if (self.project_store[project].status == STATUS_TRAINING and self.project_store[project].current_training_processes == 0): self.project_store[project].status = STATUS_READY return model_dir def training_errback(failure): logger.warning(failure) self._current_training_processes -= 1 self.project_store[project].current_training_processes -= 1 self.project_store[project].status = STATUS_FAILED self.project_store[project].error_message = str(failure) return failure logger.debug("New training queued") self._current_training_processes += 1 self.project_store[project].current_training_processes += 1 # tensorflow training is not executed in a separate thread on python 2, # as this may cause training to freeze if six.PY2 and self._tf_in_pipeline(train_config): try: logger.warning("Training a pipeline with a tensorflow " "component. This blocks the server during " "training.") model_path = do_train_in_worker(train_config, data_file, path=self.project_dir, project=project, fixed_model_name=model_name, storage=self.remote_storage) model_dir = os.path.basename(os.path.normpath(model_path)) training_callback(model_dir) return model_dir except TrainingException as e: logger.warning(e) target_project = self.project_store.get( e.failed_target_project) if target_project: target_project.status = STATUS_READY raise e else: result = self.pool.submit(do_train_in_worker, train_config, data_file, path=self.project_dir, project=project, fixed_model_name=model_name, storage=self.remote_storage) result = deferred_from_future(result) result.addCallback(training_callback) result.addErrback(training_errback) return result
def get_intent(profile, text): system = profile.intent.get('system', 'fuzzywuzzy') if system == 'rasa': rasa_config = profile.intent[system] # Use rasaNLU global intent_projects project = intent_projects.get(profile.name, None) if project is None: import rasa_nlu from rasa_nlu.project import Project project_dir = profile.read_path(rasa_config['project_dir']) project_name = rasa_config['project_name'] project = Project(project=project_name, project_dir=project_dir) intent_projects[profile.name] = project return project.parse(text) elif system == 'remote': remote_url = profile.intent[system]['url'] headers = {'Content-Type': 'text/plain'} # Pass profile name through params = {'profile': profile.name, 'nohass': True} response = requests.post(remote_url, headers=headers, data=text, params=params) response.raise_for_status() # Return intent directly return response.json() else: fuzzy_config = profile.intent[system] # Use fuzzywuzzy global intent_examples if not profile.name in intent_examples: examples_path = profile.read_path(fuzzy_config['examples_json']) with open(examples_path, 'r') as examples_file: intent_examples[profile.name] = json.load(examples_file) text, intent_name, slots = best_intent(intent_examples[profile.name], text) # Try to match RasaNLU format for future compatibility intent = { 'text': text, 'intent': { 'name': intent_name, }, 'entities': [{ 'entity': name, 'value': values[0] } for name, values in slots.items()] } return intent
def get_project(project_name, project_dir): from rasa_nlu.project import Project return Project(project=project_name, project_dir=project_dir)