def parse(self, data): project = data.get("project") or RasaNLUConfig.DEFAULT_PROJECT_NAME model = data.get("model") if project not in self.project_store: projects = os.listdir(self.config['path']) if project not in projects: raise InvalidProjectError( "No project found with name '{}'.".format(project)) else: try: self.project_store[project] = Project( self.config, self.component_builder, project) except Exception as e: raise InvalidProjectError( "Unable to load project '{}'. Error: {}".format( project, e)) response, used_model = self.project_store[project].parse( data['text'], data.get('time', None), model) if self.responses: self.responses.info(user_input=response, project=project, model=used_model) return self.format_response(response)
def parse(self, data): project = data.get("project", RasaNLUModelConfig.DEFAULT_PROJECT_NAME) model = data.get("model") if project not in self.project_store: projects = self._list_projects(self.project_dir) cloud_provided_projects = self._list_projects_in_cloud() projects.extend(cloud_provided_projects) if project not in projects: raise InvalidProjectError( "No project found with name '{}'.".format(project)) else: try: self.project_store[project] = Project( self.component_builder, project, self.project_dir, self.remote_storage) except Exception as e: raise InvalidProjectError( "Unable to load project '{}'. Error: {}".format( project, e)) time = data.get('time') response, used_model = self.project_store[project].parse( data['text'], time, model) if self.responses: self.responses.info('', user_input=response, project=project, model=used_model) return self.format_response(response)
def unload_model(self, project, model): # type: (Text, Text) -> Dict[Text] """Unload a model from server memory.""" if project is None: raise InvalidProjectError("No project specified".format(project)) elif project not in self.project_store: raise InvalidProjectError("Project {} could not " "be found".format(project)) try: unloaded_model = self.project_store[project].unload(model) return unloaded_model except KeyError: raise InvalidProjectError("Failed to unload model {} " "for project {}.".format(model, project))
def evaluate(self, data, project=None, model=None): # type: (Text, Optional[Text], Optional[Text]) -> Dict[Text, Any] """Perform a model evaluation.""" project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME model = model or None file_name = utils.create_temporary_file(data, "_training_data") if project not in self.project_store: raise InvalidProjectError("Project {} could not " "be found".format(project)) model_name = self.project_store[project]._dynamic_load_model(model) self.project_store[project]._loader_lock.acquire() try: if not self.project_store[project]._models.get(model_name): interpreter = self.project_store[project]. \ _interpreter_for_model(model_name) self.project_store[project]._models[model_name] = interpreter finally: self.project_store[project]._loader_lock.release() return run_evaluation( data_path=file_name, model=self.project_store[project]._models[model_name], errors_filename=None)
def start_train_process(self, data_file: Text, project: Text, train_config: RasaNLUModelConfig, model_name: Optional[Text] = None ) -> Deferred: """Start a model training.""" if not project: raise InvalidProjectError("Missing project name to train") if self._training_processes <= self._current_training_processes: raise MaxTrainingError if project in self.project_store: self.project_store[project].status = STATUS_TRAINING elif project not in self.project_store: self.project_store[project] = Project( self.component_builder, project, self.project_dir, self.remote_storage) self.project_store[project].status = STATUS_TRAINING def training_callback(model_path): model_dir = os.path.basename(os.path.normpath(model_path)) self.project_store[project].update(model_dir) self._current_training_processes -= 1 self.project_store[project].current_training_processes -= 1 if (self.project_store[project].status == STATUS_TRAINING and self.project_store[project].current_training_processes == 0): self.project_store[project].status = STATUS_READY return model_path def training_errback(failure): logger.warning(failure) self._current_training_processes -= 1 self.project_store[project].current_training_processes -= 1 self.project_store[project].status = STATUS_FAILED self.project_store[project].error_message = str(failure) return failure logger.debug("New training queued") self._current_training_processes += 1 self.project_store[project].current_training_processes += 1 result = self.pool.submit(do_train_in_worker, train_config, data_file, path=self.project_dir, project=project, fixed_model_name=model_name, storage=self.remote_storage) result = deferred_from_future(result) result.addCallback(training_callback) result.addErrback(training_errback) return result
def start_train_process(self, data, config_values): # type: (Text, Dict[Text, Any]) -> Deferred """Start a model training.""" if PY3: f = tempfile.NamedTemporaryFile("w+", suffix="_training_data", delete=False, encoding="utf-8") f.write(data) else: f = tempfile.NamedTemporaryFile("w+", suffix="_training_data", delete=False) f.write(data.encode("utf-8")) f.close() # TODO: fix config handling _config = self.config.as_dict() for key, val in config_values.items(): _config[key] = val _config["data"] = f.name train_config = RasaNLUConfig(cmdline_args=_config) project = _config.get("project") if not project: raise InvalidProjectError("Missing project name to train") elif project in self.project_store: if self.project_store[project].status == 1: raise AlreadyTrainingError else: self.project_store[project].status = 1 elif project not in self.project_store: self.project_store[project] = Project(self.config, self.component_builder, project) self.project_store[project].status = 1 def training_callback(model_path): model_dir = os.path.basename(os.path.normpath(model_path)) self.project_store[project].update(model_dir) return model_dir def training_errback(failure): target_project = self.project_store.get( failure.value.failed_target_project) if target_project: target_project.status = 0 return failure logger.debug("New training queued") result = self.pool.submit(do_train_in_worker, train_config) result = deferred_from_future(result) result.addCallback(training_callback) result.addErrback(training_errback) return result
def unload_model(self, project, model): # type: (Text, Text) -> Dict[Text] """Unload a model from server memory.""" if project is None: raise InvalidProjectError("No project specified".format(project)) if not self.project_manager.project_exists(project): raise InvalidProjectError("Project {} could not " "be found".format(project)) project_instance = self.project_manager.load_project(project) try: unloaded_model = project_instance.unload(model) return unloaded_model except KeyError: raise InvalidProjectError("Failed to unload model {} " "for project {}.".format(model, project))
def _try_load_project(self, project): # type: (str) -> None # Try to load project from local storage, if failed # an InvalidProjectError exception will raise try: self.project_store[project] = Project(self.component_builder, project, self.project_dir, self.remote_storage) except Exception as e: raise InvalidProjectError( "Unable to load project '{}'. Error: {}".format(project, e))
def start_train_process( self, data_file, # type: Text project, # type: Text train_config, # type: RasaNLUModelConfig model_name=None # type: Optional[Text] ): # type: (...) -> Deferred """Start a model training.""" if not project: raise InvalidProjectError("Missing project name to train") if project in self.project_store: if self.project_store[project].status == 1: raise AlreadyTrainingError else: self.project_store[project].status = 1 elif project not in self.project_store: self.project_store[project] = Project(self.component_builder, project, self.project_dir, self.remote_storage) self.project_store[project].status = 1 def training_callback(model_path): model_dir = os.path.basename(os.path.normpath(model_path)) self.project_store[project].update(model_dir) return model_dir def training_errback(failure): logger.warn(failure) target_project = self.project_store.get( failure.value.failed_target_project) if target_project: target_project.status = 0 return failure logger.debug("New training queued") result = self.pool.submit(do_train_in_worker, train_config, data_file, path=self.project_dir, project=project, fixed_model_name=model_name, storage=self.remote_storage) result = deferred_from_future(result) result.addCallback(training_callback) result.addErrback(training_errback) return result
def evaluate(self, data, project=None, model=None): # type: (Text, Optional[Text], Optional[Text]) -> Dict[Text, Any] """Perform a model evaluation.""" project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME model = model or None file_name = utils.create_temporary_file(data, "_training_data") test_data = load_data(file_name) if project not in self.project_store: raise InvalidProjectError("Project {} could not " "be found".format(project)) preds_json = self.parse_training_examples(test_data.intent_examples, project, model) predictions = [ {"text": e.text, "intent": e.data.get("intent"), "predicted": p.get("intent", {}).get("name"), "confidence": p.get("intent", {}).get("confidence")} for e, p in zip(test_data.intent_examples, preds_json) ] y_true = [e.data.get("intent") for e in test_data.intent_examples] y_true = clean_intent_labels(y_true) y_pred = [p.get("intent", {}).get("name") for p in preds_json] y_pred = clean_intent_labels(y_pred) report, precision, f1, accuracy = get_evaluation_metrics(y_true, y_pred) return { "intent_evaluation": { "report": report, "predictions": predictions, "precision": precision, "f1_score": f1, "accuracy": accuracy} }
def load_project(self, project=None): # type: (Union[None, str]) -> Project # Try to load a project from memory first, if not exists # try load from local file system or cloud by name, # if project not exists or can not be loaded correctly, # a InvalidProjectError exception will raise project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME if project not in self.project_store: projects = self._list_projects(self.project_dir) cloud_provided_projects = self._list_projects_in_cloud() projects.extend(cloud_provided_projects) if project not in projects: raise InvalidProjectError( "No project found with name '{}'.".format(project)) else: self._try_load_project(project) return self.project_store[project]
def start_train_process( self, data_file, # type: Text project, # type: Text train_config, # type: RasaNLUModelConfig model_name=None # type: Optional[Text] ): # type: (...) -> Deferred """Start a model training.""" if not project: raise InvalidProjectError("Missing project name to train") if self._training_processes <= self._current_training_processes: raise MaxTrainingError if project in self.project_store: self.project_store[project].status = STATUS_TRAINING elif project not in self.project_store: self.project_store[project] = Project(self.component_builder, project, self.project_dir, self.remote_storage) self.project_store[project].status = STATUS_TRAINING def training_callback(model_path): model_dir = os.path.basename(os.path.normpath(model_path)) self.project_store[project].update(model_dir) self._current_training_processes -= 1 self.project_store[project].current_training_processes -= 1 if (self.project_store[project].status == STATUS_TRAINING and self.project_store[project].current_training_processes == 0): self.project_store[project].status = STATUS_READY return model_dir def training_errback(failure): logger.warning(failure) self._current_training_processes -= 1 self.project_store[project].current_training_processes -= 1 self.project_store[project].status = STATUS_FAILED self.project_store[project].error_message = str(failure) return failure logger.debug("New training queued") self._current_training_processes += 1 self.project_store[project].current_training_processes += 1 # tensorflow training is not executed in a separate thread on python 2, # as this may cause training to freeze if six.PY2 and self._tf_in_pipeline(train_config): try: logger.warning("Training a pipeline with a tensorflow " "component. This blocks the server during " "training.") model_path = do_train_in_worker(train_config, data_file, path=self.project_dir, project=project, fixed_model_name=model_name, storage=self.remote_storage) model_dir = os.path.basename(os.path.normpath(model_path)) training_callback(model_dir) return model_dir except TrainingException as e: logger.warning(e) target_project = self.project_store.get( e.failed_target_project) if target_project: target_project.status = STATUS_READY raise e else: result = self.pool.submit(do_train_in_worker, train_config, data_file, path=self.project_dir, project=project, fixed_model_name=model_name, storage=self.remote_storage) result = deferred_from_future(result) result.addCallback(training_callback) result.addErrback(training_errback) return result