def create(cls, component_config: Dict[Text, Any],
config: RasaNLUModelConfig) -> "SpacyNLP":
import spacy
component_config = override_defaults(cls.defaults, component_config)
spacy_model_name = component_config.get("model")
# if no model is specified, we fall back to the language string
if not spacy_model_name:
spacy_model_name = config.language
component_config["model"] = config.language
logger.info("Trying to load spacy model with name '{}'".format(
spacy_model_name))
try:
nlp = spacy.load(spacy_model_name, disable=["parser"])
except OSError:
raise InvalidModelError(
"Model '{}' is not a linked spaCy model. "
"Please download and/or link a spaCy model, "
"e.g. by running:\npython -m spacy download "
"en_core_web_md\npython -m spacy link "
"en_core_web_md en".format(spacy_model_name))
cls.ensure_proper_language_model(nlp)
return cls(component_config, nlp)
def unload_model(self, model: Text):
"""Unload a model from server memory."""
if not self.nlu_model.is_loaded(model):
raise InvalidModelError(
"Model with name '{}' is not loaded.".format(model))
self.nlu_model.unload()
async def evaluate(self,
data_file: Text,
model: Optional[Text] = None) -> Dict[Text, Any]:
"""Perform a model evaluation."""
if not self.nlu_model.is_loaded(model):
raise InvalidModelError(
"Model with name '{}' is not loaded.".format(model))
logger.debug(
"Evaluation request received for model '{}'.".format(model))
if self._worker_processes <= self._current_worker_processes:
raise MaxWorkerProcessError
if self.nlu_model.name == FALLBACK_MODEL_NAME:
raise UnsupportedModelError("No model is loaded. Cannot evaluate.")
loop = asyncio.get_event_loop()
self._current_worker_processes += 1
pool = ProcessPoolExecutor(max_workers=self._worker_processes)
task = loop.run_in_executor(pool, run_evaluation, data_file,
self.nlu_model.path)
try:
return await task
finally:
self._current_worker_processes -= 1
pool.shutdown()
async def load_model(self, model_path: Text):
# model_path can point to a directory containing any number of tar.gz model
# files or to one specific model file. If it is pointing to a directory, the
# latest model in that directory is taken.
if model_path is None:
logger.warning("Could not load any model. Using fallback model.")
self.nlu_model = NLUModel.fallback_model(self.component_builder)
return
try:
if os.path.exists(model_path):
self.nlu_model = NLUModel.load_local_model(
model_path, self.component_builder)
elif self.model_server is not None:
self.nlu_model = await load_from_server(
self.component_builder,
self.model_server,
self.wait_time_between_pulls,
)
elif self.remote_storage is not None:
self.nlu_model = NLUModel.load_from_remote_storage(
self.remote_storage, self.component_builder, model_path)
else:
raise InvalidModelError(
"Model in '{}' could not be loaded.".format(model_path))
logger.debug("Loaded model '{}'".format(self.nlu_model.name))
except Exception as e:
logger.error("Could not load model due to {}.".format(e))
raise
def load_model(spacy_model_name: Text) -> "Language":
"""Try loading the model, catching the OSError if missing."""
import spacy
try:
return spacy.load(spacy_model_name, disable=["parser"])
except OSError:
raise InvalidModelError(
"Model '{}' is not a linked spaCy model. "
"Please download and/or link a spaCy model, "
"e.g. by running:\npython -m spacy download "
"en_core_web_md\npython -m spacy link "
"en_core_web_md en".format(spacy_model_name))
def load_model(spacy_model_name: Text) -> "Language":
"""Try loading the model, catching the OSError if missing."""
import spacy
try:
return spacy.load(spacy_model_name, disable=["parser"])
except OSError:
raise InvalidModelError(
f"Please confirm that {spacy_model_name} is an available spaCy model. "
f"You need to download one upfront. For example:\npython -m spacy download "
f"en_core_web_md\n"
f"More informaton can be found on {DOCS_URL_COMPONENTS}#spacynlp"
)
def load_model(spacy_model_name: Text) -> Language:
"""Try loading the model, catching the OSError if missing."""
import spacy
if not spacy_model_name:
raise InvalidModelError(
f"Missing model configuration for `SpacyNLP` in `config.yml`.\n"
f"You must pass a model to the `SpacyNLP` component explicitly.\n"
f"For example:\n"
f"- name: SpacyNLP\n"
f" model: en_core_web_md\n"
f"More informaton can be found on {DOCS_URL_COMPONENTS}#spacynlp"
)
try:
return spacy.load(spacy_model_name, disable=["parser"])
except OSError:
raise InvalidModelError(
f"Please confirm that {spacy_model_name} is an available spaCy model. "
f"You need to download one upfront. For example:\n"
f"python -m spacy download en_core_web_md\n"
f"More informaton can be found on {DOCS_URL_COMPONENTS}#spacynlp"
)
def cache_key(cls, component_meta: Dict[Text, Any],
model_metadata: "Metadata") -> Optional[Text]:
spacy_model_name = component_meta.get("model")
if not spacy_model_name:
raise InvalidModelError(
f"Missing model configuration for `SpacyNLP` in `config.yml`.\n"
f"You must pass a model to the `SpacyNLP` component explicitly.\n"
f"For example:\n"
f"- name: SpacyNLP\n"
f" model: en_core_web_md\n"
f"More informaton can be found on {DOCS_URL_COMPONENTS}#spacynlp"
)
return f"{cls.name}-{spacy_model_name}"
def _check_model_fallback(
spacy_model_name: Union[str, None], language_name: str, warn: bool = False
) -> Text:
"""This method checks if the `spacy_model_name` is missing.
If it is missing, we will attempt a fallback. This feature is a measure
to support spaCy 3.0 without breaking on users. In the future
spaCy will no longer support `spacy link`.
"""
if not spacy_model_name:
fallback_mapping = {
"zh": "zh_core_web_md",
"da": "da_core_news_md",
"nl": "nl_core_news_md",
"en": "en_core_web_md",
"fr": "fr_core_news_md",
"de": "de_core_news_sm",
"el": "el_core_news_md",
"it": "it_core_news_md",
"ja": "ja_core_news_md",
"lt": "lt_core_news_md",
"mk": "mk_core_news_md",
"nb": "nb_core_news_md",
"pl": "pl_core_news_md",
"pt": "pt_core_news_md",
"ro": "ro_core_news_md",
"ru": "ru_core_news_md",
"es": "es_core_news_md",
}
if language_name not in fallback_mapping.keys():
raise InvalidModelError(
f"There is no fallback model for language '{language_name}'. "
f"Please add a `model` property to `SpacyNLP` manually to prevent this. "
f"More informaton can be found on {DOCS_URL_COMPONENTS}#spacynlp"
)
spacy_model_name = fallback_mapping[language_name]
if warn:
message = (
f"SpaCy model is not properly configured! Please add a `model` property to `SpacyNLP`. "
f"Will use '{spacy_model_name}' as a fallback spaCy model. "
f"This fallback will be deprecated in Rasa 3.0"
)
rasa.shared.utils.io.raise_deprecation_warning(
message=message, docs=f"{DOCS_URL_COMPONENTS}#spacynlp"
)
return spacy_model_name
