def test_should_be_serializable(self):
# Given
parser = CustomEntityParser.build(
DATASET, CustomEntityParserUsage.WITHOUT_STEMS, resources=dict())
self.tmp_file_path.mkdir()
parser_path = self.tmp_file_path / "custom_entity_parser"
parser.persist(parser_path)
loaded_parser = CustomEntityParser.from_path(parser_path)
# When
scope = ["dummy_entity_1"]
text = "dummy_entity_1 dummy_1"
result = loaded_parser.parse(text, scope=scope)
# Then
expected_entities = [{
"value": "dummy_entity_1",
"resolved_value": "dummy_entity_1",
"range": {
"start": 0,
"end": 14
},
"entity_kind": "dummy_entity_1"
}, {
"value": "dummy_1",
"resolved_value": "dummy_entity_1",
"range": {
"start": 15,
"end": 22
},
"entity_kind": "dummy_entity_1"
}]
self.assertListEqual(expected_entities, result)
def from_path(cls, path, **shared):
"""Loads a :class:`SnipsNLUEngine` instance from a directory path
The data at the given path must have been generated using
:func:`~SnipsNLUEngine.persist`
Args:
path (str): The path where the nlu engine is stored
Raises:
LoadingError: when some files are missing
IncompatibleModelError: when trying to load an engine model which
is not compatible with the current version of the lib
"""
directory_path = Path(path)
model_path = directory_path / "nlu_engine.json"
if not model_path.exists():
raise LoadingError("Missing nlu engine model file: %s" %
model_path.name)
with model_path.open(encoding="utf8") as f:
model = json.load(f)
model_version = model.get("model_version")
if model_version is None or model_version != __model_version__:
raise IncompatibleModelError(model_version)
dataset_metadata = model["dataset_metadata"]
if shared.get(RESOURCES) is None and dataset_metadata is not None:
language = dataset_metadata["language_code"]
resources_dir = directory_path / "resources" / language
if resources_dir.is_dir():
resources = load_resources_from_dir(resources_dir)
shared[RESOURCES] = resources
if shared.get(BUILTIN_ENTITY_PARSER) is None:
path = model["builtin_entity_parser"]
if path is not None:
parser_path = directory_path / path
shared[BUILTIN_ENTITY_PARSER] = BuiltinEntityParser.from_path(
parser_path)
if shared.get(CUSTOM_ENTITY_PARSER) is None:
path = model["custom_entity_parser"]
if path is not None:
parser_path = directory_path / path
shared[CUSTOM_ENTITY_PARSER] = CustomEntityParser.from_path(
parser_path)
config = cls.config_type.from_dict(model["config"])
nlu_engine = cls(config=config, **shared)
nlu_engine.dataset_metadata = dataset_metadata
intent_parsers = []
for parser_idx, parser_name in enumerate(model["intent_parsers"]):
parser_config = config.intent_parsers_configs[parser_idx]
intent_parser_path = directory_path / parser_name
intent_parser = IntentParser.load_from_path(
intent_parser_path, parser_config.unit_name, **shared)
intent_parsers.append(intent_parser)
nlu_engine.intent_parsers = intent_parsers
return nlu_engine
def from_path(cls, path, **shared):
"""Load a :class:`SnipsNLUEngine` instance from a directory path
The data at the given path must have been generated using
:func:`~SnipsNLUEngine.persist`
Args:
path (str): The path where the nlu engine is
stored.
"""
directory_path = Path(path)
model_path = directory_path / "nlu_engine.json"
if not model_path.exists():
raise OSError("Missing nlu engine model file: %s" %
model_path.name)
with model_path.open(encoding="utf8") as f:
model = json.load(f)
model_version = model.get("model_version")
if model_version is None or model_version != __model_version__:
raise ValueError(
"Incompatible data model: persisted object=%s, python lib=%s" %
(model_version, __model_version__))
dataset_metadata = model["dataset_metadata"]
if dataset_metadata is not None:
language = dataset_metadata["language_code"]
resources_dir = directory_path / "resources" / language
if resources_dir.is_dir():
load_resources_from_dir(resources_dir)
if shared.get(BUILTIN_ENTITY_PARSER) is None:
path = model["builtin_entity_parser"]
if path is not None:
parser_path = directory_path / path
shared[BUILTIN_ENTITY_PARSER] = BuiltinEntityParser.from_path(
parser_path)
if shared.get(CUSTOM_ENTITY_PARSER) is None:
path = model["custom_entity_parser"]
if path is not None:
parser_path = directory_path / path
shared[CUSTOM_ENTITY_PARSER] = CustomEntityParser.from_path(
parser_path)
nlu_engine = cls(config=model["config"], **shared)
# pylint:disable=protected-access
nlu_engine._dataset_metadata = dataset_metadata
# pylint:enable=protected-access
intent_parsers = []
for intent_parser_name in model["intent_parsers"]:
intent_parser_path = directory_path / intent_parser_name
intent_parser = load_processing_unit(intent_parser_path, **shared)
intent_parsers.append(intent_parser)
nlu_engine.intent_parsers = intent_parsers
return nlu_engine
def test_should_be_serializable(self):
# Given
parser = CustomEntityParser.build(
DATASET, CustomEntityParserUsage.WITHOUT_STEMS, resources=dict())
self.tmp_file_path.mkdir()
parser_path = self.tmp_file_path / "custom_entity_parser"
parser.persist(parser_path)
loaded_parser = CustomEntityParser.from_path(parser_path)
# When
scope = ["dummy_entity_1"]
text = "dummy_entity_1 dummy_1"
result = loaded_parser.parse(text, scope=scope)
# Then
expected_entities = [{
"value": "dummy_entity_1",
"resolved_value": "dummy_entity_1",
"range": {
"start": 0,
"end": 14
},
"entity_kind": "dummy_entity_1"
}, {
"value": "dummy_1",
"resolved_value": "dummy_entity_1",
"range": {
"start": 15,
"end": 22
},
"entity_kind": "dummy_entity_1"
}]
self.assertListEqual(expected_entities, result)
license_path = parser_path / "parser" / "parser_1" / "LICENSE"
self.assertTrue(license_path.exists())
with license_path.open(encoding="utf8") as f:
license_content = f.read()
self.assertEqual("some license content here", license_content)