def test_should_be_serializable(self): # Given parser = CustomEntityParser.build( DATASET, CustomEntityParserUsage.WITHOUT_STEMS, resources=dict()) self.tmp_file_path.mkdir() parser_path = self.tmp_file_path / "custom_entity_parser" parser.persist(parser_path) loaded_parser = CustomEntityParser.from_path(parser_path) # When scope = ["dummy_entity_1"] text = "dummy_entity_1 dummy_1" result = loaded_parser.parse(text, scope=scope) # Then expected_entities = [{ "value": "dummy_entity_1", "resolved_value": "dummy_entity_1", "range": { "start": 0, "end": 14 }, "entity_kind": "dummy_entity_1" }, { "value": "dummy_1", "resolved_value": "dummy_entity_1", "range": { "start": 15, "end": 22 }, "entity_kind": "dummy_entity_1" }] self.assertListEqual(expected_entities, result)
def from_path(cls, path, **shared): """Loads a :class:`SnipsNLUEngine` instance from a directory path The data at the given path must have been generated using :func:`~SnipsNLUEngine.persist` Args: path (str): The path where the nlu engine is stored Raises: LoadingError: when some files are missing IncompatibleModelError: when trying to load an engine model which is not compatible with the current version of the lib """ directory_path = Path(path) model_path = directory_path / "nlu_engine.json" if not model_path.exists(): raise LoadingError("Missing nlu engine model file: %s" % model_path.name) with model_path.open(encoding="utf8") as f: model = json.load(f) model_version = model.get("model_version") if model_version is None or model_version != __model_version__: raise IncompatibleModelError(model_version) dataset_metadata = model["dataset_metadata"] if shared.get(RESOURCES) is None and dataset_metadata is not None: language = dataset_metadata["language_code"] resources_dir = directory_path / "resources" / language if resources_dir.is_dir(): resources = load_resources_from_dir(resources_dir) shared[RESOURCES] = resources if shared.get(BUILTIN_ENTITY_PARSER) is None: path = model["builtin_entity_parser"] if path is not None: parser_path = directory_path / path shared[BUILTIN_ENTITY_PARSER] = BuiltinEntityParser.from_path( parser_path) if shared.get(CUSTOM_ENTITY_PARSER) is None: path = model["custom_entity_parser"] if path is not None: parser_path = directory_path / path shared[CUSTOM_ENTITY_PARSER] = CustomEntityParser.from_path( parser_path) config = cls.config_type.from_dict(model["config"]) nlu_engine = cls(config=config, **shared) nlu_engine.dataset_metadata = dataset_metadata intent_parsers = [] for parser_idx, parser_name in enumerate(model["intent_parsers"]): parser_config = config.intent_parsers_configs[parser_idx] intent_parser_path = directory_path / parser_name intent_parser = IntentParser.load_from_path( intent_parser_path, parser_config.unit_name, **shared) intent_parsers.append(intent_parser) nlu_engine.intent_parsers = intent_parsers return nlu_engine
def from_path(cls, path, **shared): """Load a :class:`SnipsNLUEngine` instance from a directory path The data at the given path must have been generated using :func:`~SnipsNLUEngine.persist` Args: path (str): The path where the nlu engine is stored. """ directory_path = Path(path) model_path = directory_path / "nlu_engine.json" if not model_path.exists(): raise OSError("Missing nlu engine model file: %s" % model_path.name) with model_path.open(encoding="utf8") as f: model = json.load(f) model_version = model.get("model_version") if model_version is None or model_version != __model_version__: raise ValueError( "Incompatible data model: persisted object=%s, python lib=%s" % (model_version, __model_version__)) dataset_metadata = model["dataset_metadata"] if dataset_metadata is not None: language = dataset_metadata["language_code"] resources_dir = directory_path / "resources" / language if resources_dir.is_dir(): load_resources_from_dir(resources_dir) if shared.get(BUILTIN_ENTITY_PARSER) is None: path = model["builtin_entity_parser"] if path is not None: parser_path = directory_path / path shared[BUILTIN_ENTITY_PARSER] = BuiltinEntityParser.from_path( parser_path) if shared.get(CUSTOM_ENTITY_PARSER) is None: path = model["custom_entity_parser"] if path is not None: parser_path = directory_path / path shared[CUSTOM_ENTITY_PARSER] = CustomEntityParser.from_path( parser_path) nlu_engine = cls(config=model["config"], **shared) # pylint:disable=protected-access nlu_engine._dataset_metadata = dataset_metadata # pylint:enable=protected-access intent_parsers = [] for intent_parser_name in model["intent_parsers"]: intent_parser_path = directory_path / intent_parser_name intent_parser = load_processing_unit(intent_parser_path, **shared) intent_parsers.append(intent_parser) nlu_engine.intent_parsers = intent_parsers return nlu_engine
def test_should_be_serializable(self): # Given parser = CustomEntityParser.build( DATASET, CustomEntityParserUsage.WITHOUT_STEMS, resources=dict()) self.tmp_file_path.mkdir() parser_path = self.tmp_file_path / "custom_entity_parser" parser.persist(parser_path) loaded_parser = CustomEntityParser.from_path(parser_path) # When scope = ["dummy_entity_1"] text = "dummy_entity_1 dummy_1" result = loaded_parser.parse(text, scope=scope) # Then expected_entities = [{ "value": "dummy_entity_1", "resolved_value": "dummy_entity_1", "range": { "start": 0, "end": 14 }, "entity_kind": "dummy_entity_1" }, { "value": "dummy_1", "resolved_value": "dummy_entity_1", "range": { "start": 15, "end": 22 }, "entity_kind": "dummy_entity_1" }] self.assertListEqual(expected_entities, result) license_path = parser_path / "parser" / "parser_1" / "LICENSE" self.assertTrue(license_path.exists()) with license_path.open(encoding="utf8") as f: license_content = f.read() self.assertEqual("some license content here", license_content)