def test_read_options():
cfg = copy.deepcopy(_test_config)
cfg['inference_options'] = copy.deepcopy(_test_inference_options)
with pytest.raises(ValueError):
options = {"bpreprocess": {"domain": "Technology"}}
config.read_options(cfg, options)
options = {"bpreprocess": {"domain": "IT"}}
config.get_options(cfg, options)
assert cfg["bpreprocess"]["classifiers"][1]["value"] == "IT"
def test_read_options_v2():
cfg = copy.deepcopy(_test_config_v2)
cfg['inference_options'] = copy.deepcopy(_test_inference_options_v2)
options = {"bpreprocess": {"domain": "IT"}}
assert config.read_options(cfg, options) == {
"my-domain-op": {
"value": "IT"
}
}
def test_read_options():
cfg = copy.deepcopy(_test_config)
cfg['inference_options'] = copy.deepcopy(_test_inference_options)
with pytest.raises(ValueError):
options = {"bpreprocess": {"domain": "Technology"}}
config.read_options(cfg, options)
options = {"bpreprocess": {"domain": "IT"}}
assert config.read_options(cfg, options) == {
"bpreprocess": {
"classifiers": [{
"name": "politeness"
}, {
"name": "domain",
"value": "IT",
}]
}
}
def finalize_config(config, override=None, options=None):
"""Finalizes the configuration with possible override and options."""
if config is None:
supported_features = None
else:
supported_features = config.get('supported_features')
if config_util.is_v2_config(config):
if override:
raise ValueError(
"Configuration override is not supported for V2 "
"configurations")
if options:
options = config_util.read_options(config, options)
config = None
else:
if override or options:
config = copy.deepcopy(config)
if override:
config_util.merge_config(config, override)
if options:
config_util.read_options(config, options)
options = None
return config, options, supported_features
def _build_pipeline(
self,
config,
preprocess_exit_step=None,
shared_state=None,
):
self._ops = []
self._config = config
inference = config.get("inference", {})
if inference and self._process_type == ProcessType.TRAINING:
raise RuntimeError(
"'inference' field can only be specified in translation")
self._inference_config = inference.get("overrides")
self._inference_options = inference.get("options")
if self._inference_options:
self._inference_options = read_options(config,
self._inference_options)
preprocess_config = config.get("preprocess")
if preprocess_config:
self._add_op_list(
preprocess_config,
exit_step=preprocess_exit_step,
shared_state=shared_state,
)
if self._process_type == ProcessType.POSTPROCESS:
# Reverse preprocessing operators.
self._ops = list(reversed(self._ops))
# Reverse start and build states.
self.start_state, self.build_state = self.build_state, self.start_state
# Flag current pipeline state as 'postprocess_only'.
# Subsequent operators may need to be aware that they come from 'postprocess' configuration.
self.build_state["postprocess_only"] = True
# Add pure postprocessing operators.
postprocess_config = config.get("postprocess")
if postprocess_config:
self._add_op_list(postprocess_config)
def preprocess_example(preprocessor,
index,
raw_example,
config=None,
config_override=None):
"""Applies preprocessing function on example."""
if not isinstance(raw_example, dict):
raise InvalidRequest("example %d is not a JSON object" % index)
source_text = raw_example.get("text")
if source_text is None:
raise InvalidRequest("missing text field in example %d" % index)
mode = raw_example.get("mode", "default")
options = None
example_config_override = raw_example.get("config")
# Resolve example options.
if config is not None:
example_options = raw_example.get("options")
if example_options:
options_or_override = config_util.read_options(
config, example_options)
if config_util.is_v2_config(config):
options = options_or_override
else:
example_config_override = config_util.merge_config(
example_config_override or {}, options_or_override)
# Merge example-level config override into batch-level config override.
if example_config_override:
if config_override:
config_override = config_util.merge_config(
copy.deepcopy(config_override), example_config_override)
else:
config_override = example_config_override
target_prefix = raw_example.get("target_prefix")
target_fuzzy = raw_example.get("fuzzy")
if target_prefix is not None and target_fuzzy is not None:
raise InvalidRequest(
"Using both a target prefix and a fuzzy target is currently unsupported"
)
target_text = None
target_name = None
if target_prefix is not None:
target_text = target_prefix
elif target_fuzzy is not None:
supported_features = config.get(
"supported_features") if config else None
if supported_features is not None and supported_features.get(
"NFA", False):
target_text = target_fuzzy
target_name = "fuzzy"
else:
logger.warning(
"The fuzzy target is ignored because this model does not "
"support Neural Fuzzy Adaptation")
if preprocessor is None:
source_tokens = source_text
target_tokens = None
metadata = None
else:
source_tokens, target_tokens, metadata = preprocessor.process_input(
source_text,
target=target_text,
target_name=target_name,
config=config_override,
options=options,
)
# Move to the general multiparts representation.
if not source_tokens or not isinstance(source_tokens[0], list):
source_tokens = [source_tokens]
target_tokens = [target_tokens]
metadata = [metadata]
return TranslationExample(
index=index,
config=config_override,
options=options,
source_tokens=source_tokens,
target_tokens=target_tokens,
mode=mode,
metadata=metadata,
)
