def test_read_options(): cfg = copy.deepcopy(_test_config) cfg['inference_options'] = copy.deepcopy(_test_inference_options) with pytest.raises(ValueError): options = {"bpreprocess": {"domain": "Technology"}} config.read_options(cfg, options) options = {"bpreprocess": {"domain": "IT"}} config.get_options(cfg, options) assert cfg["bpreprocess"]["classifiers"][1]["value"] == "IT"
def test_read_options_v2(): cfg = copy.deepcopy(_test_config_v2) cfg['inference_options'] = copy.deepcopy(_test_inference_options_v2) options = {"bpreprocess": {"domain": "IT"}} assert config.read_options(cfg, options) == { "my-domain-op": { "value": "IT" } }
def test_read_options(): cfg = copy.deepcopy(_test_config) cfg['inference_options'] = copy.deepcopy(_test_inference_options) with pytest.raises(ValueError): options = {"bpreprocess": {"domain": "Technology"}} config.read_options(cfg, options) options = {"bpreprocess": {"domain": "IT"}} assert config.read_options(cfg, options) == { "bpreprocess": { "classifiers": [{ "name": "politeness" }, { "name": "domain", "value": "IT", }] } }
def finalize_config(config, override=None, options=None): """Finalizes the configuration with possible override and options.""" if config is None: supported_features = None else: supported_features = config.get('supported_features') if config_util.is_v2_config(config): if override: raise ValueError( "Configuration override is not supported for V2 " "configurations") if options: options = config_util.read_options(config, options) config = None else: if override or options: config = copy.deepcopy(config) if override: config_util.merge_config(config, override) if options: config_util.read_options(config, options) options = None return config, options, supported_features
def _build_pipeline( self, config, preprocess_exit_step=None, shared_state=None, ): self._ops = [] self._config = config inference = config.get("inference", {}) if inference and self._process_type == ProcessType.TRAINING: raise RuntimeError( "'inference' field can only be specified in translation") self._inference_config = inference.get("overrides") self._inference_options = inference.get("options") if self._inference_options: self._inference_options = read_options(config, self._inference_options) preprocess_config = config.get("preprocess") if preprocess_config: self._add_op_list( preprocess_config, exit_step=preprocess_exit_step, shared_state=shared_state, ) if self._process_type == ProcessType.POSTPROCESS: # Reverse preprocessing operators. self._ops = list(reversed(self._ops)) # Reverse start and build states. self.start_state, self.build_state = self.build_state, self.start_state # Flag current pipeline state as 'postprocess_only'. # Subsequent operators may need to be aware that they come from 'postprocess' configuration. self.build_state["postprocess_only"] = True # Add pure postprocessing operators. postprocess_config = config.get("postprocess") if postprocess_config: self._add_op_list(postprocess_config)
def preprocess_example(preprocessor, index, raw_example, config=None, config_override=None): """Applies preprocessing function on example.""" if not isinstance(raw_example, dict): raise InvalidRequest("example %d is not a JSON object" % index) source_text = raw_example.get("text") if source_text is None: raise InvalidRequest("missing text field in example %d" % index) mode = raw_example.get("mode", "default") options = None example_config_override = raw_example.get("config") # Resolve example options. if config is not None: example_options = raw_example.get("options") if example_options: options_or_override = config_util.read_options( config, example_options) if config_util.is_v2_config(config): options = options_or_override else: example_config_override = config_util.merge_config( example_config_override or {}, options_or_override) # Merge example-level config override into batch-level config override. if example_config_override: if config_override: config_override = config_util.merge_config( copy.deepcopy(config_override), example_config_override) else: config_override = example_config_override target_prefix = raw_example.get("target_prefix") target_fuzzy = raw_example.get("fuzzy") if target_prefix is not None and target_fuzzy is not None: raise InvalidRequest( "Using both a target prefix and a fuzzy target is currently unsupported" ) target_text = None target_name = None if target_prefix is not None: target_text = target_prefix elif target_fuzzy is not None: supported_features = config.get( "supported_features") if config else None if supported_features is not None and supported_features.get( "NFA", False): target_text = target_fuzzy target_name = "fuzzy" else: logger.warning( "The fuzzy target is ignored because this model does not " "support Neural Fuzzy Adaptation") if preprocessor is None: source_tokens = source_text target_tokens = None metadata = None else: source_tokens, target_tokens, metadata = preprocessor.process_input( source_text, target=target_text, target_name=target_name, config=config_override, options=options, ) # Move to the general multiparts representation. if not source_tokens or not isinstance(source_tokens[0], list): source_tokens = [source_tokens] target_tokens = [target_tokens] metadata = [metadata] return TranslationExample( index=index, config=config_override, options=options, source_tokens=source_tokens, target_tokens=target_tokens, mode=mode, metadata=metadata, )