def init_from_config(self, configs: Dict): """ Parse the configuration sections from the input config, into a list of [processor, config] Initialize the pipeline with the configurations """ if "Reader" not in configs or configs["Reader"] is None: raise KeyError('No reader in the configuration') reader_config = configs["Reader"] reader, reader_hparams = create_class_with_kwargs( class_name=reader_config["type"], class_args=reader_config.get("kwargs", {}), h_params=reader_config.get("hparams", {})) self.set_reader(reader, reader_hparams) # HParams cannot create HParams from the inner dict of list if "Processors" in configs and configs["Processors"] is not None: for processor_configs in configs["Processors"]: p, processor_hparams = create_class_with_kwargs( class_name=processor_configs["type"], class_args=processor_configs.get("kwargs", {}), h_params=processor_configs.get("hparams", {})) selector_hparams = processor_hparams.selector selector_class = get_class(selector_hparams['type']) selector_kwargs = selector_hparams["kwargs"] selector = selector_class(**selector_kwargs) self.add_processor(p, processor_hparams, selector) self.initialize()
def initialize(self, resources: Resources, configs: Config): super().initialize(resources, configs) # Replace explicit class with configuration class name. self.index = create_class_with_kwargs( self.configs.indexer_class, class_args={"config": self.configs.indexer_configs}, ) self.index.load(self.configs.model_dir) self.k = self.configs.k or 5
def init_from_config(self, configs: Dict): """ Initialize the pipeline with the configurations Args: configs: The configurations used to create the pipeline. Returns: """ if "Reader" not in configs or configs["Reader"] is None: raise KeyError('No reader in the configuration') reader_config = configs["Reader"] reader, reader_hparams = create_class_with_kwargs( class_name=reader_config["type"], class_args=reader_config.get("kwargs", {}), h_params=reader_config.get("hparams", {})) self.set_reader(reader, reader_hparams) if "Processors" in configs and configs["Processors"] is not None: for processor_configs in configs["Processors"]: p_class = get_class(processor_configs["type"]) if processor_configs.get("kwargs"): processor_kwargs = processor_configs["kwargs"] else: processor_kwargs = {} p = p_class(**processor_kwargs) hparams: Dict = {} if processor_configs.get("hparams"): # Extract the hparams section and build hparams processor_hparams = processor_configs["hparams"] if processor_hparams.get("config_path"): filebased_hparams = yaml.safe_load( open(processor_hparams["config_path"])) else: filebased_hparams = {} hparams.update(filebased_hparams) if processor_hparams.get("overwrite_configs"): overwrite_hparams = processor_hparams[ "overwrite_configs"] else: overwrite_hparams = {} hparams.update(overwrite_hparams) default_processor_hparams = p_class.default_hparams() processor_hparams = HParams(hparams, default_processor_hparams) self.add_processor(p, processor_hparams) self.initialize()
def init_from_config(self, configs: List): r"""Initialized the pipeline (ontology and processors) from the given configurations. Args: configs: The configs used to initialize the pipeline. """ is_first: bool = True for component_config in configs: component = create_class_with_kwargs( class_name=component_config['type'], class_args=component_config.get('kwargs', {}), ) if is_first: if not isinstance(component, BaseReader): raise ProcessorConfigError( "The first component of a pipeline must be a reader.") self.set_reader(component, component_config.get('configs', {})) is_first = False else: # Can be processor, caster, or evaluator self.add(component, component_config.get('configs', {}))
def initialize(self, resources: Resources, configs: Config): super().initialize(resources, configs) self.index = create_class_with_kwargs( self.configs.indexer_class, class_args={"config": self.configs.index_configs}, )