def init_from_config(self, configs: Dict):
"""
Parse the configuration sections from the input config,
into a list of [processor, config]
Initialize the pipeline with the configurations
"""
if "Reader" not in configs or configs["Reader"] is None:
raise KeyError('No reader in the configuration')
reader_config = configs["Reader"]
reader, reader_hparams = create_class_with_kwargs(
class_name=reader_config["type"],
class_args=reader_config.get("kwargs", {}),
h_params=reader_config.get("hparams", {}))
self.set_reader(reader, reader_hparams)
# HParams cannot create HParams from the inner dict of list
if "Processors" in configs and configs["Processors"] is not None:
for processor_configs in configs["Processors"]:
p, processor_hparams = create_class_with_kwargs(
class_name=processor_configs["type"],
class_args=processor_configs.get("kwargs", {}),
h_params=processor_configs.get("hparams", {}))
selector_hparams = processor_hparams.selector
selector_class = get_class(selector_hparams['type'])
selector_kwargs = selector_hparams["kwargs"]
selector = selector_class(**selector_kwargs)
self.add_processor(p, processor_hparams, selector)
self.initialize()
def initialize(self, resources: Resources, configs: Config):
super().initialize(resources, configs)
# Replace explicit class with configuration class name.
self.index = create_class_with_kwargs(
self.configs.indexer_class,
class_args={"config": self.configs.indexer_configs},
)
self.index.load(self.configs.model_dir)
self.k = self.configs.k or 5
def init_from_config(self, configs: Dict):
"""
Initialize the pipeline with the configurations
Args:
configs: The configurations used to create the pipeline.
Returns:
"""
if "Reader" not in configs or configs["Reader"] is None:
raise KeyError('No reader in the configuration')
reader_config = configs["Reader"]
reader, reader_hparams = create_class_with_kwargs(
class_name=reader_config["type"],
class_args=reader_config.get("kwargs", {}),
h_params=reader_config.get("hparams", {}))
self.set_reader(reader, reader_hparams)
if "Processors" in configs and configs["Processors"] is not None:
for processor_configs in configs["Processors"]:
p_class = get_class(processor_configs["type"])
if processor_configs.get("kwargs"):
processor_kwargs = processor_configs["kwargs"]
else:
processor_kwargs = {}
p = p_class(**processor_kwargs)
hparams: Dict = {}
if processor_configs.get("hparams"):
# Extract the hparams section and build hparams
processor_hparams = processor_configs["hparams"]
if processor_hparams.get("config_path"):
filebased_hparams = yaml.safe_load(
open(processor_hparams["config_path"]))
else:
filebased_hparams = {}
hparams.update(filebased_hparams)
if processor_hparams.get("overwrite_configs"):
overwrite_hparams = processor_hparams[
"overwrite_configs"]
else:
overwrite_hparams = {}
hparams.update(overwrite_hparams)
default_processor_hparams = p_class.default_hparams()
processor_hparams = HParams(hparams, default_processor_hparams)
self.add_processor(p, processor_hparams)
self.initialize()
def init_from_config(self, configs: List):
r"""Initialized the pipeline (ontology and processors) from the
given configurations.
Args:
configs: The configs used to initialize the pipeline.
"""
is_first: bool = True
for component_config in configs:
component = create_class_with_kwargs(
class_name=component_config['type'],
class_args=component_config.get('kwargs', {}),
)
if is_first:
if not isinstance(component, BaseReader):
raise ProcessorConfigError(
"The first component of a pipeline must be a reader.")
self.set_reader(component, component_config.get('configs', {}))
is_first = False
else:
# Can be processor, caster, or evaluator
self.add(component, component_config.get('configs', {}))
def initialize(self, resources: Resources, configs: Config):
super().initialize(resources, configs)
self.index = create_class_with_kwargs(
self.configs.indexer_class,
class_args={"config": self.configs.index_configs},
)
