def run(self, data, config=None, pipeline=None):
"""Run child pipeline if `self.field` is ``True``."""
if self.field in data and data[self.field]:
pipeline = self.pipeline(config)
pipeline.start = data
pipe_output = pipeline.run()
data.update(pipe_output)
return data
def run(self, data, config=None, pipeline=None):
"""Run a branch of the pipeline state."""
pipeline = self.pipeline(config)
if self.shallow:
pipeline.start = copy.copy(data)
else:
pipeline.start = copy.deepcopy(data)
branch_results = pipeline.run()
for field in self.keep:
if field in branch_results:
data[field] = branch_results[field]
return data
def run(self, data, config=None, pipeline=None):
"""Run child pipeline, ignoring errors."""
try:
pipeline = self.pipeline(config)
pipeline.start = data
pipe_output = pipeline.run()
return pipe_output
except Exception as error:
self.err_logger.log(
self.log_level,
"TryCatch: Ignoring exception %s",
error, exc_info=True)
if self.pass_through:
return data
return None
def run(self, data, config=None, pipeline=None):
"""Iterate over given field, applying a child pipeline."""
# If we were given a list of items to copy, it is equivalent to a
# dictionary with the new names equal to the old.
to_copy = self.copy
if not hasattr(to_copy, "items"):
to_copy = {i: i for i in to_copy}
pipeline = self.pipeline(config)
pipe_output = []
for i, item in enumerate(data[self.field]):
self.logger.debug("Runing pipeline %d / %d",
i, len(data[self.field]))
# Copy top-level elements into child elements
for field, new_name in to_copy.items():
item[new_name] = data[field]
pipeline.start = item
pipeline_results = pipeline.run()
if pipeline_results is not None and not self.discard:
if isinstance(pipeline_results, list):
pipe_output.extend(pipeline_results)
else:
pipe_output.append(pipeline_results)
# Components in the child pipeline may return copies of the
# original elements, so we need to clean up extra fields in the
# original elements as well as the returned elements.
for new_name in to_copy.values():
del item[new_name]
# Clean up copied elements in results
for result in pipe_output:
for new_name in to_copy.values():
if new_name in result:
del result[new_name]
data[self.field] = pipe_output
return data
def run(self, data, config=None, pipeline=None):
"""Alias pipeline state into child configuraton."""
config = self.generate_config(data, config)
pipeline = self.pipeline(config, data)
return pipeline.run()
def main(): # IGNORE:C0111
'''Command line options.'''
# Before doing anything, tell the loggers to capture warnings.
logging.captureWarnings(True)
try:
parser = arg_parser()
args = parser.parse_args()
if args.dump:
return 0
pipe_config = phyre_engine.pipeline.PipelineConfig(
resolve_yml(
load_default_config(),
lambda doc: doc,
{"ENV": os.environ}))
# Parse pipeline descriptor from YAML file
with open(args.pipeline, "r") as yml_in:
pipeline_desc = yaml.load(yml_in, yaml.ImplicitLoader)
# Merge default configuration with the current unresolved config
pipe_config = pipe_config.merge_params(
phyre_engine.pipeline.PipelineConfig(
pipeline_desc.unresolved["pipeline"].get("config", {})))
pipeline_desc.unresolved["pipeline"]["config"] = pipe_config
pipeline_desc = resolve_yml(
pipeline_desc,
lambda doc: doc["pipeline"]["config"],
{"ENV": os.environ})
# Set default start values"
pipeline_desc["pipeline"].setdefault("start", {})
if args.start is not None:
pipeline_desc["pipeline"]["start"].update(args.start)
# Set up logging if a logging section was given in the pipeline
init_logging(pipeline_desc["pipeline"]["config"])
# Add extra configuration keys to the pipeline config
for dotted_key, value in args.config.items():
apply_dotted_key(pipeline_desc["pipeline"]["config"],
dotted_key, value)
# Load a pipeline
pipeline = phyre_engine.pipeline.Pipeline.load(
pipeline_desc["pipeline"])
# Check that the parameters each component requires are present. This
# is non-deterministic, so this can be silenced with the
# --no-static-check command-line option.
if not (args.no_static_check
or pipeline.config.get("no_static_check", False)):
static_validate(pipeline)
pipeline.run()
return 0
except KeyboardInterrupt:
### handle keyboard interrupt ###
return 0
except phyre_engine.pipeline.Pipeline.ValidationError as error:
logging.getLogger(ROOT_LOGGER).error(
"Component %s expected the missing keys %s in the pipeline state",
error.component.qualname, error.missing)
except Exception as error:
logging.getLogger(ROOT_LOGGER).error(
"Uncaught exception encountered: exiting.",
exc_info=error)
raise error