def _put_data_to_clipboard(self, clipboard_check):
"""
In the pipeline configuration, as part of the "steps" list, we have
hints on which data each Step produces and which data it consumes. In
order to transfer these pieces of data in-memory between steps we have
a simple architecture.
We have a dictionary at the Pipeline level where data is put and
possibly updated. This is the clipboard. Before executing each Step,
the data the Step needs as input is put in instance variables whose
name (and optionally type) are given in the input parameter of the
steps section of the pipeline configuration. After the Step completes,
instance variables of Step (and defined in the output parameter of the
same configuration block) are put in the clipboard.
"""
for clipboard_info in self.output_info:
# clipboard_info is a tuple. It either has a single element, which
# is the name of the clipboard key to fetch, or two elements: the
# clipboard key (same as before) and the corresponding object type
# for optional type checking (clipboard_check == True).
# Either way, put data in self.inbox in the order it is defined in
# self.input_info.
if(not (isinstance(clipboard_info, tuple) or
isinstance(clipboard_info, list)) or
not len(clipboard_info) in (0, 1, 2)):
raise(Exception('Step %s: malformed output info %s.' \
% (self.name, clipboard_info)))
if(not clipboard_info):
# Nothing to do.
continue
elif(len(clipboard_info) == 2):
[key_name, class_name] = clipboard_info
elif(len(clipboard_info) == 1):
key_name = clipboard_info[0]
class_name = None
# Fetch value from self.<key_name> but check types first if we need
# to.
value = getattr(self, key_name)
if(clipboard_check):
self.log.debug('Checking clipboard output types.')
if(not class_name):
self.log.debug('No class information to do the check.')
else:
cls = utilities.import_class(class_name)
assert(isinstance(value, cls))
self.log.debug('Clipboard output types are OK.')
# Now update the clipboard.
self._clipboard[key_name] = value
return
def process(self):
# Handle self.output_info
for (var_name, class_name) in self.output_info:
# Import class_name.
cls = utilities.import_class(class_name)
# Create a dummy instance of cls.
dummy_object = cls()
# Write it in an instance variable.
setattr(self, var_name, dummy_object)
# Log what we did.
self.log.info('Created a new instance of %s as %s.' \
% (class_name, var_name))
return(0)
def process(self):
# Handle self.output_info
for (var_name, class_name) in self.output_info:
# Import class_name.
cls = utilities.import_class(class_name)
# Create a dummy instance of cls.
dummy_object = cls()
# Write it in an instance variable.
setattr(self, var_name, dummy_object)
# Log what we did.
self.log.info('Created a new instance of %s as %s.' \
% (class_name, var_name))
return (0)
def from_parsed_config(cls, pipeline_config, pipeline):
"""
Create a Step instance from a parsed Pipeline configuration file
`pipeline_config` which specifies the Step steps, data directories as
well as the Step configuration file path.
If a file called <self.name>.spec is found in the same directory as the
Step class source code, then the Step configuration file is validated
against the spec file.
"""
# First understand which Step (sub)class we need to instantiate. The
# class name is given in full Python package notation, e.g.
# package.subPackage.subsubpackage.className
# this means that
# 1. We HAVE to be able to say
# from package.subPackage.subsubpackage import className
# 2. The resulting Python class MUST be a subclass of Step.
step_class = utilities.import_class(pipeline_config['python_class'],
subclassof=cls)
# Now, we have the right Python class for our Step, we just need to
# get to the corresponding config file and we are done.
step_config = {}
step_config_file = pipeline_config.get('config_file', None)
# Do we have a spec file? If so, do parameter and input/output key
# validation as well. If not keep going.
step_spec_file = utilities.find_spec_file(step_class)
if(not step_spec_file):
pipeline.log.debug("No spec file for Step %s." \
% (pipeline_config['name']))
else:
pipeline.log.debug("Step %s specfile: %s" \
% (pipeline_config['name'], step_spec_file))
# Now do the actual parsing and, if we do have a spec file, validate as
# well.
if(step_config_file):
step_config = config_parser.loads(step_config_file,
specfile=step_spec_file)
parameters = step_config.get('parameters', {})
# Now we have everything we need to create a Step instance.
return(step_class(name=pipeline_config['name'],
pipeline=pipeline,
input_info=pipeline_config.get('input', []),
output_info=pipeline_config.get('output', []),
**parameters))
def process(self):
# Handle self.input_info
for (var_name, class_name) in self.input_info:
thing = getattr(self, var_name)
# Log what we found.
self.log.info('Found an instance of %s as %s' \
% (thing.__class__.__name__, var_name))
# Handle self.output_info
for (var_name, class_name) in self.output_info:
# Import class_name.
cls = utilities.import_class(class_name)
# Create a dummy instance of cls.
dummy_object = cls()
# Write it as instance variable.
setattr(self, var_name, dummy_object)
# Log what we did.
self.log.info('Added a new instance of %s as %s.' \
% (class_name, var_name))
return(0)
def process(self):
# Handle self.input_info
for (var_name, class_name) in self.input_info:
thing = getattr(self, var_name)
# Log what we found.
self.log.info('Found an instance of %s as %s' \
% (thing.__class__.__name__, var_name))
# Handle self.output_info
for (var_name, class_name) in self.output_info:
# Import class_name.
cls = utilities.import_class(class_name)
# Create a dummy instance of cls.
dummy_object = cls()
# Write it as instance variable.
setattr(self, var_name, dummy_object)
# Log what we did.
self.log.info('Added a new instance of %s as %s.' \
% (class_name, var_name))
return (0)
