def __call__(self, inputs=None, outputs=None, arguments=None,
includes=None, local=False, environment=None, collect=False):
abstraction = CurrentAbstraction()
nest = CurrentNest()
# Engine Functions define inputs and output member attributes
try:
inputs = inputs or self.inputs
outputs = outputs or self.outputs
except AttributeError:
pass
inputs = parse_input_list(inputs)
outputs = parse_output_list(outputs, inputs)
includes = parse_input_list(includes) + parse_input_list(self.includes)
command = self.command_format(inputs, outputs, arguments)
options = Options(environment=dict(self.environment), collect=inputs if collect else None)
if local:
options.local = True
if environment:
options.environment.update(environment)
if nest.batch:
options.batch = nest.batch
nest.schedule(abstraction, self, command,
list(inputs) + list(includes), outputs, options, nest.symbol)
return outputs
def __call__(self,
inputs=None,
outputs=None,
arguments=None,
includes=None,
local=False,
environment=None,
collect=False):
abstraction = CurrentAbstraction()
nest = CurrentNest()
# Engine Functions define inputs and output member attributes
try:
inputs = inputs or self.inputs
outputs = outputs or self.outputs
except AttributeError:
pass
inputs = parse_input_list(inputs)
outputs = parse_output_list(outputs, inputs)
includes = parse_input_list(includes) + parse_input_list(self.includes)
command = self.command_format(inputs, outputs, arguments)
options = Options(environment=dict(self.environment),
collect=inputs if collect else None)
if local:
options.local = True
if environment:
options.environment.update(environment)
nest.schedule(abstraction, self, command,
list(inputs) + list(includes), outputs, options)
return outputs
def _generate(self):
with self:
debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self))
function = parse_function(self.function)
inputs = parse_input_list(self.inputs)
includes = parse_input_list(self.includes)
output = self.outputs
nest = CurrentNest()
if not os.path.isabs(output):
output = os.path.join(nest.work_dir, output)
while len(inputs) > self.group:
next_inputs = []
for group in groups(inputs, self.group):
output_file = next(nest.stash)
next_inputs.append(output_file)
with Options(local=self.options.local,
collect=group if self.collect else None):
yield function(group, output_file, None, includes)
inputs = next_inputs
with Options(local=self.options.local,
collect=inputs if self.collect else None):
yield function(inputs, output, None, includes)
def parse_output_list(output_list=None, input_list=None):
""" Return an :func:`~weaver.util.iterable` object of output files.
If `output_list` is ``None``, then return ``[]``. If `output_list` is a
string template, then use it to generate a list of :class:`File`
objects. If `output_list` is already an :func:`~weaver.util.iterable`,
then map :class:`File` to `output_list` and return it.
This means that `output_list` must be one of the following:
1. ``None`` to leave it to the caller to generate an output file object.
2. A string object to be used as a template.
3. An :func:`~weaver.util.iterable` object (ex. list, iterator, etc.).
If `output_list` is a string template, then it may have the following
fields:
- `{fullpath}`, `{FULL}` -- Full input file path.
- `{basename}`, `{BASE}` -- Base input file name.
- `{fullpath_woext}`, `{FULLWE}` -- Full input file path without extension
- `{basename_woext}`, `{BASEWE}` -- Base input file name without extension
"""
debug(D_DATA, 'Parsing output list')
if output_list is None:
return []
if isinstance(output_list, str) or isinstance(output_list, File):
# If input list is empty or output list is not a format string, then
# return list of single output file.
# TODO: support single {stash}
if not input_list or not '{' in str(output_list):
return [MakeFile(output_list)]
nest = CurrentNest()
return [
MakeFile(
str(output_list).format(
fullpath=input,
FULL=input,
i='{0:05X}'.format(i),
NUMBER='{0:05X}'.format(i),
stash=next(nest.stash) if '{stash}' in output_list else '',
fullpath_woext=os.path.splitext(input)[0],
FULL_WOEXT=os.path.splitext(input)[0],
basename=os.path.basename(input),
BASE=os.path.basename(input),
basename_woext=os.path.splitext(
os.path.basename(input))[0],
BASE_WOEXT=os.path.splitext(os.path.basename(input))[0]))
for i, input in enumerate(parse_string_list(input_list))
]
if iterable(output_list):
return [MakeFile(o) for o in parse_object_list(output_list)]
raise WeaverError(
D_DATA, 'Could not parse output argument: {0}'.format(output_list))
def _generate(self):
with self:
debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self))
mapper = parse_function(self.mapper, PythonMapper)
inputs = parse_input_list(self.inputs)
includes = parse_input_list(self.includes)
output = self.outputs
nest = CurrentNest()
for map_input in groups(inputs, self.group):
map_output = next(nest.stash)
with Options(local=self.options.local, collect=map_input if self.collect else None):
yield mapper(map_input, map_output, includes)
def __init__(self,
work_dir=None,
dag_path=None,
stash=None,
barrier=None,
wrapper=None,
track_imports=True,
track_exports=True):
self.work_dir = work_dir or '.'
self.tasks = []
self.parent = CurrentNest()
if self.parent:
self.work_dir = os.path.join(self.parent.work_dir, self.work_dir)
self.stash = stash or Stash(root=os.path.join(self.work_dir, '_Stash'))
if not os.path.exists(self.work_dir):
make_directory(self.work_dir)
Makeflow.__init__(self,
wrapper=wrapper,
track_imports=track_imports,
track_exports=track_exports)
self.dag_path = dag_path or os.path.join(self.work_dir, 'Makeflow')
self.dag_file = open(self.dag_path, 'w')
self.includes.add(self.dag_path)
# TODO: fix work_dir so it can be translated by makeflow_link
if barrier:
self.includes.update(parse_input_list(barrier))
# Since Abstractions and SubNests are not compiled immediately, these
# objects must regster with their parent Nest, who will compile them in
# the order that they are registered to ensure proper semantics.
self.futures = []
if self.parent:
debug(
D_NEST,
'Register child {0} with parent {1}'.format(self, self.parent))
self.parent.futures.append((self, True))
debug(D_NEST, 'Created {0}'.format(self))
def __init__(self, path, nest=None):
self.path = str(path) # __str__ method returns path to file
self.nest = nest or CurrentNest()
debug(D_DATA, 'Created File: {0}'.format(self.path))
def Export(variables):
CurrentNest().exports.update(parse_string_list(variables))
# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
def Define(key, value, export=False):
if export:
Export(key)
CurrentNest().variables[key] = value
def __init__(self):
Nest.__init__(self, work_dir='.', dag_path=next(CurrentNest().stash),
stash=CurrentNest().stash)
def Export(variables):
CurrentNest().exports.update(parse_string_list(variables))
def __init__(self, cache_path=None, cursor=None):
self.c = cursor or ObjectCursor()
self.nest = CurrentNest()
self.cache_path = cache_path or next(self.nest.stash)
debug(D_DATASET, 'Created Dataset {0}'.format(self))