def __call__(self, inputs=None, outputs=None, arguments=None, includes=None, local=False, environment=None, collect=False): abstraction = CurrentAbstraction() nest = CurrentNest() # Engine Functions define inputs and output member attributes try: inputs = inputs or self.inputs outputs = outputs or self.outputs except AttributeError: pass inputs = parse_input_list(inputs) outputs = parse_output_list(outputs, inputs) includes = parse_input_list(includes) + parse_input_list(self.includes) command = self.command_format(inputs, outputs, arguments) options = Options(environment=dict(self.environment), collect=inputs if collect else None) if local: options.local = True if environment: options.environment.update(environment) if nest.batch: options.batch = nest.batch nest.schedule(abstraction, self, command, list(inputs) + list(includes), outputs, options, nest.symbol) return outputs
def __call__(self, inputs=None, outputs=None, arguments=None, includes=None, local=False, environment=None, collect=False): abstraction = CurrentAbstraction() nest = CurrentNest() # Engine Functions define inputs and output member attributes try: inputs = inputs or self.inputs outputs = outputs or self.outputs except AttributeError: pass inputs = parse_input_list(inputs) outputs = parse_output_list(outputs, inputs) includes = parse_input_list(includes) + parse_input_list(self.includes) command = self.command_format(inputs, outputs, arguments) options = Options(environment=dict(self.environment), collect=inputs if collect else None) if local: options.local = True if environment: options.environment.update(environment) nest.schedule(abstraction, self, command, list(inputs) + list(includes), outputs, options) return outputs
def _generate(self): with self: debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self)) function = parse_function(self.function) inputs = parse_input_list(self.inputs) includes = parse_input_list(self.includes) output = self.outputs nest = CurrentNest() if not os.path.isabs(output): output = os.path.join(nest.work_dir, output) while len(inputs) > self.group: next_inputs = [] for group in groups(inputs, self.group): output_file = next(nest.stash) next_inputs.append(output_file) with Options(local=self.options.local, collect=group if self.collect else None): yield function(group, output_file, None, includes) inputs = next_inputs with Options(local=self.options.local, collect=inputs if self.collect else None): yield function(inputs, output, None, includes)
def parse_output_list(output_list=None, input_list=None): """ Return an :func:`~weaver.util.iterable` object of output files. If `output_list` is ``None``, then return ``[]``. If `output_list` is a string template, then use it to generate a list of :class:`File` objects. If `output_list` is already an :func:`~weaver.util.iterable`, then map :class:`File` to `output_list` and return it. This means that `output_list` must be one of the following: 1. ``None`` to leave it to the caller to generate an output file object. 2. A string object to be used as a template. 3. An :func:`~weaver.util.iterable` object (ex. list, iterator, etc.). If `output_list` is a string template, then it may have the following fields: - `{fullpath}`, `{FULL}` -- Full input file path. - `{basename}`, `{BASE}` -- Base input file name. - `{fullpath_woext}`, `{FULLWE}` -- Full input file path without extension - `{basename_woext}`, `{BASEWE}` -- Base input file name without extension """ debug(D_DATA, 'Parsing output list') if output_list is None: return [] if isinstance(output_list, str) or isinstance(output_list, File): # If input list is empty or output list is not a format string, then # return list of single output file. # TODO: support single {stash} if not input_list or not '{' in str(output_list): return [MakeFile(output_list)] nest = CurrentNest() return [ MakeFile( str(output_list).format( fullpath=input, FULL=input, i='{0:05X}'.format(i), NUMBER='{0:05X}'.format(i), stash=next(nest.stash) if '{stash}' in output_list else '', fullpath_woext=os.path.splitext(input)[0], FULL_WOEXT=os.path.splitext(input)[0], basename=os.path.basename(input), BASE=os.path.basename(input), basename_woext=os.path.splitext( os.path.basename(input))[0], BASE_WOEXT=os.path.splitext(os.path.basename(input))[0])) for i, input in enumerate(parse_string_list(input_list)) ] if iterable(output_list): return [MakeFile(o) for o in parse_object_list(output_list)] raise WeaverError( D_DATA, 'Could not parse output argument: {0}'.format(output_list))
def _generate(self): with self: debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self)) mapper = parse_function(self.mapper, PythonMapper) inputs = parse_input_list(self.inputs) includes = parse_input_list(self.includes) output = self.outputs nest = CurrentNest() for map_input in groups(inputs, self.group): map_output = next(nest.stash) with Options(local=self.options.local, collect=map_input if self.collect else None): yield mapper(map_input, map_output, includes)
def __init__(self, work_dir=None, dag_path=None, stash=None, barrier=None, wrapper=None, track_imports=True, track_exports=True): self.work_dir = work_dir or '.' self.tasks = [] self.parent = CurrentNest() if self.parent: self.work_dir = os.path.join(self.parent.work_dir, self.work_dir) self.stash = stash or Stash(root=os.path.join(self.work_dir, '_Stash')) if not os.path.exists(self.work_dir): make_directory(self.work_dir) Makeflow.__init__(self, wrapper=wrapper, track_imports=track_imports, track_exports=track_exports) self.dag_path = dag_path or os.path.join(self.work_dir, 'Makeflow') self.dag_file = open(self.dag_path, 'w') self.includes.add(self.dag_path) # TODO: fix work_dir so it can be translated by makeflow_link if barrier: self.includes.update(parse_input_list(barrier)) # Since Abstractions and SubNests are not compiled immediately, these # objects must regster with their parent Nest, who will compile them in # the order that they are registered to ensure proper semantics. self.futures = [] if self.parent: debug( D_NEST, 'Register child {0} with parent {1}'.format(self, self.parent)) self.parent.futures.append((self, True)) debug(D_NEST, 'Created {0}'.format(self))
def __init__(self, path, nest=None): self.path = str(path) # __str__ method returns path to file self.nest = nest or CurrentNest() debug(D_DATA, 'Created File: {0}'.format(self.path))
def Export(variables): CurrentNest().exports.update(parse_string_list(variables)) # vim: set sts=4 sw=4 ts=8 expandtab ft=python:
def Define(key, value, export=False): if export: Export(key) CurrentNest().variables[key] = value
def __init__(self): Nest.__init__(self, work_dir='.', dag_path=next(CurrentNest().stash), stash=CurrentNest().stash)
def Export(variables): CurrentNest().exports.update(parse_string_list(variables))
def __init__(self, cache_path=None, cursor=None): self.c = cursor or ObjectCursor() self.nest = CurrentNest() self.cache_path = cache_path or next(self.nest.stash) debug(D_DATASET, 'Created Dataset {0}'.format(self))