def _generate(self): with self: debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self)) function = parse_function(self.function) inputs_a = parse_input_list(self.inputs_a) inputs_b = parse_input_list(self.inputs_b) includes = parse_input_list(self.includes) # If native is enabled, then use allpairs_master, otherwise # generate tasks as part of the DAG. # # Note: parse_output_list flattens inputs, so we need to manually # translate pairs into a single string. if self.native: # Store inputs A and B lists as required by allpairs_master inputs_a_file = next(self.nest.stash) with open(inputs_a_file, 'w') as fs: for input_file in map(str, inputs_a): fs.write(input_file + '\n') inputs_b_file = next(self.nest.stash) with open(inputs_b_file, 'w') as fs: for input_file in map(str, inputs_b): fs.write(input_file + '\n') inputs = [inputs_a_file, inputs_b_file] outputs = parse_output_list(self.outputs, ['_'.join( [os.path.basename(str(s)) for s in p]) for p in inputs]) # Schedule allpairs_master with Options(local=True, collect=[i] if self.collect else None): allpairs_master = parse_function( 'allpairs_master -p {0} {{IN}} {{ARG}} > {{OUT}}'.format(self.port)) yield allpairs_master(inputs, outputs, function.path, includes + [function.path]) else: inputs = list(itertools.product(inputs_a, inputs_b)) outputs = parse_output_list(self.outputs, ['_'.join( [os.path.basename(str(s)) for s in p]) for p in inputs]) # We use a wrapper script to collect the output of the # comparison and put in {INPUT_A} {INPUT_B} {OUTPUT} format, as # used by allpairs_master. for i, o in zip(inputs, outputs): tmp_output = next(self.nest.stash) with Options(local=self.options.local, collect=[i] if self.collect else None): output = function(i, tmp_output, None, includes) # Wrapper script should run locally and we should always # try to collect the temporary intermediate output file. with Options(local=True, collect=[tmp_output]): yield AllPairsCompareWrapper(output, o, [os.path.basename(str(p)) for p in i], None)
def _generate(self): with self: debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self)) function = parse_function(self.function) inputs_a = parse_input_list(self.inputs_a) inputs_b = parse_input_list(self.inputs_b) includes = parse_input_list(self.includes) # If native is enabled, then use allpairs_master, otherwise # generate tasks as part of the DAG. # # Note: parse_output_list flattens inputs, so we need to manually # translate pairs into a single string. if self.native: # Store inputs A and B lists as required by allpairs_master inputs_a_file = next(self.nest.stash) with open(inputs_a_file, 'w') as fs: for input_file in map(str, inputs_a): fs.write(input_file + '\n') inputs_b_file = next(self.nest.stash) with open(inputs_b_file, 'w') as fs: for input_file in map(str, inputs_b): fs.write(input_file + '\n') inputs = [inputs_a_file, inputs_b_file] outputs = parse_output_list(self.outputs, map(lambda p: '_'.join( map(lambda s: os.path.basename(str(s)), p)),inputs)) # Schedule allpairs_master with Options(local=True, collect=[i] if self.collect else None): allpairs_master = parse_function( 'allpairs_master -p {0} {{IN}} {{ARG}} > {{OUT}}'.format(self.port)) yield allpairs_master(inputs, outputs, function.path, includes + [function.path]) else: inputs = list(itertools.product(inputs_a, inputs_b)) outputs = parse_output_list(self.outputs, map(lambda p: '_'.join( map(lambda s: os.path.basename(str(s)), p)),inputs)) # We use a wrapper script to collect the output of the # comparison and put in {INPUT_A} {INPUT_B} {OUTPUT} format, as # used by allpairs_master. for i, o in zip(inputs, outputs): tmp_output = next(self.nest.stash) with Options(local=self.options.local, collect=[i] if self.collect else None): output = function(i, tmp_output, None, includes) # Wrapper script should run locally and we should always # try to collect the temporary intermediate output file. with Options(local=True, collect=[tmp_output]): yield AllPairsCompareWrapper(output, o, map(lambda p: os.path.basename(str(p)), i), None)
def _generate(self): with self: debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self)) function = parse_function(self.function) inputs = parse_input_list(self.inputs) includes = parse_input_list(self.includes) output = self.outputs nest = CurrentNest() if not os.path.isabs(output): output = os.path.join(nest.work_dir, output) while len(inputs) > self.group: next_inputs = [] for group in groups(inputs, self.group): output_file = next(nest.stash) next_inputs.append(output_file) with Options(local=self.options.local, collect=group if self.collect else None): yield function(group, output_file, None, includes) inputs = next_inputs with Options(local=self.options.local, collect=inputs if self.collect else None): yield function(inputs, output, None, includes)
def _generate(self): with self: debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self)) function = parse_function(self.function) includes = parse_input_list(self.includes) # First format inputs and figure out the number of iteration to perform group_size = 0 inputs = [] if isinstance(self.inputs, list): # If inputs is a matrix if isinstance(self.inputs[0], list): for i, ingroup in enumerate(self.inputs): inputs.append(parse_input_list(ingroup)) if group_size == 0: group_size = len(ingroup) if len(ingroup) != group_size: raise IOError("Iteration group size are different between inputs!") # If inputs is a simple list else: group_size = len(self.inputs) inputs = parse_input_list(self.inputs) # If inputs is a string else: group_size = 1 inputs = parse_input_list(self.inputs) for iter in range(group_size): iteration_inputs = [] if isinstance(inputs[0], list): for i, input in enumerate(inputs): iteration_inputs.append(input[iter]) else: iteration_inputs.append(inputs[iter]) input_pattern = self._longestCommonSubstr(list(map(os.path.basename, list(map(str, iteration_inputs))))) iteration_outputs = [] if isinstance(self.outputs, list): # If outputs is a matrix if isinstance(self.outputs[0], list): for i, outgroup in enumerate(self.outputs): iteration_outputs.append(outgroup[iter]) # If inputs is a simple list and a motif table elif isinstance(self.outputs[0], str) and '{' in self.outputs[0]: for motif in self.outputs: iteration_outputs.extend(parse_output_list(motif, input_pattern)) # If a simple string table elif isinstance(self.outputs[0], str): iteration_outputs = parse_output_list(self.outputs[iter], input_pattern) # If inputs is a string else: iteration_outputs = parse_output_list(self.outputs, input_pattern) with Options(local=self.options.local): yield function(iteration_inputs, iteration_outputs, None, includes)
def _generate(self): with self: debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self)) function = parse_function(self.function) inputs = parse_input_list(self.inputs) outputs = parse_output_list(self.outputs, inputs) includes = parse_input_list(self.includes) for i, o in zip(inputs, outputs): with Options(local=self.options.local, collect=[i] if self.collect else None): yield function(i, o, None, includes)
def _generate(self): with self: debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self)) mapper = parse_function(self.mapper, PythonMapper) inputs = parse_input_list(self.inputs) includes = parse_input_list(self.includes) output = self.outputs nest = CurrentNest() for map_input in groups(inputs, self.group): map_output = next(nest.stash) with Options(local=self.options.local, collect=map_input if self.collect else None): yield mapper(map_input, map_output, includes)
def _generate(self): with self: debug(D_ABSTRACTION, 'Generating Abstraction {0}'.format(self)) function = parse_function(self.function) includes = parse_input_list(self.includes) # First format inputs and figure out the number of iteration to perform group_size = 0 inputs = [] if isinstance(self.inputs, list): # If inputs is a matrix if isinstance(self.inputs[0], list): for i, ingroup in enumerate(self.inputs): inputs.append(parse_input_list(ingroup)) if group_size == 0: group_size = len(ingroup) if len(ingroup) != group_size: raise IOError( "Iteration group size are different between inputs!" ) # If inputs is a simple list else: group_size = len(self.inputs) inputs = parse_input_list(self.inputs) # If inputs is a string else: group_size = 1 inputs = parse_input_list(self.inputs) for iter in range(group_size): iteration_inputs = [] if isinstance(inputs[0], list): for i, input in enumerate(inputs): iteration_inputs.append(input[iter]) else: iteration_inputs.append(inputs[iter]) input_pattern = self._longestCommonSubstr( list( map(os.path.basename, list(map(str, iteration_inputs))))) iteration_outputs = [] if isinstance(self.outputs, list): # If outputs is a matrix if isinstance(self.outputs[0], list): for i, outgroup in enumerate(self.outputs): iteration_outputs.append(outgroup[iter]) # If inputs is a simple list and a motif table elif isinstance(self.outputs[0], str) and '{' in self.outputs[0]: for motif in self.outputs: iteration_outputs.extend( parse_output_list(motif, input_pattern)) # If a simple string table elif isinstance(self.outputs[0], str): iteration_outputs = parse_output_list( self.outputs[iter], input_pattern) # If inputs is a string else: iteration_outputs = parse_output_list( self.outputs, input_pattern) with Options(local=self.options.local): yield function(iteration_inputs, iteration_outputs, self.arguments, includes)
# We use a wrapper script to collect the output of the # comparison and put in {INPUT_A} {INPUT_B} {OUTPUT} format, as # used by allpairs_master. for i, o in zip(inputs, outputs): tmp_output = next(self.nest.stash) with Options(local=self.options.local, collect=[i] if self.collect else None): output = function(i, tmp_output, None, includes) # Wrapper script should run locally and we should always # try to collect the temporary intermediate output file. with Options(local=True, collect=[tmp_output]): yield AllPairsCompareWrapper(output, o, map(lambda p: os.path.basename(str(p)), i), None) AllPairsCompareWrapper = parse_function('printf "%s\\t%s\\t%s\\n" {ARG} `cat {IN}` > {OUT}') # Iterate Abstraction class Iterate(Abstraction): """ Weaver Iterate Abstraction. This Abstraction enables the following pattern of execution: Iterate(f, limit or range, outputs) In this case, the :class:`Function` *f* is applied for the given *limit* or *range* to generate the corresponding *outputs*. """
for i, o in zip(inputs, outputs): tmp_output = next(self.nest.stash) with Options(local=self.options.local, collect=[i] if self.collect else None): output = function(i, tmp_output, None, includes) # Wrapper script should run locally and we should always # try to collect the temporary intermediate output file. with Options(local=True, collect=[tmp_output]): yield AllPairsCompareWrapper( output, o, map(lambda p: os.path.basename(str(p)), i), None) AllPairsCompareWrapper = parse_function( 'printf "%s\\t%s\\t%s\\n" {ARG} `cat {IN}` > {OUT}') # Iterate Abstraction class Iterate(Abstraction): """ Weaver Iterate Abstraction. This Abstraction enables the following pattern of execution: Iterate(f, limit or range, outputs) In this case, the :class:`Function` *f* is applied for the given *limit* or *range* to generate the corresponding *outputs*. """