def check_filename_pattern(self, glob_pattern, constraints): """ Check that added constraints are also found in the input pattern.""" generated_cons = FileCreator.constraints_from_pattern(glob_pattern) gen_names = [cons.key for cons in generated_cons] for cons in constraints: if cons.key not in gen_names: raise ConstraintNotFoundError("Constraint {} is not found in output pattern {}". format(cons.key, glob_pattern))
def check_filename_pattern(self, glob_pattern, constraints): """ Check that added constraints are also found in the input pattern.""" generated_cons = FileCreator.constraints_from_pattern(glob_pattern) gen_names = [cons.key for cons in generated_cons] for cons in constraints: if cons.key not in gen_names: raise ConstraintNotFoundError( "Constraint {} is not found in output pattern {}".format( cons.key, glob_pattern))
def __init__(self, inputlist, output_pattern, shell_command, extra_constraints=set([]), map_dict={}, cons_keywords={}, positional_args=[], execution_options={}): """ The class takes in a DataSet object, constraints to change and the path to an executable. It has an self.execute() method that returns a FileCreator to be used as input to the next module. Extra constraints to be applied to the output are given by extra_constraints. map_dict is a dictionary linking constraint names in the input DataSets to new constraints in the output. e.g. if map_dict = {"obs-model" ("model", 0)} then the "model" constraint in the input position 0 is renamed to be the "obs-model" Constraint in the output. """ self.inputlist = inputlist self.cons_keywords = cons_keywords self.positional_args = positional_args self.execution_options = execution_options self.extra_constraints = extra_constraints self.shell_command = shell_command # Make a list of mappings between renamed constraints of the # input and constraints in the output. self.map_dict = map_dict self.mapped_con_names = [cons_name for cons_name in map_dict] # The initial Constraints for the output are built from the # output file pattern. self.output_constraints = set(FileCreator.constraints_from_pattern(output_pattern)) # Apply mappings to copy the required Constraints from the input that will be used # in the output to the final collection of output constraints. self.apply_mappings() # Apply extra constraints given in constructor. self.fill_constraints_from_extras() # Fill the empty output constraints from the input DataSets. self.fill_from_input() module_logger.debug("Final output constraints are: {0}".format(self.output_constraints)) # Make a file_creator from the new, fixed constraints. self.file_creator = FileCreator(output_pattern, self.output_constraints)
def __init__(self, inputlist, output_pattern, shell_command, extra_constraints=None, map_dict=None, cons_keywords=None, positional_args=None, execution_options=None, kw_string=None, merge_output=None): """ Arguments: inputlist: A list of input DataSets to get files from. output_pattern: A filename pattern to use for data output. shell_command: The base shell command for the process to run. Optional: extra_constraints: Extra constraints to be applied to the output. map_dict: a dictionary linking constraint names in the input DataSets to new constraints in the output. e.g. if map_dict = {"obs-model": ("model", 0)} then the "model" constraint in the input position 0 is renamed to be the "obs-model" Constraint in the output. cons_keywords: Used in building the command to be run, if a constraint has to be used as a keyword argument. positional_args: Used in building the command to be run, if a constraint has to be used as a positional argument. execution_options: A dictionary to pass options like required queues, walltime, required modules etc. to the process unit. Currently only required_modules is implemented. kw_string: A string used for composite constraint keyword arguments, i.e. using multiple attribute values in a single keyword argument. example - kw_string="--title $model_$variable" """ if map_dict: self.map_dict = map_dict else: self.map_dict = {} self.merge_output = merge_output self.mapped_con_names = [cons_name for cons_name in self.map_dict] self.inputlist = inputlist self.shell_command = shell_command # To avoid mutable defaults problems, set # Nones to empty dicts. if execution_options: self.execution_options = execution_options else: self.execution_options = {} if cons_keywords: self.cons_keywords = cons_keywords else: self.cons_keywords = {} if positional_args: self.positional_args = positional_args else: self.positional_args = {} if kw_string: self.kw_string = kw_string else: self.kw_string = None # The initial Constraints are built from the output file pattern. pattern_constraints = set(FileCreator.constraints_from_pattern(output_pattern)) mapped_constraints = self.apply_mappings(pattern_constraints) # Apply extra constraints given in the constructor. filled_constraints = self.fill_constraints_from_extras(mapped_constraints, extra_constraints) # Finallly fill the empty output constraints from the input DataSets. self.final_constraints = self.fill_from_input(self.inputlist, filled_constraints) module_logger.debug("Final output constraints are: {0}".format(self.final_constraints)) for ds in inputlist: module_logger.debug("Input constraints are: {}" .format(ds.constraints)) # Make a file_creator from the new, fixed constraints. self.file_creator = FileCreator(output_pattern, self.final_constraints)
def __init__(self, inputlist, output_pattern, shell_command, extra_constraints=None, map_dict=None, cons_keywords=None, positional_args=None, execution_options=None, kw_string=None, merge_output=None): """ Arguments: inputlist: A list of input DataSets to get files from. output_pattern: A filename pattern to use for data output. shell_command: The base shell command for the process to run. Optional: extra_constraints: Extra constraints to be applied to the output. map_dict: a dictionary linking constraint names in the input DataSets to new constraints in the output. e.g. if map_dict = {"obs-model": ("model", 0)} then the "model" constraint in the input position 0 is renamed to be the "obs-model" Constraint in the output. cons_keywords: Used in building the command to be run, if a constraint has to be used as a keyword argument. positional_args: Used in building the command to be run, if a constraint has to be used as a positional argument. execution_options: A dictionary to pass options like required queues, walltime, required modules etc. to the process unit. Currently only required_modules is implemented. kw_string: A string used for composite constraint keyword arguments, i.e. using multiple attribute values in a single keyword argument. example - kw_string="--title $model_$variable" """ if map_dict: self.map_dict = map_dict else: self.map_dict = {} self.merge_output = merge_output self.mapped_con_names = [cons_name for cons_name in self.map_dict] self.inputlist = inputlist self.shell_command = shell_command # To avoid mutable defaults problems, set # Nones to empty dicts. if execution_options: self.execution_options = execution_options else: self.execution_options = {} if cons_keywords: self.cons_keywords = cons_keywords else: self.cons_keywords = {} if positional_args: self.positional_args = positional_args else: self.positional_args = {} if kw_string: self.kw_string = kw_string else: self.kw_string = None # The initial Constraints are built from the output file pattern. pattern_constraints = set(FileCreator.constraints_from_pattern(output_pattern)) mapped_constraints = self.apply_mappings(pattern_constraints) # Apply extra constraints given in the constructor. filled_constraints = self.fill_constraints_from_extras(mapped_constraints, extra_constraints) # Finallly fill the empty output constraints from the input DataSets. self.final_constraints = self.fill_from_input(self.inputlist, filled_constraints) module_logger.debug("Final output constraints are: {0}".format(self.final_constraints)) for ds in inputlist: module_logger.debug("Input constraints are: {}" .format(ds.constraints)) # Make a file_creator from the new, fixed constraints. self.file_creator = FileCreator(output_pattern, self.final_constraints)