def test_files_method(self):
''' A file creator .files method should only return files that are valid combinations exist. '''
cons_set = set([
Constraint('model', ['ACCESS1-0', 'ACCESS1-3']),
Constraint('experiment', ['rcp45', 'rcp85'])
])
this_file_creator = FileCreator(
"/a/fake/pattern/%model%_%experiment%.nc",
extra_constraints=cons_set)
# Now tell the file creator which files are real!
# ACCESS1-3 has no rcp85 experiment in this case.
file_1 = this_file_creator.get_files(
{
'model': 'ACCESS1-0',
'experiment': 'rcp45'
},
check=False,
update=True)
file_2 = this_file_creator.get_files(
{
'model': 'ACCESS1-0',
'experiment': 'rcp85'
},
check=False,
update=True)
file_3 = this_file_creator.get_files(
{
'model': 'ACCESS1-3',
'experiment': 'rcp45'
},
check=False,
update=True)
# Ensure that the FileCreator has returned a file
# for each combination.
for file_thing in [file_1, file_2, file_3]:
self.assertTrue(file_thing)
all_files = [file_thing for file_thing in this_file_creator.files]
# There should only be 3 valid file combinations returned.
self.assertEqual(len(all_files), 3)
def __init__(self, inputlist, output_pattern, shell_command,
extra_constraints=None, map_dict=None, cons_keywords=None,
positional_args=None, execution_options=None, kw_string=None,
merge_output=None):
"""
Arguments:
inputlist: A list of input DataSets to get files from.
output_pattern: A filename pattern to use for data output.
shell_command: The base shell command for the process to run.
Optional:
extra_constraints: Extra constraints to be applied to the output.
map_dict: a dictionary linking constraint names in the input DataSets
to new constraints in the output. e.g.
if map_dict = {"obs-model": ("model", 0)} then the "model" constraint in the
input position 0 is renamed to be the "obs-model" Constraint in the output.
cons_keywords: Used in building the command to be run, if a constraint has to
be used as a keyword argument.
positional_args: Used in building the command to be run, if a constraint
has to be used as a positional argument.
execution_options: A dictionary to pass options like required queues, walltime,
required modules etc. to the process unit. Currently only
required_modules is implemented.
kw_string: A string used for composite constraint keyword arguments, i.e.
using multiple attribute values in a single keyword argument.
example - kw_string="--title $model_$variable"
"""
if map_dict:
self.map_dict = map_dict
else:
self.map_dict = {}
self.merge_output = merge_output
self.mapped_con_names = [cons_name for cons_name in self.map_dict]
self.inputlist = inputlist
self.shell_command = shell_command
# To avoid mutable defaults problems, set
# Nones to empty dicts.
if execution_options:
self.execution_options = execution_options
else:
self.execution_options = {}
if cons_keywords:
self.cons_keywords = cons_keywords
else:
self.cons_keywords = {}
if positional_args:
self.positional_args = positional_args
else:
self.positional_args = {}
if kw_string:
self.kw_string = kw_string
else:
self.kw_string = None
# The initial Constraints are built from the output file pattern.
pattern_constraints = set(FileCreator.constraints_from_pattern(output_pattern))
mapped_constraints = self.apply_mappings(pattern_constraints)
# Apply extra constraints given in the constructor.
filled_constraints = self.fill_constraints_from_extras(mapped_constraints,
extra_constraints)
# Finallly fill the empty output constraints from the input DataSets.
self.final_constraints = self.fill_from_input(self.inputlist, filled_constraints)
module_logger.debug("Final output constraints are: {0}".format(self.final_constraints))
for ds in inputlist:
module_logger.debug("Input constraints are: {}"
.format(ds.constraints))
# Make a file_creator from the new, fixed constraints.
self.file_creator = FileCreator(output_pattern, self.final_constraints)
