def test_change_mapping(self):
""" Test using multiple input datasets, like if you were calculating a change. """
with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
fake_file_1 = '/a/fake/file_1956_red.nc'
fake_file_2 = '/a/fake/file_1981_red.nc'
mock_glob.return_value = [fake_file_1, fake_file_2]
first_pattern_ds = PatternDataSet("/a/fake/file_%date%_%colour%.nc",
set([Constraint('date', ['1956'])]))
second_pattern_ds = PatternDataSet("/a/fake/file_%date%_%colour%.nc",
set([Constraint('date', ['1981'])]))
# Overwrite the valid combinations for these mock datasets.
first_pattern_ds.valid_combinations = set([frozenset([Constraint('colour', ['red']),
Constraint('date', ['1956'])])])
second_pattern_ds.valid_combinations = set([frozenset([Constraint('colour', ['red']),
Constraint('date', ['1981'])])])
the_process_unit = ProcessUnit([first_pattern_ds, second_pattern_ds],
"/a/final/output/file_%start_date%_%end_date%_%colour%.txt",
'echo', map_dict={'start_date': ('date', 0),
'end_date': ('date', 1)})
ds_result = the_process_unit.execute(simulate=True)
outfiles = [file_thing for file_thing in ds_result.files]
self.assertEqual(len(outfiles), 1)
expected_string = self.script_header + "mkdir -p /a/final/output\necho /a/fake/file_1956_red.nc /a/fake/file_1981_red.nc /a/final/output/file_1956_1981_red.txt\n"
self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())
def compute(self):
in_dataset1 = self.getInputFromPort('in_dataset1')
in_dataset2 = self.getInputFromPort('in_dataset2')
operation = self.getInputFromPort('operation')
self.positional_args = [
(operation, 0, 'raw'),
]
self.keyword_args = {}
new_constraints_for_output = set([
Constraint('extra_info', [operation]),
Constraint('suffix', ['nc']),
])
this_process = ProcessUnit([in_dataset1, in_dataset2],
self.out_pattern,
self.command,
new_constraints_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args,
cons_keywords=self.keyword_args,
merge_output=['model', 'institute'])
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def test_mapping_after_passing(self):
""" Mapping of constraints should work if applied to the output of an earlier ProcessUnit. """
# First create a basic process unit.
with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
mock_glob.return_value = ['/a/fake/file/red_1986_2005_kangaroo.nc']
first_patternds = PatternDataSet('/a/fake/file/%colour%_%start%_%end%_%animal%.nc')
first_process = ProcessUnit([first_patternds], '/a/second/file_pattern/%colour%_%start%_%end%_%animal%.nc',
'echo')
result = first_process.execute(simulate=True)
# Then take the output of that process and apply a mapping.
second_process = ProcessUnit([result], '/a/final/pattern/%colour%_%hist_start%_%hist_end%_%animal%.txt',
'echo', map_dict={'hist_start': ('start', 0),
'hist_end': ('end', 0)})
final_out = second_process.execute(simulate=True)
expected_string = (self.script_header + "mkdir -p /a/final/output\necho" +
"/a/second/file_pattern/red_1986_2005_kangaroo.nc " +
"/a/final/pattern/red_1986_2005_kangaroo.nc")
self.assertEqual(expected_string, second_process.scheduler.job.to_str())
def compute(self):
# Required input
future_dataset = self.getInputFromPort("future_dataset")
baseline_dataset = self.getInputFromPort("baseline_dataset")
# Execute the process.
new_constraints = [Constraint('change_type', ['abs-change'])]
this_process = ProcessUnit([future_dataset, baseline_dataset],
self.out_pattern,
self.command,
extra_constraints=new_constraints,
execution_options=self._execution_options,
map_dict={'fut_start': ('year_start', 0),
'fut_end': ('year_end', 0),
'hist_start': ('year_start', 1),
'hist_end': ('year_end', 1)})
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, e.output)
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def test_simple_mapping(self):
""" Test using constraints from the input in the output pattern. """
with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
mock_file_1 = '/a/fake/ACCESS1-0_tas_netCDF'
mock_file_2 = '/a/fake/MIROC_tas_netCDF'
mock_file_3 = '/a/fake/ACCESS1-0_pr_netCDF'
mock_glob.return_value = [mock_file_1, mock_file_2, mock_file_3]
a_pattern_ds = PatternDataSet(self.input_pattern)
the_process_unit = ProcessUnit([a_pattern_ds], self.output_pattern,
'echo', map_dict={'obs_model': ('gcm_model', 0)})
ds_result = the_process_unit.execute(simulate=True)
outfiles = [file_thing for file_thing in ds_result.files]
self.assertEqual(len(outfiles), 3)
expected_string = (self.script_header + "mkdir -p /a/fake/output\necho /a/fake/MIROC_tas_netCDF /a/fake/output/MIROC_tas_netCDF\n" +
"echo /a/fake/ACCESS1-0_tas_netCDF /a/fake/output/ACCESS1-0_tas_netCDF\n" +
"echo /a/fake/ACCESS1-0_pr_netCDF /a/fake/output/ACCESS1-0_pr_netCDF\n")
self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())
def compute(self):
in_dataset = self.getInputFromPort('cod_dataset')
command = "${CWSL_CTOOLS}/sdm/sdmrun.py"
sdm_config = configuration.cwsl_ctools_path + "/sdm/default.cfg"
positional_args = [("dxt-gridded", 0, "raw"),
("-c " + sdm_config, 0, "raw")]
# The data is written out to the default
# location.
output_pattern = os.path.join(
configuration.user_basepath,
FileCreator.default_pattern(in_dataset.constraints) + ".nc")
this_process = ProcessUnit([in_dataset],
output_pattern,
command,
in_dataset.constraints,
execution_options=self._required_modules,
positional_args=positional_args)
this_process.execute(simulate=configuration.simulate_execution)
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def test_overwrite_constraints(self):
""" Test to ensure that Constraints are correctly overwritten when data is processed. """
extra_cons = set([
Constraint('extras', ['other_things']),
Constraint('fake', ['OVERWRITE'])
])
a_process_unit = ProcessUnit(
[self.a_pattern_ds],
"/foo/%fake%/%file%/%pattern%_%extras%.txt",
"echo",
extra_constraints=extra_cons)
output_ds = a_process_unit.execute(simulate=True)
expected_cons = set([
Constraint('extras', ['other_things']),
Constraint('fake', ['OVERWRITE']),
Constraint('file', ['file_1']),
Constraint('pattern', ['pattern_1'])
])
self.assertEqual(expected_cons, output_ds.constraints)
expected_string = self.script_header + "mkdir -p /foo/OVERWRITE/file_1\necho /a/fake_1/file_1/pattern_1 /foo/OVERWRITE/file_1/pattern_1_other_things.txt\n"
self.assertEqual(expected_string,
a_process_unit.scheduler.job.to_str())
def compute(self):
in_dataset = self.getInputFromPort('in_dataset')
method = self.getInputFromPort('method')
grid = self.getInputFromPort('grid')
self.positional_args = [(method, 0, 'raw'), (grid, 1, 'raw'), ]
self.keyword_args = {}
grid = grid.split('/')[-1]
if len(grid.split('.')) > 1: # i.e. a weights file as opposed to pre-defined grid
grid_constraint = method+'-'+grid.split('.')[0]
else:
grid_constraint = method+'-'+grid
new_constraints_for_output = set([Constraint('grid_info', [grid_constraint]),
Constraint('suffix', ['nc']),
])
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
new_constraints_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args,
cons_keywords=self.keyword_args)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def compute(self):
in_dataset = self.getInputFromPort('in_dataset')
west_lon = self.getInputFromPort('west_lon')
east_lon = self.getInputFromPort('east_lon')
south_lat = self.getInputFromPort('south_lat')
north_lat = self.getInputFromPort('north_lat')
self.positional_args = [('west_lon', 0),
('east_lon', 1),
('south_lat', 2),
('north_lat', 3)]
self.keyword_args = {}
new_constraints_for_output = set([Constraint('west_lon', [west_lon]),
Constraint('east_lon', [east_lon]),
Constraint('south_lat', [south_lat],
Constraint('north_lat', [north_lat]))
])
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
new_constraints_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args,
cons_keywords=self.keyword_args)
try:
this_process.execute(simulate=configuration.simulate_execution)
except subprocess.CalledProcessError, e:
raise vistrails_module.ModuleError(self, e.output)
def compute(self):
in_dataset1 = self.getInputFromPort('in_dataset1')
in_dataset2 = self.getInputFromPort('in_dataset2')
operation = self.getInputFromPort('operation')
self.positional_args = [(operation, 0, 'raw'), ]
self.keyword_args = {}
new_constraints_for_output = set([Constraint('extra_info', [operation]),
Constraint('suffix', ['nc']),
])
this_process = ProcessUnit([in_dataset1, in_dataset2],
self.out_pattern,
self.command,
new_constraints_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args,
cons_keywords=self.keyword_args,
merge_output=['model', 'institute'])
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def compute(self):
# Required input
future_dataset = self.getInputFromPort("future_dataset")
baseline_dataset = self.getInputFromPort("baseline_dataset")
# Execute the process.
new_constraints = [Constraint('change_type', ['abs-change'])]
this_process = ProcessUnit(
[future_dataset, baseline_dataset],
self.out_pattern,
self.command,
extra_constraints=new_constraints,
execution_options=self._execution_options,
map_dict={
'fut_start': ('year_start', 0),
'fut_end': ('year_end', 0),
'hist_start': ('year_start', 1),
'hist_end': ('year_end', 1)
})
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, e.output)
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def compute(self):
in_dataset = self.getInputFromPort('in_dataset')
method = self.getInputFromPort('method')
self.positional_args = [(method, 0, 'raw'), ]
self.keyword_args = {}
if len(method.split(',')) > 1:
agg_constraint = "".join(method.split(','))
else:
agg_constraint = method
new_constraints_for_output = set([Constraint('timeagg_info', [agg_constraint]),
Constraint('suffix', ['nc']),
])
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
new_constraints_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args,
cons_keywords=self.keyword_args)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def compute(self):
in_dataset = self.getInputFromPort('in_dataset')
try:
# Add extra constraints if necessary.
added_constraints = self.getInputFromPort('added_constraints')
except vistrails_module.ModuleError:
added_constraints = None
# Change the file_type constraint from nc to xml and add
# any added constraints.
cons_for_output = set([Constraint('suffix', ['xml'])])
if added_constraints:
cons_for_output = set.union(cons_for_output,
set(added_constraints))
# Execute the xml_to_nc process.
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
cons_for_output,
execution_options=self._execution_options)
try:
this_process.execute(simulate=configuration.simulate_execution)
except subprocess.CalledProcessError, e:
raise vistrails_module.ModuleError(self, e.output)
def compute(self):
in_dataset = self.getInputFromPort('in_dataset')
try:
# Add extra constraints if necessary.
added_constraints = self.getInputFromPort('added_constraints')
except vistrails_module.ModuleError:
added_constraints = None
command = self.getInputFromPort("command")
output_pattern = self.getInputFromPort("output_pattern")
cons_for_output = set()
if added_constraints:
cons_for_output = set.union(cons_for_output,
set(added_constraints))
# Do the stuff.
this_process = ProcessUnit([in_dataset], self.output_pattern, command,
cons_for_output)
this_process.execute(simulate=configuration.simulate_execution)
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
self.setResult('out_constraints', str(process_output.constraints))
# Unload the modules at the end.
self.module_loader.unload(self.required_modules)
def test_model_correllation_2(self):
""" This test is to try combining Constraints from two different DataSets.
This uses the new merge_output keyword option.
"""
mock_file_1 = ["/red_echidna"]
mock_file_2 = ["/blue_echidna"]
input_pattern = "/%colour%_%animal%"
# Create our mock DataSets.
with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
mock_glob.return_value = mock_file_1
test_ds_1 = PatternDataSet(input_pattern)
with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
mock_glob.return_value = mock_file_2
test_ds_2 = PatternDataSet(input_pattern)
# A ProcessUnit which merges the Constraint on colour.
the_process = ProcessUnit([test_ds_1, test_ds_2], "/tmp/%animal%_%colour%.file",
"echo", merge_output=["colour"])
output_ds = the_process.execute(simulate=False)
outfiles = [metafile for metafile in output_ds.files]
self.assertEqual(len(outfiles), 1)
self.assertEqual(outfiles[0].full_path, "/tmp/echidna_red-blue.file")
def compute(self):
in_dataset = self.getInputFromPort("in_dataset")
method = self.getInputFromPort("method")
self.positional_args = [(method, 0, "raw")]
self.keyword_args = {}
if len(method.split(",")) > 1:
agg_constraint = "".join(method.split(","))
else:
agg_constraint = method
new_constraints_for_output = set([Constraint("latagg_info", [agg_constraint]), Constraint("suffix", ["nc"])])
this_process = ProcessUnit(
[in_dataset],
self.out_pattern,
self.command,
new_constraints_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args,
cons_keywords=self.keyword_args,
)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult("out_dataset", process_output)
def compute(self):
in_dataset = self.getInputFromPort('in_dataset')
try:
# Add extra constraints if necessary.
added_constraints = self.getInputFromPort('added_constraints')
except vistrails_module.ModuleError:
added_constraints = None
command = self.getInputFromPort("command")
output_pattern = self.getInputFromPort("output_pattern")
cons_for_output = set()
if added_constraints:
cons_for_output = set.union(cons_for_output,
set(added_constraints))
# Do the stuff.
this_process = ProcessUnit([in_dataset],
self.output_pattern,
command,
cons_for_output)
this_process.execute(simulate=configuration.simulate_execution)
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
self.setResult('out_constraints', str(process_output.constraints))
# Unload the modules at the end.
self.module_loader.unload(self.required_modules)
def compute(self):
# Required input
in_dataset = self.getInputFromPort("in_dataset")
variable_name = self.getInputFromPort("variable_name")
self.positional_args=[(variable_name, 0, 'raw')]
cons_for_output = set([Constraint('suffix', ['png'])])
# Execute plotting process.
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
cons_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args,
kw_string="--title '${model}_${experiment}'")
try:
process_output = this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
self.setResult('out_dataset', process_output)
def compute(self):
# Required input
in_dataset = self.getInputFromPort("in_dataset")
new_cons = set([Constraint('extra_info', ['nino34']),
Constraint('latsouth_info', ['5S']),
Constraint('latnorth_info', ['5N']),
Constraint('latagg_info', ['fldavg']),
Constraint('lonwest_info', ['190E']),
Constraint('loneast_info', ['240E']),
Constraint('lonagg_info', ['fldavg']),
Constraint('leveltop_info', ['surface']),
Constraint('levelbottom_info', ['surface']),
Constraint('anomaly_info', ['anom']),
])
# Execute the process.
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
new_cons,
positional_args=self.positional_args,
execution_options=self._execution_options)
try:
this_process.execute(simulate=configuration.simulate_execution)
except subprocess.CalledProcessError as e:
raise vistrails_module.ModuleError(self, e.output)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def compute(self):
in_dataset1 = self.getInputFromPort('in_dataset1')
in_dataset2 = self.getInputFromPort('in_dataset2')
self.positional_args = []
self.keyword_args = {}
new_constraints_for_output = set([Constraint('extra_info', ['fldcor']),
Constraint('suffix', ['nc'])])
merge_val = self.getInputFromPort('merge_constraints')
if merge_val:
extra_merge = [cons_name.strip() for cons_name in merge_val.split(',')]
else:
extra_merge = []
this_process = ProcessUnit([in_dataset1, in_dataset2],
self.out_pattern,
self.command,
new_constraints_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args,
cons_keywords=self.keyword_args,
merge_output=extra_merge)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def compute(self):
in_dataset = self.getInputFromPort('in_dataset')
method = self.getInputFromPort('method')
self.positional_args = [(method, 0, 'raw'), ]
self.keyword_args = {}
if len(method.split(',')) > 1:
agg_constraint = "".join(method.split(','))
else:
agg_constraint = method
new_constraints_for_output = set([Constraint('timeagg_info', [agg_constraint]),
Constraint('suffix', ['nc']),
])
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
new_constraints_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args,
cons_keywords=self.keyword_args)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def compute(self):
# Required input
in_dataset = self.getInputFromPort("in_dataset")
year_start = self.getInputFromPort("start_year")
year_end = self.getInputFromPort("end_year")
new_cons = set([Constraint('year_start', [year_start]),
Constraint('year_end', [year_end]),
Constraint('suffix', ['nc']),
Constraint('grid', ['native'])])
cons_for_output = new_cons
# Execute the seas_vars process.
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
cons_for_output,
positional_args=self.positional_args,
cons_keywords=self.keyword_args,
execution_options=self._execution_options)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception, e:
raise vistrails_module.ModuleError(self, e.output)
def compute(self):
# Required input
in_dataset = self.getInputFromPort("in_dataset")
year_start = self.getInputFromPort("year_start")
year_end = self.getInputFromPort("year_end")
seas_agg = self.getInputFromPort("seas_agg")
new_cons = set([Constraint('year_start', [year_start]),
Constraint('year_end', [year_end]),
Constraint('seas_agg', [seas_agg]),
Constraint('suffix', ['nc']),
Constraint('grid', ['native'])])
# Optional added constraints.
try:
# Add extra constraints if necessary.
added_constraints = self.getInputFromPort('added_constraints')
cons_for_output = new_cons.intersection(added_constraints)
except vistrails_module.ModuleError:
cons_for_output = new_cons
#self.command = self.command + ' tos tos ' + year_start + ' ' + year_end + ' ' + seas_agg
# Execute the seas_vars process.
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
cons_for_output,
positional_args=self.positional_args,
execution_options=self._execution_options)
this_process.execute(simulate=configuration.simulate_execution)
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def compute(self):
in_dataset = self.getInputFromPort('cod_dataset')
command = "${CWSL_CTOOLS}/sdm/sdmrun.py"
sdm_config = configuration.cwsl_ctools_path + "/sdm/default.cfg"
positional_args = [("dxt-gridded", 0, "raw"),
("-c " + sdm_config, 0, "raw")]
# The data is written out to the default
# location.
output_pattern = os.path.join(configuration.user_basepath,
FileCreator.default_pattern(in_dataset.constraints) + ".nc")
this_process = ProcessUnit([in_dataset],
output_pattern,
command,
in_dataset.constraints,
execution_options=self._required_modules,
positional_args=positional_args)
this_process.execute(simulate=configuration.simulate_execution)
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def compute(self):
# Required input
in_dataset = self.getInputFromPort("in_dataset")
variable_name = self.getInputFromPort("variable_name")
self.positional_args=[(variable_name,0,'raw'),('model',1)]
new_cons = set([Constraint('variable', [variable_name]),
Constraint('suffix', ['png']),
])
# Optional added constraints.
try:
# Add extra constraints if necessary.
added_constraints = self.getInputFromPort('added_constraints')
cons_for_output = new_cons.intersection(added_constraints)
except vistrails_module.ModuleError:
cons_for_output = new_cons
# Execute the seas_vars process.
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
cons_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception, e:
raise vistrails_module.ModuleError(self, e.output)
def test_kwstrings(self):
""" Test to check that multi-constraint keyword arguments can be created. """
the_process_unit = ProcessUnit([self.a_pattern_ds], '/a/new/pattern/%fake%/%file%/%pattern%.file',
'echo', kw_string="--title $fake-$file")
output = the_process_unit.execute(simulate=True)
expected_string = self.script_header + 'mkdir -p /a/new/pattern/fake_1/file_1\necho test_file1 /a/new/pattern/fake_1/file_1/pattern_1.file --title fake_1-file_1\n'
self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())
def test_positionalargs_2(self):
""" Test that positional arguments work if the constraint is part of the input only. """
the_process_unit = ProcessUnit([self.a_pattern_ds], '/another/%file%/%pattern%.txt',
'echo', positional_args=[('fake', 0)])
ds_result = the_process_unit.execute(simulate=True)
outfiles = [file_thing for file_thing in ds_result.files]
self.assertEqual(len(outfiles), 1)
expected_string = self.script_header + "mkdir -p /another_file_1\necho fake_1 test_file1 /another/file_1/pattern_1.txt\n"
self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())
def test_positionalargs_3(self):
""" Test that positional arguments work if multiple extra constraints found only in the output are added. """
extra_cons = set([Constraint('animal', ['moose', 'kangaroo']),
Constraint('colour', ['blue'])])
the_process_unit = ProcessUnit([self.a_pattern_ds], '/another/%file%/%pattern%_%animal%_%colour%.txt',
'echo', extra_constraints=extra_cons,
positional_args=[('animal', 0), ('colour', 1)])
ds_result = the_process_unit.execute(simulate=True)
expected_string = self.script_header + 'mkdir -p /another/file_1\necho moose blue test_file1 /another/file_1/pattern_1_moose_blue.txt\necho kangaroo blue test_file1 /another/file_1/pattern_1_kangaroo_blue.txt\n'
self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())
def test_positionalargs_4(self):
""" Test that positional arguments work if multiple extra constraints found only in the output are added. """
extra_cons = set([Constraint('animal', ['moose', 'kangaroo']),
Constraint('colour', ['blue'])])
the_process_unit = ProcessUnit([self.a_pattern_ds], '/another/%file%/%pattern%_%animal%_%colour%.txt',
'echo', extra_constraints=extra_cons,
positional_args=[('animal', 0), ('colour', 1)])
ds_result = the_process_unit.execute(simulate=True)
expected_string = self.script_header + 'mkdir -p /another/file_1\necho moose blue test_file1 /another/file_1/pattern_1_moose_blue.txt\necho kangaroo blue test_file1 /another/file_1/pattern_1_kangaroo_blue.txt\n'
self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())
def test_positionalargs_2(self):
""" Test that positional arguments work if the constraint is part of the input only. """
the_process_unit = ProcessUnit([self.a_pattern_ds], '/another/%file%/%pattern%.txt',
'echo', positional_args=[('fake', 0)])
ds_result = the_process_unit.execute(simulate=True)
outfiles = [file_thing for file_thing in ds_result.files]
self.assertEqual(len(outfiles), 1)
expected_string = self.script_header + "mkdir -p /another/file_1\necho fake_1 test_file1 /another/file_1/pattern_1.txt\n"
self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())
def test_mapping(self):
""" Test to allow simple mapping of a constraint in the input to one in the output. """
# Input PatternDS has constraints fake, file and pattern.
# Use fake from first input as animal constraint.
the_process_unit = ProcessUnit([self.a_pattern_ds], '/a/new/pattern/%animal%/%file%/%pattern%.file',
'echo', map_dict={'animal': ('fake', 0)})
output = the_process_unit.execute(simulate=True)
all_files = [thing for thing in output.files]
self.assertEqual(len(all_files), 1)
self.assertEqual(all_files[0].full_path, '/a/new/pattern/fake_1/file_1/pattern_1.file')
def test_execution(self):
""" Test that a process unit can execute a basic process without falling over. """
# This process will echo the input and output file name to stdout.
the_process_unit = ProcessUnit([self.a_pattern_ds], '/another/%file%/%pattern%.txt',
'echo')
ds_result = the_process_unit.execute(simulate=True)
outfiles = [file_thing for file_thing in ds_result.files]
self.assertEqual(len(outfiles), 1)
expected_string = self.script_header + "mkdir -p /another/file_1\necho test_file1 /another/file_1/pattern_1.txt\n"
self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())
def test_execution(self):
""" Test that a process unit can execute a basic process without falling over. """
# This process will echo the input and output file name to stdout.
the_process_unit = ProcessUnit([self.a_pattern_ds], '/another/%file%/%pattern%.txt',
'echo')
ds_result = the_process_unit.execute(simulate=True)
outfiles = [file_thing for file_thing in ds_result.files]
self.assertEqual(len(outfiles), 1)
expected_string = self.script_header + "mkdir -p /another/file_1\necho test_file1 /another/file_1/pattern_1.txt\n"
self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())
def compute(self):
# Required input
in_dataset = self.getInputFromPort("in_dataset")
# Set up the output command for this module, adding extra options.
positional_args = []
anom_label = 'anom-wrt-all'
arg_number = 0
try:
clim_bounds = self.getInputFromPort('clim_bounds')
positional_args += [('-b', arg_number, 'raw'),
(clim_bounds, arg_number + 1, 'raw')]
start_date, end_date = clim_bounds.split(',')
anom_label = 'anom-wrt-' + start_date + '-' + end_date
arg_number += 2
except vistrails_module.ModuleError as e:
pass
try:
timescale = self.getInputFromPort('timescale')
positional_args += [('-t', arg_number, 'raw'),
(timescale, arg_number + 1, 'raw')]
anom_label = timescale + anom_label
except vistrails_module.ModuleError as e:
pass
cons_for_output = set([
Constraint('suffix', ['nc']),
Constraint('anomaly_info', [anom_label])
])
# Execute the process.
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
cons_for_output,
positional_args=positional_args,
execution_options=self._execution_options)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def test_positionalargs_3(self):
""" Test that positional arguments work if the constraint is part of the output only. """
extra_cons = set([Constraint('animal', ['moose', 'kangaroo'])])
the_process_unit = ProcessUnit([self.a_pattern_ds], '/another/%file%/%pattern%_%animal%.txt',
'echo', extra_constraints=extra_cons,
positional_args=[('animal', 0)])
ds_result = the_process_unit.execute(simulate=True)
outfiles = [file_thing for file_thing in ds_result.files]
self.assertEqual(len(outfiles), 2)
expected_string = self.script_header + 'mkdir -p /another/file_1\necho moose test_file1 /another/file_1/pattern_1_moose.txt\necho kangaroo test_file1 /another/file_1/pattern_1_kangaroo.txt\n'
self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())
def test_positionalargs_2(self):
""" Test that positional arguments work if the constraint is part of the output only. """
extra_cons = set([Constraint('animal', ['moose', 'kangaroo'])])
the_process_unit = ProcessUnit([self.a_pattern_ds], '/another/%file%/%pattern%_%animal%.txt',
'echo', extra_constraints=extra_cons,
positional_args=[('animal', 0)])
ds_result = the_process_unit.execute(simulate=True)
outfiles = [file_thing for file_thing in ds_result.files]
self.assertEqual(len(outfiles), 2)
expected_string = self.script_header + 'mkdir -p /another/file_1\necho moose test_file1 /another/file_1/pattern_1_moose.txt\necho kangaroo test_file1 /another/file_1/pattern_1_kangaroo.txt\n'
self.assertEqual(expected_string, the_process_unit.scheduler.job.to_str())
def test_add_then_overwrite(self):
""" Test to make sure that adding and then overwriting constraints in later process units works. """
extra_con = set([Constraint('an_extra', ['new_value'])])
a_process_unit = ProcessUnit([self.a_pattern_ds], "/%fake%/%file%/%pattern%/%an_extra%.txt",
"echo", extra_constraints=extra_con)
first_output = a_process_unit.execute(simulate=True)
# Now make a new output with an new value of %pattern%.
new_process_unit = ProcessUnit([first_output], "/%fake%/%file%/%pattern%/%an_extra%.txt",
"echo", extra_constraints=set([Constraint('pattern', ['OVERWRITE_PATTERN'])]))
new_process_unit.execute(simulate=True)
expected_string = self.script_header + "mkdir -p /fake_1/file_1/OVERWRITE_PATTERN\necho /fake_1/file_1/pattern_1/new_value.txt /fake_1/file_1/OVERWRITE_PATTERN/new_value.txt\n"
self.assertEqual(expected_string, new_process_unit.scheduler.job.to_str())
def compute(self):
# Required input
in_dataset = self.getInputFromPort("in_dataset")
# Set up the output command for this module, adding extra options.
positional_args = []
anom_label = 'anom-wrt-all'
arg_number = 0
try:
clim_bounds = self.getInputFromPort('clim_bounds')
positional_args += [('-b', arg_number, 'raw'),
(clim_bounds, arg_number+1, 'raw')]
start_date, end_date = clim_bounds.split(',')
anom_label = 'anom-wrt-'+start_date+'-'+end_date
arg_number += 2
except vistrails_module.ModuleError as e:
pass
try:
timescale = self.getInputFromPort('timescale')
positional_args += [('-t', arg_number, 'raw'),
(timescale, arg_number+1, 'raw')]
anom_label = timescale+anom_label
except vistrails_module.ModuleError as e:
pass
cons_for_output = set([Constraint('suffix', ['nc']),
Constraint('anomaly_info', [anom_label])])
# Execute the process.
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
cons_for_output,
positional_args=positional_args,
execution_options=self._execution_options)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def test_overwrites(self):
""" Test that datasets/file creators always maintain their correct constraints during constraint overwrites. """
extra_con = set([Constraint('fake', ['OVERWRITE'])])
the_process_unit = ProcessUnit([self.a_pattern_ds], '/%fake%/%file%/%pattern%.txt',
'echo', extra_constraints=extra_con)
ds_result = the_process_unit.execute(simulate=True)
expected_in_cons = set([Constraint('fake', ['fake_1']),
Constraint('file', ['file_1']),
Constraint('pattern', ['pattern_1'])])
expected_out_cons = set([Constraint('fake', ['OVERWRITE']),
Constraint('file', ['file_1']),
Constraint('pattern', ['pattern_1'])])
self.assertEqual(expected_in_cons, self.a_pattern_ds.constraints)
self.assertEqual(expected_out_cons, ds_result.constraints)
def test_overwrite_constraints(self):
""" Test to ensure that Constraints are correctly overwritten when data is processed. """
extra_cons = set([Constraint('extras', ['other_things']),
Constraint('fake', ['OVERWRITE'])])
a_process_unit = ProcessUnit([self.a_pattern_ds], "/foo/%fake%/%file%/%pattern%_%extras%.txt",
"echo", extra_constraints=extra_cons)
output_ds = a_process_unit.execute(simulate=True)
expected_cons = set([Constraint('extras', ['other_things']),
Constraint('fake', ['OVERWRITE']),
Constraint('file', ['file_1']),
Constraint('pattern', ['pattern_1'])])
self.assertEqual(expected_cons, output_ds.constraints)
expected_string = self.script_header + "mkdir -p /foo/OVERWRITE/file_1\necho /a/fake_1/file_1/pattern_1 /foo/OVERWRITE/file_1/pattern_1_other_things.txt\n"
self.assertEqual(expected_string, a_process_unit.scheduler.job.to_str())
def test_overwrites(self):
""" Test that datasets/file creators always maintain their correct constraints during constraint overwrites. """
extra_con = set([Constraint('fake', ['OVERWRITE'])])
the_process_unit = ProcessUnit([self.a_pattern_ds], '/%fake%/%file%/%pattern%.txt',
'echo', extra_constraints=extra_con)
ds_result = the_process_unit.execute(simulate=True)
expected_in_cons = set([Constraint('fake', ['fake_1']),
Constraint('file', ['file_1']),
Constraint('pattern', ['pattern_1'])])
expected_out_cons = set([Constraint('fake', ['OVERWRITE']),
Constraint('file', ['file_1']),
Constraint('pattern', ['pattern_1'])])
self.assertEqual(expected_in_cons, self.a_pattern_ds.constraints)
self.assertEqual(expected_out_cons, ds_result.constraints)
def compute(self):
in_dataset = self.getInputFromPort('in_dataset')
# Change the file_type constraint from nc to xml
cons_for_output = set([Constraint('suffix', ['xml'])])
# Execute the cdscan
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
cons_for_output,
execution_options=self._execution_options)
try:
this_process.execute(simulate=configuration.simulate_execution)
except subprocess.CalledProcessError, e:
raise vistrails_module.ModuleError(self, e.output)
def compute(self):
in_dataset = self.getInputFromPort('in_dataset')
method = self.getInputFromPort('method')
seas_list = {'mon': 'ymon', 'seas': 'yseas', 'ann': 'tim'}
### loop over seas_list to generate all 3 season files ###
for seas in seas_list.keys():
self.positional_args = [
('%s%s' % (seas_list[seas], method), 0, 'raw'),
]
self.keyword_args = {}
if len(method.split(',')) > 1:
agg_constraint = "".join(method.split(','))
else:
agg_constraint = method
new_constraints_for_output = set([
Constraint(
'timeagg_info',
['%s%s' % (seas_list[seas], method)],
),
Constraint('suffix', ['nc']),
])
this_process = ProcessUnit(
[in_dataset],
self.out_pattern,
self.command,
new_constraints_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args,
cons_keywords=self.keyword_args)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset_%s' % seas, process_output)
def compute(self):
in_dataset = self.getInputFromPort('cod_dataset')
command = "echo This is the command to run."
# The data is written out to the default
# location.
output_pattern = FileCreator.default_pattern(in_dataset.constraints,
temp=True)
this_process = ProcessUnit([in_dataset],
output_pattern,
command,
in_dataset.constraints,
execution_options=self._required_modules)
this_process.execute(simulate=configuration.simulate_execution)
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def compute(self):
in_dataset = self.getInputFromPort('in_dataset')
method = self.getInputFromPort('method')
grid = self.getInputFromPort('grid')
self.positional_args = [
(method, 0, 'raw'),
(grid, 1, 'raw'),
]
self.keyword_args = {}
grid = grid.split('/')[-1]
if len(grid.split('.')
) > 1: # i.e. a weights file as opposed to pre-defined grid
grid_constraint = method + '-' + grid.split('.')[0]
else:
grid_constraint = method + '-' + grid
new_constraints_for_output = set([
Constraint('grid_info', [grid_constraint]),
Constraint('suffix', ['nc']),
])
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
new_constraints_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args,
cons_keywords=self.keyword_args)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def test_changefile_generation(self):
""" This test works to cover the common case when you want to calculate changes.
For example, comparing two dataset by time.
"""
mock_files_1 = ["/model1_1986_rain", "/model2_1986_rain",
"/model3_1986_rain", "/model4_1986_rain",
"/model1_1986_temp"]
mock_files_2 = ["/model1_2015_rain", "/model2_2015_rain",
"/model3_2015_rain", "/model4_2015_rain",
"/model1_2015_temp"]
input_pattern = "/%model%_%date%_%variable%"
# Create our mock DataSets.
with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
mock_glob.return_value = mock_files_1
test_ds_1 = PatternDataSet(input_pattern)
with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
mock_glob.return_value = mock_files_2
test_ds_2 = PatternDataSet(input_pattern)
# A ProcessUnit which merges the Constraint on colour.
the_process = ProcessUnit([test_ds_1, test_ds_2], "/tmp/%model%_%date%_%variable%",
"echo", merge_output=["date"])
output_ds = the_process.execute(simulate=True)
outfile_names = [metafile.full_path for metafile in output_ds.files]
expected_files = ["/tmp/model1_1986-2015_rain",
"/tmp/model2_1986-2015_rain",
"/tmp/model3_1986-2015_rain",
"/tmp/model4_1986-2015_rain",
"/tmp/model1_1986-2015_temp"]
self.assertItemsEqual(expected_files, outfile_names)
def test_model_correllation(self):
with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
mock_glob.return_value = self.mock_obs_files
test_obsds = PatternDataSet(self.observational_pattern)
with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
mock_glob.return_value = self.mock_model_files
test_model_ds = PatternDataSet(self.model_pattern)
output_pattern = "/%variable%_%obs_model%_%model%.nc"
our_process = ProcessUnit([test_obsds, test_model_ds],
output_pattern, "echo")
output = our_process.execute()
all_outs = [thing.full_path for thing in output.files]
good_names = ["/tas_HadISST_BadModel.nc", "/tas_AWAP_BadModel.nc",
"/tas_HadISST_GoodModel.nc", "/tas_AWAP_GoodModel.nc"]
self.assertItemsEqual(good_names, all_outs)
def compute(self):
in_dataset1 = self.getInputFromPort('in_dataset1')
in_dataset2 = self.getInputFromPort('in_dataset2')
self.positional_args = []
self.keyword_args = {}
new_constraints_for_output = set([
Constraint('extra_info', ['timcor']),
Constraint('suffix', ['nc']),
])
merge_val = self.getInputFromPort('merge_constraints')
if merge_val:
extra_merge = [
cons_name.strip() for cons_name in merge_val.split(',')
]
else:
extra_merge = []
this_process = ProcessUnit([in_dataset1, in_dataset2],
self.out_pattern,
self.command,
new_constraints_for_output,
execution_options=self._execution_options,
positional_args=self.positional_args,
cons_keywords=self.keyword_args,
merge_output=extra_merge)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def compute(self):
in_dataset = self.getInputFromPort('in_dataset')
x_value = self.getInputFromPort('x_value')
y_value = self.getInputFromPort('y_value')
command = "echo"
# The data is written out to the default location.
output_pattern = FileCreator.default_pattern(in_dataset.constraints,
temp=True) + ".json"
this_process = ProcessUnit([in_dataset],
output_pattern,
command,
in_dataset.constraints,
execution_options=self._required_modules,
positional_args=[(x_value, 1, 'raw'),
(y_value, 2, 'raw')])
this_process.execute(simulate=configuration.simulate_execution)
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
def test_model_correllation_3(self):
""" This test is to try combining multiple DataSets, each with many files. """
mock_files_1 = ["/red_echidna", "/blue_echidna", "/green_echidna"]
mock_files_2 = ["/blue_echidna", "/red_echidna", "/green_echidna"]
input_pattern = "/%colour%_%animal%"
# Create our mock DataSets.
with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
mock_glob.return_value = mock_files_1
test_ds_1 = PatternDataSet(input_pattern)
with mock.patch('cwsl.core.pattern_dataset.PatternDataSet.glob_fs') as mock_glob:
mock_glob.return_value = mock_files_2
test_ds_2 = PatternDataSet(input_pattern)
# A ProcessUnit which merges the Constraint on colour.
the_process = ProcessUnit([test_ds_1, test_ds_2], "/tmp/%animal%_%colour%.file",
"echo", merge_output=["colour"])
output_ds = the_process.execute(simulate=True)
outfile_names = [metafile.full_path for metafile in output_ds.files]
expected_outfiles = ["/tmp/echidna_red-red.file",
"/tmp/echidna_red-blue.file",
"/tmp/echidna_red-green.file",
"/tmp/echidna_blue-red.file",
"/tmp/echidna_blue-blue.file",
"/tmp/echidna_blue-green.file",
"/tmp/echidna_green-red.file",
"/tmp/echidna_green-blue.file",
"/tmp/echidna_green-green.file"]
self.assertItemsEqual(expected_outfiles, outfile_names)
def test_add_then_overwrite(self):
""" Test to make sure that adding and then overwriting constraints in later process units works. """
extra_con = set([Constraint('an_extra', ['new_value'])])
a_process_unit = ProcessUnit([self.a_pattern_ds],
"/%fake%/%file%/%pattern%/%an_extra%.txt",
"echo",
extra_constraints=extra_con)
first_output = a_process_unit.execute(simulate=True)
# Now make a new output with an new value of %pattern%.
new_process_unit = ProcessUnit(
[first_output],
"/%fake%/%file%/%pattern%/%an_extra%.txt",
"echo",
extra_constraints=set(
[Constraint('pattern', ['OVERWRITE_PATTERN'])]))
new_process_unit.execute(simulate=True)
expected_string = self.script_header + "mkdir -p /fake_1/file_1/OVERWRITE_PATTERN\necho /fake_1/file_1/pattern_1/new_value.txt /fake_1/file_1/OVERWRITE_PATTERN/new_value.txt\n"
self.assertEqual(expected_string,
new_process_unit.scheduler.job.to_str())
def test_mix_types(self):
""" Test to ensure that mixing keyword and positional arguments works as expected. """
the_process_unit_1 = ProcessUnit([self.a_pattern_ds], '/%fake%/%file%/%pattern%.txt',
'echo', cons_keywords={'fake_name': 'fake'})
result_1 = the_process_unit_1.execute(simulate=True)
expected_string_1 = self.script_header + 'mkdir -p /fake_1/file_1\necho test_file1 /fake_1/file_1/pattern_1.txt --fake_name fake_1\n'
self.assertEqual(expected_string_1, the_process_unit_1.scheduler.job.to_str())
# Now try both keyword and positional.
the_process_unit_2 = ProcessUnit([self.a_pattern_ds], '/%fake%/%file%/%pattern%.txt',
'echo', cons_keywords={'fake_name': 'fake'},
positional_args=[('--input', 0, 'raw'), ('--output', 2, 'raw')])
result_2 = the_process_unit_2.execute(simulate=True)
expected_string_2 = self.script_header + 'mkdir -p /fake_1/file_1\necho --input test_file1 --output /fake_1/file_1/pattern_1.txt --fake_name fake_1\n'
self.assertEqual(expected_string_2, the_process_unit_2.scheduler.job.to_str())
def test_files_method(self):
""" Test that the .files method works for a ProcessUnit output
This is to pick up a bug with multiple FileCreators feeding into
each other."""
outpattern = '/a/new/%fake%/%file%/%pattern%'
finalpattern = '/a/final/%fake%/%file%/%pattern%'
first_process = ProcessUnit([self.a_pattern_ds], outpattern, "echo")
first_output = first_process.execute(simulate=True)
second_process = ProcessUnit([first_output], finalpattern, "echo")
second_output = second_process.execute(simulate=True)
final_files = [thing for thing in second_output.files]
self.assertEqual(len(final_files), 1)
final_names = [thing.full_path for thing in final_files]
self.assertItemsEqual(["/a/final/fake_1/file_1/pattern_1"],
final_names)
def compute(self):
# Required input
in_dataset = self.getInputFromPort("in_dataset")
variable = self.getInputFromPort("variable")
#self.positional_args=[(variable_name, 0, 'raw')]
plot_type = self.getInputFromPort("plot_type")
#self.positional_args=[(plot_type, 1, 'raw')]
title = self.getInputFromPort("title")
#self.positional_args=[(title, 2, 'raw')]
region = self.getInputFromPort("region")
#self.positional_args=[(plot_type, 3, 'raw')]
colormap = self.getInputFromPort("colormap")
#self.positional_args=[(colormap, 4, 'raw')]
ticks = self.getInputFromPort("ticks")
#self.positional_args=[(plot_type, 5, 'raw')]
conv_units = self.getInputFromPort("conv_units")
#self.positional_args=[(conv_units, 6, 'raw')]
cons_for_output = set([Constraint('suffix', ['png'])])
run_opts = ''
if variable:
run_opts = run_opts + ' --variable %s' %variable
if plot_type:
run_opts = run_opts + ' --plot_type %s' %plot_type
if title:
run_opts = run_opts + ' --title %s' %title
if region:
run_opts = run_opts + ' --region %s' %region
if colormap:
run_opts = run_opts + ' --colourmap %s' %colormap
if ticks:
run_opts = run_opts + " --ticks '%s'" %ticks
if conv_units:
run_opts = run_opts + " --conv_units '%s'" %conv_units
# Execute plotting process.
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
cons_for_output,
execution_options=self._execution_options,
#positional_args=self.positional_args,
kw_string=run_opts)
#kw_string="--title '${model}_${experiment}'")
try:
process_output = this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
self.setResult('out_dataset', process_output)
def compute(self):
# Required input
in_dataset = self.getInputFromPort("in_dataset")
# Set up the output command for this module, adding extra options.
positional_args = [('variable', 0)]
port_names = [
"timestart", "timeend", "lonwest", "loneast", "latsouth",
"latnorth", "levelbottom", "leveltop"
]
port_vals = {}
for name in port_names:
try:
port_vals[name + "_info"] = self.getInputFromPort(name)
except vistrails_module.ModuleError as e:
port_vals[name + "_info"] = None
arg_number = 3
cons_for_output = set([Constraint('suffix', ['nc'])])
if port_vals["timestart_info"] and port_vals["timeend_info"]:
positional_args += [('--time_bounds', arg_number, 'raw'),
('timestart_info', arg_number + 1),
('timeend_info', arg_number + 2)]
arg_number += 3
cons_for_output |= set([
Constraint('timestart_info', [port_vals["timestart_info"]]),
Constraint('timeend_info', [port_vals["timeend_info"]])
])
if port_vals["loneast_info"] and port_vals["lonwest_info"]:
positional_args += [('--lon_bounds', arg_number, 'raw'),
('lonwest_info', arg_number + 1),
('loneast_info', arg_number + 2)]
arg_number += 3
lonwest_text = longitude_label(port_vals["lonwest_info"])
loneast_text = longitude_label(port_vals["loneast_info"])
cons_for_output |= set([
Constraint('lonwest_info', [lonwest_text]),
Constraint('loneast_info', [loneast_text])
])
if port_vals["latsouth_info"] and port_vals["latnorth_info"]:
positional_args += [('--lat_bounds', arg_number, 'raw'),
('latsouth_info', arg_number + 1),
('latnorth_info', arg_number + 2)]
arg_number += 3
latsouth_text = latitude_label(port_vals["latsouth_info"])
latnorth_text = latitude_label(port_vals["latnorth_info"])
cons_for_output |= set([
Constraint('latsouth_info', [latsouth_text]),
Constraint('latnorth_info', [latnorth_text])
])
if port_vals["levelbottom_info"] and port_vals["leveltop_info"]:
positional_args += [('--level_bounds', arg_number, 'raw'),
('levelbottom_info', arg_number + 1),
('leveltop_info', arg_number + 2)]
arg_number += 3
cons_for_output |= set([
Constraint('levelbottom_info',
[port_vals["levelbottom_info"]]),
Constraint('leveltop_info', [port_vals["leveltop_info"]])
])
# Execute the xml_to_nc process.
this_process = ProcessUnit([in_dataset],
self.out_pattern,
self.command,
cons_for_output,
positional_args=positional_args,
execution_options=self._execution_options)
try:
this_process.execute(simulate=configuration.simulate_execution)
except Exception as e:
raise vistrails_module.ModuleError(self, repr(e))
process_output = this_process.file_creator
self.setResult('out_dataset', process_output)
