def execute(self, simulate=False):
""" This method runs the actual process.
This method returns a FileCreator to be used
as input to the next VisTrails module.
"""
# Check that cws_ctools_path is set
if not configuration.cwsl_ctools_path or not os.path.exists(configuration.cwsl_ctools_path):
raise Exception("cwsl_ctools_path is not set in package options")
# We now create a looper to compare all the input Datasets with
# the output FileCreator.
this_looper = ArgumentCreator(self.inputlist, self.file_creator, self.merge_output)
# TODO determine scheduler from user options.
scheduler = SimpleExecManager(noexec=simulate)
if self.execution_options.has_key('required_modules'):
scheduler.add_module_deps(self.execution_options['required_modules'])
# Add environment variables to the script and the current environment.
scheduler.add_environment_variables({'CWSL_CTOOLS':configuration.cwsl_ctools_path})
os.environ['CWSL_CTOOLS'] = configuration.cwsl_ctools_path
scheduler.add_python_paths([os.path.join(configuration.cwsl_ctools_path,'pythonlib')])
# For every valid possible combination, apply any positional and
# keyword args, then add the command to the scheduler.
for combination in this_looper:
if combination:
in_files, out_files = self.get_fullnames((combination[0], combination[1]))
this_dict = combination[2]
base_cmd_list = [self.shell_command] + in_files + out_files
# Now apply any keyword arguments and positional args.
keyword_command_list = self.apply_keyword_args(base_cmd_list, this_dict)
positional_list = self.apply_positional_args(keyword_command_list, this_dict)
final_command_list = self.apply_kwstring(positional_list, this_dict)
# Generate the annotation string.
try:
annotation = utils.build_metadata(final_command_list)
except NameError:
annotation = None
# The subprocess / queue submission is done here.
scheduler.add_cmd(final_command_list, out_files, annotation=annotation)
scheduler.submit()
# The scheduler is kept for testing purposes.
self.scheduler = scheduler
return self.file_creator
def test_schedulecommand(self):
""" Test that the scheduler can create a script for a simple command. """
in_files = ['infile_1']
out_files = ['outfile_1']
this_manager = SimpleExecManager(noexec=True)
this_manager.add_cmd(['echo'] + in_files + out_files, out_files)
this_manager.submit()
expected_string = """#!/bin/sh\nset -e\n\nmodule purge\nmkdir -p \necho infile_1 outfile_1\n"""
self.assertEqual(this_manager.job.to_str(), expected_string)
def test_command_annotation(self):
""" Test that the scheduler can correctly add annotations. """
in_files = ['infile_1.nc']
out_files = ['outfile_1.nc']
this_manager = SimpleExecManager(noexec=True)
this_manager.add_cmd(['echo'] + in_files + out_files, out_files, annotation="This is an annotation")
this_manager.submit()
expected_string = """#!/bin/sh\nset -e\n\nmodule purge\nmodule load nco\nmkdir -p \necho infile_1.nc outfile_1.nc\nncatted -O -a vistrails_history,global,a,c,"This is an annotation" outfile_1.nc\n"""
self.assertEqual(this_manager.job.to_str(), expected_string)
def test_schedulecommand(self):
""" Test that the scheduler can create a script for a simple command. """
in_files = ['infile_1']
out_files = ['outfile_1']
this_manager = SimpleExecManager(noexec=True)
this_manager.add_cmd(['echo'] + in_files + out_files, out_files)
this_manager.submit()
expected_string = """#!/bin/sh\nset -e\n\nmodule purge\nmkdir -p \necho infile_1 outfile_1\n"""
self.assertEqual(this_manager.job.to_str(), expected_string)
def test_command_annotation(self):
""" Test that the scheduler can correctly add annotations. """
in_files = ['infile_1.nc']
out_files = ['outfile_1.nc']
this_manager = SimpleExecManager(noexec=True)
this_manager.add_cmd(['echo'] + in_files + out_files,
out_files,
annotation="This is an annotation")
this_manager.submit()
expected_string = """#!/bin/sh\nset -e\n\nmodule purge\nmodule load nco\nmkdir -p \necho infile_1.nc outfile_1.nc\nncatted -O -a vistrails_history,global,a,c,"This is an annotation" outfile_1.nc\n"""
self.assertEqual(this_manager.job.to_str(), expected_string)
def execute(self, simulate=False):
""" This method runs the actual process.
This method returns a FileCreator to be used
as input to the next VisTrails module.
"""
# Check that cws_ctools_path is set
if not configuration.cwsl_ctools_path:
raise Exception("cwsl_ctools_path is not set in package options")
configuration.cwsl_ctools_path = os.path.expandvars(configuration.cwsl_ctools_path)
if not os.path.exists(configuration.cwsl_ctools_path):
raise Exception("Path: {} for cwsl_ctools_path does not exist"
.format(configuration.cwsl_ctools_path))
# We now create a looper to compare all the input Datasets with
# the output FileCreator.
this_looper = ArgumentCreator(self.inputlist, self.file_creator, self.merge_output)
# TODO determine scheduler from user options.
scheduler = SimpleExecManager(noexec=simulate)
if self.execution_options.has_key('required_modules'):
scheduler.add_module_deps(self.execution_options['required_modules'])
# Add environment variables to the script and the current environment.
scheduler.add_environment_variables({'CWSL_CTOOLS':configuration.cwsl_ctools_path})
os.environ['CWSL_CTOOLS'] = configuration.cwsl_ctools_path
scheduler.add_python_paths([os.path.join(configuration.cwsl_ctools_path,'pythonlib')])
# For every valid possible combination, apply any positional and
# keyword args, then add the command to the scheduler.
for combination in this_looper:
if combination:
in_files, out_files = self.get_fullnames((combination[0], combination[1]))
this_dict = combination[2]
base_cmd_list = [self.shell_command] + in_files + out_files
# Now apply any keyword arguments and positional args.
keyword_command_list = self.apply_keyword_args(base_cmd_list, this_dict)
positional_list = self.apply_positional_args(keyword_command_list, this_dict)
final_command_list = self.apply_kwstring(positional_list, this_dict)
# Generate the annotation string.
try:
annotation = utils.build_metadata(final_command_list)
except NameError:
annotation = None
# The subprocess / queue submission is done here.
scheduler.add_cmd(final_command_list, out_files, annotation=annotation)
scheduler.submit()
# The scheduler is kept for testing purposes.
self.scheduler = scheduler
return self.file_creator