def _split_jobs(self, cmd, cmd_def, splits, work_dir, stdout=None,
stderr=None, stdin=None,):
''''I creates one job for every split.
Every job has a cmd, work_dir and streams, this info is in the jobs dict
with the keys: cmds, work_dirs, streams
'''
#too many arguments, but similar interface to our __init__
#pylint: disable-msg=R0913
#pylint: disable-msg=R0914
#the main job streams
main_job_streams = get_streams_from_cmd(cmd, cmd_def, stdout=stdout,
stderr=stderr, stdin=stdin)
self._job['streams'] = main_job_streams
streams, work_dirs = self._split_streams(main_job_streams, splits,
work_dir.name)
#now we have to create a new cmd with the right in and out streams for
#every split
cmds, stdins, stdouts, stderrs = self._create_cmds(cmd, streams)
jobs = {'cmds': cmds, 'work_dirs': work_dirs, 'streams': streams,
'stdins':stdins, 'stdouts':stdouts, 'stderrs':stderrs}
return jobs
def _create_condor_job_file(self, cmd, cmd_def, log_file, runner_conf,
stdout, stderr, stdin):
'Given a cmd and the cmd_def it returns the condor job file'
#streams
streams = get_streams_from_cmd(cmd, cmd_def, stdout=stdout,
stderr=stderr, stdin=stdin)
#we need some parameters to write the condor file
parameters = {}
#the executable
binary = cmd[0]
#the binary should be an absolute path
if not os.path.isabs(binary):
#the path to the binary could be relative
if os.sep in binary:
#we make the path absolute
binary = os.path.abspath(binary)
else:
#we have to look in the system $PATH
binary = call(['which', binary])[0].strip()
parameters['executable'] = binary
parameters['log_file'] = log_file
#the cmd shouldn't have absolute path in the files because they will be
#transfered to another node in the condor working dir and they wouldn't
#be found with an absolute path
cmd_no_path = self._remove_paths_from_cmd(cmd, streams, runner_conf)
parameters['arguments'] = ' '.join(cmd_no_path[1:])
if stdout is not None:
parameters['stdout'] = stdout
if stderr is not None:
parameters['stderr'] = stderr
if stdin is not None:
parameters['stdin'] = stdin
transfer_bin = False
if 'transfer_executable' in runner_conf:
transfer_bin = runner_conf['transfer_executable']
parameters['transfer_executable'] = transfer_bin
transfer_files = runner_conf['transfer_executable']
parameters['transfer_files'] = str(transfer_files)
if 'requirements' in runner_conf:
parameters['requirements'] = runner_conf['requirements']
in_fnames = []
for stream in streams:
if stream['io'] == 'in':
fname = None
if 'fname' in stream:
fname = stream['fname']
else:
fname = stream['fhand'].name
in_fnames.append(fname)
parameters['input_fnames'] = in_fnames
#now we can create the job file
condor_job_file = NamedTemporaryFile()
write_condor_job_file(condor_job_file, parameters=parameters)
return condor_job_file
def test_arguments():
'It test that it works with cmd arguments, not options'
#the options we want is in the pre_argv, after the binary
cmd = ['hola', 'hola.txt', '-i', 'caracola.txt']
cmd_def = [{'options': 1, 'io': 'in'}]
expected_streams = [{'fname': 'hola.txt', 'io':'in',
'cmd_location':1}]
streams = get_streams_from_cmd(cmd, cmd_def=cmd_def)
_check_streams(streams, expected_streams)
#the option we want is at the end of the cmd
cmd = ['hola', '-i', 'caracola.txt', 'hola.txt']
cmd_def = [{'options': -1, 'io': 'in'}]
expected_streams = [{'fname': 'hola.txt', 'io':'in',
'cmd_location':3}]
streams = get_streams_from_cmd(cmd, cmd_def=cmd_def)
_check_streams(streams, expected_streams)
def test_simple_case():
'It tests the most simple cases'
#a simple case
cmd = ['hola', '-i', 'caracola.txt']
cmd_def = [{'options': ('-i', '--input'), 'io': 'in'}]
expected_streams = [{'fname': 'caracola.txt', 'io':'in',
'cmd_location':2}]
streams = get_streams_from_cmd(cmd, cmd_def=cmd_def)
_check_streams(streams, expected_streams)
#a parameter not found in the cmd
cmd = ['hola', '-i', 'caracola.txt']
cmd_def = [{'options': ('-i', '--input'), 'io': 'in'},
{'options': ('-j', '--input2'), 'io': 'in'}]
expected_streams = [{'fname': 'caracola.txt', 'io':'in',
'cmd_location': 2}]
streams = get_streams_from_cmd(cmd, cmd_def=cmd_def)
_check_streams(streams, expected_streams)
def test_stdin():
'We want stdin, stdout and stderr as streams'
#stdin
cmd = ['hola']
cmd_def = [{'options':STDIN, 'io': 'in'},
{'options':STDOUT, 'io': 'out'}]
stdout = 'stdout' #in the real world they will be files
stderr = 'stderr'
stdin = 'stdin'
expected_streams = [{'fhand': stdin, 'io':'in', 'cmd_location':STDIN,
'options': stdin},
{'fhand': stdout, 'io':'out', 'cmd_location':STDOUT,
'options':stdout},
{'fhand': stderr, 'io':'out', 'cmd_location':STDERR,
'options':stderr}]
streams = get_streams_from_cmd(cmd, cmd_def=cmd_def, stdout=stdout,
stderr=stderr, stdin=stdin)
_check_streams(streams, expected_streams)
assert 'fname' not in streams[0]
assert 'fname' not in streams[1]
assert 'fname' not in streams[2]
