def _split_jobs(self, cmd, cmd_def, splits, work_dir, stdout=None, stderr=None, stdin=None,): ''''I creates one job for every split. Every job has a cmd, work_dir and streams, this info is in the jobs dict with the keys: cmds, work_dirs, streams ''' #too many arguments, but similar interface to our __init__ #pylint: disable-msg=R0913 #pylint: disable-msg=R0914 #the main job streams main_job_streams = get_streams_from_cmd(cmd, cmd_def, stdout=stdout, stderr=stderr, stdin=stdin) self._job['streams'] = main_job_streams streams, work_dirs = self._split_streams(main_job_streams, splits, work_dir.name) #now we have to create a new cmd with the right in and out streams for #every split cmds, stdins, stdouts, stderrs = self._create_cmds(cmd, streams) jobs = {'cmds': cmds, 'work_dirs': work_dirs, 'streams': streams, 'stdins':stdins, 'stdouts':stdouts, 'stderrs':stderrs} return jobs
def _create_condor_job_file(self, cmd, cmd_def, log_file, runner_conf, stdout, stderr, stdin): 'Given a cmd and the cmd_def it returns the condor job file' #streams streams = get_streams_from_cmd(cmd, cmd_def, stdout=stdout, stderr=stderr, stdin=stdin) #we need some parameters to write the condor file parameters = {} #the executable binary = cmd[0] #the binary should be an absolute path if not os.path.isabs(binary): #the path to the binary could be relative if os.sep in binary: #we make the path absolute binary = os.path.abspath(binary) else: #we have to look in the system $PATH binary = call(['which', binary])[0].strip() parameters['executable'] = binary parameters['log_file'] = log_file #the cmd shouldn't have absolute path in the files because they will be #transfered to another node in the condor working dir and they wouldn't #be found with an absolute path cmd_no_path = self._remove_paths_from_cmd(cmd, streams, runner_conf) parameters['arguments'] = ' '.join(cmd_no_path[1:]) if stdout is not None: parameters['stdout'] = stdout if stderr is not None: parameters['stderr'] = stderr if stdin is not None: parameters['stdin'] = stdin transfer_bin = False if 'transfer_executable' in runner_conf: transfer_bin = runner_conf['transfer_executable'] parameters['transfer_executable'] = transfer_bin transfer_files = runner_conf['transfer_executable'] parameters['transfer_files'] = str(transfer_files) if 'requirements' in runner_conf: parameters['requirements'] = runner_conf['requirements'] in_fnames = [] for stream in streams: if stream['io'] == 'in': fname = None if 'fname' in stream: fname = stream['fname'] else: fname = stream['fhand'].name in_fnames.append(fname) parameters['input_fnames'] = in_fnames #now we can create the job file condor_job_file = NamedTemporaryFile() write_condor_job_file(condor_job_file, parameters=parameters) return condor_job_file
def test_arguments(): 'It test that it works with cmd arguments, not options' #the options we want is in the pre_argv, after the binary cmd = ['hola', 'hola.txt', '-i', 'caracola.txt'] cmd_def = [{'options': 1, 'io': 'in'}] expected_streams = [{'fname': 'hola.txt', 'io':'in', 'cmd_location':1}] streams = get_streams_from_cmd(cmd, cmd_def=cmd_def) _check_streams(streams, expected_streams) #the option we want is at the end of the cmd cmd = ['hola', '-i', 'caracola.txt', 'hola.txt'] cmd_def = [{'options': -1, 'io': 'in'}] expected_streams = [{'fname': 'hola.txt', 'io':'in', 'cmd_location':3}] streams = get_streams_from_cmd(cmd, cmd_def=cmd_def) _check_streams(streams, expected_streams)
def test_simple_case(): 'It tests the most simple cases' #a simple case cmd = ['hola', '-i', 'caracola.txt'] cmd_def = [{'options': ('-i', '--input'), 'io': 'in'}] expected_streams = [{'fname': 'caracola.txt', 'io':'in', 'cmd_location':2}] streams = get_streams_from_cmd(cmd, cmd_def=cmd_def) _check_streams(streams, expected_streams) #a parameter not found in the cmd cmd = ['hola', '-i', 'caracola.txt'] cmd_def = [{'options': ('-i', '--input'), 'io': 'in'}, {'options': ('-j', '--input2'), 'io': 'in'}] expected_streams = [{'fname': 'caracola.txt', 'io':'in', 'cmd_location': 2}] streams = get_streams_from_cmd(cmd, cmd_def=cmd_def) _check_streams(streams, expected_streams)
def test_stdin(): 'We want stdin, stdout and stderr as streams' #stdin cmd = ['hola'] cmd_def = [{'options':STDIN, 'io': 'in'}, {'options':STDOUT, 'io': 'out'}] stdout = 'stdout' #in the real world they will be files stderr = 'stderr' stdin = 'stdin' expected_streams = [{'fhand': stdin, 'io':'in', 'cmd_location':STDIN, 'options': stdin}, {'fhand': stdout, 'io':'out', 'cmd_location':STDOUT, 'options':stdout}, {'fhand': stderr, 'io':'out', 'cmd_location':STDERR, 'options':stderr}] streams = get_streams_from_cmd(cmd, cmd_def=cmd_def, stdout=stdout, stderr=stderr, stdin=stdin) _check_streams(streams, expected_streams) assert 'fname' not in streams[0] assert 'fname' not in streams[1] assert 'fname' not in streams[2]