def _writeJobScript(statement, job_memory, job_name, shellfile): # disabled - problems with quoting # tmpfile.write( '''echo 'statement=%s' >> %s\n''' % # (shellquote(statement), shellfile) ) # module list outputs to stderr, so merge stderr and stdout script = '''#!/bin/bash -e \n echo "%(job_name)s : START -> ${0}" >> %(shellfile)s set | sed 's/^/%(job_name)s : /' &>> %(shellfile)s set +o errexit module list 2>&1 | sed 's/^/%(job_name)s: /' &>> %(shellfile)s set -o errexit hostname | sed 's/^/%(job_name)s: /' &>> %(shellfile)s cat /proc/meminfo | sed 's/^/%(job_name)s: /' &>> %(shellfile)s echo "%(job_name)s : END -> ${0}" >> %(shellfile)s ''' % locals() # restrict virtual memory # Note that there are resources in SGE which could do this directly # such as v_hmem. # Note that limiting resident set sizes (RSS) with ulimit is not # possible in newer kernels. script += "ulimit -v %i\n" % IOTools.human2bytes(job_memory) script += expandStatement(statement, ignore_pipe_errors=ignore_pipe_errors) script += "\n" job_path = getTempFilename(dir=PARAMS["workingdir"]) with open(job_path, "w") as script_file: script_file.write(script) return (job_path)
def _writeJobScript(statement, job_memory, job_name, shellfile): # disabled - problems with quoting # tmpfile.write( '''echo 'statement=%s' >> %s\n''' % # (shellquote(statement), shellfile) ) # module list outputs to stderr, so merge stderr and stdout script = '''#!/bin/bash\n echo "%(job_name)s : START -> ${0}" >> %(shellfile)s set | sed 's/^/%(job_name)s : /' &>> %(shellfile)s module list 2>&1 | sed 's/^/%(job_name)s: /' &>> %(shellfile)s hostname | sed 's/^/%(job_name)s: /' &>> %(shellfile)s cat /proc/meminfo | sed 's/^/%(job_name)s: /' &>> %(shellfile)s echo "%(job_name)s : END -> ${0}" >> %(shellfile)s ''' % locals() # restrict virtual memory # Note that there are resources in SGE which could do this directly # such as v_hmem. # Note that limiting resident set sizes (RSS) with ulimit is not # possible in newer kernels. script += "ulimit -v %i\n" % IOTools.human2bytes(job_memory) script += expandStatement(statement, ignore_pipe_errors=ignore_pipe_errors) script += "\n" job_path = getTempFilename(dir=PARAMS["workingdir"]) with open(job_path, "w") as script_file: script_file.write(script) return(job_path)
def joinStatements(statements, infile): '''join a chain of statements into a single statement. Each statement contains an @IN@ or a @OUT@ placeholder or both. These will be replaced by the names of successive temporary files. In the first statement, @IN@ is replaced with `infile`. The last statement should move @IN@ to outfile. Arguments --------- statements : list A list of command line statements. infile : string Filename of the first data set. Returns ------- statement : string A single command line statement. ''' prefix = getTempFilename() pattern = "%s_%%i" % prefix result = [] for x, statement in enumerate(statements): if x == 0: s = re.sub("@IN@", infile, statement) else: s = re.sub("@IN@", pattern % x, statement) s = re.sub("@OUT@", pattern % (x + 1), s).strip() if s.endswith(";"): s = s[:-1] result.append(s) assert prefix != "" result.append("rm -f %s*" % prefix) result = "; checkpoint ; ".join(result) return result
def _pickle_args(args, kwargs): ''' Pickle a set of function arguments. Removes any kwargs that are arguements to submit first. Returns a tuple, the first member of which is the key word arguements to submit, the second is a file name with the picked call arguements ''' use_args = [ "to_cluster", "logfile", "job_options", "job_queue", "job_threads", "job_memory" ] submit_args = {} for arg in use_args: if arg in kwargs: submit_args[arg] = kwargs[arg] del kwargs[arg] args_file = getTempFilename(shared=True) pickle.dump([args, kwargs], open(args_file, "wb")) return (submit_args, args_file)
def _pickle_args(args, kwargs): ''' Pickle a set of function arguments. Removes any kwargs that are arguements to submit first. Returns a tuple, the first member of which is the key word arguements to submit, the second is a file name with the picked call arguements ''' use_args = ["to_cluster", "logfile", "job_options", "job_queue", "job_threads", "job_memory"] submit_args = {} for arg in use_args: if arg in kwargs: submit_args[arg] = kwargs[arg] del kwargs[arg] args_file = getTempFilename(shared=True) pickle.dump([args, kwargs], open(args_file, "wb")) return (submit_args, args_file)