def _writeJobScript(statement, job_memory, job_name, shellfile):
# disabled - problems with quoting
# tmpfile.write( '''echo 'statement=%s' >> %s\n''' %
# (shellquote(statement), shellfile) )
# module list outputs to stderr, so merge stderr and stdout
script = '''#!/bin/bash -e \n
echo "%(job_name)s : START -> ${0}" >> %(shellfile)s
set | sed 's/^/%(job_name)s : /' &>> %(shellfile)s
set +o errexit
module list 2>&1 | sed 's/^/%(job_name)s: /' &>> %(shellfile)s
set -o errexit
hostname | sed 's/^/%(job_name)s: /' &>> %(shellfile)s
cat /proc/meminfo | sed 's/^/%(job_name)s: /' &>> %(shellfile)s
echo "%(job_name)s : END -> ${0}" >> %(shellfile)s
''' % locals()
# restrict virtual memory
# Note that there are resources in SGE which could do this directly
# such as v_hmem.
# Note that limiting resident set sizes (RSS) with ulimit is not
# possible in newer kernels.
script += "ulimit -v %i\n" % IOTools.human2bytes(job_memory)
script += expandStatement(statement,
ignore_pipe_errors=ignore_pipe_errors)
script += "\n"
job_path = getTempFilename(dir=PARAMS["workingdir"])
with open(job_path, "w") as script_file:
script_file.write(script)
return (job_path)
def _writeJobScript(statement, job_memory, job_name, shellfile):
# disabled - problems with quoting
# tmpfile.write( '''echo 'statement=%s' >> %s\n''' %
# (shellquote(statement), shellfile) )
# module list outputs to stderr, so merge stderr and stdout
script = '''#!/bin/bash\n
echo "%(job_name)s : START -> ${0}" >> %(shellfile)s
set | sed 's/^/%(job_name)s : /' &>> %(shellfile)s
module list 2>&1 | sed 's/^/%(job_name)s: /' &>> %(shellfile)s
hostname | sed 's/^/%(job_name)s: /' &>> %(shellfile)s
cat /proc/meminfo | sed 's/^/%(job_name)s: /' &>> %(shellfile)s
echo "%(job_name)s : END -> ${0}" >> %(shellfile)s
''' % locals()
# restrict virtual memory
# Note that there are resources in SGE which could do this directly
# such as v_hmem.
# Note that limiting resident set sizes (RSS) with ulimit is not
# possible in newer kernels.
script += "ulimit -v %i\n" % IOTools.human2bytes(job_memory)
script += expandStatement(statement,
ignore_pipe_errors=ignore_pipe_errors)
script += "\n"
job_path = getTempFilename(dir=PARAMS["workingdir"])
with open(job_path, "w") as script_file:
script_file.write(script)
return(job_path)
def joinStatements(statements, infile):
'''join a chain of statements into a single statement.
Each statement contains an @IN@ or a @OUT@ placeholder or both.
These will be replaced by the names of successive temporary files.
In the first statement, @IN@ is replaced with `infile`.
The last statement should move @IN@ to outfile.
Arguments
---------
statements : list
A list of command line statements.
infile : string
Filename of the first data set.
Returns
-------
statement : string
A single command line statement.
'''
prefix = getTempFilename()
pattern = "%s_%%i" % prefix
result = []
for x, statement in enumerate(statements):
if x == 0:
s = re.sub("@IN@", infile, statement)
else:
s = re.sub("@IN@", pattern % x, statement)
s = re.sub("@OUT@", pattern % (x + 1), s).strip()
if s.endswith(";"):
s = s[:-1]
result.append(s)
assert prefix != ""
result.append("rm -f %s*" % prefix)
result = "; checkpoint ; ".join(result)
return result
def joinStatements(statements, infile):
'''join a chain of statements into a single statement.
Each statement contains an @IN@ or a @OUT@ placeholder or both.
These will be replaced by the names of successive temporary files.
In the first statement, @IN@ is replaced with `infile`.
The last statement should move @IN@ to outfile.
Arguments
---------
statements : list
A list of command line statements.
infile : string
Filename of the first data set.
Returns
-------
statement : string
A single command line statement.
'''
prefix = getTempFilename()
pattern = "%s_%%i" % prefix
result = []
for x, statement in enumerate(statements):
if x == 0:
s = re.sub("@IN@", infile, statement)
else:
s = re.sub("@IN@", pattern % x, statement)
s = re.sub("@OUT@", pattern % (x + 1), s).strip()
if s.endswith(";"):
s = s[:-1]
result.append(s)
assert prefix != ""
result.append("rm -f %s*" % prefix)
result = "; checkpoint ; ".join(result)
return result
def _pickle_args(args, kwargs):
''' Pickle a set of function arguments. Removes any kwargs that are
arguements to submit first. Returns a tuple, the first member of which
is the key word arguements to submit, the second is a file name
with the picked call arguements '''
use_args = [
"to_cluster", "logfile", "job_options", "job_queue", "job_threads",
"job_memory"
]
submit_args = {}
for arg in use_args:
if arg in kwargs:
submit_args[arg] = kwargs[arg]
del kwargs[arg]
args_file = getTempFilename(shared=True)
pickle.dump([args, kwargs], open(args_file, "wb"))
return (submit_args, args_file)
def _pickle_args(args, kwargs):
''' Pickle a set of function arguments. Removes any kwargs that are
arguements to submit first. Returns a tuple, the first member of which
is the key word arguements to submit, the second is a file name
with the picked call arguements '''
use_args = ["to_cluster",
"logfile",
"job_options",
"job_queue",
"job_threads",
"job_memory"]
submit_args = {}
for arg in use_args:
if arg in kwargs:
submit_args[arg] = kwargs[arg]
del kwargs[arg]
args_file = getTempFilename(shared=True)
pickle.dump([args, kwargs], open(args_file, "wb"))
return (submit_args, args_file)