def alt(jobid, priority=None):
"""Alter a job in the SGE queue
Parameters
----------
jobid : int
The job id to remove
priority : int, optional
The priority level of the job
Notes
-----
This function is currently still under development does not provide
the full range of ``qalter`` flags.
Todo
----
* Add more functionality
"""
cmd = ["qalter"]
if priority:
cmd += ["-p", str(priority)]
logger.debug("Altered priority for job %s by %s", str(jobid),
str(priority))
cmd += [str(jobid)]
cexec(cmd)
def kill(jobid):
"""Remove a job from the LSF queue
Parameters
----------
jobid : int
The job id to remove
Raises
------
RuntimeError
Execution exited with non-zero return code
"""
stdout = cexec(["bkill", str(jobid)], permit_nonzero=True)
# Large arrays can take time to be full deleted, give it a few moments
if "is in progress" in stdout:
stdout = cexec(["bkill", "-b", str(jobid)], permit_nonzero=True)
import time
time.sleep(10)
if any(
text in stdout for text in
["has already finished", "is being terminated", "is in progress"]):
logger.debug("Removed job %d from the queue", jobid)
else:
msg = "Execution of '{0}' exited with non-zero return code: {1}".format(
"bkill " + str(jobid), stdout)
raise RuntimeError(msg)
def shelxc(self):
cmd = ["shelxc", self.name]
stdin = """CELL {0}
SPAG {1}
SAD {2}"""
sca_out = os.path.join(self.work_dir, "{0}.sca".format(self.name))
stdin = stdin.format(self._cell_parameters, self._space_group, os.path.relpath(sca_out))
cexec(cmd, stdin=stdin)
def shelxc(self):
cmd = ["shelxc" + EXE_EXT, self.name]
stdin = """CELL {0}
SPAG {1}
SAD {2}"""
sca_out = os.path.join(self.work_dir, "{0}.sca".format(self.name))
stdin = stdin.format(self._cell_parameters, self._space_group, os.path.relpath(sca_out))
cexec(cmd, stdin=stdin)
def _generate_hklpck0(self):
mtz_labels = simbad.util.mtz_util.GetLabels(self.mtz)
logger.info("Preparing files for AMORE rotation function")
stdin = self.sortfun_stdin_template.format(f=mtz_labels.f,
sigf=mtz_labels.sigf)
hklpck0 = os.path.join(self.work_dir, 'spmipch.hkl')
cmd = [self.amore_exe, 'hklin', self.mtz, 'hklpck0', hklpck0]
pyjob.cexec(cmd, stdin=stdin)
return hklpck0
def reindex(hklin, hklout, sg):
"""Function to reindex input hkl using pointless"""
cmd = ["pointless" + EXE_EXT, "hklin", hklin, "hklout", hklout]
stdin = """
spacegroup {0}
"""
stdin = stdin.format(sg)
cexec(cmd, stdin=stdin)
def hold(jobid):
"""Hold a job in the SGE queue
Parameters
----------
jobid : int
The job id to remove
"""
cexec(["qhold", str(jobid)])
logger.debug("Holding back job %d from the queue", jobid)
def kill(jobid):
"""Remove a job from the SGE queue
Parameters
----------
jobid : int
The job id to remove
"""
cexec(["qdel", str(jobid)])
logger.debug("Removed job %d from the queue", jobid)
def rls(jobid):
"""Release a job from the SGE queue
Parameters
----------
jobid : int
The job id to remove
"""
cexec(["qrls", str(jobid)])
logger.debug("Released job %d from the queue", jobid)
def mtz2sca(self):
sca_out = os.path.join(self.work_dir, "{0}.sca".format(self.name))
cmd = ["mtz2sca" + EXE_EXT, self.mtz, sca_out]
if self.mtz_labels.iplus:
cmd += ['-p', self.mtz_labels.iplus,
'-P', self.mtz_labels.sigiplus,
'-m', self.mtz_labels.iminus,
'-M', self.mtz_labels.sigiminus]
elif self.mtz_labels.fplus:
cmd += ['-p', self.mtz_labels.fplus,
'-P', self.mtz_labels.sigfplus,
'-m', self.mtz_labels.fminus,
'-M', self.mtz_labels.sigfminus]
cexec(cmd)
def stat(jobid):
"""Obtain information about a job id
Parameters
----------
jobid : int
The job id to remove
Returns
-------
dict
A dictionary with job specific data
"""
stdout = cexec(["qstat", "-j", str(jobid)], permit_nonzero=True)
data = {}
line_split = re.compile(":\s+")
for line in stdout.split(os.linesep):
line = line.strip()
if 'jobs do not exist' in line:
return data
if not line or "=" * 30 in line:
continue
else:
kv = line_split.split(line, 1)
if len(kv) == 2:
data[kv[0]] = kv[1]
return data
def sheetbend(exe, hklin, pdbin, pdbout, ncyc, logfile):
"""Function to run refinement using sheetbend
Parameters
----------
hklin : str
Path to the input hkl file
pdbin : str
Path to the input pdb
pdbout : str
Path to the output pdb
ncyc : int
Number of cycles to run
logfile : str
Path to the output log
Returns
-------
file
Output pdb file
file
Output log file
"""
mtz_labels = mtz_util.GetLabels(hklin)
colin = "{0},{1}".format(mtz_labels.f, mtz_labels.sigf)
cmd = [
exe, '--pdbin', pdbin, '--mtzin', hklin, '--pdbout', pdbout,
'--colin-fo', colin, '-cycles',
str(ncyc), '-resolution-by-cycle', '6,3'
]
stdout = cexec(cmd)
with open(logfile, 'w') as f_out:
f_out.write(stdout)
def refmac(hklin, hklout, pdbin, pdbout, logfile, key):
"""Function to run refinement using REFMAC
Parameters
----------
hklin : str
Path to the input hkl file
hklout : str
Path to the output hkl file
pdbin : str
Path to the input pdb file
pdbout : str
Path to the output pdb file
logfile : str
Path to the output logfile
key : str
REFMAC key words
Returns
-------
file
Output hkl file
file
Output pdb file
file
Output log file
"""
cmd = ['refmac5', 'hklin', hklin, 'hklout', hklout, 'xyzin', pdbin, 'xyzout', pdbout]
stdout = cexec(cmd, stdin=key)
with open(logfile, 'w') as f_out:
f_out.write(stdout)
def refmac(hklin, hklout, pdbin, pdbout, logfile, key):
"""Function to run refinement using REFMAC
Parameters
----------
hklin : str
Path to the input hkl file
hklout : str
Path to the output hkl file
pdbin : str
Path to the input pdb file
pdbout : str
Path to the output pdb file
logfile : str
Path to the output logfile
key : str
REFMAC key words
Returns
-------
file
Output hkl file
file
Output pdb file
file
Output log file
"""
cmd = [
'refmac5' + EXE_EXT, 'hklin', hklin, 'hklout', hklout, 'xyzin',
pdbin, 'xyzout', pdbout
]
stdout = cexec(cmd, stdin=key)
with open(logfile, 'w') as f_out:
f_out.write(stdout)
def test_cexec_5(self):
cmd = [
sys.executable, "-c",
"import os, sys; print(os.getcwd()); sys.exit(0)"
]
directory = os.path.join(os.getcwd(), 'pyjob')
stdout = cexec(cmd, directory=directory)
self.assertEqual(directory, stdout)
def _ensure_exec_available(exe):
"""Ensure that the specified executable is available in the system
Parameters
----------
exe : str
The executable to test
Raises
------
:exc:`~pyjob.exception.PyJobError`
The executable cannot be found
"""
try:
cexec([exe])
except PyJobExecutableNotFoundError:
raise PyJobError(
f"Cannot find executable {exe}. Please ensure environment is set up correctly."
)
def test_cexec_4(self):
if sys.version_info < (3, 0):
cmd = [
sys.executable, "-c",
"import sys; print(raw_input()); sys.exit(0)"
]
else:
cmd = [
sys.executable, "-c", "import sys; print(input()); sys.exit(0)"
]
stdout = cexec(cmd, stdin="hello")
self.assertEqual("hello", stdout)
def __init__(self):
StrictVersion.__init__(self)
ccp4_major_minor = os.path.join(os.environ["CCP4"], "lib", "ccp4", "MAJOR_MINOR")
if os.path.isfile(ccp4_major_minor):
with open(ccp4_major_minor, "r") as f_in:
tversion = f_in.read().strip()
else:
logger = logging.getLogger(__name__)
logger.debug("Detecting CCP4 version via executing pdbcur")
stdout = cexec(['pdbcur' + EXE_EXT], permit_nonzero=True)
tversion = None
for line in stdout.split(os.linesep):
if line.startswith(' ### CCP4'):
tversion = line.split()[2].rstrip(':')
if tversion is None:
raise RuntimeError("Cannot determine CCP4 version")
self.parse(tversion)
def __init__(self):
StrictVersion.__init__(self)
ccp4_major_minor = os.path.join(os.environ["CCP4"], "lib", "ccp4",
"MAJOR_MINOR")
if os.path.isfile(ccp4_major_minor):
with open(ccp4_major_minor, "r") as f_in:
tversion = f_in.read().strip()
else:
logger = logging.getLogger(__name__)
logger.debug("Detecting CCP4 version via executing pdbcur")
stdout = cexec(['pdbcur' + EXE_EXT], permit_nonzero=True)
tversion = None
for line in stdout.split(os.linesep):
if line.startswith(' ### CCP4'):
tversion = line.split()[2].rstrip(':')
if tversion is None:
raise RuntimeError("Cannot determine CCP4 version")
self.parse(tversion)
def molrep(key, logfile):
"""Function to run molecular replacement using MOLREP
Parameters
----------
key : str
MOLREP keywords
logfile :
Path to output log file
Returns
-------
file
The output pdb from MOLREP
file
The output log file
"""
cmd = ["molrep"]
stdout = cexec(cmd, stdin=key)
with open(logfile, 'w') as f_out:
f_out.write(stdout)
def molrep(key, logfile):
"""Function to run molecular replacement using MOLREP
Parameters
----------
key : str
MOLREP keywords
logfile :
Path to output log file
Returns
-------
file
The output pdb from MOLREP
file
The output log file
"""
cmd = ["molrep" + EXE_EXT]
stdout = cexec(cmd, stdin=key)
with open(logfile, 'w') as f_out:
f_out.write(stdout)
def stat(jobid):
"""Obtain information about a job id
Parameters
----------
jobid : int
The job id to remove
Returns
-------
dict
A dictionary with job specific data
Todo
----
* Extract the correct information
"""
stdout = cexec(["bjobs", "-l", str(jobid)])
if "Done successfully" in stdout:
return {}
else:
return {'job_number': jobid, 'status': "Running"}
def sub(command,
array=None,
deps=None,
directory=None,
hold=False,
log=None,
name=None,
priority=None,
queue=None,
runtime=None,
shell=None,
threads=None,
*args,
**kwargs):
"""Submit a job to the LSF queue
Parameters
----------
command : list
A list with the final command
array : list, optional
A list of array instructions in form of [min, max, (step)].
``step`` is optional.
deps : list, optional
A list of dependency job ids
directory : str, optional
A path to a directory to run the job in
hold : bool, optional
Submit but __hold__ the job
log : str, optional
The path to a logfile for stdout
name : str, optional
The name of the job
pe_opts : list, optional
Job-specific keywords [Unused]
priority : int, optional
The priority level of the job
queue : str, optional
The queue to submit the job to
runtime : int, optional
The maximum runtime of the job in seconds
shell : str, optional
The absolute path to the shell to run the job in
threads : int, optional
The maximum number of threads available to a job
"""
cmd = ["bsub", "-cwd", directory if directory else os.getcwd()]
if array:
name = "pyjob" if name is None else name
if len(array) == 3:
cmd += [
"-J", "{0}[{1}-{2}%{3}]".format(name, array[0], array[1],
array[2])
]
elif len(array) == 2:
cmd += ["-J", "{0}[{1}-{2}]".format(name, array[0], array[1])]
name = None # Reset this!
if deps:
cmd += [
"-w", " && ".join(["done(%s)" % dep for dep in map(str, deps)])
]
if hold:
cmd += ["-H"]
if log:
cmd += ["-o", log]
if name:
cmd += ["-J", '"{0}"'.format(name)]
if priority:
cmd += ["-sp", str(priority)]
if queue:
cmd += ["-q", queue]
if shell:
cmd += ["-L", shell]
if threads:
cmd += ["-R", '"span[ptile={0}]"'.format(threads)]
if runtime:
cmd += ["-W", str(runtime)]
command_f = command[0] if isinstance(command, list) else command
stdout = cexec(cmd, stdin=open(command_f).read(), directory=directory)
jobid = int(stdout.split()[1][1:-1])
logger.debug("Job %d successfully submitted to the LSF queue", jobid)
return jobid
def test_cexec_3(self):
cmd = [
sys.executable, "-c", "import sys; print(\"hello\"); sys.exit(0)"
]
stdout = cexec(cmd, permit_nonzero=True)
self.assertEqual("hello", stdout)
def test_cexec_1(self):
cmd = [
sys.executable, "-c", "import sys; print(\"hello\"); sys.exit(0)"
]
stdout = cexec(cmd)
self.assertEqual("hello", stdout)
def run(self):
"""Method representing the process's activity"""
for job in iter(self.queue.get, None):
stdout = cexec([job], directory=self.directory, permit_nonzero=self.permit_nonzero)
with open(job.rsplit('.', 1)[0] + '.log', 'w') as f_out:
f_out.write(stdout)
def sub(command,
array=None,
deps=None,
directory=None,
hold=False,
log=None,
name=None,
pe_opts=None,
priority=None,
queue=None,
runtime=None,
shell=None,
threads=None,
*args,
**kwargs):
"""Submit a job to the SGE queue
Parameters
----------
command : list
A list with the final command
array : list, optional
A list of array instructions in form of [min, max, (step)].
``step`` is optional.
deps : list, optional
A list of dependency job ids
directory : str, optional
A path to a directory to run the job in
hold : bool, optional
Submit but __hold__ the job
log : str, optional
The path to a logfile for stdout
name : str, optional
The name of the job
pe_opts : list, optional
Job-specific keywords
priority : int, optional
The priority level of the job
queue : str, optional
The queue to submit the job to
runtime : int, optional
The maximum runtime of the job in seconds
shell : str, optional
The absolute path to the shell to run the job in
threads : int, optional
The maximum number of threads available to a job
"""
cmd = ["qsub", "-cwd", "-V", "-w", "e", "-j", "y"]
if array and len(array) == 3:
cmd += [
"-t", "{0}-{1}".format(array[0], array[1]), "-tc",
str(array[2])
]
elif array and len(array) == 2:
cmd += ["-t", "{0}-{1}".format(array[0], array[1])]
if deps:
cmd += ["-hold_jid", "{0}".format(",".join(map(str, deps)))]
if hold:
cmd += ["-h"]
if log:
cmd += ["-o", log]
if name:
cmd += ["-N", name]
if pe_opts:
cmd += ["-pe"] + pe_opts.split()
if priority:
cmd += ["-p", str(priority)]
if queue:
cmd += ["-q", queue]
if runtime:
cmd += ["-l", "h_rt={0}".format(runtime)]
if shell:
cmd += ["-S", shell]
if threads:
cmd += ["-pe mpi", str(threads)]
cmd += command if isinstance(command, list) else [command]
stdout = cexec(cmd, directory=directory)
jobid = int(stdout.split()[2].split(".")[0]) if array else int(
stdout.split()[2])
logger.debug("Job %d successfully submitted to the SGE queue", jobid)
return jobid
def _run(self):
pyjob.cexec(self.cmd, stdin=self.keywords)
def anode(self, input_model):
shutil.copyfile(input_model, os.path.join(self.work_dir, self.name + ".pdb"))
cmd = ["anode" + EXE_EXT, self.name]
cexec(cmd)
def mtz2sca(self):
sca_out = os.path.join(self.work_dir, "{0}.sca".format(self.name))
cmd = ["mtz2sca", self.mtz, sca_out]
cexec(cmd)
def anode(self, input_model):
shutil.copyfile(input_model, os.path.join(self.work_dir, self.name + ".pdb"))
cmd = ["anode", self.name]
cexec(cmd)
