def info(self):
""":obj:`~pyjob.slurm.SlurmTask` information"""
if self.pid is None:
return {}
try:
cexec(["squeue", "-j", str(self.pid)])
except (PyJobExecutableNotFoundError, Exception):
return {}
else:
return {"job_number": self.pid, "status": "Running"}
def run(self):
"""Method representing the :obj:`~pyjob.local.LocalProcess` activity"""
for job in iter(self.queue.get, None):
if self.kill_switch.is_set():
continue
if self.chdir:
directory = os.path.dirname(job)
else:
directory = self.directory
log = os.path.splitext(job)[0] + ".log"
with open(log, "w") as f:
cexec([job],
cwd=directory,
stdout=f,
permit_nonzero=self.permit_nonzero)
def test_5(self):
cmd = [
sys.executable, "-c",
"import os, sys; print(os.getcwd()); sys.exit(0)"
]
directory = os.path.join(os.getcwd())
stdout = cexec(cmd, cwd=directory)
assert stdout == directory
def test_6(self):
cmd = [sys.executable, "-c", 'import sys; print("hello"); sys.exit(0)']
fname = "test.log"
with open(fname, "w") as f:
stdout = cexec(cmd, stdout=f)
assert stdout is None
with open(fname, "r") as f:
assert f.read().strip() == "hello"
pytest.helpers.unlink([fname])
def _run(self):
"""Method to initialise :obj:`~pyjob.pbs.PortableBatchSystemTask` execution"""
self.runscript = self._create_runscript()
self.runscript.write()
self.pid = cexec(["qsub", self.runscript.path], cwd=self.directory)
logger.debug(
"%s [%d] submission script is %s",
self.__class__.__qualname__,
self.pid,
self.runscript.path,
)
def info(self):
""":obj:`~pyjob.lsf.LoadSharingFacilityTask` information"""
if self.pid is None:
return {}
try:
stdout = cexec(["bjobs", "-l", str(self.pid)], permit_nonzero=True)
except PyJobExecutableNotFoundError:
return {}
if "Done successfully" in stdout:
return {}
else:
return {"job_number": self.pid, "status": "Running"}
def test_4(self):
if sys.version_info < (3, 0):
cmd = [
sys.executable, "-c",
"import sys; print(raw_input()); sys.exit(0)"
]
else:
cmd = [
sys.executable, "-c", "import sys; print(input()); sys.exit(0)"
]
stdout = cexec(cmd, stdin="hello")
assert stdout == "hello"
def _run(self):
"""Method to initialise :obj:`~pyjob.lsf.LoadSharingFacilityTask` execution"""
self.runscript = self._create_runscript()
self.runscript.write()
stdout = cexec(["bsub"], stdin=str(self.runscript), cwd=self.directory)
self.pid = int(stdout.split()[1][1:-1])
logger.debug(
"%s [%d] submission script is %s",
self.__class__.__qualname__,
self.pid,
self.runscript.path,
)
def _run(self):
"""Method to initialise :obj:`~pyjob.slurm.SlurmTask` execution"""
self.runscript = self._create_runscript()
self.runscript.write()
stdout = cexec(["sbatch", self.runscript.path], cwd=self.directory)
self.pid = int(stdout.strip().split()[-1])
logger.debug(
"%s [%d] submission script is %s",
self.__class__.__qualname__,
self.pid,
self.runscript.path,
)
def kill(self):
"""Immediately terminate the :obj:`~pyjob.lsf.LoadSharingFacilityTask`
Raises
------
:exc:`RuntimeError`
Cannot delete :obj:`~pyjob.lsf.LoadSharingFacilityTask`
"""
if self.pid is None:
return
stdout = cexec(["bkill", str(self.pid)], permit_nonzero=True)
if "is in progress" in stdout:
stdout = cexec(["bkill", "-b", str(self.pid)], permit_nonzero=True)
time.sleep(10)
if any(text in stdout for text in [
"has already finished",
"is being terminated",
"is in progress",
]):
logger.debug("Terminated task: %d", self.pid)
else:
raise RuntimeError("Cannot delete task!")
def _run(self):
"""Method to initialise :obj:`~pyjob.sge.SunGridEngineTask` execution"""
self.runscript = self._create_runscript()
self.runscript.write()
stdout = cexec(["qsub", self.runscript.path], cwd=self.directory)
for line in stdout.split("\n"):
line = line.strip()
if re.match(RE_PID_MATCH, line):
if len(self.script) > 1:
self.pid = int(line.split()[2].split(".")[0])
else:
self.pid = int(line.split()[2])
logger.debug(
"%s [%d] submission script is %s",
self.__class__.__qualname__,
self.pid,
self.runscript.path,
)
def test_7(self):
cmd = [
sys.executable,
"-c",
'import os, sys; print(os.getcwd()); sys.exit("error message")',
]
directory = os.path.join(os.getcwd())
with open("stdout.log", "w") as fstdout, open("stderr.log",
"w") as fstderr:
stdout = cexec(cmd,
stdout=fstdout,
stderr=fstderr,
permit_nonzero=True)
assert stdout is None
with open("stdout.log", "r") as f:
assert f.read().strip() == directory
with open("stderr.log", "r") as f:
assert f.read().strip() == "error message"
pytest.helpers.unlink(["stdout.log", "stderr.log"])
def info(self):
""":obj:`~pyjob.sge.SunGridEngineTask` information"""
if self.pid is None:
return {}
try:
stdout = cexec(["qstat", "-j", str(self.pid)], permit_nonzero=True)
except PyJobExecutableNotFoundError:
return {}
data = {}
for line in stdout.splitlines():
line = line.strip()
if "jobs do not exist" in line:
return data
if not line or "=" * 30 in line:
continue
else:
kv = RE_LINE_SPLIT.split(line, 1)
if len(kv) == 2:
data[kv[0]] = kv[1]
return data
def get_sge_avail_configs(cls, param):
"""Get the set of available configurations for a given SGE parameter
Parameters
----------
param : :obj:~SGEConfigParameter
The parameter to be tested
Returns
-------
set
A set with the available configurations for the parameter of interest
Raises
------
:exc:`ValueError`
Parameter is not found in :obj:~SGEConfigParameter
"""
if param in cls._sge_avail_configs_by_env:
return cls._sge_avail_configs_by_env[param]
if SGEConfigParameter(param) == SGEConfigParameter.ENVIRONMENT:
cmd = ["qconf", "-spl"]
elif SGEConfigParameter(param) == SGEConfigParameter.QUEUE:
cmd = ["qconf", "-sql"]
else:
raise ValueError("Requested SGE parameter is not supported!")
stdout = cexec(cmd, permit_nonzero=True)
config = []
for line in stdout.splitlines():
line = line.split()
if len(line) > 1:
break
else:
config.append(line[0].encode("utf-8"))
cls._sge_avail_configs_by_env[param] = set(config)
return cls._sge_avail_configs_by_env[param]
def info(self):
""":obj:`~pyjob.pbs.PortableBatchSystemTask` information"""
if self.pid is None:
return {}
try:
stdout = cexec(["qstat", "-f", str(self.pid)], permit_nonzero=True)
except PyJobExecutableNotFoundError:
return {}
all_lines = stdout.splitlines()
data = {}
key, job_id = RE_LINE_SPLIT_1.split(all_lines[0], 1)
data[key] = job_id
for line in all_lines[1:]:
line = line.strip()
if "Unknown queue destination" in line:
return data
else:
kv = RE_LINE_SPLIT_2.split(line, 1)
if len(kv) == 2:
data[kv[0]] = kv[1]
return data
def kill(self):
"""Immediately terminate the :obj:`~pyjob.slurm.SlurmTask`"""
if self.pid is None:
return
cexec(["scancel", str(self.pid)])
logger.debug("Terminated task: %d", self.pid)
def test_1(self):
stdout = cexec(
[sys.executable, "-c", 'import sys; print("hello"); sys.exit(0)'])
assert stdout == "hello"
def test_2(self):
with pytest.raises(PyJobExecutionError):
cexec([sys.executable, "-c", "import sys; sys.exit(1)"])
def test_3(self):
cmd = [sys.executable, "-c", 'import sys; print("hello"); sys.exit(1)']
stdout = cexec(cmd, permit_nonzero=True)
assert stdout == "hello"
def test_8(self):
with pytest.raises(PyJobExecutableNotFoundError):
cexec(["fjezfsdkj"])
def kill(self):
"""Immediately terminate the :obj:`~pyjob.pbs.PortableBatchSystemTask`"""
if self.pid is None:
return
cexec(["qdel", str(self.pid)])
logger.debug("Terminated task: %d", self.pid)
def kill(self):
"""Immediately terminate the :obj:`~pyjob.sge.SunGridEngineTask`"""
if self.pid is None:
return
cexec(["qdel", str(self.pid)])
logger.debug("Terminated task: %d", self.pid)
