def info(self): """:obj:`~pyjob.slurm.SlurmTask` information""" if self.pid is None: return {} try: cexec(["squeue", "-j", str(self.pid)]) except (PyJobExecutableNotFoundError, Exception): return {} else: return {"job_number": self.pid, "status": "Running"}
def run(self): """Method representing the :obj:`~pyjob.local.LocalProcess` activity""" for job in iter(self.queue.get, None): if self.kill_switch.is_set(): continue if self.chdir: directory = os.path.dirname(job) else: directory = self.directory log = os.path.splitext(job)[0] + ".log" with open(log, "w") as f: cexec([job], cwd=directory, stdout=f, permit_nonzero=self.permit_nonzero)
def test_5(self): cmd = [ sys.executable, "-c", "import os, sys; print(os.getcwd()); sys.exit(0)" ] directory = os.path.join(os.getcwd()) stdout = cexec(cmd, cwd=directory) assert stdout == directory
def test_6(self): cmd = [sys.executable, "-c", 'import sys; print("hello"); sys.exit(0)'] fname = "test.log" with open(fname, "w") as f: stdout = cexec(cmd, stdout=f) assert stdout is None with open(fname, "r") as f: assert f.read().strip() == "hello" pytest.helpers.unlink([fname])
def _run(self): """Method to initialise :obj:`~pyjob.pbs.PortableBatchSystemTask` execution""" self.runscript = self._create_runscript() self.runscript.write() self.pid = cexec(["qsub", self.runscript.path], cwd=self.directory) logger.debug( "%s [%d] submission script is %s", self.__class__.__qualname__, self.pid, self.runscript.path, )
def info(self): """:obj:`~pyjob.lsf.LoadSharingFacilityTask` information""" if self.pid is None: return {} try: stdout = cexec(["bjobs", "-l", str(self.pid)], permit_nonzero=True) except PyJobExecutableNotFoundError: return {} if "Done successfully" in stdout: return {} else: return {"job_number": self.pid, "status": "Running"}
def test_4(self): if sys.version_info < (3, 0): cmd = [ sys.executable, "-c", "import sys; print(raw_input()); sys.exit(0)" ] else: cmd = [ sys.executable, "-c", "import sys; print(input()); sys.exit(0)" ] stdout = cexec(cmd, stdin="hello") assert stdout == "hello"
def _run(self): """Method to initialise :obj:`~pyjob.lsf.LoadSharingFacilityTask` execution""" self.runscript = self._create_runscript() self.runscript.write() stdout = cexec(["bsub"], stdin=str(self.runscript), cwd=self.directory) self.pid = int(stdout.split()[1][1:-1]) logger.debug( "%s [%d] submission script is %s", self.__class__.__qualname__, self.pid, self.runscript.path, )
def _run(self): """Method to initialise :obj:`~pyjob.slurm.SlurmTask` execution""" self.runscript = self._create_runscript() self.runscript.write() stdout = cexec(["sbatch", self.runscript.path], cwd=self.directory) self.pid = int(stdout.strip().split()[-1]) logger.debug( "%s [%d] submission script is %s", self.__class__.__qualname__, self.pid, self.runscript.path, )
def kill(self): """Immediately terminate the :obj:`~pyjob.lsf.LoadSharingFacilityTask` Raises ------ :exc:`RuntimeError` Cannot delete :obj:`~pyjob.lsf.LoadSharingFacilityTask` """ if self.pid is None: return stdout = cexec(["bkill", str(self.pid)], permit_nonzero=True) if "is in progress" in stdout: stdout = cexec(["bkill", "-b", str(self.pid)], permit_nonzero=True) time.sleep(10) if any(text in stdout for text in [ "has already finished", "is being terminated", "is in progress", ]): logger.debug("Terminated task: %d", self.pid) else: raise RuntimeError("Cannot delete task!")
def _run(self): """Method to initialise :obj:`~pyjob.sge.SunGridEngineTask` execution""" self.runscript = self._create_runscript() self.runscript.write() stdout = cexec(["qsub", self.runscript.path], cwd=self.directory) for line in stdout.split("\n"): line = line.strip() if re.match(RE_PID_MATCH, line): if len(self.script) > 1: self.pid = int(line.split()[2].split(".")[0]) else: self.pid = int(line.split()[2]) logger.debug( "%s [%d] submission script is %s", self.__class__.__qualname__, self.pid, self.runscript.path, )
def test_7(self): cmd = [ sys.executable, "-c", 'import os, sys; print(os.getcwd()); sys.exit("error message")', ] directory = os.path.join(os.getcwd()) with open("stdout.log", "w") as fstdout, open("stderr.log", "w") as fstderr: stdout = cexec(cmd, stdout=fstdout, stderr=fstderr, permit_nonzero=True) assert stdout is None with open("stdout.log", "r") as f: assert f.read().strip() == directory with open("stderr.log", "r") as f: assert f.read().strip() == "error message" pytest.helpers.unlink(["stdout.log", "stderr.log"])
def info(self): """:obj:`~pyjob.sge.SunGridEngineTask` information""" if self.pid is None: return {} try: stdout = cexec(["qstat", "-j", str(self.pid)], permit_nonzero=True) except PyJobExecutableNotFoundError: return {} data = {} for line in stdout.splitlines(): line = line.strip() if "jobs do not exist" in line: return data if not line or "=" * 30 in line: continue else: kv = RE_LINE_SPLIT.split(line, 1) if len(kv) == 2: data[kv[0]] = kv[1] return data
def get_sge_avail_configs(cls, param): """Get the set of available configurations for a given SGE parameter Parameters ---------- param : :obj:~SGEConfigParameter The parameter to be tested Returns ------- set A set with the available configurations for the parameter of interest Raises ------ :exc:`ValueError` Parameter is not found in :obj:~SGEConfigParameter """ if param in cls._sge_avail_configs_by_env: return cls._sge_avail_configs_by_env[param] if SGEConfigParameter(param) == SGEConfigParameter.ENVIRONMENT: cmd = ["qconf", "-spl"] elif SGEConfigParameter(param) == SGEConfigParameter.QUEUE: cmd = ["qconf", "-sql"] else: raise ValueError("Requested SGE parameter is not supported!") stdout = cexec(cmd, permit_nonzero=True) config = [] for line in stdout.splitlines(): line = line.split() if len(line) > 1: break else: config.append(line[0].encode("utf-8")) cls._sge_avail_configs_by_env[param] = set(config) return cls._sge_avail_configs_by_env[param]
def info(self): """:obj:`~pyjob.pbs.PortableBatchSystemTask` information""" if self.pid is None: return {} try: stdout = cexec(["qstat", "-f", str(self.pid)], permit_nonzero=True) except PyJobExecutableNotFoundError: return {} all_lines = stdout.splitlines() data = {} key, job_id = RE_LINE_SPLIT_1.split(all_lines[0], 1) data[key] = job_id for line in all_lines[1:]: line = line.strip() if "Unknown queue destination" in line: return data else: kv = RE_LINE_SPLIT_2.split(line, 1) if len(kv) == 2: data[kv[0]] = kv[1] return data
def kill(self): """Immediately terminate the :obj:`~pyjob.slurm.SlurmTask`""" if self.pid is None: return cexec(["scancel", str(self.pid)]) logger.debug("Terminated task: %d", self.pid)
def test_1(self): stdout = cexec( [sys.executable, "-c", 'import sys; print("hello"); sys.exit(0)']) assert stdout == "hello"
def test_2(self): with pytest.raises(PyJobExecutionError): cexec([sys.executable, "-c", "import sys; sys.exit(1)"])
def test_3(self): cmd = [sys.executable, "-c", 'import sys; print("hello"); sys.exit(1)'] stdout = cexec(cmd, permit_nonzero=True) assert stdout == "hello"
def test_8(self): with pytest.raises(PyJobExecutableNotFoundError): cexec(["fjezfsdkj"])
def kill(self): """Immediately terminate the :obj:`~pyjob.pbs.PortableBatchSystemTask`""" if self.pid is None: return cexec(["qdel", str(self.pid)]) logger.debug("Terminated task: %d", self.pid)
def kill(self): """Immediately terminate the :obj:`~pyjob.sge.SunGridEngineTask`""" if self.pid is None: return cexec(["qdel", str(self.pid)]) logger.debug("Terminated task: %d", self.pid)