def alt(jobid, priority=None): """Alter a job in the SGE queue Parameters ---------- jobid : int The job id to remove priority : int, optional The priority level of the job Notes ----- This function is currently still under development does not provide the full range of ``qalter`` flags. Todo ---- * Add more functionality """ cmd = ["qalter"] if priority: cmd += ["-p", str(priority)] logger.debug("Altered priority for job %s by %s", str(jobid), str(priority)) cmd += [str(jobid)] cexec(cmd)
def kill(jobid): """Remove a job from the LSF queue Parameters ---------- jobid : int The job id to remove Raises ------ RuntimeError Execution exited with non-zero return code """ stdout = cexec(["bkill", str(jobid)], permit_nonzero=True) # Large arrays can take time to be full deleted, give it a few moments if "is in progress" in stdout: stdout = cexec(["bkill", "-b", str(jobid)], permit_nonzero=True) import time time.sleep(10) if any( text in stdout for text in ["has already finished", "is being terminated", "is in progress"]): logger.debug("Removed job %d from the queue", jobid) else: msg = "Execution of '{0}' exited with non-zero return code: {1}".format( "bkill " + str(jobid), stdout) raise RuntimeError(msg)
def shelxc(self): cmd = ["shelxc", self.name] stdin = """CELL {0} SPAG {1} SAD {2}""" sca_out = os.path.join(self.work_dir, "{0}.sca".format(self.name)) stdin = stdin.format(self._cell_parameters, self._space_group, os.path.relpath(sca_out)) cexec(cmd, stdin=stdin)
def shelxc(self): cmd = ["shelxc" + EXE_EXT, self.name] stdin = """CELL {0} SPAG {1} SAD {2}""" sca_out = os.path.join(self.work_dir, "{0}.sca".format(self.name)) stdin = stdin.format(self._cell_parameters, self._space_group, os.path.relpath(sca_out)) cexec(cmd, stdin=stdin)
def _generate_hklpck0(self): mtz_labels = simbad.util.mtz_util.GetLabels(self.mtz) logger.info("Preparing files for AMORE rotation function") stdin = self.sortfun_stdin_template.format(f=mtz_labels.f, sigf=mtz_labels.sigf) hklpck0 = os.path.join(self.work_dir, 'spmipch.hkl') cmd = [self.amore_exe, 'hklin', self.mtz, 'hklpck0', hklpck0] pyjob.cexec(cmd, stdin=stdin) return hklpck0
def reindex(hklin, hklout, sg): """Function to reindex input hkl using pointless""" cmd = ["pointless" + EXE_EXT, "hklin", hklin, "hklout", hklout] stdin = """ spacegroup {0} """ stdin = stdin.format(sg) cexec(cmd, stdin=stdin)
def hold(jobid): """Hold a job in the SGE queue Parameters ---------- jobid : int The job id to remove """ cexec(["qhold", str(jobid)]) logger.debug("Holding back job %d from the queue", jobid)
def kill(jobid): """Remove a job from the SGE queue Parameters ---------- jobid : int The job id to remove """ cexec(["qdel", str(jobid)]) logger.debug("Removed job %d from the queue", jobid)
def rls(jobid): """Release a job from the SGE queue Parameters ---------- jobid : int The job id to remove """ cexec(["qrls", str(jobid)]) logger.debug("Released job %d from the queue", jobid)
def mtz2sca(self): sca_out = os.path.join(self.work_dir, "{0}.sca".format(self.name)) cmd = ["mtz2sca" + EXE_EXT, self.mtz, sca_out] if self.mtz_labels.iplus: cmd += ['-p', self.mtz_labels.iplus, '-P', self.mtz_labels.sigiplus, '-m', self.mtz_labels.iminus, '-M', self.mtz_labels.sigiminus] elif self.mtz_labels.fplus: cmd += ['-p', self.mtz_labels.fplus, '-P', self.mtz_labels.sigfplus, '-m', self.mtz_labels.fminus, '-M', self.mtz_labels.sigfminus] cexec(cmd)
def stat(jobid): """Obtain information about a job id Parameters ---------- jobid : int The job id to remove Returns ------- dict A dictionary with job specific data """ stdout = cexec(["qstat", "-j", str(jobid)], permit_nonzero=True) data = {} line_split = re.compile(":\s+") for line in stdout.split(os.linesep): line = line.strip() if 'jobs do not exist' in line: return data if not line or "=" * 30 in line: continue else: kv = line_split.split(line, 1) if len(kv) == 2: data[kv[0]] = kv[1] return data
def sheetbend(exe, hklin, pdbin, pdbout, ncyc, logfile): """Function to run refinement using sheetbend Parameters ---------- hklin : str Path to the input hkl file pdbin : str Path to the input pdb pdbout : str Path to the output pdb ncyc : int Number of cycles to run logfile : str Path to the output log Returns ------- file Output pdb file file Output log file """ mtz_labels = mtz_util.GetLabels(hklin) colin = "{0},{1}".format(mtz_labels.f, mtz_labels.sigf) cmd = [ exe, '--pdbin', pdbin, '--mtzin', hklin, '--pdbout', pdbout, '--colin-fo', colin, '-cycles', str(ncyc), '-resolution-by-cycle', '6,3' ] stdout = cexec(cmd) with open(logfile, 'w') as f_out: f_out.write(stdout)
def refmac(hklin, hklout, pdbin, pdbout, logfile, key): """Function to run refinement using REFMAC Parameters ---------- hklin : str Path to the input hkl file hklout : str Path to the output hkl file pdbin : str Path to the input pdb file pdbout : str Path to the output pdb file logfile : str Path to the output logfile key : str REFMAC key words Returns ------- file Output hkl file file Output pdb file file Output log file """ cmd = ['refmac5', 'hklin', hklin, 'hklout', hklout, 'xyzin', pdbin, 'xyzout', pdbout] stdout = cexec(cmd, stdin=key) with open(logfile, 'w') as f_out: f_out.write(stdout)
def refmac(hklin, hklout, pdbin, pdbout, logfile, key): """Function to run refinement using REFMAC Parameters ---------- hklin : str Path to the input hkl file hklout : str Path to the output hkl file pdbin : str Path to the input pdb file pdbout : str Path to the output pdb file logfile : str Path to the output logfile key : str REFMAC key words Returns ------- file Output hkl file file Output pdb file file Output log file """ cmd = [ 'refmac5' + EXE_EXT, 'hklin', hklin, 'hklout', hklout, 'xyzin', pdbin, 'xyzout', pdbout ] stdout = cexec(cmd, stdin=key) with open(logfile, 'w') as f_out: f_out.write(stdout)
def test_cexec_5(self): cmd = [ sys.executable, "-c", "import os, sys; print(os.getcwd()); sys.exit(0)" ] directory = os.path.join(os.getcwd(), 'pyjob') stdout = cexec(cmd, directory=directory) self.assertEqual(directory, stdout)
def _ensure_exec_available(exe): """Ensure that the specified executable is available in the system Parameters ---------- exe : str The executable to test Raises ------ :exc:`~pyjob.exception.PyJobError` The executable cannot be found """ try: cexec([exe]) except PyJobExecutableNotFoundError: raise PyJobError( f"Cannot find executable {exe}. Please ensure environment is set up correctly." )
def test_cexec_4(self): if sys.version_info < (3, 0): cmd = [ sys.executable, "-c", "import sys; print(raw_input()); sys.exit(0)" ] else: cmd = [ sys.executable, "-c", "import sys; print(input()); sys.exit(0)" ] stdout = cexec(cmd, stdin="hello") self.assertEqual("hello", stdout)
def __init__(self): StrictVersion.__init__(self) ccp4_major_minor = os.path.join(os.environ["CCP4"], "lib", "ccp4", "MAJOR_MINOR") if os.path.isfile(ccp4_major_minor): with open(ccp4_major_minor, "r") as f_in: tversion = f_in.read().strip() else: logger = logging.getLogger(__name__) logger.debug("Detecting CCP4 version via executing pdbcur") stdout = cexec(['pdbcur' + EXE_EXT], permit_nonzero=True) tversion = None for line in stdout.split(os.linesep): if line.startswith(' ### CCP4'): tversion = line.split()[2].rstrip(':') if tversion is None: raise RuntimeError("Cannot determine CCP4 version") self.parse(tversion)
def molrep(key, logfile): """Function to run molecular replacement using MOLREP Parameters ---------- key : str MOLREP keywords logfile : Path to output log file Returns ------- file The output pdb from MOLREP file The output log file """ cmd = ["molrep"] stdout = cexec(cmd, stdin=key) with open(logfile, 'w') as f_out: f_out.write(stdout)
def molrep(key, logfile): """Function to run molecular replacement using MOLREP Parameters ---------- key : str MOLREP keywords logfile : Path to output log file Returns ------- file The output pdb from MOLREP file The output log file """ cmd = ["molrep" + EXE_EXT] stdout = cexec(cmd, stdin=key) with open(logfile, 'w') as f_out: f_out.write(stdout)
def stat(jobid): """Obtain information about a job id Parameters ---------- jobid : int The job id to remove Returns ------- dict A dictionary with job specific data Todo ---- * Extract the correct information """ stdout = cexec(["bjobs", "-l", str(jobid)]) if "Done successfully" in stdout: return {} else: return {'job_number': jobid, 'status': "Running"}
def sub(command, array=None, deps=None, directory=None, hold=False, log=None, name=None, priority=None, queue=None, runtime=None, shell=None, threads=None, *args, **kwargs): """Submit a job to the LSF queue Parameters ---------- command : list A list with the final command array : list, optional A list of array instructions in form of [min, max, (step)]. ``step`` is optional. deps : list, optional A list of dependency job ids directory : str, optional A path to a directory to run the job in hold : bool, optional Submit but __hold__ the job log : str, optional The path to a logfile for stdout name : str, optional The name of the job pe_opts : list, optional Job-specific keywords [Unused] priority : int, optional The priority level of the job queue : str, optional The queue to submit the job to runtime : int, optional The maximum runtime of the job in seconds shell : str, optional The absolute path to the shell to run the job in threads : int, optional The maximum number of threads available to a job """ cmd = ["bsub", "-cwd", directory if directory else os.getcwd()] if array: name = "pyjob" if name is None else name if len(array) == 3: cmd += [ "-J", "{0}[{1}-{2}%{3}]".format(name, array[0], array[1], array[2]) ] elif len(array) == 2: cmd += ["-J", "{0}[{1}-{2}]".format(name, array[0], array[1])] name = None # Reset this! if deps: cmd += [ "-w", " && ".join(["done(%s)" % dep for dep in map(str, deps)]) ] if hold: cmd += ["-H"] if log: cmd += ["-o", log] if name: cmd += ["-J", '"{0}"'.format(name)] if priority: cmd += ["-sp", str(priority)] if queue: cmd += ["-q", queue] if shell: cmd += ["-L", shell] if threads: cmd += ["-R", '"span[ptile={0}]"'.format(threads)] if runtime: cmd += ["-W", str(runtime)] command_f = command[0] if isinstance(command, list) else command stdout = cexec(cmd, stdin=open(command_f).read(), directory=directory) jobid = int(stdout.split()[1][1:-1]) logger.debug("Job %d successfully submitted to the LSF queue", jobid) return jobid
def test_cexec_3(self): cmd = [ sys.executable, "-c", "import sys; print(\"hello\"); sys.exit(0)" ] stdout = cexec(cmd, permit_nonzero=True) self.assertEqual("hello", stdout)
def test_cexec_1(self): cmd = [ sys.executable, "-c", "import sys; print(\"hello\"); sys.exit(0)" ] stdout = cexec(cmd) self.assertEqual("hello", stdout)
def run(self): """Method representing the process's activity""" for job in iter(self.queue.get, None): stdout = cexec([job], directory=self.directory, permit_nonzero=self.permit_nonzero) with open(job.rsplit('.', 1)[0] + '.log', 'w') as f_out: f_out.write(stdout)
def sub(command, array=None, deps=None, directory=None, hold=False, log=None, name=None, pe_opts=None, priority=None, queue=None, runtime=None, shell=None, threads=None, *args, **kwargs): """Submit a job to the SGE queue Parameters ---------- command : list A list with the final command array : list, optional A list of array instructions in form of [min, max, (step)]. ``step`` is optional. deps : list, optional A list of dependency job ids directory : str, optional A path to a directory to run the job in hold : bool, optional Submit but __hold__ the job log : str, optional The path to a logfile for stdout name : str, optional The name of the job pe_opts : list, optional Job-specific keywords priority : int, optional The priority level of the job queue : str, optional The queue to submit the job to runtime : int, optional The maximum runtime of the job in seconds shell : str, optional The absolute path to the shell to run the job in threads : int, optional The maximum number of threads available to a job """ cmd = ["qsub", "-cwd", "-V", "-w", "e", "-j", "y"] if array and len(array) == 3: cmd += [ "-t", "{0}-{1}".format(array[0], array[1]), "-tc", str(array[2]) ] elif array and len(array) == 2: cmd += ["-t", "{0}-{1}".format(array[0], array[1])] if deps: cmd += ["-hold_jid", "{0}".format(",".join(map(str, deps)))] if hold: cmd += ["-h"] if log: cmd += ["-o", log] if name: cmd += ["-N", name] if pe_opts: cmd += ["-pe"] + pe_opts.split() if priority: cmd += ["-p", str(priority)] if queue: cmd += ["-q", queue] if runtime: cmd += ["-l", "h_rt={0}".format(runtime)] if shell: cmd += ["-S", shell] if threads: cmd += ["-pe mpi", str(threads)] cmd += command if isinstance(command, list) else [command] stdout = cexec(cmd, directory=directory) jobid = int(stdout.split()[2].split(".")[0]) if array else int( stdout.split()[2]) logger.debug("Job %d successfully submitted to the SGE queue", jobid) return jobid
def _run(self): pyjob.cexec(self.cmd, stdin=self.keywords)
def anode(self, input_model): shutil.copyfile(input_model, os.path.join(self.work_dir, self.name + ".pdb")) cmd = ["anode" + EXE_EXT, self.name] cexec(cmd)
def mtz2sca(self): sca_out = os.path.join(self.work_dir, "{0}.sca".format(self.name)) cmd = ["mtz2sca", self.mtz, sca_out] cexec(cmd)
def anode(self, input_model): shutil.copyfile(input_model, os.path.join(self.work_dir, self.name + ".pdb")) cmd = ["anode", self.name] cexec(cmd)