class BcbioSLURMEngineSetLauncher(SLURMLauncher, launcher.BatchClusterAppMixin): """Custom launcher handling heterogeneous clusters on SLURM """ batch_file_name = Unicode(unicode("SLURM_engine" + str(uuid.uuid4()))) machines = traitlets.Integer(0, config=True) cores = traitlets.Integer(1, config=True) numengines = traitlets.Integer(1, config=True) mem = traitlets.Unicode("", config=True) tag = traitlets.Unicode("", config=True) account = traitlets.Unicode("", config=True) timelimit = traitlets.Unicode("", config=True) resources = traitlets.Unicode("", config=True) default_template = traitlets.Unicode("""#!/bin/sh #SBATCH -p {queue} #SBATCH -J {tag}-e[1-{n}] #SBATCH -o bcbio-ipengine.out.%%j #SBATCH -e bcbio-ipengine.err.%%j #SBATCH --cpus-per-task={cores} #SBATCH --array=1-{n} #SBATCH -A {account} #SBATCH -t {timelimit} {machines} {mem} {resources} {cmd} """) def start(self, n): self.context["cores"] = self.cores * self.numengines if self.mem: self.context["mem"] = "#SBATCH --mem=%s" % int( float(self.mem) * 1024.0 * self.numengines) else: self.context["mem"] = "#SBATCH --mem=%d" % int( DEFAULT_MEM_PER_CPU * self.cores * self.numengines) self.context["tag"] = self.tag if self.tag else "bcbio" self.context["machines"] = ("#SBATCH %s" % (self.machines) if int(self.machines) > 0 else "") self.context["account"] = self.account self.context["timelimit"] = self.timelimit self.context["resources"] = "\n".join([ "#SBATCH --%s" % r.strip() for r in str(self.resources).split(";") if r.strip() ]) self.context["cmd"] = get_engine_commands(self.context, self.numengines) return super(BcbioSLURMEngineSetLauncher, self).start(n)
class FileRepeat(ToolBase): """ This is an example tool that takes in an OpaqueFileSource, OpaqueFileSink, and an integer. The contents of the first file are repeated that number of times into the output file. """ src_file = traitlets.Instance(klass=OpaqueFileSource, tooltip='source file', label='input') snk_file = traitlets.Instance(klass=OpaqueFileSink, tooltip='source file', label='output') repeat_count = traitlets.Integer(1, tooltip='number of times to repeat', label='input') def run(self): # grab the backing file of the source. In general, source/sinks # need to be activated before they can be used, but you don't need to # in this case because you just need to get a string out. src_fname = self.src_file.backing_file # grab the backing file for the destination file snk_fname = self.snk_file.backing_file # open the output file for writing with open(snk_fname, 'w') as snk: # loop over the number of times to repeat for j in range(self.repeat_count): # open the source read only with open(src_fname, 'r') as src: # copy all of the lines to the new file for ln in src.readlines(): snk.write(ln)
class BcbioLSFEngineSetLauncher(launcher.LSFEngineSetLauncher): """Custom launcher handling heterogeneous clusters on LSF. """ batch_file_name = Unicode(unicode("lsf_engine" + str(uuid.uuid4()))) cores = traitlets.Integer(1, config=True) mem = traitlets.Unicode("", config=True) tag = traitlets.Unicode("", config=True) resources = traitlets.Unicode("", config=True) job_array_template = Unicode('') queue_template = Unicode('') default_template = traitlets.Unicode("""#!/bin/sh #BSUB -q {queue} #BSUB -J {tag}-e[1-{n}] #BSUB -oo bcbio-ipengine.bsub.%%J #BSUB -n {cores} #BSUB -R "span[hosts=1]" {mem} {resources} %s %s --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" """ % (' '.join(map(pipes.quote, engine_cmd_argv)), ' '.join(timeout_params))) def start(self, n): self.context["cores"] = self.cores if self.mem: # scale memory to kb mem = int(float(self.mem) * 1024.0 * 1024.0) self.context["mem"] = "#BSUB -M %s" % mem else: self.context["mem"] = "" self.context["tag"] = self.tag if self.tag else "bcbio" self.context["resources"] = _format_lsf_resources(self.resources) return super(BcbioLSFEngineSetLauncher, self).start(n)
class BcbioPBSEngineSetLauncher(launcher.PBSEngineSetLauncher): """Custom launcher handling heterogeneous clusters on PBS. """ batch_file_name = Unicode(unicode("pbs_engines" + str(uuid.uuid4()))) cores = traitlets.Integer(1, config=True) mem = traitlets.Unicode("", config=True) tag = traitlets.Unicode("", config=True) pename = traitlets.Unicode("", config=True) resources = traitlets.Unicode("", config=True) default_template = traitlets.Unicode("""#PBS -V #PBS -j oe #PBS -S /bin/sh #PBS -q {queue} #PBS -N {tag}-e #PBS -t 1-{n} {mem} %s %s --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" """ % (' '.join(map(pipes.quote, engine_cmd_argv)), ' '.join(timeout_params))) def start(self, n): self.context["cores"] = self.cores self.context["pename"] = str(self.pename) if self.mem: self.context["mem"] = "#PBS -l mem=%smb" % int(float(self.mem) * 1024) else: self.context["mem"] = "" self.context["tag"] = self.tag if self.tag else "bcbio" return super(BcbioPBSEngineSetLauncher, self).start(n)
class BcbioTORQUEControllerLauncher(TORQUELauncher, launcher.BatchClusterAppMixin): """Launch a controller using Torque.""" batch_file_name = Unicode(unicode("torque_controller" + str(uuid.uuid4())), config=True, help="batch file name for the engine(s) job.") cores = traitlets.Integer(1, config=True) tag = traitlets.Unicode("", config=True) resources = traitlets.Unicode("", config=True) default_template = Unicode("""#!/bin/sh #PBS -V #PBS -N {tag}-c #PBS -j oe #PBS -l nodes=1:ppn={cores} {resources} cd $PBS_O_WORKDIR %s --ip=* --log-to-file --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" %s """ % (' '.join(map(pipes.quote, controller_cmd_argv)), ' '.join(controller_params))) def start(self): """Start the controller by profile or profile_dir.""" try: self.context["cores"] = self.cores self.context["tag"] = self.tag if self.tag else "bcbio" self.context["resources"] = "\n".join( _prep_torque_resources(self.resources)) return super(BcbioTORQUEControllerLauncher, self).start(1) except: self.log.exception("Controller start failed")
class BcbioOLDSLURMEngineSetLauncher(SLURMLauncher, launcher.BatchClusterAppMixin): """Launch engines using SLURM for version < 2.6""" machines = traitlets.Integer(1, config=True) account = traitlets.Unicode("", config=True) timelimit = traitlets.Unicode("", config=True) batch_file_name = Unicode(unicode("SLURM_engines" + str(uuid.uuid4())), config=True, help="batch file name for the engine(s) job.") default_template = Unicode(u"""#!/bin/sh #SBATCH -A {account} #SBATCH --job-name ipengine #SBATCH -N {machines} #SBATCH -t {timelimit} srun -N {machines} -n {n} %s %s --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" """ % (' '.join(map(pipes.quote, engine_cmd_argv)), ' '.join(timeout_params))) def start(self, n): """Start n engines by profile or profile_dir.""" self.context["machines"] = self.machines self.context["account"] = self.account self.context["timelimit"] = self.timelimit return super(BcbioOLDSLURMEngineSetLauncher, self).start(n)
class BcbioSLURMControllerLauncher(SLURMLauncher, launcher.BatchClusterAppMixin): batch_file_name = Unicode(unicode("SLURM_controller" + str(uuid.uuid4()))) account = traitlets.Unicode("", config=True) cores = traitlets.Integer(1, config=True) timelimit = traitlets.Unicode("", config=True) mem = traitlets.Unicode("", config=True) tag = traitlets.Unicode("", config=True) resources = traitlets.Unicode("", config=True) default_template = traitlets.Unicode("""#!/bin/sh #SBATCH -J {tag}-c #SBATCH -o bcbio-ipcontroller.out.%%j #SBATCH -e bcbio-ipcontroller.err.%%j #SBATCH -A {account} #SBATCH -t {timelimit} #SBATCH --cpus-per-task={cores} {mem} {resources} %s --ip=* --log-to-file --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" %s """ % (' '.join(map(pipes.quote, controller_cmd_argv)), ' '.join(controller_params))) def start(self): self.context["account"] = self.account self.context["timelimit"] = self.timelimit self.context["cores"] = self.cores self.context["mem"] = "#SBATCH --mem=%d" % (12 * DEFAULT_MEM_PER_CPU) self.context["tag"] = self.tag if self.tag else "bcbio" self.context["resources"] = "\n".join([ "#SBATCH --%s" % r.strip() for r in str(self.resources).split(";") if r.strip() ]) return super(BcbioSLURMControllerLauncher, self).start(1)
class BcbioSGEControllerLauncher(launcher.SGEControllerLauncher): batch_file_name = Unicode(unicode("sge_controller" + str(uuid.uuid4()))) tag = traitlets.Unicode("", config=True) cores = traitlets.Integer(1, config=True) pename = traitlets.Unicode("", config=True) resources = traitlets.Unicode("", config=True) queue_template = Unicode('') default_template = traitlets.Unicode(u"""#$ -V #$ -cwd #$ -w w #$ -S /bin/sh #$ -N {tag}-c {cores} {queue} {resources} {exports} %s --ip=* --log-to-file --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" %s """ % (' '.join(map(pipes.quote, controller_cmd_argv)), ' '.join(controller_params))) def start(self): self.context["cores"] = "#$ -pe %s %s" % ( self.pename, self.cores) if self.cores > 1 else "" self.context["tag"] = self.tag if self.tag else "bcbio" self.context["resources"] = "\n".join([ _prep_sge_resource(r) for r in str(self.resources).split(";") if r.strip() ]) if self.queue: self.context["queue"] = "#$ -q %s" % self.queue else: self.context["queue"] = "" self.context["exports"] = _local_environment_exports() return super(BcbioSGEControllerLauncher, self).start()
class BcbioTORQUEEngineSetLauncher(TORQUELauncher, launcher.BatchClusterAppMixin): """Launch Engines using Torque""" cores = traitlets.Integer(1, config=True) mem = traitlets.Unicode("", config=True) tag = traitlets.Unicode("", config=True) resources = traitlets.Unicode("", config=True) batch_file_name = Unicode(unicode("torque_engines" + str(uuid.uuid4())), config=True, help="batch file name for the engine(s) job.") default_template = Unicode(u"""#!/bin/sh #PBS -V #PBS -j oe #PBS -N {tag}-e #PBS -t 1-{n} #PBS -l nodes=1:ppn={cores} {mem} {resources} cd $PBS_O_WORKDIR %s %s --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" """ % (' '.join(map(pipes.quote, engine_cmd_argv)), ' '.join(timeout_params))) def start(self, n): """Start n engines by profile or profile_dir.""" try: self.context["cores"] = self.cores if self.mem: self.context["mem"] = "#PBS -l mem=%smb" % int(float(self.mem) * 1024) else: self.context["mem"] = "" self.context["tag"] = self.tag if self.tag else "bcbio" self.context["resources"] = "\n".join(_prep_torque_resources(self.resources)) return super(BcbioTORQUEEngineSetLauncher, self).start(n) except: self.log.exception("Engine start failed")
class BcbioPBSPROEngineSetLauncher(PBSPROLauncher, launcher.BatchClusterAppMixin): """Launch Engines using PBSPro""" batch_file_name = Unicode(u'pbspro_engines', config=True, help="batch file name for the engine(s) job.") tag = traitlets.Unicode("", config=True) cores = traitlets.Integer(1, config=True) mem = traitlets.Unicode("", config=True) array_cmd = traitlets.Unicode("", config=True) default_template = Unicode(u"""#!/bin/sh #PBS -V #PBS -N {tag}-e {resources} {array_cmd} cd $PBS_O_WORKDIR %s %s --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" """ % (' '.join(map(pipes.quote, engine_cmd_argv)), ' '.join(timeout_params))) def start(self, n): """Start n engines by profile or profile_dir.""" resources = "#PBS -l select=1:ncpus=%d" % self.cores if self.mem: resources += ":mem=%smb" % int(float(self.mem) * 1024) self.context["resources"] = resources self.context["cores"] = self.cores self.context["tag"] = self.tag if self.tag else "bcbio" self.context["array_cmd"] = "" if n == 1 else "#PBS -J 1-%i" % n return super(BcbioPBSPROEngineSetLauncher, self).start(n)
class BcbioSLURMEngineSetLauncher(SLURMLauncher, launcher.BatchClusterAppMixin): """Custom launcher handling heterogeneous clusters on SLURM """ batch_file_name = Unicode(unicode("SLURM_engine" + str(uuid.uuid4()))) machines = traitlets.Integer(1, config=True) cores = traitlets.Integer(1, config=True) mem = traitlets.Unicode("", config=True) tag = traitlets.Unicode("", config=True) account = traitlets.Unicode("", config=True) timelimit = traitlets.Unicode("", config=True) resources = traitlets.Unicode("", config=True) default_template = traitlets.Unicode("""#!/bin/sh #SBATCH -p {queue} #SBATCH -J {tag}-e[1-{n}] #SBATCH -o bcbio-ipengine.out.%%j #SBATCH -e bcbio-ipengine.err.%%j #SBATCH --cpus-per-task={cores} #SBATCH --array=1-{n} #SBATCH -A {account} #SBATCH -t {timelimit} #SBATCH -N {machines} {mem} {resources} %s %s --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" """ % (' '.join(map(pipes.quote, engine_cmd_argv)), ' '.join(timeout_params))) def start(self, n): self.context["cores"] = self.cores if self.mem: self.context["mem"] = "#SBATCH --mem=%s" % int(float(self.mem) * 1024.0) else: self.context["mem"] = "#SBATCH --mem=%d" % int(DEFAULT_MEM_PER_CPU * self.cores) self.context["tag"] = self.tag if self.tag else "bcbio" self.context["machines"] = self.machines self.context["account"] = self.account self.context["timelimit"] = self.timelimit self.context["resources"] = "\n".join(["#SBATCH --%s" % r.strip() for r in str(self.resources).split(";") if r.strip()]) return super(BcbioSLURMEngineSetLauncher, self).start(n)
class BcbioLSFEngineSetLauncher(launcher.LSFEngineSetLauncher): """Custom launcher handling heterogeneous clusters on LSF. """ batch_file_name = Unicode(unicode("lsf_engine" + str(uuid.uuid4()))) cores = traitlets.Integer(1, config=True) numengines = traitlets.Integer(1, config=True) mem = traitlets.Unicode("", config=True) tag = traitlets.Unicode("", config=True) resources = traitlets.Unicode("", config=True) job_array_template = Unicode('') queue_template = Unicode('') default_template = traitlets.Unicode("""#!/bin/sh #BSUB -q {queue} #BSUB -J {tag}-e[1-{n}] #BSUB -oo bcbio-ipengine.bsub.%%J #BSUB -n {cores} #BSUB -R "span[hosts=1]" {mem} {resources} {cmd} """) def start(self, n): self.context["cores"] = self.cores * self.numengines if self.mem: # lsf.conf can specify nonstandard units for memory reservation lsf_unit = lsf.get_lsf_units(resource=True) mem = utils.convert_mb(float(self.mem) * 1024, lsf_unit) # check if memory reservation is per core or per job if lsf.per_core_reservation(): mem = mem / self.cores mem = mem * self.numengines self.context["mem"] = '#BSUB -R "rusage[mem=%s]"' % mem else: self.context["mem"] = "" self.context["tag"] = self.tag if self.tag else "bcbio" self.context["resources"] = _format_lsf_resources(self.resources) self.context["cmd"] = get_engine_commands(self.context, self.numengines) return super(BcbioLSFEngineSetLauncher, self).start(n)
class BcbioSGEEngineSetLauncher(launcher.SGEEngineSetLauncher): """Custom launcher handling heterogeneous clusters on SGE. """ batch_file_name = Unicode(unicode("sge_engine" + str(uuid.uuid4()))) cores = traitlets.Integer(1, config=True) pename = traitlets.Unicode("", config=True) resources = traitlets.Unicode("", config=True) mem = traitlets.Unicode("", config=True) memtype = traitlets.Unicode("mem_free", config=True) tag = traitlets.Unicode("", config=True) queue_template = Unicode('') default_template = traitlets.Unicode("""#$ -V #$ -cwd #$ -w w #$ -j y #$ -S /bin/sh #$ -N {tag}-e #$ -t 1-{n} #$ -pe {pename} {cores} {queue} {mem} {resources} {exports} echo \($SGE_TASK_ID - 1\) \* 0.5 | bc | xargs sleep %s %s --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" """ % (' '.join(map(pipes.quote, engine_cmd_argv)), ' '.join(timeout_params))) def start(self, n): self.context["cores"] = self.cores if self.mem: if self.memtype == "rss": self.context["mem"] = "#$ -l rss=%sM" % int( float(self.mem) * 1024 / self.cores) else: self.context["mem"] = "#$ -l mem_free=%sM" % int( float(self.mem) * 1024) else: self.context["mem"] = "" if self.queue: self.context["queue"] = "#$ -q %s" % self.queue else: self.context["queue"] = "" self.context["tag"] = self.tag if self.tag else "bcbio" self.context["pename"] = str(self.pename) self.context["resources"] = "\n".join([ _prep_sge_resource(r) for r in str(self.resources).split(";") if r.strip() ]) self.context["exports"] = _local_environment_exports() return super(BcbioSGEEngineSetLauncher, self).start(n)
class BcbioLSFEngineSetLauncher(launcher.LSFEngineSetLauncher): """Custom launcher handling heterogeneous clusters on LSF. """ cores = traitlets.Integer(1, config=True) default_template = traitlets.Unicode("""#!/bin/sh #BSUB -q {queue} #BSUB -J bcbio-ipengine[1-{n}] #BSUB -oo bcbio-ipengine.bsub.%%J #BSUB -n {cores} #BSUB -R "span[hosts=1]" %s %s --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" """ % (' '.join(map( pipes.quote, launcher.ipengine_cmd_argv)), ' '.join(timeout_params))) def start(self, n): self.context["cores"] = self.cores return super(BcbioLSFEngineSetLauncher, self).start(n)
class BcbioSGEEngineSetLauncher(launcher.SGEEngineSetLauncher): """Custom launcher handling heterogeneous clusters on SGE. """ cores = traitlets.Integer(1, config=True) default_template = traitlets.Unicode("""#$ -V #$ -cwd #$ -b y #$ -j y #$ -S /bin/sh #$ -q {queue} #$ -N bcbio-ipengine #$ -t 1-{n} #$ -pe threaded {cores} %s %s --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" """ % (' '.join(map(pipes.quote, launcher.ipengine_cmd_argv)), ' '.join(timeout_params))) def start(self, n): self.context["cores"] = self.cores return super(BcbioSGEEngineSetLauncher, self).start(n)
class BcbioLSFControllerLauncher(launcher.LSFControllerLauncher): batch_file_name = Unicode(unicode("lsf_controller" + str(uuid.uuid4()))) tag = traitlets.Unicode("", config=True) cores = traitlets.Integer(1, config=True) resources = traitlets.Unicode("", config=True) job_array_template = Unicode('') queue_template = Unicode('') default_template = traitlets.Unicode("""#!/bin/sh #BSUB -q {queue} #BSUB -J {tag}-c #BSUB -n {cores} #BSUB -oo bcbio-ipcontroller.bsub.%%J {resources} %s --ip=* --log-to-file --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" %s """ % (' '.join(map(pipes.quote, controller_cmd_argv)), ' '.join(controller_params))) def start(self): self.context["cores"] = self.cores self.context["tag"] = self.tag if self.tag else "bcbio" self.context["resources"] = _format_lsf_resources(self.resources) return super(BcbioLSFControllerLauncher, self).start()
class BcbioPBSPROControllerLauncher(PBSPROLauncher, launcher.BatchClusterAppMixin): """Launch a controller using PBSPro.""" batch_file_name = Unicode(u'pbspro_controller', config=True, help="batch file name for the controller job.") tag = traitlets.Unicode("", config=True) cores = traitlets.Integer(1, config=True) default_template = Unicode("""#!/bin/sh #PBS -V #PBS -N {tag}-c #PBS -l select=1:ncpus={cores} cd $PBS_O_WORKDIR %s --ip=* --log-to-file --profile-dir="{profile_dir}" --cluster-id="{cluster_id}" %s """ % (' '.join(map(pipes.quote, controller_cmd_argv)), ' '.join(controller_params))) def start(self): """Start the controller by profile or profile_dir.""" self.context["cores"] = self.cores self.context["tag"] = self.tag if self.tag else "bcbio" return super(BcbioPBSPROControllerLauncher, self).start(1)