def CopyInfoFiles(thisOutPath, thisImputeDir, thisTempDir, thisSampleListName,
thisCopyInfoScript):
s = drmaa.Session()
s.initialize()
print('creating job template for copy_info_files')
jt = s.createJobTemplate()
thisBsubOut = thisTempDir + '/copy_info_files_%J.out'
thisBsubErr = thisTempDir + '/copy_info_files_%J.err'
jt.nativeSpecification = '-R \'rusage[mem=4096]\' ' + \
'-M 8192 ' + \
'-q pcpgmwgs ' + \
'-J copy_info_files ' + \
'-o ' + thisBsubOut + ' ' \
'-e ' + thisBsubErr
jt.remoteCommand = thisCopyInfoScript
jt.args = [thisOutPath, thisImputeDir, thisSampleListName]
jobid = s.runJob(jt)
print('copy_info_files has been submitted with id %s' % jobid)
print('cleaning up')
s.deleteJobTemplate(jt)
s.exit()
return jobid
def _sge(self):
j = 0
Run.RUNNINGTASK['sge'] = []
Run.RUNNINGTASK['drmaa'] = self.drmaa = drmaa.Session()
self.drmaa.initialize()
jt = self.drmaa.createJobTemplate()
jt.jobEnvironment = os.environ.copy()
jt.nativeSpecification = self.option
while j < len(self.unfinished_tasks):
task = self.unfinished_tasks[j]
if j < self.max_pa_jobs or self._check_running <= self.max_pa_jobs:
jt.remoteCommand = task
jt.outputPath = ':' + task + '.o'
jt.errorPath = ':' + task + '.e'
jt.workingDirectory = os.path.dirname(task)
jobid = self.drmaa.runJob(jt)
Run.RUNNINGTASK['sge'].append(jobid)
log.info('Throw jobID:[' + jobid + '] jobCmd:[' + task +
'] in the ' + self.job_type + '_cycle.')
j += 1
else:
time.sleep(self.interval)
else:
while (1):
if self._check_running:
time.sleep(self.interval)
else:
break
self.drmaa.deleteJobTemplate(jt)
self.drmaa.exit()
def submit(self):
job_status = []
s = drmaa.Session()
s.initialize()
jt = s.createJobTemplate()
jt.jobEnvironment = os.environ.copy()
jt.nativeSpecification = self.grid_opts
jt.remoteCommand = self.script
jt.workingDirectory = self.work_dir
jt.outputPath = self.out_path
# jt.errorPath = self.err_path
logger.info('submiting {}'.format([self.script, self.grid_opts]))
jt.joinFiles = self.join_files
joblist = s.runBulkJobs(jt, 1, len(self.cmd_list), 1)
jobid = joblist[0].split('.')[0]
_qsub_log(jobid, self.work_dir, self.script)
#jid = s.runJob(jt)
s.synchronize(joblist, drmaa.Session.TIMEOUT_WAIT_FOREVER, False)
logger.info('waiting for {} tasks'.format(len(joblist)))
for curjob in joblist:
try:
retval = s.wait(curjob, drmaa.Session.TIMEOUT_WAIT_FOREVER)
except Exception as e:
logger.warn('Job {}: {}'.format(curjob, e))
job_status += [(None, 1)]
continue
logger.info(
'Task: {0} finished with status {1}, exit code {2}'.format(
retval.jobId, retval.hasExited, retval.exitStatus))
task_id, status = retval.jobId, (
not retval.hasExited) + retval.exitStatus
job_status += [(task_id, status)]
s.deleteJobTemplate(jt)
s.exit()
return job_status
def __init__(self, tempdir=None):
self.session = drmaa.Session()
self.session.initialize()
self.jobs_current = []
self.jobs_done = []
self.jobs_error = []
self.tempdir = tempdir
def __init__(self, **resources):
"""Initialize a GridEngineScheduler instance
Only one instance may run per Python process, since the underlying drmaa
layer is a singleton.
Keyword Args:
Resources to be passed to qsub commands. These override any
arguments that were given to the constructor:
`-l` command:
h_cpu: maximum time expressed in format '02:00:00' (2 hours)
h_vmem: maximum memory allocation before job is killed in format '10G' (10GB)
virtual_free: memory free on host BEFORE job can be allocated
`-pe` command:
pe_type: either 'smp' or 'ompi' for shared memory or distributed memory.
n_slots: the number of slots to request to the grid engine.
"""
import drmaa
self.drmaa = drmaa
# pass-through options to the jobs
self.resources = settings.DEFAULT_RESOURCES
self.resources.update(resources)
self.session = drmaa.Session()
self.session.initialize()
self.sgeids = []
def submit_job(job_name=None,
command=None,
add_args=None,
cwd=None,
queue='shudra.q',
hold_jobs=None):
# Start SGE session
s = drmaa.Session()
s.initialize()
jt = s.createJobTemplate()
jt.jobName = job_name
jt.remoteCommand = command
if add_args is not None:
jt.args = add_args
jt.joinFiles = True
jt.nativeSpecification = '-shell y -V -j y -S /bin/bash -q ' + queue
jt.workingDirectory = cwd
if hold_jobs is not None:
jt.nativeSpecification += ' -hold_jid ' + hold_jobs
jobid = s.runJob(jt)
print 'INFO: Job ' + job_name + ' has been submitted with id ' + jobid
s.deleteJobTemplate(jt)
# Close SGE session
s.exit()
# Return SGE job id
return jobid
def mRun(self):
""" can we chain 2 jobs in a runJob without a script?
"""
job_path = self.machexe
s = drmaa.Session()
s.initialize()
alljids = []
for clist in self.clists:
if type(clist) == type([]):
clist = ['%s %s' % (self.machexe, x)
for x in clist] # prepend mach
batch = '\n'.join(clist)
path = self.machexe
jt = self.init_job_template(jt=s.createJobTemplate(),
path=job_path,
args=clist,
as_bulk=False)
jobid = s.runJob(jt)
alljids.append(jobid)
sleep(1)
logging.info('# submitted jobids: %s' % alljids)
s.synchronize(alljids, drmaa.Session.TIMEOUT_WAIT_FOREVER, False)
for jid in all_jids:
logging.info('jid %s:' % str(jid))
jinfo = s.wait(jid, drmaa.Session.TIMEOUT_WAIT_FOREVER)
alog = """ id: %(jobId)s
resource usage: %(resourceUsage)s
""" % jinfo._asdict()
logging.info(alog)
s.exit()
def inner(*args, **kwargs):
global s
# self = args[0]
s = drmaa.Session()
s.initialize()
func(*args, **kwargs)
s.exit()
def __call__(self, *args, **kwargs):
fn = self.fn
print('Building job for {fn}'.format(fn=fn))
s = drmaa.Session()
s.initialize()
try:
jt = s.createJobTemplate()
jt.workingDirectory = WORKING_DIR
jt.outputPath = WORKING_DIR
jt.joinFiles = True
jt.jobName = fn
jt.remoteCommand = os.path.join(os.getcwd(), 'wrapper.sh')
jt.args = [PATH, fn]
jt.nativeSpecification = RESOURCES
job_id = s.runJob(jt)
print('Submitted job: {job_id}'.format(job_id=job_id))
info = s.wait(job_id, drmaa.Session.TIMEOUT_WAIT_FOREVER)
print('Completed job: {job_id}'.format(job_id=job_id))
print("""\
id: %(jobId)s
exited: %(hasExited)s
signaled: %(hasSignal)s
with signal (id signaled): %(terminatedSignal)s
dumped core: %(hasCoreDump)s
aborted: %(wasAborted)s
resource usage:
%(resourceUsage)s
""" % info._asdict())
finally:
s.exit()
def test_defaults(self):
with drmaa.Session() as session:
outputDirectoryLocation = getPath("tests/output/manager")
logDirectoryLocation = getPath("tests/output/manager/log")
defaultSpecification = "-l h_vmem=16G -pe smp 8"
jobManager = JobManagerDRMAA(outputDirectoryLocation,
logDirectoryLocation, session,
defaultSpecification)
self.assertEquals(jobManager.session, session)
self.assertEquals(jobManager.outputDirectoryLocation,
outputDirectoryLocation)
self.assertEquals(jobManager.countSpecification,
defaultSpecification)
self.assertEquals(jobManager.aggregateSpecification,
defaultSpecification)
self.assertEquals(jobManager.extractSpecification,
defaultSpecification)
self.assertEquals(jobManager.databaseSpecification,
defaultSpecification)
self.assertEquals(jobManager.filterSpecification,
defaultSpecification)
def main():
"""Submit an array job, wait for them to finish, and collect results.
This is slightly different from the example from sunsource because it only
waits for the jobs that it submitted.
"""
s = drmaa.Session()
s.initialize()
print 'Creating job template'
jt = s.createJobTemplate()
jt.remoteCommand = os.getcwd() + '/sleeper.sh'
jt.args = ['42', 'Simon says:']
jt.joinFiles = True
joblist = s.runBulkJobs(jt, 1, 30, 2)
print 'Your job has been submitted with id ' + str(joblist)
s.synchronize(joblist, drmaa.Session.TIMEOUT_WAIT_FOREVER, False)
for curjob in joblist:
print 'Collecting job ' + curjob
retval = s.wait(curjob, drmaa.Session.TIMEOUT_WAIT_FOREVER)
print 'Job: ' + str(retval.jobId) + ' finished with status ' + str(
retval.hasExited)
print 'Cleaning up'
s.deleteJobTemplate(jt)
s.exit()
def __init__(self, nworkers=1, **resources):
"""Initialize a GridEngineScheduler instance
Only one instance may run per Python process, since the underlying drmaa
layer is a singleton.
Parameters
----------
nworkers: int
number of workers (can be updated later)
Resources: dict
to be passed to the -l command of qsub.
e.g.:
h_cpu: maximum time expressed in format '02:00:00' (2 hours)
h_vmem: maximum memory allocation before job is killed in format '10G' (10GB)
virtual_free: memory free on host BEFORE job can be allocated
"""
import drmaa
self.drmaa = drmaa
self.nworkers = nworkers
# pass-through options to the jobs
self.resources = DEFAULT_RESOURCES
self.resources.update(resources)
self.session = drmaa.Session()
self.session.initialize()
self.sgeids = []
self.address = HOSTNAME
self.cluster = None
def __init__(self,
stop_on_first_error: bool = False,
max_jobs: int = None,
skip_already_done=False):
super().__init__(stop_on_first_error, max_jobs, skip_already_done)
self._session = drmaa.Session()
self._session.initialize()
def __init__(self, workflow, dag, cores,
jobname="snakejob.{rulename}.{jobid}.sh",
printreason=False,
quiet=False,
printshellcmds=False,
drmaa_args="",
latency_wait=3,
benchmark_repeats=1,
cluster_config=None,
max_jobs_per_second=None,
restart_times=0):
super().__init__(workflow, dag, cores,
jobname=jobname,
printreason=printreason,
quiet=quiet,
printshellcmds=printshellcmds,
latency_wait=latency_wait,
benchmark_repeats=benchmark_repeats,
cluster_config=cluster_config,
max_jobs_per_second=max_jobs_per_second,
restart_times=restart_times)
try:
import drmaa
except ImportError:
raise WorkflowError(
"Python support for DRMAA is not installed. "
"Please install it, e.g. with easy_install3 --user drmaa")
except RuntimeError as e:
raise WorkflowError("Error loading drmaa support:\n{}".format(e))
self.session = drmaa.Session()
self.drmaa_args = drmaa_args
self.session.initialize()
self.submitted = list()
def main():
"""Submit a job.
Note, need file called sleeper.sh in current directory.
"""
s = drmaa.Session()
s.initialize()
print 'Creating job template'
jt = s.createJobTemplate()
jt.remoteCommand = os.getcwd() + '/sleeper.sh'
jt.args = ['42', 'Simon says:']
#jt.nativeSpecification = "--type besteffort --project yop --name poy --hold"
#jt.nativeSpecification = "-q yop --name poy -O s1 -E lksdf " -> OK
jt.nativeSpecification = "--type besteffort "
#jt.nativeSpecification = "--hold" -> OK
#hardWallclockTimeLimit = _h.Attribute(_c.WCT_HLIMIT, _h.IntConverter)
#'Hard' Wallclock time limit, in seconds.
jt.joinFiles = True #not supported yet support in OAR
jobid = s.runJob(jt)
print 'Your job has been submitted with id ' + jobid
print 'Cleaning up'
s.deleteJobTemplate(jt)
s.exit()
def main():
'''Initialise the pipeline, then run it'''
# Parse command line arguments
options = parse_command_line()
# Initialise the logger
logger = Logger(__name__, options.log_file, options.verbose)
# Log the command line used to run the pipeline
logger.info(' '.join(sys.argv))
drmaa_session = None
try:
# Set up the DRMAA session for running cluster jobs
import drmaa
drmaa_session = drmaa.Session()
drmaa_session.initialize()
except Exception as e:
print("{progname} error using DRMAA library".format(
progname=program_name),
file=sys.stdout)
print("Error message: {msg}".format(msg=e.message, file=sys.stdout))
exit(error_codes.DRMAA_ERROR)
# Parse the configuration file, and initialise global state
config = Config(options.config)
config.validate()
state = State(options=options,
config=config,
logger=logger,
drmaa_session=drmaa_session)
# Build the pipeline workflow
pipeline = make_pipeline(state)
# Run (or print) the pipeline
cmdline.run(options)
if drmaa_session is not None:
# Shut down the DRMAA session
drmaa_session.exit()
def start(self):
psutil.cpu_percent()
time.sleep(1)
self.keep_running = True
t = threading.Thread(target=self.finalize_tasks)
t.daemon = True
t.start()
logger.info('Starting Task Server')
logger.info("using code at: " + __file__)
logger.info("Path to do_ Scripts : %s" % self.path_to_do_scripts)
logger.info("Data_dir : %s" % self.data_dir)
logger.info("Port : %s" % self.port)
if self.sg.cluster_scheduler == 1:
logger.info("Initilizing DRMAA interface to cluster scheduler")
import drmaa
self.drmaa_session = drmaa.Session(
) # Start the interface session to DRMAA to control GridEngine
self.drmaa_session.initialize()
try:
# Setup a thread that just updates the last checkin time for this still every 5min
timer_thread = threading.Thread(target=self.checkin_timer)
timer_thread.daemon = True # Make it a daemon so that when ctrl-c happens this thread goes away
timer_thread.start() # Start heartbeat
self.serve_forever() # Start the lisetenser server
finally:
self.shutdown()
return
def startSession():
"""start and initialize the global DRMAA session."""
global GLOBAL_SESSION
GLOBAL_SESSION = drmaa.Session()
GLOBAL_SESSION.initialize()
return GLOBAL_SESSION
def main(options, args):
samples = glob.glob(sys.argv[1] + "/*_barcode/*1_fastq.txt")
outdir = "fastq_screen"
for sample1 in samples:
sample2 = sample1.replace("1_fastq", "2_fastq")
s = drmaa.Session()
s.initialize()
jt = s.createJobTemplate()
jt.remoteCommand = 'fastq_screen'
jt.args = ['--illumina', '--multilib', '--subset', '2000000', sample1, \
'--paired', sample2, '--outdir', os.path.join(sys.argv[1], outdir)]
# TODO: job name is always (null), must fix slurm_drmaa C library and its
# custom parsing (substitute "slurmdrmaa_parse_native"
# for GNU GetOpt on slurm_drmaa/util.c)
jt.job_name = "fastq_screen %s" % os.path.basename(sample1)
jt.nativeSpecification = "-A a2010002 -p node -t 00:30:00"
jobid = s.runJob(jt)
print 'Your job has been submitted with id ' + jobid
s.deleteJobTemplate(jt)
s.exit()
exit(0)
def run_queries(self):
provean_queries = self._create_queries()
session = None
if self.cluster_mode.lower() == 'sge':
session = drmaa.Session()
session.initialize()
jobs = []
for record, queries in provean_queries.items():
for query in queries:
if self.cluster_mode.lower() == 'none':
query.run_local()
else:
jobs.append(
query.run_cluster(session=session,
cluster_mode=self.cluster_mode,
mem_per_thread=self.mem_per_thread))
for job in jobs:
job.wait()
for record, queries in provean_queries.items():
scores = []
for query in queries:
scores.append(query.process())
record.INFO['provean_protein_id'] = query.ensembl_id
record.INFO['provean_score'] = min(scores)
if min(scores) < -2.5:
record.INFO['provean_pred'] = 'Deleterious'
else:
record.INFO['provean_pred'] = 'Neutral'
if session is not None:
session.exit()
def test_default_specification(self):
defaultSpecification = "-l h_vmem=2G -pe smp 1"
parameters = DefaultArgs().parameters
parameters[Neptune.DEFAULT_SPECIFICATION] = defaultSpecification
with drmaa.Session() as session:
jobManager = buildDRMAAJobManager(session, defaultSpecification)
execution = Execution(jobManager, parameters)
self.assertEquals(execution.jobManager.countSpecification,
defaultSpecification)
self.assertEquals(execution.jobManager.aggregateSpecification,
defaultSpecification)
self.assertEquals(execution.jobManager.extractSpecification,
defaultSpecification)
self.assertEquals(execution.jobManager.databaseSpecification,
defaultSpecification)
self.assertEquals(execution.jobManager.filterSpecification,
defaultSpecification)
self.assertEquals(execution.jobManager.consolidateSpecification,
defaultSpecification)
def runpipe():
s = drmaa.Session()
s.initialize()
jt = s.createJobTemplate()
jt.remoteCommand = cmd_args[0]
jt.args = cmd_args[1:]
jt.jobName = job_args['jobname']
if os.path.isdir(job_args['outputPath']):
jt.outputPath = ":{}".format(os.path.join(os.path.abspath(job_args['outputPath']), jt.jobName + "-drmaa.out"))
else:
jt.outputPath = ":{}".format(os.path.abspath(job_args['outputPath']))
if os.path.isdir(job_args['errorPath']):
jt.errorPath = ":{}".format(os.path.join(os.path.abspath(job_args['errorPath']), jt.jobName + "-drmaa.err"))
else:
jt.errorPath = ":{}".format(os.path.abspath(job_args['errorPath']))
jt.workingDirectory = os.path.abspath(job_args['workingDirectory'])
jt.nativeSpecification = "-t {time} -p {partition} -A {account} {extra}".format(**job_args)
if kw.get('email', None):
jt.email=[kw.get('email')]
self.app.log.info("Submitting job with native specification {}".format(jt.nativeSpecification))
self.app.log.info("Working directory: {}".format(jt.workingDirectory))
self.app.log.info("Output logging: {}".format(jt.outputPath))
self.app.log.info("Error logging: {}".format(jt.errorPath))
self._meta.jobid = s.runJob(jt)
self.app.log.info('Your job has been submitted with id ' + self._meta.jobid)
if kw.get('saveJobId', False):
self._save_job_id(**job_args)
s.deleteJobTemplate(jt)
s.exit()
def __init__(self, lsfargs=None):
# finicky, has core dumped in initialize.
# inits a drmaa session.
# !must be done in correct order or things
# get squirrely.
'''# Order:
# -create this obj with/without job args
# for constructor,
# -if not in constructor, create jobTemplate
# with self.runner()
# -submit
# -wait, release whatuva '''
self.session = drmaa.Session()
if lsfargs is not None:
''' arguments passed to LSF in a string, like -q pcpgm -o outfile '''
self.job = self.runner(lsfargs)
self.args = lsfargs
else:
self.job = None
# finicky, has core dumped in initialize.
self.session.initialize()
def main(cmd, args, slurm_parameters):
slurm_project, slurm_run_time, slurm_partition, slurm_job_name, slurm_output, slurm_error, slurm_opts = slurm_parameters
slurm_args = "-A %s -t %s -p %s -J %s" % (slurm_project, slurm_run_time,
slurm_partition, slurm_job_name)
# if not slurm_output is None:
# slurm_args += " -o %s" % slurm_output
# if not slurm_error is None:
# slurm_args += " -e %s" % slurm_error
if not slurm_opts is None:
slurm_args += " %s" % slurm_opts
print "Command: %s" % cmd
print "Args: %s" % args
print "slurm args: %s" % slurm_args
if options.dry_run:
sys.exit()
s = drmaa.Session()
s.initialize()
jt = s.createJobTemplate()
jt.remoteCommand = cmd
jt.args = args
jt.nativeSpecification = slurm_args
jobid = s.runJob(jt)
s.deleteJobTemplate(jt)
s.exit()
print "Your job was submitted with jobid %s" % jobid
def submit_jobs(jobs, home_address, white_list=""):
"""
Method used to send a list of jobs onto the cluster.
@param jobs: list of jobs to be executed
@type jobs: list<Job>
"""
session = drmaa.Session()
session.initialize()
jobids = []
for job in jobs:
#print job.name, job.mem, job.walltime
# set job white list
job.white_list = white_list
# remember address of submission host
job.home_address = home_address
# append jobs
jobid = append_job_to_session(session, job)
jobids.append(jobid)
sid = session.contact
session.exit()
return (sid, jobids)
def main():
jobID, uid = validate_paramters()
set_user(uid)
s = drmaa.Session()
s.initialize()
s.control(jobID, drmaa.JobControlAction.TERMINATE)
s.exit()
def get_drmaa_session():
global _drmaa_session
if _drmaa_session is None:
import drmaa
_drmaa_session = drmaa.Session()
_drmaa_session.initialize()
return _drmaa_session
def run_shelxd_drmaa(njobs, job_settings):
'''Run shelxd on cluster with settings given in dictionary, containing:
nrefl = 1 + floor(nref / 100000) - space to allocate
ncpu - number of cpus to use
wd - working directory'''
import drmaa
with drmaa.Session() as session:
job = session.createJobTemplate()
batches = range(0, len(job_settings), njobs)
for idx in batches:
jobs = []
for _settings in job_settings[idx:idx+njobs]:
nrefl = _settings['nrefl']
ncpu = _settings['ncpu']
wd = _settings['wd']
setup_job_drmaa(job,
'shelxd', ['-L%d' % nrefl, 'sad_fa', '-t%d' % ncpu],
[], wd, ncpu, timeout = 600)
jobs.append(session.runJob(job))
session.synchronize(jobs, drmaa.Session.TIMEOUT_WAIT_FOREVER, True)
session.deleteJobTemplate(job)
return
def get_slurm_jobstatus(jobid):
"""Get the status for a jobid
"""
s = drmaa.Session()
s.initialize()
status = s.jobStatus(str(jobid))
s.exit()
return status
def __enter__(self):
self.local_queue.__enter__()
self.session = drmaa.Session()
self.session.initialize()
return self
