def main(qdir: str):
print(ColorText('\nStarting baypass scheduler ...').bold())
# get the threshold for slurm jobs
qthresh = 999 if os.environ['CC_CLUSTER'] == 'cedar' else 950
# get current slurm queue
slurmjobs = getsq()
# determine if any jobs can be scheduled
qlimit = qthresh - len(slurmjobs)
if qlimit <= 0:
print('\tQueue is full, exiting baypass_scheduler.py')
exit()
# reserve scheduling permissions to the current job
scheduler = start_scheduler(qdir)
# get qfiles
qfiles = fs(qdir, endswith='.sh')
# fill up the slurm queue
pids = sbatch(qfiles, limit=qthresh)
# remove reservation
delsched(scheduler)
pass
def main(scheddir):
# write a file and reserve scheduling to this call of the scheduler, or pass if another scheduler is running
startscheduler(scheduler) # reserve right away
x = len(getsq())
print('queue length = ', x)
if x < qthresh: # if there is room in the queue
print('scheduler not running')
print('queue length less than thresh')
nsbatch = qthresh - x # how many should I submit?
print('nsbatch =', nsbatch)
print(len(fs(scheddir)))
files = [
f for f in fs(scheddir) if 'scheduler.txt' not in f
and '.out' not in f and 'workingdir' not in f
][0:nsbatch]
if len(files) > 0:
print('submitting %s jobs' % str(len(files)))
print(files)
sbatchjobs(files)
else:
print('no files to sbatch')
else:
print('scheduler was not running, but no room in queue')
balance_queue = op.join(os.environ['HOME'],
'gatk_pipeline/balance_queue.py')
subprocess.call([sys.executable, balance_queue, 'scatter', parentdir])
delsched(scheduler)
def main(scheddir):
# write a file and reserve scheduling to this call of the scheduler, or pass if another scheduler is running
startscheduler(scheduler) # reserve right away
x = len(getsq())
print('queue length = ', x)
if x < qthresh: # if there is room in the queue
print('scheduler not running')
print('queue length less than thresh')
nsbatch = qthresh - x # how many should I submit?
print('nsbatch =', nsbatch)
print(len(fs(scheddir)))
files = [
f for f in fs(scheddir) if 'scheduler.txt' not in f
and '.out' not in f and 'workingdir' not in f
][0:nsbatch]
if len(files) > 0:
print('submitting %s jobs' % str(len(files)))
print(files)
sbatchjobs(files)
else:
print('no files to sbatch')
else:
print('scheduler was not running, but no room in queue')
pipedir = os.popen('echo $HOME/gatk_pipeline').read().replace("\n", "")
os.system('python %s scatter' % (op.join(pipedir, 'balance_queue.py')))
delsched(scheduler)
def bigbrother(rescheduler):
print('reschduler = ', rescheduler)
# if the scheduler controller has died, remove the scheduler
with open(rescheduler, 'r') as o:
bigtext = o.read().replace("\n", "")
os.system('echo %s' % bigtext)
pid = bigtext.split()[-1]
if not pid == '=':
sq = getsq(states=['running'])
pids = getpids(sq)
if not pid in pids:
os.system(
'echo controller was not running, so the scheduler was destroyed'
)
delrescheduler(rescheduler, True)
else:
os.system('echo controller is running, allowing it to proceed')
if line.startswith('gatk HaplotypeCaller'):
vcf = line.split()[-5]
trushfile = vcf2sh(vcf)
linkname = op.join(scheddir, op.basename(trushfile))
# add job back to the queue
addlink((trushfile, linkname))
# remove worker from workingdir
removeworker(scheddir, trushfile)
break
# identify outs that aren't running
createdrescheduler = False
sq = getsq(states=['running'])
print('len(sq) = ', len(sq))
#print(sq)
pids = getpids(sq)
#print(pids)
runs = []
for out in outs:
pid = op.basename(out).split(".out")[0].split("_")[-1]
if pid not in pids:
runs.append(out)
outs = runs
os.system('echo running rescheduler.py')
if len(outs) > 0:
if not op.exists(rescheduler):
# reserve the rescheduler
def check_queue(outs, pooldir):
"""Get jobs from the queue, except those that are closing (assumes jobs haven't failed)."""
print('checking queue')
sq = getsq(grepping=['crisp_bedfile', op.basename(pooldir)])
if len(sq) > 0:
checkpids(outs, sq)
