def process_switching(filenames, exepath, queues, nminutes):
# define the switcher
switcher = whackamole.Switcher(queues, switch)
# create the jobs
jobs = []
for i, datapath in enumerate(filenames):
jobs.append(Job(datapath, exepath, nminutes, i))
# submit all of the jobs to the debug queue
number_to_job = {}
for job in jobs:
job.submit('debug')
number_to_job[job.b.job_number] = job
# tell the switcher that all jobs were submitted to the debug queue
switcher.on_submission({'debug': set(number_to_job)})
# go until all jobs have finished
while True:
time.sleep(2.0)
# query the state of the submitted jobs
jnum_state_queue_triples = list(lsf.bjobs())
# if no submitted jobs were found then we are done
if not jnum_state_queue_triples:
break
# possibly switch some queues
qname_to_jnums = collections.defaultdict(set)
for jnum, state, queue in jnum_state_queue_triples:
if state == 'PEND':
qname_to_jnums[queue].add(jnum)
switcher.on_observation(qname_to_jnums)
# return the job numbers for the finished jobs
return set(number_to_job)
def main(args):
# define the path to the exe
exepath = os.path.join(args.exe_dir, args.exe_name)
# read the expected chromosome names
with open(args.expected) as fin:
expected_names = set(nonempty_stripped_lines(fin))
# get the defined queues and put the debug queue at the front
queues = list(lsf.gen_accessible_queues())
i = queues.index('debug')
queues[0], queues[i] = queues[i], queues[0]
# define the switcher
switcher = whackamole.Switcher(queues, switch)
# create the jobs
jobs = []
for i, datapath in enumerate(args.files):
gzname = os.path.basename(datapath)
outpath = os.path.join(args.scratch, gzname + '.names.' + str(i))
jobs.append(Job(datapath, exepath, outpath))
number_to_job = {}
# submit all of the jobs to the debug queue
for job in jobs:
job.submit('debug')
number_to_job[job.b.job_number] = job
# tell the switcher that all jobs were submitted to the debug queue
switcher.on_submission({'debug': set(number_to_job)})
# initialize the set of unfinished job numbers
all_numbers = set(number_to_job)
prev_unfinished = set(number_to_job)
# go until all of the jobs have finished
while prev_unfinished:
time.sleep(2.0)
# query the state of the submitted jobs
jnum_state_queue_triples = list(lsf.bjobs())
# possibly switch some queues
qname_to_jnums = collections.defaultdict(set)
for jnum, state, queue in jnum_state_queue_triples:
if state == 'PEND':
qname_to_jnums[queue].add(jnum)
switcher.on_observation(qname_to_jnums)
# check for finished jobs
if jnum_state_queue_triples:
jnums, states, queues = zip(*jnum_state_queue_triples)
else:
jnums, states, queues = [], [], []
curr_unfinished = all_numbers & set(jnums)
newly_finished = prev_unfinished - curr_unfinished
for jnum in newly_finished:
print 'job', jnum, 'has finished'
number_to_job[jnum].validate(expected_names)
prev_unfinished = curr_unfinished
def process_no_switching(filenames, exepath, qname, nminutes):
# create the jobs
jobs = []
for i, datapath in enumerate(filenames):
jobs.append(Job(datapath, exepath, nminutes, i))
# submit all of the jobs to the specified queue
jnums = set()
for job in jobs:
job.submit(qname)
jnums.add(job.b.job_number)
# go until all jobs have finished
while True:
time.sleep(2.0)
# query the state of the submitted jobs
jnum_state_queue_triples = list(lsf.bjobs())
# if no submitted jobs were found then we are done
if not jnum_state_queue_triples:
break
# return the job numbers for the finished jobs
return jnums
