示例#1
0
def process_switching(filenames, exepath, queues, nminutes):
    # define the switcher
    switcher = whackamole.Switcher(queues, switch)
    # create the jobs
    jobs = []
    for i, datapath in enumerate(filenames):
        jobs.append(Job(datapath, exepath, nminutes, i))
    # submit all of the jobs to the debug queue
    number_to_job = {}
    for job in jobs:
        job.submit('debug')
        number_to_job[job.b.job_number] = job
    # tell the switcher that all jobs were submitted to the debug queue
    switcher.on_submission({'debug': set(number_to_job)})
    # go until all jobs have finished
    while True:
        time.sleep(2.0)
        # query the state of the submitted jobs
        jnum_state_queue_triples = list(lsf.bjobs())
        # if no submitted jobs were found then we are done
        if not jnum_state_queue_triples:
            break
        # possibly switch some queues
        qname_to_jnums = collections.defaultdict(set)
        for jnum, state, queue in jnum_state_queue_triples:
            if state == 'PEND':
                qname_to_jnums[queue].add(jnum)
        switcher.on_observation(qname_to_jnums)
    # return the job numbers for the finished jobs
    return set(number_to_job)
def main(args):
    # define the path to the exe
    exepath = os.path.join(args.exe_dir, args.exe_name)
    # read the expected chromosome names
    with open(args.expected) as fin:
        expected_names = set(nonempty_stripped_lines(fin))
    # get the defined queues and put the debug queue at the front
    queues = list(lsf.gen_accessible_queues())
    i = queues.index('debug')
    queues[0], queues[i] = queues[i], queues[0]
    # define the switcher
    switcher = whackamole.Switcher(queues, switch)
    # create the jobs
    jobs = []
    for i, datapath in enumerate(args.files):
        gzname = os.path.basename(datapath)
        outpath = os.path.join(args.scratch, gzname + '.names.' + str(i))
        jobs.append(Job(datapath, exepath, outpath))
    number_to_job = {}
    # submit all of the jobs to the debug queue
    for job in jobs:
        job.submit('debug')
        number_to_job[job.b.job_number] = job
    # tell the switcher that all jobs were submitted to the debug queue
    switcher.on_submission({'debug': set(number_to_job)})
    # initialize the set of unfinished job numbers
    all_numbers = set(number_to_job)
    prev_unfinished = set(number_to_job)
    # go until all of the jobs have finished
    while prev_unfinished:
        time.sleep(2.0)
        # query the state of the submitted jobs
        jnum_state_queue_triples = list(lsf.bjobs())
        # possibly switch some queues
        qname_to_jnums = collections.defaultdict(set)
        for jnum, state, queue in jnum_state_queue_triples:
            if state == 'PEND':
                qname_to_jnums[queue].add(jnum)
        switcher.on_observation(qname_to_jnums)
        # check for finished jobs
        if jnum_state_queue_triples:
            jnums, states, queues = zip(*jnum_state_queue_triples)
        else:
            jnums, states, queues = [], [], []
        curr_unfinished = all_numbers & set(jnums)
        newly_finished = prev_unfinished - curr_unfinished
        for jnum in newly_finished:
            print 'job', jnum, 'has finished'
            number_to_job[jnum].validate(expected_names)
        prev_unfinished = curr_unfinished
示例#3
0
def process_no_switching(filenames, exepath, qname, nminutes):
    # create the jobs
    jobs = []
    for i, datapath in enumerate(filenames):
        jobs.append(Job(datapath, exepath, nminutes, i))
    # submit all of the jobs to the specified queue
    jnums = set()
    for job in jobs:
        job.submit(qname)
        jnums.add(job.b.job_number)
    # go until all jobs have finished
    while True:
        time.sleep(2.0)
        # query the state of the submitted jobs
        jnum_state_queue_triples = list(lsf.bjobs())
        # if no submitted jobs were found then we are done
        if not jnum_state_queue_triples:
            break
    # return the job numbers for the finished jobs
    return jnums