Esempio n. 1
0
    def queue_job( self, job_wrapper ):
        # prepare the job
        if not self.prepare_job( job_wrapper ):
            return

        # command line has been added to the wrapper by prepare_job()
        command_line = job_wrapper.runner_command_line

        stderr = stdout = ''

        # Persist the destination
        job_wrapper.set_job_destination(job_wrapper.job_destination)

        # This is the job's exit code, which will depend on the tasks'
        # exit code. The overall job's exit code will be one of two values:
        # o if the job is successful, then the last task scanned will be
        #   used to determine the exit code. Note that this is not the same
        #   thing as the last task to complete, which could be added later.
        # o if a task fails, then the job will fail and the failing task's
        #   exit code will become the job's exit code.
        job_exit_code = None

        try:
            job_wrapper.change_state( model.Job.states.RUNNING )
            self.sa_session.flush()
            # Split with the defined method.
            parallelism = job_wrapper.get_parallelism()
            try:
                splitter = getattr(__import__('galaxy.jobs.splitters', globals(), locals(), [parallelism.method]), parallelism.method)
            except:
                job_wrapper.change_state( model.Job.states.ERROR )
                job_wrapper.fail("Job Splitting Failed, no match for '%s'" % parallelism)
                return
            tasks = splitter.do_split(job_wrapper)
            # Not an option for now.  Task objects don't *do* anything
            # useful yet, but we'll want them tracked outside this thread
            # to do anything.
            # if track_tasks_in_database:
            task_wrappers = []
            for task in tasks:
                self.sa_session.add(task)
            self.sa_session.flush()
            # Must flush prior to the creation and queueing of task wrappers.
            for task in tasks:
                tw = TaskWrapper(task, job_wrapper.queue)
                task_wrappers.append(tw)
                self.app.job_manager.job_handler.dispatcher.put(tw)
            tasks_complete = False
            count_complete = 0
            sleep_time = 1
            # sleep/loop until no more progress can be made. That is when
            # all tasks are one of { OK, ERROR, DELETED }. If a task
            completed_states = [ model.Task.states.OK,
                                 model.Task.states.ERROR,
                                 model.Task.states.DELETED ]

            # TODO: Should we report an error (and not merge outputs) if
            # one of the subtasks errored out?  Should we prevent any that
            # are pending from being started in that case?
            # SM: I'm
            # If any task has an error, then we will stop all of them
            # immediately. Tasks that are in the QUEUED state will be
            # moved to the DELETED state. The task's runner should
            # ignore tasks that are not in the QUEUED state.
            # Deleted tasks are not included right now.
            #
            while tasks_complete is False:
                count_complete = 0
                tasks_complete = True
                for tw in task_wrappers:
                    task_state = tw.get_state()
                    if ( model.Task.states.ERROR == task_state ):
                        job_exit_code = tw.get_exit_code()
                        log.debug( "Canceling job %d: Task %s returned an error"
                                   % ( tw.job_id, tw.task_id ) )
                        self._cancel_job( job_wrapper, task_wrappers )
                        tasks_complete = True
                        break
                    elif task_state not in completed_states:
                        tasks_complete = False
                    else:
                        job_exit_code = tw.get_exit_code()
                        count_complete = count_complete + 1
                if tasks_complete is False:
                    sleep( sleep_time )
                    if sleep_time < 8:
                        sleep_time *= 2
            job_wrapper.reclaim_ownership()      # if running as the actual user, change ownership before merging.
            log.debug('execution finished - beginning merge: %s' % command_line)
            stdout, stderr = splitter.do_merge(job_wrapper, task_wrappers)
        except Exception:
            job_wrapper.fail( "failure running job", exception=True )
            log.exception("failure running job %d" % job_wrapper.job_id)
            return

        # run the metadata setting script here
        # this is terminate-able when output dataset/job is deleted
        # so that long running set_meta()s can be canceled without having to reboot the server
        self._handle_metadata_externally(job_wrapper, resolve_requirements=True )
        # Finish the job
        try:
            job_wrapper.finish( stdout, stderr, job_exit_code )
        except:
            log.exception("Job wrapper finish method failed")
            job_wrapper.fail("Unable to finish job", exception=True)
Esempio n. 2
0
 def _wrapper(self):
     return TaskWrapper(self.task, self.queue)
Esempio n. 3
0
    def run_job(self, job_wrapper):
        job_wrapper.set_runner('tasks:///', None)
        stderr = stdout = command_line = ''
        # Prepare the job to run
        try:
            job_wrapper.prepare()
            command_line = job_wrapper.get_command_line()
        except:
            job_wrapper.fail("failure preparing job", exception=True)
            log.exception("failure running job %d" % job_wrapper.job_id)
            return

        # This is the job's exit code, which will depend on the tasks'
        # exit code. The overall job's exit code will be one of two values:
        # o if the job is successful, then the last task scanned will be
        #   used to determine the exit code. Note that this is not the same
        #   thing as the last task to complete, which could be added later.
        # o if a task fails, then the job will fail and the failing task's
        #   exit code will become the job's exit code.
        job_exit_code = None

        # If we were able to get a command line, run the job.  ( must be passed to tasks )
        if command_line:
            try:
                job_wrapper.change_state(model.Job.states.RUNNING)
                self.sa_session.flush()
                # Split with the defined method.
                parallelism = job_wrapper.get_parallelism()
                try:
                    splitter = getattr(
                        __import__('galaxy.jobs.splitters', globals(),
                                   locals(), [parallelism.method]),
                        parallelism.method)
                except:
                    job_wrapper.change_state(model.Job.states.ERROR)
                    job_wrapper.fail(
                        "Job Splitting Failed, no match for '%s'" %
                        parallelism)
                    return
                tasks = splitter.do_split(job_wrapper)
                # Not an option for now.  Task objects don't *do* anything
                # useful yet, but we'll want them tracked outside this thread
                # to do anything.
                # if track_tasks_in_database:
                task_wrappers = []
                for task in tasks:
                    self.sa_session.add(task)
                self.sa_session.flush()
                # Must flush prior to the creation and queueing of task wrappers.
                for task in tasks:
                    tw = TaskWrapper(task, job_wrapper.queue)
                    task_wrappers.append(tw)
                    self.app.job_manager.job_handler.dispatcher.put(tw)
                tasks_complete = False
                count_complete = 0
                sleep_time = 1
                # sleep/loop until no more progress can be made. That is when
                # all tasks are one of { OK, ERROR, DELETED }. If a task
                completed_states = [ model.Task.states.OK, \
                                     model.Task.states.ERROR, \
                                     model.Task.states.DELETED ]

                # TODO: Should we report an error (and not merge outputs) if
                # one of the subtasks errored out?  Should we prevent any that
                # are pending from being started in that case?
                # SM: I'm
                # If any task has an error, then we will stop all of them
                # immediately. Tasks that are in the QUEUED state will be
                # moved to the DELETED state. The task's runner should
                # ignore tasks that are not in the QUEUED state.
                # Deleted tasks are not included right now.
                #
                while tasks_complete is False:
                    count_complete = 0
                    tasks_complete = True
                    for tw in task_wrappers:
                        task_state = tw.get_state()
                        if (model.Task.states.ERROR == task_state):
                            job_exit_code = tw.get_exit_code()
                            log.debug(
                                "Canceling job %d: Task %s returned an error" %
                                (tw.job_id, tw.task_id))
                            self.cancel_job(job_wrapper, task_wrappers)
                            tasks_complete = True
                            break
                        elif not task_state in completed_states:
                            tasks_complete = False
                        else:
                            job_exit_code = tw.get_exit_code()
                            count_complete = count_complete + 1
                    if tasks_complete is False:
                        sleep(sleep_time)
                        if sleep_time < 8:
                            sleep_time *= 2
                import time
                job_wrapper.reclaim_ownership(
                )  # if running as the actual user, change ownership before merging.
                log.debug('execution finished - beginning merge: %s' %
                          command_line)
                stdout, stderr = splitter.do_merge(job_wrapper, task_wrappers)
            except Exception:
                job_wrapper.fail("failure running job", exception=True)
                log.exception("failure running job %d" % job_wrapper.job_id)
                return

        #run the metadata setting script here
        #this is terminate-able when output dataset/job is deleted
        #so that long running set_meta()s can be canceled without having to reboot the server
        if job_wrapper.get_state() not in [
                model.Job.states.ERROR, model.Job.states.DELETED
        ] and self.app.config.set_metadata_externally and job_wrapper.output_paths:
            external_metadata_script = job_wrapper.setup_external_metadata(
                output_fnames=job_wrapper.get_output_fnames(),
                set_extension=True,
                kwds={'overwrite': False}
            )  #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
            log.debug('executing external set_meta script for job %d: %s' %
                      (job_wrapper.job_id, external_metadata_script))
            external_metadata_proc = subprocess.Popen(
                args=external_metadata_script,
                shell=True,
                env=os.environ,
                preexec_fn=os.setpgrp)
            job_wrapper.external_output_metadata.set_job_runner_external_pid(
                external_metadata_proc.pid, self.sa_session)
            external_metadata_proc.wait()
            log.debug('execution of external set_meta finished for job %d' %
                      job_wrapper.job_id)

        # Finish the job
        try:
            job_wrapper.finish(stdout, stderr, job_exit_code)
        except:
            log.exception("Job wrapper finish method failed")
            job_wrapper.fail("Unable to finish job", exception=True)
Esempio n. 4
0
    def run_job( self, job_wrapper ):
        job_wrapper.set_runner( 'tasks:///', None )
        stderr = stdout = command_line = ''
        # Prepare the job to run
        try:
            job_wrapper.prepare()
            command_line = job_wrapper.get_command_line()
        except:
            job_wrapper.fail( "failure preparing job", exception=True )
            log.exception("failure running job %d" % job_wrapper.job_id)
            return
        # If we were able to get a command line, run the job.  ( must be passed to tasks )
        if command_line:
            try:
                # DBTODO read tool info and use the right kind of parallelism.  
                # For now, the only splitter is the 'basic' one
                job_wrapper.change_state( model.Job.states.RUNNING )
                self.sa_session.flush()
                # Split with the tool-defined method.
                try:
                    splitter = getattr(__import__('galaxy.jobs.splitters',  globals(),  locals(),  [job_wrapper.tool.parallelism.method]),  job_wrapper.tool.parallelism.method)
                except:   
                    job_wrapper.change_state( model.Job.states.ERROR )
                    job_wrapper.fail("Job Splitting Failed, no match for '%s'" % job_wrapper.tool.parallelism)
                    return
                tasks = splitter.do_split(job_wrapper)

                # Not an option for now.  Task objects don't *do* anything useful yet, but we'll want them tracked outside this thread to do anything.
                # if track_tasks_in_database:
                task_wrappers = []
                for task in tasks:
                    self.sa_session.add(task)
                self.sa_session.flush()
                
                # Must flush prior to the creation and queueing of task wrappers.
                for task in tasks:
                    tw = TaskWrapper(task, job_wrapper.queue)
                    task_wrappers.append(tw)
                    self.app.job_manager.job_handler.dispatcher.put(tw)
                tasks_incomplete = False
                count_complete = 0
                sleep_time = 1
                # sleep/loop until no more progress can be made. That is when
                # all tasks are one of { OK, ERROR, DELETED }
                completed_states = [ model.Task.states.OK, \
                                    model.Task.states.ERROR, \
                                    model.Task.states.DELETED ]
                # TODO: Should we report an error (and not merge outputs) if one of the subtasks errored out?
                # Should we prevent any that are pending from being started in that case?
                while tasks_incomplete is False:
                    count_complete = 0
                    tasks_incomplete = True
                    for tw in task_wrappers:
                        task_state = tw.get_state()
                        if not task_state in completed_states:
                            tasks_incomplete = False
                        else:
                            count_complete = count_complete + 1
                    if tasks_incomplete is False:
                        # log.debug('Tasks complete: %s. Sleeping %s' % (count_complete, sleep_time))
                        sleep( sleep_time )
                        if sleep_time < 8:
                            sleep_time *= 2
                
                import time

                job_wrapper.reclaim_ownership()      # if running as the actual user, change ownership before merging.

                log.debug('execution finished - beginning merge: %s' % command_line)
                stdout,  stderr = splitter.do_merge(job_wrapper,  task_wrappers)
                
            except Exception:
                job_wrapper.fail( "failure running job", exception=True )
                log.exception("failure running job %d" % job_wrapper.job_id)
                return

        #run the metadata setting script here
        #this is terminate-able when output dataset/job is deleted
        #so that long running set_meta()s can be canceled without having to reboot the server
        if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally and job_wrapper.output_paths:
            external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(),
                                                                            set_extension = True,
                                                                            kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
            log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) )
            external_metadata_proc = subprocess.Popen( args = external_metadata_script, 
                                         shell = True, 
                                         env = os.environ,
                                         preexec_fn = os.setpgrp )
            job_wrapper.external_output_metadata.set_job_runner_external_pid( external_metadata_proc.pid, self.sa_session )
            external_metadata_proc.wait()
            log.debug( 'execution of external set_meta finished for job %d' % job_wrapper.job_id )
        
        # Finish the job                
        try:
            job_wrapper.finish( stdout, stderr )
        except:
            log.exception("Job wrapper finish method failed")
            job_wrapper.fail("Unable to finish job", exception=True)
Esempio n. 5
0
    def run_job( self, job_wrapper ):
        job_wrapper.set_runner( 'tasks:///', None )
        stderr = stdout = command_line = ''
        # Prepare the job to run
        try:
            job_wrapper.prepare()
            command_line = job_wrapper.get_command_line()
        except:
            job_wrapper.fail( "failure preparing job", exception=True )
            log.exception("failure running job %d" % job_wrapper.job_id)
            return

        # This is the job's exit code, which will depend on the tasks'
        # exit code. The overall job's exit code will be one of two values: 
        # o if the job is successful, then the last task scanned will be
        #   used to determine the exit code. Note that this is not the same
        #   thing as the last task to complete, which could be added later.
        # o if a task fails, then the job will fail and the failing task's
        #   exit code will become the job's exit code.
        job_exit_code = None

        # If we were able to get a command line, run the job.  ( must be passed to tasks )
        if command_line:
            try:
                job_wrapper.change_state( model.Job.states.RUNNING )
                self.sa_session.flush()
                # Split with the defined method.
                parallelism = job_wrapper.get_parallelism()
                try:
                    splitter = getattr(__import__('galaxy.jobs.splitters',  globals(),  locals(),  [parallelism.method]),  parallelism.method)
                except:
                    job_wrapper.change_state( model.Job.states.ERROR )
                    job_wrapper.fail("Job Splitting Failed, no match for '%s'" % parallelism)
                    return
                tasks = splitter.do_split(job_wrapper)
                # Not an option for now.  Task objects don't *do* anything 
                # useful yet, but we'll want them tracked outside this thread 
                # to do anything.
                # if track_tasks_in_database:
                task_wrappers = []
                for task in tasks:
                    self.sa_session.add(task)
                self.sa_session.flush()
                # Must flush prior to the creation and queueing of task wrappers.
                for task in tasks:
                    tw = TaskWrapper(task, job_wrapper.queue)
                    task_wrappers.append(tw)
                    self.app.job_manager.job_handler.dispatcher.put(tw)
                tasks_complete = False
                count_complete = 0
                sleep_time = 1
                # sleep/loop until no more progress can be made. That is when
                # all tasks are one of { OK, ERROR, DELETED }. If a task  
                completed_states = [ model.Task.states.OK, \
                                     model.Task.states.ERROR, \
                                     model.Task.states.DELETED ]

                # TODO: Should we report an error (and not merge outputs) if 
                # one of the subtasks errored out?  Should we prevent any that 
                # are pending from being started in that case?
                # SM: I'm 
                # If any task has an error, then we will stop all of them 
                # immediately. Tasks that are in the QUEUED state will be
                # moved to the DELETED state. The task's runner should 
                # ignore tasks that are not in the QUEUED state. 
                # Deleted tasks are not included right now.
                # 
                while tasks_complete is False:
                    count_complete = 0
                    tasks_complete = True
                    for tw in task_wrappers:
                        task_state = tw.get_state()
                        if ( model.Task.states.ERROR == task_state ):
                            job_exit_code = tw.get_exit_code()
                            log.debug( "Canceling job %d: Task %s returned an error"
                                     % ( tw.job_id, tw.task_id ) )
                            self.cancel_job( job_wrapper, task_wrappers )
                            tasks_complete = True
                            break
                        elif not task_state in completed_states:
                            tasks_complete = False
                        else:
                            job_exit_code = tw.get_exit_code()
                            count_complete = count_complete + 1
                    if tasks_complete is False:
                        sleep( sleep_time )
                        if sleep_time < 8:
                            sleep_time *= 2
                import time
                job_wrapper.reclaim_ownership()      # if running as the actual user, change ownership before merging.
                log.debug('execution finished - beginning merge: %s' % command_line)
                stdout,  stderr = splitter.do_merge(job_wrapper,  task_wrappers)
            except Exception:
                job_wrapper.fail( "failure running job", exception=True )
                log.exception("failure running job %d" % job_wrapper.job_id)
                return

        #run the metadata setting script here
        #this is terminate-able when output dataset/job is deleted
        #so that long running set_meta()s can be canceled without having to reboot the server
        if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally and job_wrapper.output_paths:
            external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(),
                                                                            set_extension = True,
                                                                            kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
            log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) )
            external_metadata_proc = subprocess.Popen( args = external_metadata_script,
                                         shell = True,
                                         env = os.environ,
                                         preexec_fn = os.setpgrp )
            job_wrapper.external_output_metadata.set_job_runner_external_pid( external_metadata_proc.pid, self.sa_session )
            external_metadata_proc.wait()
            log.debug( 'execution of external set_meta finished for job %d' % job_wrapper.job_id )

        # Finish the job
        try:
            job_wrapper.finish( stdout, stderr, job_exit_code )
        except:
            log.exception("Job wrapper finish method failed")
            job_wrapper.fail("Unable to finish job", exception=True)
Esempio n. 6
0
    def run_job(self, job_wrapper):
        job_wrapper.set_runner('tasks:///', None)
        stderr = stdout = command_line = ''
        # Prepare the job to run
        try:
            job_wrapper.prepare()
            command_line = job_wrapper.get_command_line()
        except:
            job_wrapper.fail("failure preparing job", exception=True)
            log.exception("failure running job %d" % job_wrapper.job_id)
            return
        # If we were able to get a command line, run the job.  ( must be passed to tasks )
        if command_line:
            try:
                # DBTODO read tool info and use the right kind of parallelism.
                # For now, the only splitter is the 'basic' one
                job_wrapper.change_state(model.Job.states.RUNNING)
                self.sa_session.flush()
                # Split with the tool-defined method.
                try:
                    splitter = getattr(
                        __import__('galaxy.jobs.splitters', globals(),
                                   locals(),
                                   [job_wrapper.tool.parallelism.method]),
                        job_wrapper.tool.parallelism.method)
                except:
                    job_wrapper.change_state(model.Job.states.ERROR)
                    job_wrapper.fail(
                        "Job Splitting Failed, no match for '%s'" %
                        job_wrapper.tool.parallelism)
                    return
                tasks = splitter.do_split(job_wrapper)

                # Not an option for now.  Task objects don't *do* anything useful yet, but we'll want them tracked outside this thread to do anything.
                # if track_tasks_in_database:
                task_wrappers = []
                for task in tasks:
                    self.sa_session.add(task)
                self.sa_session.flush()

                # Must flush prior to the creation and queueing of task wrappers.
                for task in tasks:
                    tw = TaskWrapper(task, job_wrapper.queue)
                    task_wrappers.append(tw)
                    self.app.job_manager.job_handler.dispatcher.put(tw)
                tasks_incomplete = False
                count_complete = 0
                sleep_time = 1
                # sleep/loop until no more progress can be made. That is when
                # all tasks are one of { OK, ERROR, DELETED }
                completed_states = [ model.Task.states.OK, \
                                    model.Task.states.ERROR, \
                                    model.Task.states.DELETED ]
                # TODO: Should we report an error (and not merge outputs) if one of the subtasks errored out?
                # Should we prevent any that are pending from being started in that case?
                while tasks_incomplete is False:
                    count_complete = 0
                    tasks_incomplete = True
                    for tw in task_wrappers:
                        task_state = tw.get_state()
                        if not task_state in completed_states:
                            tasks_incomplete = False
                        else:
                            count_complete = count_complete + 1
                    if tasks_incomplete is False:
                        # log.debug('Tasks complete: %s. Sleeping %s' % (count_complete, sleep_time))
                        sleep(sleep_time)
                        if sleep_time < 8:
                            sleep_time *= 2

                import time

                job_wrapper.reclaim_ownership(
                )  # if running as the actual user, change ownership before merging.

                log.debug('execution finished - beginning merge: %s' %
                          command_line)
                stdout, stderr = splitter.do_merge(job_wrapper, task_wrappers)

            except Exception:
                job_wrapper.fail("failure running job", exception=True)
                log.exception("failure running job %d" % job_wrapper.job_id)
                return

        #run the metadata setting script here
        #this is terminate-able when output dataset/job is deleted
        #so that long running set_meta()s can be canceled without having to reboot the server
        if job_wrapper.get_state() not in [
                model.Job.states.ERROR, model.Job.states.DELETED
        ] and self.app.config.set_metadata_externally and job_wrapper.output_paths:
            external_metadata_script = job_wrapper.setup_external_metadata(
                output_fnames=job_wrapper.get_output_fnames(),
                set_extension=True,
                kwds={'overwrite': False}
            )  #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
            log.debug('executing external set_meta script for job %d: %s' %
                      (job_wrapper.job_id, external_metadata_script))
            external_metadata_proc = subprocess.Popen(
                args=external_metadata_script,
                shell=True,
                env=os.environ,
                preexec_fn=os.setpgrp)
            job_wrapper.external_output_metadata.set_job_runner_external_pid(
                external_metadata_proc.pid, self.sa_session)
            external_metadata_proc.wait()
            log.debug('execution of external set_meta finished for job %d' %
                      job_wrapper.job_id)

        # Finish the job
        try:
            job_wrapper.finish(stdout, stderr)
        except:
            log.exception("Job wrapper finish method failed")
            job_wrapper.fail("Unable to finish job", exception=True)
Esempio n. 7
0
    def queue_job(self, job_wrapper):
        # prepare the job
        if not self.prepare_job(job_wrapper):
            return

        # command line has been added to the wrapper by prepare_job()
        command_line = job_wrapper.runner_command_line

        stderr = stdout = ''

        # Persist the destination
        job_wrapper.set_job_destination(job_wrapper.job_destination)

        # This is the job's exit code, which will depend on the tasks'
        # exit code. The overall job's exit code will be one of two values:
        # o if the job is successful, then the last task scanned will be
        #   used to determine the exit code. Note that this is not the same
        #   thing as the last task to complete, which could be added later.
        # o if a task fails, then the job will fail and the failing task's
        #   exit code will become the job's exit code.
        job_exit_code = None

        try:
            job_wrapper.change_state(model.Job.states.RUNNING)
            self.sa_session.flush()
            # Split with the defined method.
            parallelism = job_wrapper.get_parallelism()
            try:
                splitter = getattr(
                    __import__('galaxy.jobs.splitters', globals(), locals(),
                               [parallelism.method]), parallelism.method)
            except:
                job_wrapper.change_state(model.Job.states.ERROR)
                job_wrapper.fail("Job Splitting Failed, no match for '%s'" %
                                 parallelism)
                return
            tasks = splitter.do_split(job_wrapper)
            # Not an option for now.  Task objects don't *do* anything
            # useful yet, but we'll want them tracked outside this thread
            # to do anything.
            # if track_tasks_in_database:
            task_wrappers = []
            for task in tasks:
                self.sa_session.add(task)
            self.sa_session.flush()
            # Must flush prior to the creation and queueing of task wrappers.
            for task in tasks:
                tw = TaskWrapper(task, job_wrapper.queue)
                task_wrappers.append(tw)
                self.app.job_manager.job_handler.dispatcher.put(tw)
            tasks_complete = False
            count_complete = 0
            sleep_time = 1
            # sleep/loop until no more progress can be made. That is when
            # all tasks are one of { OK, ERROR, DELETED }. If a task
            completed_states = [
                model.Task.states.OK, model.Task.states.ERROR,
                model.Task.states.DELETED
            ]

            # TODO: Should we report an error (and not merge outputs) if
            # one of the subtasks errored out?  Should we prevent any that
            # are pending from being started in that case?
            # SM: I'm
            # If any task has an error, then we will stop all of them
            # immediately. Tasks that are in the QUEUED state will be
            # moved to the DELETED state. The task's runner should
            # ignore tasks that are not in the QUEUED state.
            # Deleted tasks are not included right now.
            #
            while tasks_complete is False:
                count_complete = 0
                tasks_complete = True
                for tw in task_wrappers:
                    task_state = tw.get_state()
                    if (model.Task.states.ERROR == task_state):
                        job_exit_code = tw.get_exit_code()
                        log.debug(
                            "Canceling job %d: Task %s returned an error" %
                            (tw.job_id, tw.task_id))
                        self._cancel_job(job_wrapper, task_wrappers)
                        tasks_complete = True
                        break
                    elif task_state not in completed_states:
                        tasks_complete = False
                    else:
                        job_exit_code = tw.get_exit_code()
                        count_complete = count_complete + 1
                if tasks_complete is False:
                    sleep(sleep_time)
                    if sleep_time < 8:
                        sleep_time *= 2
            job_wrapper.reclaim_ownership(
            )  # if running as the actual user, change ownership before merging.
            log.debug('execution finished - beginning merge: %s' %
                      command_line)
            stdout, stderr = splitter.do_merge(job_wrapper, task_wrappers)
        except Exception:
            job_wrapper.fail("failure running job", exception=True)
            log.exception("failure running job %d", job_wrapper.job_id)
            return

        # run the metadata setting script here
        # this is terminate-able when output dataset/job is deleted
        # so that long running set_meta()s can be canceled without having to reboot the server
        self._handle_metadata_externally(job_wrapper,
                                         resolve_requirements=True)
        # Finish the job
        try:
            job_wrapper.finish(stdout, stderr, job_exit_code)
        except:
            log.exception("Job wrapper finish method failed")
            job_wrapper.fail("Unable to finish job", exception=True)