def queue_job( self, job_wrapper ):
# prepare the job
if not self.prepare_job( job_wrapper ):
return
# command line has been added to the wrapper by prepare_job()
command_line = job_wrapper.runner_command_line
stderr = stdout = ''
# Persist the destination
job_wrapper.set_job_destination(job_wrapper.job_destination)
# This is the job's exit code, which will depend on the tasks'
# exit code. The overall job's exit code will be one of two values:
# o if the job is successful, then the last task scanned will be
# used to determine the exit code. Note that this is not the same
# thing as the last task to complete, which could be added later.
# o if a task fails, then the job will fail and the failing task's
# exit code will become the job's exit code.
job_exit_code = None
try:
job_wrapper.change_state( model.Job.states.RUNNING )
self.sa_session.flush()
# Split with the defined method.
parallelism = job_wrapper.get_parallelism()
try:
splitter = getattr(__import__('galaxy.jobs.splitters', globals(), locals(), [parallelism.method]), parallelism.method)
except:
job_wrapper.change_state( model.Job.states.ERROR )
job_wrapper.fail("Job Splitting Failed, no match for '%s'" % parallelism)
return
tasks = splitter.do_split(job_wrapper)
# Not an option for now. Task objects don't *do* anything
# useful yet, but we'll want them tracked outside this thread
# to do anything.
# if track_tasks_in_database:
task_wrappers = []
for task in tasks:
self.sa_session.add(task)
self.sa_session.flush()
# Must flush prior to the creation and queueing of task wrappers.
for task in tasks:
tw = TaskWrapper(task, job_wrapper.queue)
task_wrappers.append(tw)
self.app.job_manager.job_handler.dispatcher.put(tw)
tasks_complete = False
count_complete = 0
sleep_time = 1
# sleep/loop until no more progress can be made. That is when
# all tasks are one of { OK, ERROR, DELETED }. If a task
completed_states = [ model.Task.states.OK,
model.Task.states.ERROR,
model.Task.states.DELETED ]
# TODO: Should we report an error (and not merge outputs) if
# one of the subtasks errored out? Should we prevent any that
# are pending from being started in that case?
# SM: I'm
# If any task has an error, then we will stop all of them
# immediately. Tasks that are in the QUEUED state will be
# moved to the DELETED state. The task's runner should
# ignore tasks that are not in the QUEUED state.
# Deleted tasks are not included right now.
#
while tasks_complete is False:
count_complete = 0
tasks_complete = True
for tw in task_wrappers:
task_state = tw.get_state()
if ( model.Task.states.ERROR == task_state ):
job_exit_code = tw.get_exit_code()
log.debug( "Canceling job %d: Task %s returned an error"
% ( tw.job_id, tw.task_id ) )
self._cancel_job( job_wrapper, task_wrappers )
tasks_complete = True
break
elif task_state not in completed_states:
tasks_complete = False
else:
job_exit_code = tw.get_exit_code()
count_complete = count_complete + 1
if tasks_complete is False:
sleep( sleep_time )
if sleep_time < 8:
sleep_time *= 2
job_wrapper.reclaim_ownership() # if running as the actual user, change ownership before merging.
log.debug('execution finished - beginning merge: %s' % command_line)
stdout, stderr = splitter.do_merge(job_wrapper, task_wrappers)
except Exception:
job_wrapper.fail( "failure running job", exception=True )
log.exception("failure running job %d" % job_wrapper.job_id)
return
# run the metadata setting script here
# this is terminate-able when output dataset/job is deleted
# so that long running set_meta()s can be canceled without having to reboot the server
self._handle_metadata_externally(job_wrapper, resolve_requirements=True )
# Finish the job
try:
job_wrapper.finish( stdout, stderr, job_exit_code )
except:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
def _wrapper(self):
return TaskWrapper(self.task, self.queue)
def run_job(self, job_wrapper):
job_wrapper.set_runner('tasks:///', None)
stderr = stdout = command_line = ''
# Prepare the job to run
try:
job_wrapper.prepare()
command_line = job_wrapper.get_command_line()
except:
job_wrapper.fail("failure preparing job", exception=True)
log.exception("failure running job %d" % job_wrapper.job_id)
return
# This is the job's exit code, which will depend on the tasks'
# exit code. The overall job's exit code will be one of two values:
# o if the job is successful, then the last task scanned will be
# used to determine the exit code. Note that this is not the same
# thing as the last task to complete, which could be added later.
# o if a task fails, then the job will fail and the failing task's
# exit code will become the job's exit code.
job_exit_code = None
# If we were able to get a command line, run the job. ( must be passed to tasks )
if command_line:
try:
job_wrapper.change_state(model.Job.states.RUNNING)
self.sa_session.flush()
# Split with the defined method.
parallelism = job_wrapper.get_parallelism()
try:
splitter = getattr(
__import__('galaxy.jobs.splitters', globals(),
locals(), [parallelism.method]),
parallelism.method)
except:
job_wrapper.change_state(model.Job.states.ERROR)
job_wrapper.fail(
"Job Splitting Failed, no match for '%s'" %
parallelism)
return
tasks = splitter.do_split(job_wrapper)
# Not an option for now. Task objects don't *do* anything
# useful yet, but we'll want them tracked outside this thread
# to do anything.
# if track_tasks_in_database:
task_wrappers = []
for task in tasks:
self.sa_session.add(task)
self.sa_session.flush()
# Must flush prior to the creation and queueing of task wrappers.
for task in tasks:
tw = TaskWrapper(task, job_wrapper.queue)
task_wrappers.append(tw)
self.app.job_manager.job_handler.dispatcher.put(tw)
tasks_complete = False
count_complete = 0
sleep_time = 1
# sleep/loop until no more progress can be made. That is when
# all tasks are one of { OK, ERROR, DELETED }. If a task
completed_states = [ model.Task.states.OK, \
model.Task.states.ERROR, \
model.Task.states.DELETED ]
# TODO: Should we report an error (and not merge outputs) if
# one of the subtasks errored out? Should we prevent any that
# are pending from being started in that case?
# SM: I'm
# If any task has an error, then we will stop all of them
# immediately. Tasks that are in the QUEUED state will be
# moved to the DELETED state. The task's runner should
# ignore tasks that are not in the QUEUED state.
# Deleted tasks are not included right now.
#
while tasks_complete is False:
count_complete = 0
tasks_complete = True
for tw in task_wrappers:
task_state = tw.get_state()
if (model.Task.states.ERROR == task_state):
job_exit_code = tw.get_exit_code()
log.debug(
"Canceling job %d: Task %s returned an error" %
(tw.job_id, tw.task_id))
self.cancel_job(job_wrapper, task_wrappers)
tasks_complete = True
break
elif not task_state in completed_states:
tasks_complete = False
else:
job_exit_code = tw.get_exit_code()
count_complete = count_complete + 1
if tasks_complete is False:
sleep(sleep_time)
if sleep_time < 8:
sleep_time *= 2
import time
job_wrapper.reclaim_ownership(
) # if running as the actual user, change ownership before merging.
log.debug('execution finished - beginning merge: %s' %
command_line)
stdout, stderr = splitter.do_merge(job_wrapper, task_wrappers)
except Exception:
job_wrapper.fail("failure running job", exception=True)
log.exception("failure running job %d" % job_wrapper.job_id)
return
#run the metadata setting script here
#this is terminate-able when output dataset/job is deleted
#so that long running set_meta()s can be canceled without having to reboot the server
if job_wrapper.get_state() not in [
model.Job.states.ERROR, model.Job.states.DELETED
] and self.app.config.set_metadata_externally and job_wrapper.output_paths:
external_metadata_script = job_wrapper.setup_external_metadata(
output_fnames=job_wrapper.get_output_fnames(),
set_extension=True,
kwds={'overwrite': False}
) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
log.debug('executing external set_meta script for job %d: %s' %
(job_wrapper.job_id, external_metadata_script))
external_metadata_proc = subprocess.Popen(
args=external_metadata_script,
shell=True,
env=os.environ,
preexec_fn=os.setpgrp)
job_wrapper.external_output_metadata.set_job_runner_external_pid(
external_metadata_proc.pid, self.sa_session)
external_metadata_proc.wait()
log.debug('execution of external set_meta finished for job %d' %
job_wrapper.job_id)
# Finish the job
try:
job_wrapper.finish(stdout, stderr, job_exit_code)
except:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
def run_job( self, job_wrapper ):
job_wrapper.set_runner( 'tasks:///', None )
stderr = stdout = command_line = ''
# Prepare the job to run
try:
job_wrapper.prepare()
command_line = job_wrapper.get_command_line()
except:
job_wrapper.fail( "failure preparing job", exception=True )
log.exception("failure running job %d" % job_wrapper.job_id)
return
# If we were able to get a command line, run the job. ( must be passed to tasks )
if command_line:
try:
# DBTODO read tool info and use the right kind of parallelism.
# For now, the only splitter is the 'basic' one
job_wrapper.change_state( model.Job.states.RUNNING )
self.sa_session.flush()
# Split with the tool-defined method.
try:
splitter = getattr(__import__('galaxy.jobs.splitters', globals(), locals(), [job_wrapper.tool.parallelism.method]), job_wrapper.tool.parallelism.method)
except:
job_wrapper.change_state( model.Job.states.ERROR )
job_wrapper.fail("Job Splitting Failed, no match for '%s'" % job_wrapper.tool.parallelism)
return
tasks = splitter.do_split(job_wrapper)
# Not an option for now. Task objects don't *do* anything useful yet, but we'll want them tracked outside this thread to do anything.
# if track_tasks_in_database:
task_wrappers = []
for task in tasks:
self.sa_session.add(task)
self.sa_session.flush()
# Must flush prior to the creation and queueing of task wrappers.
for task in tasks:
tw = TaskWrapper(task, job_wrapper.queue)
task_wrappers.append(tw)
self.app.job_manager.job_handler.dispatcher.put(tw)
tasks_incomplete = False
count_complete = 0
sleep_time = 1
# sleep/loop until no more progress can be made. That is when
# all tasks are one of { OK, ERROR, DELETED }
completed_states = [ model.Task.states.OK, \
model.Task.states.ERROR, \
model.Task.states.DELETED ]
# TODO: Should we report an error (and not merge outputs) if one of the subtasks errored out?
# Should we prevent any that are pending from being started in that case?
while tasks_incomplete is False:
count_complete = 0
tasks_incomplete = True
for tw in task_wrappers:
task_state = tw.get_state()
if not task_state in completed_states:
tasks_incomplete = False
else:
count_complete = count_complete + 1
if tasks_incomplete is False:
# log.debug('Tasks complete: %s. Sleeping %s' % (count_complete, sleep_time))
sleep( sleep_time )
if sleep_time < 8:
sleep_time *= 2
import time
job_wrapper.reclaim_ownership() # if running as the actual user, change ownership before merging.
log.debug('execution finished - beginning merge: %s' % command_line)
stdout, stderr = splitter.do_merge(job_wrapper, task_wrappers)
except Exception:
job_wrapper.fail( "failure running job", exception=True )
log.exception("failure running job %d" % job_wrapper.job_id)
return
#run the metadata setting script here
#this is terminate-able when output dataset/job is deleted
#so that long running set_meta()s can be canceled without having to reboot the server
if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally and job_wrapper.output_paths:
external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(),
set_extension = True,
kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) )
external_metadata_proc = subprocess.Popen( args = external_metadata_script,
shell = True,
env = os.environ,
preexec_fn = os.setpgrp )
job_wrapper.external_output_metadata.set_job_runner_external_pid( external_metadata_proc.pid, self.sa_session )
external_metadata_proc.wait()
log.debug( 'execution of external set_meta finished for job %d' % job_wrapper.job_id )
# Finish the job
try:
job_wrapper.finish( stdout, stderr )
except:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
def run_job( self, job_wrapper ):
job_wrapper.set_runner( 'tasks:///', None )
stderr = stdout = command_line = ''
# Prepare the job to run
try:
job_wrapper.prepare()
command_line = job_wrapper.get_command_line()
except:
job_wrapper.fail( "failure preparing job", exception=True )
log.exception("failure running job %d" % job_wrapper.job_id)
return
# This is the job's exit code, which will depend on the tasks'
# exit code. The overall job's exit code will be one of two values:
# o if the job is successful, then the last task scanned will be
# used to determine the exit code. Note that this is not the same
# thing as the last task to complete, which could be added later.
# o if a task fails, then the job will fail and the failing task's
# exit code will become the job's exit code.
job_exit_code = None
# If we were able to get a command line, run the job. ( must be passed to tasks )
if command_line:
try:
job_wrapper.change_state( model.Job.states.RUNNING )
self.sa_session.flush()
# Split with the defined method.
parallelism = job_wrapper.get_parallelism()
try:
splitter = getattr(__import__('galaxy.jobs.splitters', globals(), locals(), [parallelism.method]), parallelism.method)
except:
job_wrapper.change_state( model.Job.states.ERROR )
job_wrapper.fail("Job Splitting Failed, no match for '%s'" % parallelism)
return
tasks = splitter.do_split(job_wrapper)
# Not an option for now. Task objects don't *do* anything
# useful yet, but we'll want them tracked outside this thread
# to do anything.
# if track_tasks_in_database:
task_wrappers = []
for task in tasks:
self.sa_session.add(task)
self.sa_session.flush()
# Must flush prior to the creation and queueing of task wrappers.
for task in tasks:
tw = TaskWrapper(task, job_wrapper.queue)
task_wrappers.append(tw)
self.app.job_manager.job_handler.dispatcher.put(tw)
tasks_complete = False
count_complete = 0
sleep_time = 1
# sleep/loop until no more progress can be made. That is when
# all tasks are one of { OK, ERROR, DELETED }. If a task
completed_states = [ model.Task.states.OK, \
model.Task.states.ERROR, \
model.Task.states.DELETED ]
# TODO: Should we report an error (and not merge outputs) if
# one of the subtasks errored out? Should we prevent any that
# are pending from being started in that case?
# SM: I'm
# If any task has an error, then we will stop all of them
# immediately. Tasks that are in the QUEUED state will be
# moved to the DELETED state. The task's runner should
# ignore tasks that are not in the QUEUED state.
# Deleted tasks are not included right now.
#
while tasks_complete is False:
count_complete = 0
tasks_complete = True
for tw in task_wrappers:
task_state = tw.get_state()
if ( model.Task.states.ERROR == task_state ):
job_exit_code = tw.get_exit_code()
log.debug( "Canceling job %d: Task %s returned an error"
% ( tw.job_id, tw.task_id ) )
self.cancel_job( job_wrapper, task_wrappers )
tasks_complete = True
break
elif not task_state in completed_states:
tasks_complete = False
else:
job_exit_code = tw.get_exit_code()
count_complete = count_complete + 1
if tasks_complete is False:
sleep( sleep_time )
if sleep_time < 8:
sleep_time *= 2
import time
job_wrapper.reclaim_ownership() # if running as the actual user, change ownership before merging.
log.debug('execution finished - beginning merge: %s' % command_line)
stdout, stderr = splitter.do_merge(job_wrapper, task_wrappers)
except Exception:
job_wrapper.fail( "failure running job", exception=True )
log.exception("failure running job %d" % job_wrapper.job_id)
return
#run the metadata setting script here
#this is terminate-able when output dataset/job is deleted
#so that long running set_meta()s can be canceled without having to reboot the server
if job_wrapper.get_state() not in [ model.Job.states.ERROR, model.Job.states.DELETED ] and self.app.config.set_metadata_externally and job_wrapper.output_paths:
external_metadata_script = job_wrapper.setup_external_metadata( output_fnames = job_wrapper.get_output_fnames(),
set_extension = True,
kwds = { 'overwrite' : False } ) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
log.debug( 'executing external set_meta script for job %d: %s' % ( job_wrapper.job_id, external_metadata_script ) )
external_metadata_proc = subprocess.Popen( args = external_metadata_script,
shell = True,
env = os.environ,
preexec_fn = os.setpgrp )
job_wrapper.external_output_metadata.set_job_runner_external_pid( external_metadata_proc.pid, self.sa_session )
external_metadata_proc.wait()
log.debug( 'execution of external set_meta finished for job %d' % job_wrapper.job_id )
# Finish the job
try:
job_wrapper.finish( stdout, stderr, job_exit_code )
except:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
def run_job(self, job_wrapper):
job_wrapper.set_runner('tasks:///', None)
stderr = stdout = command_line = ''
# Prepare the job to run
try:
job_wrapper.prepare()
command_line = job_wrapper.get_command_line()
except:
job_wrapper.fail("failure preparing job", exception=True)
log.exception("failure running job %d" % job_wrapper.job_id)
return
# If we were able to get a command line, run the job. ( must be passed to tasks )
if command_line:
try:
# DBTODO read tool info and use the right kind of parallelism.
# For now, the only splitter is the 'basic' one
job_wrapper.change_state(model.Job.states.RUNNING)
self.sa_session.flush()
# Split with the tool-defined method.
try:
splitter = getattr(
__import__('galaxy.jobs.splitters', globals(),
locals(),
[job_wrapper.tool.parallelism.method]),
job_wrapper.tool.parallelism.method)
except:
job_wrapper.change_state(model.Job.states.ERROR)
job_wrapper.fail(
"Job Splitting Failed, no match for '%s'" %
job_wrapper.tool.parallelism)
return
tasks = splitter.do_split(job_wrapper)
# Not an option for now. Task objects don't *do* anything useful yet, but we'll want them tracked outside this thread to do anything.
# if track_tasks_in_database:
task_wrappers = []
for task in tasks:
self.sa_session.add(task)
self.sa_session.flush()
# Must flush prior to the creation and queueing of task wrappers.
for task in tasks:
tw = TaskWrapper(task, job_wrapper.queue)
task_wrappers.append(tw)
self.app.job_manager.job_handler.dispatcher.put(tw)
tasks_incomplete = False
count_complete = 0
sleep_time = 1
# sleep/loop until no more progress can be made. That is when
# all tasks are one of { OK, ERROR, DELETED }
completed_states = [ model.Task.states.OK, \
model.Task.states.ERROR, \
model.Task.states.DELETED ]
# TODO: Should we report an error (and not merge outputs) if one of the subtasks errored out?
# Should we prevent any that are pending from being started in that case?
while tasks_incomplete is False:
count_complete = 0
tasks_incomplete = True
for tw in task_wrappers:
task_state = tw.get_state()
if not task_state in completed_states:
tasks_incomplete = False
else:
count_complete = count_complete + 1
if tasks_incomplete is False:
# log.debug('Tasks complete: %s. Sleeping %s' % (count_complete, sleep_time))
sleep(sleep_time)
if sleep_time < 8:
sleep_time *= 2
import time
job_wrapper.reclaim_ownership(
) # if running as the actual user, change ownership before merging.
log.debug('execution finished - beginning merge: %s' %
command_line)
stdout, stderr = splitter.do_merge(job_wrapper, task_wrappers)
except Exception:
job_wrapper.fail("failure running job", exception=True)
log.exception("failure running job %d" % job_wrapper.job_id)
return
#run the metadata setting script here
#this is terminate-able when output dataset/job is deleted
#so that long running set_meta()s can be canceled without having to reboot the server
if job_wrapper.get_state() not in [
model.Job.states.ERROR, model.Job.states.DELETED
] and self.app.config.set_metadata_externally and job_wrapper.output_paths:
external_metadata_script = job_wrapper.setup_external_metadata(
output_fnames=job_wrapper.get_output_fnames(),
set_extension=True,
kwds={'overwrite': False}
) #we don't want to overwrite metadata that was copied over in init_meta(), as per established behavior
log.debug('executing external set_meta script for job %d: %s' %
(job_wrapper.job_id, external_metadata_script))
external_metadata_proc = subprocess.Popen(
args=external_metadata_script,
shell=True,
env=os.environ,
preexec_fn=os.setpgrp)
job_wrapper.external_output_metadata.set_job_runner_external_pid(
external_metadata_proc.pid, self.sa_session)
external_metadata_proc.wait()
log.debug('execution of external set_meta finished for job %d' %
job_wrapper.job_id)
# Finish the job
try:
job_wrapper.finish(stdout, stderr)
except:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
def queue_job(self, job_wrapper):
# prepare the job
if not self.prepare_job(job_wrapper):
return
# command line has been added to the wrapper by prepare_job()
command_line = job_wrapper.runner_command_line
stderr = stdout = ''
# Persist the destination
job_wrapper.set_job_destination(job_wrapper.job_destination)
# This is the job's exit code, which will depend on the tasks'
# exit code. The overall job's exit code will be one of two values:
# o if the job is successful, then the last task scanned will be
# used to determine the exit code. Note that this is not the same
# thing as the last task to complete, which could be added later.
# o if a task fails, then the job will fail and the failing task's
# exit code will become the job's exit code.
job_exit_code = None
try:
job_wrapper.change_state(model.Job.states.RUNNING)
self.sa_session.flush()
# Split with the defined method.
parallelism = job_wrapper.get_parallelism()
try:
splitter = getattr(
__import__('galaxy.jobs.splitters', globals(), locals(),
[parallelism.method]), parallelism.method)
except:
job_wrapper.change_state(model.Job.states.ERROR)
job_wrapper.fail("Job Splitting Failed, no match for '%s'" %
parallelism)
return
tasks = splitter.do_split(job_wrapper)
# Not an option for now. Task objects don't *do* anything
# useful yet, but we'll want them tracked outside this thread
# to do anything.
# if track_tasks_in_database:
task_wrappers = []
for task in tasks:
self.sa_session.add(task)
self.sa_session.flush()
# Must flush prior to the creation and queueing of task wrappers.
for task in tasks:
tw = TaskWrapper(task, job_wrapper.queue)
task_wrappers.append(tw)
self.app.job_manager.job_handler.dispatcher.put(tw)
tasks_complete = False
count_complete = 0
sleep_time = 1
# sleep/loop until no more progress can be made. That is when
# all tasks are one of { OK, ERROR, DELETED }. If a task
completed_states = [
model.Task.states.OK, model.Task.states.ERROR,
model.Task.states.DELETED
]
# TODO: Should we report an error (and not merge outputs) if
# one of the subtasks errored out? Should we prevent any that
# are pending from being started in that case?
# SM: I'm
# If any task has an error, then we will stop all of them
# immediately. Tasks that are in the QUEUED state will be
# moved to the DELETED state. The task's runner should
# ignore tasks that are not in the QUEUED state.
# Deleted tasks are not included right now.
#
while tasks_complete is False:
count_complete = 0
tasks_complete = True
for tw in task_wrappers:
task_state = tw.get_state()
if (model.Task.states.ERROR == task_state):
job_exit_code = tw.get_exit_code()
log.debug(
"Canceling job %d: Task %s returned an error" %
(tw.job_id, tw.task_id))
self._cancel_job(job_wrapper, task_wrappers)
tasks_complete = True
break
elif task_state not in completed_states:
tasks_complete = False
else:
job_exit_code = tw.get_exit_code()
count_complete = count_complete + 1
if tasks_complete is False:
sleep(sleep_time)
if sleep_time < 8:
sleep_time *= 2
job_wrapper.reclaim_ownership(
) # if running as the actual user, change ownership before merging.
log.debug('execution finished - beginning merge: %s' %
command_line)
stdout, stderr = splitter.do_merge(job_wrapper, task_wrappers)
except Exception:
job_wrapper.fail("failure running job", exception=True)
log.exception("failure running job %d", job_wrapper.job_id)
return
# run the metadata setting script here
# this is terminate-able when output dataset/job is deleted
# so that long running set_meta()s can be canceled without having to reboot the server
self._handle_metadata_externally(job_wrapper,
resolve_requirements=True)
# Finish the job
try:
job_wrapper.finish(stdout, stderr, job_exit_code)
except:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)