def finish_job(self, job_state):
"""
Get the output/error for a finished job, pass to `job_wrapper.finish`
and cleanup all the job's temporary files.
"""
galaxy_id_tag = job_state.job_wrapper.get_id_tag()
external_job_id = job_state.job_id
# To ensure that files below are readable, ownership must be reclaimed first
job_state.job_wrapper.reclaim_ownership()
# wait for the files to appear
which_try = 0
collect_output_success = True
while which_try < self.app.config.retry_job_output_collection + 1:
try:
with open(job_state.output_file, "rb") as stdout_file, open(job_state.error_file, 'rb') as stderr_file:
stdout = shrink_stream_by_size(stdout_file, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True)
stderr = shrink_stream_by_size(stderr_file, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True)
break
except Exception as e:
if which_try == self.app.config.retry_job_output_collection:
stdout = ''
stderr = job_state.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER
log.error('(%s/%s) %s: %s' % (galaxy_id_tag, external_job_id, stderr, str(e)))
collect_output_success = False
else:
time.sleep(1)
which_try += 1
if not collect_output_success:
job_state.fail_message = stderr
job_state.runner_state = job_state.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER
self.mark_as_failed(job_state)
return
try:
# This should be an 8-bit exit code, but read ahead anyway:
exit_code_str = open(job_state.exit_code_file, "r").read(32)
except Exception:
# By default, the exit code is 0, which typically indicates success.
exit_code_str = "0"
try:
# Decode the exit code. If it's bogus, then just use 0.
exit_code = int(exit_code_str)
except ValueError:
log.warning("(%s/%s) Exit code '%s' invalid. Using 0." % (galaxy_id_tag, external_job_id, exit_code_str))
exit_code = 0
# clean up the job files
cleanup_job = job_state.job_wrapper.cleanup_job
if cleanup_job == "always" or (not stderr and cleanup_job == "onsuccess"):
job_state.cleanup()
try:
self._finish_or_resubmit_job(job_state, stdout, stderr, exit_code)
except Exception:
log.exception("(%s/%s) Job wrapper finish method failed" % (galaxy_id_tag, external_job_id))
job_state.job_wrapper.fail("Unable to finish job", exception=True)
def queue_job( self, job_wrapper ):
# prepare the job
if not self.prepare_job( job_wrapper ):
return
stderr = stdout = ''
exit_code = 0
# command line has been added to the wrapper by prepare_job()
command_line = self.__command_line( job_wrapper )
job_id = job_wrapper.get_id_tag()
try:
log.debug( '(%s) executing: %s' % ( job_id, command_line ) )
stdout_file = tempfile.NamedTemporaryFile( suffix='_stdout', dir=job_wrapper.working_directory )
stderr_file = tempfile.NamedTemporaryFile( suffix='_stderr', dir=job_wrapper.working_directory )
proc = subprocess.Popen( args=command_line,
shell=True,
cwd=job_wrapper.working_directory,
stdout=stdout_file,
stderr=stderr_file,
env=self._environ,
preexec_fn=os.setpgrp )
job_wrapper.set_job_destination(job_wrapper.job_destination, proc.pid)
job_wrapper.change_state( model.Job.states.RUNNING )
job_start = datetime.datetime.now()
i = 0
# Iterate until the process exits, periodically checking its limits
while proc.poll() is None:
i += 1
if (i % 20) == 0:
msg = job_wrapper.check_limits(runtime=datetime.datetime.now() - job_start)
if msg is not None:
job_wrapper.fail(msg)
log.debug('(%s) Terminating process group' % job_id)
self._terminate(proc)
return
else:
sleep(1)
# Reap the process and get the exit code.
exit_code = proc.wait()
stdout_file.seek( 0 )
stderr_file.seek( 0 )
stdout = shrink_stream_by_size( stdout_file, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
stderr = shrink_stream_by_size( stderr_file, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
stdout_file.close()
stderr_file.close()
log.debug('execution finished: %s' % command_line)
except Exception:
job_wrapper.fail( "failure running job", exception=True )
log.exception("failure running job %d" % job_wrapper.job_id)
return
self._handle_metadata_externally( job_wrapper, resolve_requirements=True )
# Finish the job!
try:
job_wrapper.finish( stdout, stderr, exit_code )
except:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
def finish_job( self, job_state ):
"""
Get the output/error for a finished job, pass to `job_wrapper.finish`
and cleanup all the job's temporary files.
"""
galaxy_id_tag = job_state.job_wrapper.get_id_tag()
external_job_id = job_state.job_id
# To ensure that files below are readable, ownership must be reclaimed first
job_state.job_wrapper.reclaim_ownership()
# wait for the files to appear
which_try = 0
while which_try < (self.app.config.retry_job_output_collection + 1):
try:
stdout = shrink_stream_by_size( file( job_state.output_file, "r" ), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
stderr = shrink_stream_by_size( file( job_state.error_file, "r" ), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
which_try = (self.app.config.retry_job_output_collection + 1)
except Exception, e:
if which_try == self.app.config.retry_job_output_collection:
stdout = ''
stderr = 'Job output not returned from cluster'
log.error( '(%s/%s) %s: %s' % ( galaxy_id_tag, external_job_id, stderr, str( e ) ) )
else:
time.sleep(1)
which_try += 1
def finish_job( self, job_state ):
"""
Get the output/error for a finished job, pass to `job_wrapper.finish`
and cleanup all the job's temporary files.
"""
galaxy_id_tag = job_state.job_wrapper.get_id_tag()
external_job_id = job_state.job_id
# To ensure that files below are readable, ownership must be reclaimed first
job_state.job_wrapper.reclaim_ownership()
# wait for the files to appear
which_try = 0
while which_try < (self.app.config.retry_job_output_collection + 1):
try:
stdout = shrink_stream_by_size( file( job_state.output_file, "r" ), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
stderr = shrink_stream_by_size( file( job_state.error_file, "r" ), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
which_try = (self.app.config.retry_job_output_collection + 1)
except Exception, e:
if which_try == self.app.config.retry_job_output_collection:
stdout = ''
stderr = 'Job output not returned from cluster'
log.error( '(%s/%s) %s: %s' % ( galaxy_id_tag, external_job_id, stderr, str( e ) ) )
else:
time.sleep(1)
which_try += 1
def finish_job(self, job_state):
"""
Get the output/error for a finished job, pass to `job_wrapper.finish`
and cleanup all the job's temporary files.
"""
galaxy_id_tag = job_state.job_wrapper.get_id_tag()
external_job_id = job_state.job_id
# To ensure that files below are readable, ownership must be reclaimed first
job_state.job_wrapper.reclaim_ownership()
# wait for the files to appear
which_try = 0
collect_output_success = True
while which_try < self.app.config.retry_job_output_collection + 1:
try:
stdout = shrink_stream_by_size(open(job_state.output_file, "r"), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True)
stderr = shrink_stream_by_size(open(job_state.error_file, "r"), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True)
break
except Exception as e:
if which_try == self.app.config.retry_job_output_collection:
stdout = ''
stderr = job_state.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER
log.error('(%s/%s) %s: %s' % (galaxy_id_tag, external_job_id, stderr, str(e)))
collect_output_success = False
else:
time.sleep(1)
which_try += 1
if not collect_output_success:
job_state.fail_message = stderr
job_state.runner_state = job_state.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER
self.mark_as_failed(job_state)
return
try:
# This should be an 8-bit exit code, but read ahead anyway:
exit_code_str = open(job_state.exit_code_file, "r").read(32)
except Exception:
# By default, the exit code is 0, which typically indicates success.
exit_code_str = "0"
try:
# Decode the exit code. If it's bogus, then just use 0.
exit_code = int(exit_code_str)
except ValueError:
log.warning("(%s/%s) Exit code '%s' invalid. Using 0." % (galaxy_id_tag, external_job_id, exit_code_str))
exit_code = 0
# clean up the job files
cleanup_job = job_state.job_wrapper.cleanup_job
if cleanup_job == "always" or (not stderr and cleanup_job == "onsuccess"):
job_state.cleanup()
try:
self._finish_or_resubmit_job(job_state, stdout, stderr, exit_code)
except Exception:
log.exception("(%s/%s) Job wrapper finish method failed" % (galaxy_id_tag, external_job_id))
job_state.job_wrapper.fail("Unable to finish job", exception=True)
def queue_job( self, job_wrapper ):
# prepare the job
include_metadata = asbool( job_wrapper.job_destination.params.get( "embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB ) )
if not self.prepare_job( job_wrapper, include_metadata=include_metadata ):
return
stderr = stdout = ''
exit_code = 0
# command line has been added to the wrapper by prepare_job()
command_line, exit_code_path = self.__command_line( job_wrapper )
job_id = job_wrapper.get_id_tag()
try:
stdout_file = tempfile.NamedTemporaryFile( suffix='_stdout', dir=job_wrapper.working_directory )
stderr_file = tempfile.NamedTemporaryFile( suffix='_stderr', dir=job_wrapper.working_directory )
log.debug( '(%s) executing job script: %s' % ( job_id, command_line ) )
proc = subprocess.Popen( args=command_line,
shell=True,
cwd=job_wrapper.working_directory,
stdout=stdout_file,
stderr=stderr_file,
env=self._environ,
preexec_fn=os.setpgrp )
job_wrapper.set_job_destination(job_wrapper.job_destination, proc.pid)
job_wrapper.change_state( model.Job.states.RUNNING )
terminated = self.__poll_if_needed( proc, job_wrapper, job_id )
if terminated:
return
# Reap the process and get the exit code.
exit_code = proc.wait()
try:
exit_code = int( open( exit_code_path, 'r' ).read() )
except Exception:
log.warn( "Failed to read exit code from path %s" % exit_code_path )
pass
stdout_file.seek( 0 )
stderr_file.seek( 0 )
stdout = shrink_stream_by_size( stdout_file, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
stderr = shrink_stream_by_size( stderr_file, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
stdout_file.close()
stderr_file.close()
log.debug('execution finished: %s' % command_line)
except Exception:
log.exception("failure running job %d" % job_wrapper.job_id)
job_wrapper.fail( "failure running job", exception=True )
return
external_metadata = not asbool( job_wrapper.job_destination.params.get( "embed_metadata_in_job", DEFAULT_EMBED_METADATA_IN_JOB ) )
if external_metadata:
self._handle_metadata_externally( job_wrapper, resolve_requirements=True )
# Finish the job!
try:
job_wrapper.finish( stdout, stderr, exit_code )
except:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
def finish_job(self, pbs_job_state):
"""
Get the output/error for a finished job, pass to `job_wrapper.finish`
and cleanup all the PBS temporary files.
"""
ofile = pbs_job_state.output_file
efile = pbs_job_state.error_file
ecfile = pbs_job_state.exit_code_file
job_file = pbs_job_state.job_file
# collect the output
try:
ofh = file(ofile, "r")
efh = file(efile, "r")
ecfh = file(ecfile, "r")
stdout = shrink_stream_by_size(ofh,
DATABASE_MAX_STRING_SIZE,
join_by="\n..\n",
left_larger=True,
beginning_on_size_error=True)
stderr = shrink_stream_by_size(efh,
DATABASE_MAX_STRING_SIZE,
join_by="\n..\n",
left_larger=True,
beginning_on_size_error=True)
# This should be an 8-bit exit code, but read ahead anyway:
exit_code_str = ecfh.read(32)
except:
stdout = ''
stderr = 'Job output not returned by PBS: the output datasets were deleted while the job was running, the job was manually dequeued or there was a cluster error.'
# By default, the exit code is 0, which usually indicates success
# (although clearly some error happened).
exit_code_str = ""
# Translate the exit code string to an integer; use 0 on failure.
try:
exit_code = int(exit_code_str)
except:
log.warning("Exit code " + exit_code_str +
" was invalid. Using 0.")
exit_code = 0
# Call on the job wrapper to complete the call:
try:
pbs_job_state.job_wrapper.finish(stdout, stderr, exit_code)
except:
log.exception("Job wrapper finish method failed")
pbs_job_state.job_wrapper.fail("Unable to finish job",
exception=True)
# clean up the pbs files
if self.app.config.cleanup_job == "always" or (
not stderr and self.app.config.cleanup_job == "onsuccess"):
self.cleanup((ofile, efile, ecfile, job_file))
def finish_job( self, job_state ):
"""
Get the output/error for a finished job, pass to `job_wrapper.finish`
and cleanup all the job's temporary files.
"""
galaxy_id_tag = job_state.job_wrapper.get_id_tag()
external_job_id = job_state.job_id
# To ensure that files below are readable, ownership must be reclaimed first
job_state.job_wrapper.reclaim_ownership()
# wait for the files to appear
which_try = 0
while which_try < (self.app.config.retry_job_output_collection + 1):
try:
stdout = shrink_stream_by_size( open( job_state.output_file, "r" ), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
stderr = shrink_stream_by_size( open( job_state.error_file, "r" ), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
which_try = (self.app.config.retry_job_output_collection + 1)
except Exception as e:
if which_try == self.app.config.retry_job_output_collection:
stdout = ''
stderr = 'Job output not returned from cluster'
log.error( '(%s/%s) %s: %s' % ( galaxy_id_tag, external_job_id, stderr, str( e ) ) )
else:
time.sleep(1)
which_try += 1
try:
# This should be an 8-bit exit code, but read ahead anyway:
exit_code_str = open( job_state.exit_code_file, "r" ).read(32)
except:
# By default, the exit code is 0, which typically indicates success.
exit_code_str = "0"
try:
# Decode the exit code. If it's bogus, then just use 0.
exit_code = int(exit_code_str)
except:
log.warning( "(%s/%s) Exit code '%s' invalid. Using 0." % ( galaxy_id_tag, external_job_id, exit_code_str ) )
exit_code = 0
# clean up the job files
cleanup_job = job_state.job_wrapper.cleanup_job
if cleanup_job == "always" or ( not stderr and cleanup_job == "onsuccess" ):
job_state.cleanup()
try:
job_state.job_wrapper.finish( stdout, stderr, exit_code )
except:
log.exception( "(%s/%s) Job wrapper finish method failed" % ( galaxy_id_tag, external_job_id ) )
job_state.job_wrapper.fail( "Unable to finish job", exception=True )
def finish_job( self, job_state ):
"""
Get the output/error for a finished job, pass to `job_wrapper.finish`
and cleanup all the job's temporary files.
"""
galaxy_id_tag = job_state.job_wrapper.get_id_tag()
external_job_id = job_state.job_id
# To ensure that files below are readable, ownership must be reclaimed first
job_state.job_wrapper.reclaim_ownership()
# wait for the files to appear
which_try = 0
while which_try < (self.app.config.retry_job_output_collection + 1):
try:
stdout = shrink_stream_by_size( file( job_state.output_file, "r" ), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
stderr = shrink_stream_by_size( file( job_state.error_file, "r" ), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
which_try = (self.app.config.retry_job_output_collection + 1)
except Exception as e:
if which_try == self.app.config.retry_job_output_collection:
stdout = ''
stderr = 'Job output not returned from cluster'
log.error( '(%s/%s) %s: %s' % ( galaxy_id_tag, external_job_id, stderr, str( e ) ) )
else:
time.sleep(1)
which_try += 1
try:
# This should be an 8-bit exit code, but read ahead anyway:
exit_code_str = file( job_state.exit_code_file, "r" ).read(32)
except:
# By default, the exit code is 0, which typically indicates success.
exit_code_str = "0"
try:
# Decode the exit code. If it's bogus, then just use 0.
exit_code = int(exit_code_str)
except:
log.warning( "(%s/%s) Exit code '%s' invalid. Using 0." % ( galaxy_id_tag, external_job_id, exit_code_str ) )
exit_code = 0
# clean up the job files
if self.app.config.cleanup_job == "always" or ( not stderr and self.app.config.cleanup_job == "onsuccess" ):
job_state.cleanup()
try:
job_state.job_wrapper.finish( stdout, stderr, exit_code )
except:
log.exception( "(%s/%s) Job wrapper finish method failed" % ( galaxy_id_tag, external_job_id ) )
job_state.job_wrapper.fail( "Unable to finish job", exception=True )
def finish_job(self, pbs_job_state):
"""
Get the output/error for a finished job, pass to `job_wrapper.finish`
and cleanup all the PBS temporary files.
"""
ofile = pbs_job_state.output_file
efile = pbs_job_state.error_file
ecfile = pbs_job_state.exit_code_file
job_file = pbs_job_state.job_file
# collect the output
try:
ofh = file(ofile, "r")
efh = file(efile, "r")
ecfh = file(ecfile, "r")
stdout = shrink_stream_by_size(
ofh, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True
)
stderr = shrink_stream_by_size(
efh, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True
)
# This should be an 8-bit exit code, but read ahead anyway:
exit_code_str = ecfh.read(32)
except:
stdout = ""
stderr = "Job output not returned by PBS: the output datasets were deleted while the job was running, the job was manually dequeued or there was a cluster error."
# By default, the exit code is 0, which usually indicates success
# (although clearly some error happened).
exit_code_str = ""
# Translate the exit code string to an integer; use 0 on failure.
try:
exit_code = int(exit_code_str)
except:
log.warning("Exit code " + exit_code_str + " was invalid. Using 0.")
exit_code = 0
# Call on the job wrapper to complete the call:
try:
pbs_job_state.job_wrapper.finish(stdout, stderr, exit_code)
except:
log.exception("Job wrapper finish method failed")
pbs_job_state.job_wrapper.fail("Unable to finish job", exception=True)
# clean up the pbs files
if self.app.config.cleanup_job == "always" or (not stderr and self.app.config.cleanup_job == "onsuccess"):
self.cleanup((ofile, efile, ecfile, job_file))
def _job_io_for_db(self, stream):
return shrink_stream_by_size(stream, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True)
def queue_job(self, job_wrapper):
if not self._prepare_job_local(job_wrapper):
return
stderr = stdout = ''
exit_code = 0
# command line has been added to the wrapper by prepare_job()
command_line, exit_code_path = self.__command_line(job_wrapper)
job_id = job_wrapper.get_id_tag()
try:
stdout_file = tempfile.NamedTemporaryFile(
mode='wb+',
suffix='_stdout',
dir=job_wrapper.working_directory)
stderr_file = tempfile.NamedTemporaryFile(
mode='wb+',
suffix='_stderr',
dir=job_wrapper.working_directory)
log.debug('(%s) executing job script: %s' % (job_id, command_line))
proc = subprocess.Popen(args=command_line,
shell=True,
cwd=job_wrapper.working_directory,
stdout=stdout_file,
stderr=stderr_file,
env=self._environ,
preexec_fn=os.setpgrp)
proc.terminated_by_shutdown = False
with self._proc_lock:
self._procs.append(proc)
try:
job_wrapper.set_job_destination(job_wrapper.job_destination,
proc.pid)
job_wrapper.change_state(model.Job.states.RUNNING)
terminated = self.__poll_if_needed(proc, job_wrapper, job_id)
if terminated:
return
# Reap the process and get the exit code.
exit_code = proc.wait()
finally:
with self._proc_lock:
self._procs.remove(proc)
try:
exit_code = int(open(exit_code_path, 'r').read())
except Exception:
log.warning("Failed to read exit code from path %s" %
exit_code_path)
pass
if proc.terminated_by_shutdown:
self._fail_job_local(job_wrapper,
"job terminated by Galaxy shutdown")
return
stdout_file.seek(0)
stderr_file.seek(0)
stdout = shrink_stream_by_size(stdout_file,
DATABASE_MAX_STRING_SIZE,
join_by="\n..\n",
left_larger=True,
beginning_on_size_error=True)
stderr = shrink_stream_by_size(stderr_file,
DATABASE_MAX_STRING_SIZE,
join_by="\n..\n",
left_larger=True,
beginning_on_size_error=True)
stdout_file.close()
stderr_file.close()
log.debug('execution finished: %s' % command_line)
except Exception:
log.exception("failure running job %d", job_wrapper.job_id)
self._fail_job_local(job_wrapper, "failure running job")
return
self._handle_metadata_if_needed(job_wrapper)
job_destination = job_wrapper.job_destination
job_state = JobState(job_wrapper, job_destination)
job_state.stop_job = False
# Finish the job!
try:
self._finish_or_resubmit_job(job_state, stdout, stderr, exit_code)
except Exception:
log.exception("Job wrapper finish method failed")
self._fail_job_local(job_wrapper, "Unable to finish job")
def queue_job(self, job_wrapper):
if not self._prepare_job_local(job_wrapper):
return
stderr = stdout = ''
exit_code = 0
# command line has been added to the wrapper by prepare_job()
command_line, exit_code_path = self.__command_line(job_wrapper)
job_id = job_wrapper.get_id_tag()
try:
stdout_file = tempfile.NamedTemporaryFile(mode='wb+', suffix='_stdout', dir=job_wrapper.working_directory)
stderr_file = tempfile.NamedTemporaryFile(mode='wb+', suffix='_stderr', dir=job_wrapper.working_directory)
log.debug('(%s) executing job script: %s' % (job_id, command_line))
proc = subprocess.Popen(args=command_line,
shell=True,
cwd=job_wrapper.working_directory,
stdout=stdout_file,
stderr=stderr_file,
env=self._environ,
preexec_fn=os.setpgrp)
proc.terminated_by_shutdown = False
with self._proc_lock:
self._procs.append(proc)
try:
job_wrapper.set_job_destination(job_wrapper.job_destination, proc.pid)
job_wrapper.change_state(model.Job.states.RUNNING)
terminated = self.__poll_if_needed(proc, job_wrapper, job_id)
if terminated:
return
# Reap the process and get the exit code.
exit_code = proc.wait()
finally:
with self._proc_lock:
self._procs.remove(proc)
try:
exit_code = int(open(exit_code_path, 'r').read())
except Exception:
log.warning("Failed to read exit code from path %s" % exit_code_path)
pass
if proc.terminated_by_shutdown:
self._fail_job_local(job_wrapper, "job terminated by Galaxy shutdown")
return
stdout_file.seek(0)
stderr_file.seek(0)
stdout = shrink_stream_by_size(stdout_file, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True)
stderr = shrink_stream_by_size(stderr_file, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True)
stdout_file.close()
stderr_file.close()
log.debug('execution finished: %s' % command_line)
except Exception:
log.exception("failure running job %d", job_wrapper.job_id)
self._fail_job_local(job_wrapper, "failure running job")
return
self._handle_metadata_if_needed(job_wrapper)
job_destination = job_wrapper.job_destination
job_state = JobState(job_wrapper, job_destination)
job_state.stop_job = False
# Finish the job!
try:
self._finish_or_resubmit_job(job_state, stdout, stderr, exit_code)
except Exception:
log.exception("Job wrapper finish method failed")
self._fail_job_local(job_wrapper, "Unable to finish job")
def queue_job(self, job_wrapper):
# prepare the job
if not self.prepare_job(job_wrapper):
return
stderr = stdout = ''
exit_code = 0
# command line has been added to the wrapper by prepare_job()
command_line = job_wrapper.runner_command_line
job_id = job_wrapper.get_id_tag()
try:
log.debug('(%s) executing: %s' % (job_id, command_line))
stdout_file = tempfile.NamedTemporaryFile(
suffix='_stdout', dir=job_wrapper.working_directory)
stderr_file = tempfile.NamedTemporaryFile(
suffix='_stderr', dir=job_wrapper.working_directory)
proc = subprocess.Popen(args=command_line,
shell=True,
cwd=job_wrapper.working_directory,
stdout=stdout_file,
stderr=stderr_file,
env=os.environ,
preexec_fn=os.setpgrp)
job_wrapper.set_job_destination(job_wrapper.job_destination,
proc.pid)
job_wrapper.change_state(model.Job.states.RUNNING)
job_start = datetime.datetime.now()
i = 0
# Iterate until the process exits, periodically checking its limits
while proc.poll() is None:
i += 1
if (i % 20) == 0:
msg = job_wrapper.check_limits(
runtime=datetime.datetime.now() - job_start)
if msg is not None:
job_wrapper.fail(msg)
log.debug('(%s) Terminating process group' % job_id)
self._terminate(proc)
return
else:
sleep(1)
# Reap the process and get the exit code.
exit_code = proc.wait()
stdout_file.seek(0)
stderr_file.seek(0)
stdout = shrink_stream_by_size(stdout_file,
DATABASE_MAX_STRING_SIZE,
join_by="\n..\n",
left_larger=True,
beginning_on_size_error=True)
stderr = shrink_stream_by_size(stderr_file,
DATABASE_MAX_STRING_SIZE,
join_by="\n..\n",
left_larger=True,
beginning_on_size_error=True)
stdout_file.close()
stderr_file.close()
log.debug('execution finished: %s' % command_line)
except Exception:
job_wrapper.fail("failure running job", exception=True)
log.exception("failure running job %d" % job_wrapper.job_id)
return
self._handle_metadata_externally(job_wrapper)
# Finish the job!
try:
job_wrapper.finish(stdout, stderr, exit_code)
except:
log.exception("Job wrapper finish method failed")
job_wrapper.fail("Unable to finish job", exception=True)
