def failed_post(queues, traces, args):
"""
Get a Job object from the "failed_payloads" queue. Set the pilot state to "stakeout" and the stageout field to
"log", and add the Job object to the "data_out" queue.
:param queues: internal queues for job handling.
:param traces: tuple containing internal pilot states.
:param args: Pilot arguments (e.g. containing queue name, queuedata dictionary, etc).
:return:
"""
while not args.graceful_stop.is_set():
time.sleep(0.5)
# finished payloads
try:
job = queues.failed_payloads.get(block=True, timeout=1)
except queue.Empty:
time.sleep(0.1)
continue
logger.debug('adding log for log stageout')
job.stageout = 'log' # only stage-out log file
#queues.data_out.put(job)
set_pilot_state(job=job, state='stageout')
put_in_queue(job, queues.data_out)
# proceed to set the job_aborted flag?
if threads_aborted():
logger.debug('will proceed to set job_aborted')
args.job_aborted.set()
else:
logger.debug('will not set job_aborted yet')
logger.info('[payload] failed_post thread has finished')
def failed_post(queues, traces, args):
"""
(add description)
:param queues:
:param traces:
:param args:
:return:
"""
while not args.graceful_stop.is_set():
time.sleep(0.5)
# finished payloads
try:
job = queues.failed_payloads.get(block=True, timeout=1)
except queue.Empty:
time.sleep(0.1)
continue
log = get_logger(job.jobid, logger)
log.debug('adding log for log stageout')
job.stageout = 'log' # only stage-out log file
#queues.data_out.put(job)
set_pilot_state(job=job, state='stageout')
put_in_queue(job, queues.data_out)
logger.info('[payload] failed_post thread has finished')
def validate_pre(queues, traces, args):
"""
Get a Job object from the "payloads" queue and validate it.
If the payload is successfully validated (user defined), the Job object is placed in the "validated_payloads" queue,
otherwise it is placed in the "failed_payloads" queue.
:param queues: internal queues for job handling.
:param traces: tuple containing internal pilot states.
:param args: Pilot arguments (e.g. containing queue name, queuedata dictionary, etc).
:return:
"""
while not args.graceful_stop.is_set():
time.sleep(0.5)
try:
job = queues.payloads.get(block=True, timeout=1)
except queue.Empty:
continue
if _validate_payload(job):
#queues.validated_payloads.put(job)
put_in_queue(job, queues.validated_payloads)
else:
#queues.failed_payloads.put(job)
put_in_queue(job, queues.failed_payloads)
# proceed to set the job_aborted flag?
if threads_aborted():
logger.debug('will proceed to set job_aborted')
args.job_aborted.set()
else:
logger.debug('will not set job_aborted yet')
logger.info('[payload] validate_pre thread has finished')
def validate_post(queues, traces, args):
"""
Validate finished payloads.
If payload finished correctly, add the job to the data_out queue. If it failed, add it to the data_out queue as
well but only for log stage-out (in failed_post() below).
:param queues:
:param traces:
:param args:
:return:
"""
while not args.graceful_stop.is_set():
time.sleep(0.5)
# finished payloads
try:
job = queues.finished_payloads.get(block=True, timeout=1)
except queue.Empty:
time.sleep(0.1)
continue
log = get_logger(job.jobid, logger)
# by default, both output and log should be staged out
job.stageout = 'all'
log.debug('adding job to data_out queue')
#queues.data_out.put(job)
set_pilot_state(job=job, state='stageout')
put_in_queue(job, queues.data_out)
logger.info('[payload] validate_post thread has finished')
def validate_pre(queues, traces, args):
"""
(add description)
:param queues:
:param traces:
:param args:
:return:
"""
while not args.graceful_stop.is_set():
try:
job = queues.payloads.get(block=True, timeout=1)
except queue.Empty:
continue
if _validate_payload(job):
#queues.validated_payloads.put(job)
put_in_queue(job, queues.validated_payloads)
else:
#queues.failed_payloads.put(job)
put_in_queue(job, queues.failed_payloads)
logger.info('[payload] validate_pre thread has finished')
def validate_post(queues, traces, args):
"""
Validate finished payloads.
If payload finished correctly, add the job to the data_out queue. If it failed, add it to the data_out queue as
well but only for log stage-out (in failed_post() below).
:param queues: internal queues for job handling.
:param traces: tuple containing internal pilot states.
:param args: Pilot arguments (e.g. containing queue name, queuedata dictionary, etc).
:return:
"""
while not args.graceful_stop.is_set():
time.sleep(0.5)
# finished payloads
try:
job = queues.finished_payloads.get(block=True, timeout=1)
except queue.Empty:
time.sleep(0.1)
continue
# by default, both output and log should be staged out
job.stageout = 'all'
logger.debug('adding job to data_out queue')
#queues.data_out.put(job)
set_pilot_state(job=job, state='stageout')
put_in_queue(job, queues.data_out)
# proceed to set the job_aborted flag?
if threads_aborted():
logger.debug('will proceed to set job_aborted')
args.job_aborted.set()
else:
logger.debug('will not set job_aborted yet')
logger.info('[payload] validate_post thread has finished')
def copytool_out(queues, traces, args):
"""
Main stage-out thread.
Perform stage-out as soon as a job object can be extracted from the data_out queue.
:param queues: internal queues for job handling.
:param traces: tuple containing internal pilot states.
:param args: Pilot arguments (e.g. containing queue name, queuedata dictionary, etc).
:return:
"""
cont = True
logger.debug('entering copytool_out loop')
if args.graceful_stop.is_set():
logger.debug('graceful_stop already set')
processed_jobs = []
while cont:
time.sleep(0.5)
# abort if kill signal arrived too long time ago, ie loop is stuck
current_time = int(time.time())
if args.kill_time and current_time - args.kill_time > MAX_KILL_WAIT_TIME:
logger.warning(
'loop has run for too long time after first kill signal - will abort'
)
break
# check for abort, print useful messages and include a 1 s sleep
abort = should_abort(args, label='data:copytool_out')
try:
job = queues.data_out.get(block=True, timeout=1)
if job:
# hack to prevent stage-out to be called more than once for same job object (can apparently happen
# in multi-output jobs)
# should not be necessary unless job object is added to queues.data_out more than once - check this
# for multiple output files
if processed_jobs:
if is_already_processed(queues, processed_jobs):
continue
logger.info('will perform stage-out for job id=%s', job.jobid)
if args.abort_job.is_set():
traces.pilot['command'] = 'abort'
logger.warning(
'copytool_out detected a set abort_job pre stage-out (due to a kill signal)'
)
declare_failed_by_kill(job, queues.failed_data_out,
args.signal)
break
if _stage_out_new(job, args):
if args.abort_job.is_set():
traces.pilot['command'] = 'abort'
logger.warning(
'copytool_out detected a set abort_job post stage-out (due to a kill signal)'
)
#declare_failed_by_kill(job, queues.failed_data_out, args.signal)
break
#queues.finished_data_out.put(job)
processed_jobs.append(job.jobid)
put_in_queue(job, queues.finished_data_out)
logger.debug('job object added to finished_data_out queue')
else:
#queues.failed_data_out.put(job)
put_in_queue(job, queues.failed_data_out)
logger.debug('job object added to failed_data_out queue')
else:
logger.debug('no returned job - why no exception?')
except queue.Empty:
if abort:
cont = False
break
continue
if abort:
cont = False
break
# proceed to set the job_aborted flag?
if threads_aborted():
logger.debug('will proceed to set job_aborted')
args.job_aborted.set()
else:
logger.debug('will not set job_aborted yet')
logger.debug('[data] copytool_out thread has finished')
def copytool_in(queues, traces, args):
"""
Call the stage-in function and put the job object in the proper queue.
:param queues: internal queues for job handling.
:param traces: tuple containing internal pilot states.
:param args: Pilot arguments (e.g. containing queue name, queuedata dictionary, etc).
:return:
"""
while not args.graceful_stop.is_set():
time.sleep(0.5)
try:
# abort if kill signal arrived too long time ago, ie loop is stuck
current_time = int(time.time())
if args.kill_time and current_time - args.kill_time > MAX_KILL_WAIT_TIME:
logger.warning(
'loop has run for too long time after first kill signal - will abort'
)
break
# extract a job to stage-in its input
job = queues.data_in.get(block=True, timeout=1)
# does the user want to execute any special commands before stage-in?
pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
user = __import__('pilot.user.%s.common' % pilot_user, globals(),
locals(), [pilot_user], 0) # Python 2/3
cmd = user.get_utility_commands(job=job,
order=UTILITY_BEFORE_STAGEIN)
if cmd:
# xcache debug
#_, _stdout, _stderr = execute('pgrep -x xrootd | awk \'{print \"ps -p \"$1\" -o args --no-headers --cols 300\"}\' | sh')
#logger.debug('[before xcache start] stdout=%s', _stdout)
#logger.debug('[before xcache start] stderr=%s', _stderr)
_, stdout, stderr = execute(cmd.get('command'))
logger.debug('stdout=%s', stdout)
logger.debug('stderr=%s', stderr)
# xcache debug
#_, _stdout, _stderr = execute('pgrep -x xrootd | awk \'{print \"ps -p \"$1\" -o args --no-headers --cols 300\"}\' | sh')
#logger.debug('[after xcache start] stdout=%s', _stdout)
#logger.debug('[after xcache start] stderr=%s', _stderr)
# perform any action necessary after command execution (e.g. stdout processing)
kwargs = {
'label': cmd.get('label', 'utility'),
'output': stdout
}
user.post_prestagein_utility_command(**kwargs)
# write output to log files
write_utility_output(job.workdir, cmd.get('label', 'utility'),
stdout, stderr)
# place it in the current stage-in queue (used by the jobs' queue monitoring)
put_in_queue(job, queues.current_data_in)
# ready to set the job in running state
send_state(job, args, 'running')
# note: when sending a state change to the server, the server might respond with 'tobekilled'
if job.state == 'failed':
logger.warning(
'job state is \'failed\' - order log transfer and abort copytool_in()'
)
job.stageout = 'log' # only stage-out log file
put_in_queue(job, queues.data_out)
break
os.environ['SERVER_UPDATE'] = SERVER_UPDATE_RUNNING
if args.abort_job.is_set():
traces.pilot['command'] = 'abort'
logger.warning(
'copytool_in detected a set abort_job pre stage-in (due to a kill signal)'
)
declare_failed_by_kill(job, queues.failed_data_in, args.signal)
break
if _stage_in(args, job):
if args.abort_job.is_set():
traces.pilot['command'] = 'abort'
logger.warning(
'copytool_in detected a set abort_job post stage-in (due to a kill signal)'
)
declare_failed_by_kill(job, queues.failed_data_in,
args.signal)
break
put_in_queue(job, queues.finished_data_in)
# remove the job from the current stage-in queue
_job = queues.current_data_in.get(block=True, timeout=1)
if _job:
logger.debug(
'job %s has been removed from the current_data_in queue',
_job.jobid)
# now create input file metadata if required by the payload
if os.environ.get('PILOT_ES_EXECUTOR_TYPE',
'generic') == 'generic':
pilot_user = os.environ.get('PILOT_USER',
'generic').lower()
user = __import__('pilot.user.%s.metadata' % pilot_user,
globals(), locals(), [pilot_user],
0) # Python 2/3
file_dictionary = get_input_file_dictionary(job.indata)
xml = user.create_input_file_metadata(
file_dictionary, job.workdir)
logger.info('created input file metadata:\n%s', xml)
else:
# remove the job from the current stage-in queue
_job = queues.current_data_in.get(block=True, timeout=1)
if _job:
logger.debug(
'job %s has been removed from the current_data_in queue',
_job.jobid)
logger.warning(
'stage-in failed, adding job object to failed_data_in queue'
)
job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(
errors.STAGEINFAILED)
set_pilot_state(job=job, state="failed")
traces.pilot['error_code'] = job.piloterrorcodes[0]
put_in_queue(job, queues.failed_data_in)
# do not set graceful stop if pilot has not finished sending the final job update
# i.e. wait until SERVER_UPDATE is DONE_FINAL
check_for_final_server_update(args.update_server)
args.graceful_stop.set()
except queue.Empty:
continue
# proceed to set the job_aborted flag?
if threads_aborted():
logger.debug('will proceed to set job_aborted')
args.job_aborted.set()
else:
logger.debug('will not set job_aborted yet')
logger.debug('[data] copytool_in thread has finished')
def queue_monitoring(queues, traces, args):
"""
Monitoring of Data queues.
:param queues: internal queues for job handling.
:param traces: tuple containing internal pilot states.
:param args: Pilot arguments (e.g. containing queue name, queuedata dictionary, etc).
:return:
"""
while True: # will abort when graceful_stop has been set
time.sleep(0.5)
if traces.pilot['command'] == 'abort':
logger.warning('data queue monitor saw the abort instruction')
args.graceful_stop.set()
# abort in case graceful_stop has been set, and less than 30 s has passed since MAXTIME was reached (if set)
# (abort at the end of the loop)
abort = should_abort(args, label='data:queue_monitoring')
# monitor the failed_data_in queue
try:
job = queues.failed_data_in.get(block=True, timeout=1)
except queue.Empty:
pass
else:
# stage-out log file then add the job to the failed_jobs queue
job.stageout = "log"
# TODO: put in data_out queue instead?
if not _stage_out_new(job, args):
logger.info(
"job %s failed during stage-in and stage-out of log, adding job object to failed_data_outs queue",
job.jobid)
put_in_queue(job, queues.failed_data_out)
else:
logger.info(
"job %s failed during stage-in, adding job object to failed_jobs queue",
job.jobid)
put_in_queue(job, queues.failed_jobs)
# monitor the finished_data_out queue
try:
job = queues.finished_data_out.get(block=True, timeout=1)
except queue.Empty:
pass
else:
# use the payload/transform exitCode from the job report if it exists
if job.transexitcode == 0 and job.exitcode == 0 and job.piloterrorcodes == []:
logger.info(
'finished stage-out for finished payload, adding job to finished_jobs queue'
)
#queues.finished_jobs.put(job)
put_in_queue(job, queues.finished_jobs)
else:
logger.info('finished stage-out (of log) for failed payload')
#queues.failed_jobs.put(job)
put_in_queue(job, queues.failed_jobs)
# monitor the failed_data_out queue
try:
job = queues.failed_data_out.get(block=True, timeout=1)
except queue.Empty:
pass
else:
# attempt to upload the log in case the previous stage-out failure was not an SE error
job.stageout = "log"
set_pilot_state(job=job, state="failed")
if not _stage_out_new(job, args):
logger.info("job %s failed during stage-out", job.jobid)
put_in_queue(job, queues.failed_jobs)
if abort:
break
# proceed to set the job_aborted flag?
if threads_aborted():
logger.debug('will proceed to set job_aborted')
args.job_aborted.set()
else:
logger.debug('will not set job_aborted yet')
logger.debug('[data] queue_monitor thread has finished')
def execute_payloads(queues, traces, args): # noqa: C901
"""
Execute queued payloads.
Extract a Job object from the "validated_payloads" queue and put it in the "monitored_jobs" queue. The payload
stdout/err streams are opened and the pilot state is changed to "starting". A payload executor is selected (for
executing a normal job, an event service job or event service merge job). After the payload (or rather its executor)
is started, the thread will wait for it to finish and then check for any failures. A successfully completed job is
placed in the "finished_payloads" queue, and a failed job will be placed in the "failed_payloads" queue.
:param queues: internal queues for job handling.
:param traces: tuple containing internal pilot states.
:param args: Pilot arguments (e.g. containing queue name, queuedata dictionary, etc).
:return:
"""
job = None
while not args.graceful_stop.is_set():
time.sleep(0.5)
try:
job = queues.validated_payloads.get(block=True, timeout=1)
q_snapshot = list(queues.finished_data_in.queue)
peek = [s_job for s_job in q_snapshot if job.jobid == s_job.jobid]
if len(peek) == 0:
#queues.validated_payloads.put(job)
put_in_queue(job, queues.validated_payloads)
for i in range(10): # Python 3
if args.graceful_stop.is_set():
break
time.sleep(1)
continue
# this job is now to be monitored, so add it to the monitored_payloads queue
#queues.monitored_payloads.put(job)
put_in_queue(job, queues.monitored_payloads)
logger.info('job %s added to monitored payloads queue' % job.jobid)
try:
out = open(
os.path.join(job.workdir, config.Payload.payloadstdout),
'wb')
err = open(
os.path.join(job.workdir, config.Payload.payloadstderr),
'wb')
except Exception as e:
logger.warning('failed to open payload stdout/err: %s' % e)
out = None
err = None
send_state(job, args, 'starting')
# note: when sending a state change to the server, the server might respond with 'tobekilled'
if job.state == 'failed':
logger.warning(
'job state is \'failed\' - abort execute_payloads()')
break
payload_executor = get_payload_executor(args, job, out, err,
traces)
logger.info("Got payload executor: %s" % payload_executor)
show_memory_usage()
# run the payload and measure the execution time
job.t0 = os.times()
exit_code = payload_executor.run()
set_cpu_consumption_time(job)
job.transexitcode = exit_code % 255
out.close()
err.close()
pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
# some HPO jobs will produce new output files (following lfn name pattern), discover those and replace the job.outdata list
if job.is_hpo:
user = __import__('pilot.user.%s.common' % pilot_user,
globals(), locals(), [pilot_user],
0) # Python 2/3
try:
user.update_output_for_hpo(job)
except Exception as e:
logger.warning(
'exception caught by update_output_for_hpo(): %s' % e)
else:
for dat in job.outdata:
if not dat.guid:
dat.guid = get_guid()
logger.warning(
'guid not set: generated guid=%s for lfn=%s' %
(dat.guid, dat.lfn))
#if traces.pilot['nr_jobs'] == 1:
# logger.debug('faking job failure in first multi-job')
# job.transexitcode = 1
# exit_code = 1
# analyze and interpret the payload execution output
perform_initial_payload_error_analysis(job, exit_code)
# was an error already found?
#if job.piloterrorcodes:
# exit_code_interpret = 1
#else:
user = __import__('pilot.user.%s.diagnose' % pilot_user, globals(),
locals(), [pilot_user], 0) # Python 2/3
try:
exit_code_interpret = user.interpret(job)
except Exception as e:
logger.warning('exception caught: %s' % e)
#exit_code_interpret = -1
job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(
errors.INTERNALPILOTPROBLEM)
if job.piloterrorcodes:
exit_code_interpret = 1
if exit_code_interpret == 0 and exit_code == 0:
logger.info(
'main payload error analysis completed - did not find any errors'
)
# update output lists if zipmaps were used
#job.add_archives_to_output_lists()
# queues.finished_payloads.put(job)
put_in_queue(job, queues.finished_payloads)
else:
logger.debug(
'main payload error analysis completed - adding job to failed_payloads queue'
)
#queues.failed_payloads.put(job)
put_in_queue(job, queues.failed_payloads)
except queue.Empty:
continue
except Exception as e:
logger.fatal(
'execute payloads caught an exception (cannot recover): %s, %s'
% (e, traceback.format_exc()))
if job:
job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(
errors.PAYLOADEXECUTIONEXCEPTION)
#queues.failed_payloads.put(job)
put_in_queue(job, queues.failed_payloads)
while not args.graceful_stop.is_set():
# let stage-out of log finish, but stop running payloads as there should be a problem with the pilot
time.sleep(5)
# proceed to set the job_aborted flag?
if threads_aborted():
logger.debug('will proceed to set job_aborted')
args.job_aborted.set()
else:
logger.debug('will not set job_aborted yet')
logger.info('[payload] execute_payloads thread has finished')
def queue_monitoring(queues, traces, args):
"""
Monitoring of Data queues.
:param queues:
:param traces:
:param args:
:return:
"""
while True: # will abort when graceful_stop has been set
if traces.pilot['command'] == 'abort':
logger.warning('data queue monitor saw the abort instruction')
# abort in case graceful_stop has been set, and less than 30 s has passed since MAXTIME was reached (if set)
# (abort at the end of the loop)
abort = should_abort(args, label='data:queue_monitoring')
# monitor the failed_data_in queue
try:
job = queues.failed_data_in.get(block=True, timeout=1)
except queue.Empty:
pass
else:
log = get_logger(job.jobid)
# stage-out log file then add the job to the failed_jobs queue
job.stageout = "log"
# TODO: put in data_out queue instead?
if not _stage_out_new(job, args):
log.info("job %s failed during stage-in and stage-out of log, adding job object to failed_data_outs "
"queue" % job.jobid)
#queues.failed_data_out.put(job)
put_in_queue(job, queues.failed_data_out)
else:
log.info("job %s failed during stage-in, adding job object to failed_jobs queue" % job.jobid)
#queues.failed_jobs.put(job)
put_in_queue(job, queues.failed_jobs)
# monitor the finished_data_out queue
try:
job = queues.finished_data_out.get(block=True, timeout=1)
except queue.Empty:
pass
else:
log = get_logger(job.jobid)
# use the payload/transform exitCode from the job report if it exists
if job.transexitcode == 0 and job.exitcode == 0 and job.piloterrorcodes == []:
log.info('finished stage-out for finished payload, adding job to finished_jobs queue')
#queues.finished_jobs.put(job)
put_in_queue(job, queues.finished_jobs)
else:
log.info('finished stage-out (of log) for failed payload')
#queues.failed_jobs.put(job)
put_in_queue(job, queues.failed_jobs)
# monitor the failed_data_out queue
try:
job = queues.failed_data_out.get(block=True, timeout=1)
except queue.Empty:
pass
else:
log = get_logger(job.jobid)
# attempt to upload the log in case the previous stage-out failure was not an SE error
job.stageout = "log"
set_pilot_state(job=job, state="failed")
if not _stage_out_new(job, args):
log.info("job %s failed during stage-out of data file(s) as well as during stage-out of log, "
"adding job object to failed_jobs queue" % job.jobid)
else:
log.info("job %s failed during stage-out of data file(s) - stage-out of log succeeded, adding job "
"object to failed_jobs queue" % job.jobid)
#queues.failed_jobs.put(job)
put_in_queue(job, queues.failed_jobs)
if abort:
break
logger.debug('[data] queue_monitor thread has finished')
def copytool_out(queues, traces, args):
"""
Main stage-out thread.
Perform stage-out as soon as a job object can be extracted from the data_out queue.
:param queues: pilot queues object.
:param traces: pilot traces object.
:param args: pilot args object.
:return:
"""
cont = True
logger.debug('entering copytool_out loop')
if args.graceful_stop.is_set():
logger.debug('graceful_stop already set')
first = True
# while not args.graceful_stop.is_set() and cont:
while cont:
if first:
first = False
logger.debug('inside copytool_out() loop')
# check for abort, print useful messages and include a 1 s sleep
abort = should_abort(args, label='data:copytool_out')
if abort:
logger.debug('will abort ')
try:
job = queues.data_out.get(block=True, timeout=1)
if job:
log = get_logger(job.jobid)
log.info('will perform stage-out')
if args.abort_job.is_set():
traces.pilot['command'] = 'abort'
log.warning('copytool_out detected a set abort_job pre stage-out (due to a kill signal)')
declare_failed_by_kill(job, queues.failed_data_out, args.signal)
break
if _stage_out_new(job, args):
if args.abort_job.is_set():
traces.pilot['command'] = 'abort'
log.warning('copytool_out detected a set abort_job post stage-out (due to a kill signal)')
#declare_failed_by_kill(job, queues.failed_data_out, args.signal)
break
#queues.finished_data_out.put(job)
put_in_queue(job, queues.finished_data_out)
log.debug('job object added to finished_data_out queue')
else:
#queues.failed_data_out.put(job)
put_in_queue(job, queues.failed_data_out)
log.debug('job object added to failed_data_out queue')
else:
log.debug('no returned job - why no exception?')
except queue.Empty:
if abort:
logger.debug('aborting')
cont = False
break
continue
if abort:
logger.debug('aborting')
cont = False
break
logger.debug('[data] copytool_out thread has finished')
def copytool_in(queues, traces, args):
"""
Call the stage-in function and put the job object in the proper queue.
:param queues:
:param traces:
:param args:
:return:
"""
while not args.graceful_stop.is_set():
try:
# extract a job to stage-in its input
job = queues.data_in.get(block=True, timeout=1)
# place it in the current stage-in queue (used by the jobs' queue monitoring)
if job:
put_in_queue(job, queues.current_data_in)
# ready to set the job in running state
send_state(job, args, 'running')
os.environ['SERVER_UPDATE'] = SERVER_UPDATE_RUNNING
log = get_logger(job.jobid)
if args.abort_job.is_set():
traces.pilot['command'] = 'abort'
log.warning('copytool_in detected a set abort_job pre stage-in (due to a kill signal)')
declare_failed_by_kill(job, queues.failed_data_in, args.signal)
break
if _stage_in(args, job):
if args.abort_job.is_set():
traces.pilot['command'] = 'abort'
log.warning('copytool_in detected a set abort_job post stage-in (due to a kill signal)')
declare_failed_by_kill(job, queues.failed_data_in, args.signal)
break
#queues.finished_data_in.put(job)
put_in_queue(job, queues.finished_data_in)
# remove the job from the current stage-in queue
_job = queues.current_data_in.get(block=True, timeout=1)
if _job:
log.debug('job %s has been removed from the current_data_in queue' % _job.jobid)
# now create input file metadata if required by the payload
try:
pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
user = __import__('pilot.user.%s.metadata' % pilot_user, globals(), locals(), [pilot_user], -1)
_dir = '/srv' if job.usecontainer else job.workdir
file_dictionary = get_input_file_dictionary(job.indata, _dir)
#file_dictionary = get_input_file_dictionary(job.indata, job.workdir)
log.debug('file_dictionary=%s' % str(file_dictionary))
xml = user.create_input_file_metadata(file_dictionary, job.workdir)
log.info('created input file metadata:\n%s' % xml)
except Exception as e:
pass
else:
log.warning('stage-in failed, adding job object to failed_data_in queue')
job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(errors.STAGEINFAILED)
set_pilot_state(job=job, state="failed")
traces.pilot['error_code'] = job.piloterrorcodes[0]
#queues.failed_data_in.put(job)
put_in_queue(job, queues.failed_data_in)
# do not set graceful stop if pilot has not finished sending the final job update
# i.e. wait until SERVER_UPDATE is DONE_FINAL
check_for_final_server_update(args.update_server)
args.graceful_stop.set()
# send_state(job, args, 'failed')
except queue.Empty:
continue
logger.debug('[data] copytool_in thread has finished')
def execute_payloads(queues, traces, args):
"""
Execute queued payloads.
:param queues:
:param traces:
:param args:
:return:
"""
job = None
while not args.graceful_stop.is_set():
time.sleep(0.5)
try:
job = queues.validated_payloads.get(block=True, timeout=1)
log = get_logger(job.jobid, logger)
q_snapshot = list(queues.finished_data_in.queue)
peek = [s_job for s_job in q_snapshot if job.jobid == s_job.jobid]
if len(peek) == 0:
#queues.validated_payloads.put(job)
put_in_queue(job, queues.validated_payloads)
for i in range(10): # Python 3
if args.graceful_stop.is_set():
break
time.sleep(1)
continue
# this job is now to be monitored, so add it to the monitored_payloads queue
#queues.monitored_payloads.put(job)
put_in_queue(job, queues.monitored_payloads)
log.info('job %s added to monitored payloads queue' % job.jobid)
out = open(os.path.join(job.workdir, config.Payload.payloadstdout),
'wb')
err = open(os.path.join(job.workdir, config.Payload.payloadstderr),
'wb')
send_state(job, args, 'starting')
payload_executor = get_payload_executor(args, job, out, err,
traces)
log.info("Got payload executor: %s" % payload_executor)
# run the payload and measure the execution time
job.t0 = os.times()
exit_code = payload_executor.run()
set_cpu_consumption_time(job)
job.transexitcode = exit_code % 255
out.close()
err.close()
# analyze and interpret the payload execution output
perform_initial_payload_error_analysis(job, exit_code)
# was an error already found?
if job.piloterrorcodes:
exit_code_interpret = 1
else:
pilot_user = os.environ.get('PILOT_USER', 'generic').lower()
user = __import__('pilot.user.%s.diagnose' % pilot_user,
globals(), locals(), [pilot_user],
0) # Python 2/3
try:
exit_code_interpret = user.interpret(job)
except Exception as e:
log.warning('exception caught: %s' % e)
exit_code_interpret = -1
job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(
errors.INTERNALPILOTPROBLEM)
if exit_code_interpret == 0 and exit_code == 0:
log.info(
'main payload error analysis completed - did not find any errors'
)
# update output lists if zipmaps were used
#job.add_archives_to_output_lists()
# queues.finished_payloads.put(job)
put_in_queue(job, queues.finished_payloads)
else:
log.debug(
'main payload error analysis completed - adding job to failed_payloads queue'
)
#queues.failed_payloads.put(job)
put_in_queue(job, queues.failed_payloads)
except queue.Empty:
continue
except Exception as e:
logger.fatal(
'execute payloads caught an exception (cannot recover): %s, %s'
% (e, traceback.format_exc()))
if job:
job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(
errors.PAYLOADEXECUTIONEXCEPTION)
#queues.failed_payloads.put(job)
put_in_queue(job, queues.failed_payloads)
while not args.graceful_stop.is_set():
# let stage-out of log finish, but stop running payloads as there should be a problem with the pilot
time.sleep(5)
logger.info('[payload] execute_payloads thread has finished')
def copytool_in(queues, traces, args):
"""
Call the stage-in function and put the job object in the proper queue.
:param queues: internal queues for job handling.
:param traces: tuple containing internal pilot states.
:param args: Pilot arguments (e.g. containing queue name, queuedata dictionary, etc).
:return:
"""
while not args.graceful_stop.is_set():
time.sleep(0.5)
try:
# abort if kill signal arrived too long time ago, ie loop is stuck
current_time = int(time.time())
if args.kill_time and current_time - args.kill_time > MAX_KILL_WAIT_TIME:
logger.warning(
'loop has run for too long time after first kill signal - will abort'
)
break
# extract a job to stage-in its input
job = queues.data_in.get(block=True, timeout=1)
# place it in the current stage-in queue (used by the jobs' queue monitoring)
if job:
put_in_queue(job, queues.current_data_in)
# ready to set the job in running state
send_state(job, args, 'running')
# note: when sending a state change to the server, the server might respond with 'tobekilled'
if job.state == 'failed':
logger.warning(
'job state is \'failed\' - order log transfer and abort copytool_in()'
)
job.stageout = 'log' # only stage-out log file
put_in_queue(job, queues.data_out)
break
os.environ['SERVER_UPDATE'] = SERVER_UPDATE_RUNNING
if args.abort_job.is_set():
traces.pilot['command'] = 'abort'
logger.warning(
'copytool_in detected a set abort_job pre stage-in (due to a kill signal)'
)
declare_failed_by_kill(job, queues.failed_data_in, args.signal)
break
if _stage_in(args, job):
if args.abort_job.is_set():
traces.pilot['command'] = 'abort'
logger.warning(
'copytool_in detected a set abort_job post stage-in (due to a kill signal)'
)
declare_failed_by_kill(job, queues.failed_data_in,
args.signal)
break
put_in_queue(job, queues.finished_data_in)
# remove the job from the current stage-in queue
_job = queues.current_data_in.get(block=True, timeout=1)
if _job:
logger.debug(
'job %s has been removed from the current_data_in queue'
% _job.jobid)
# now create input file metadata if required by the payload
if config.Payload.executor_type.lower() != 'raythena':
pilot_user = os.environ.get('PILOT_USER',
'generic').lower()
user = __import__('pilot.user.%s.metadata' % pilot_user,
globals(), locals(), [pilot_user],
0) # Python 2/3
file_dictionary = get_input_file_dictionary(job.indata)
xml = user.create_input_file_metadata(
file_dictionary, job.workdir)
logger.info('created input file metadata:\n%s' % xml)
else:
logger.warning(
'stage-in failed, adding job object to failed_data_in queue'
)
job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(
errors.STAGEINFAILED)
set_pilot_state(job=job, state="failed")
traces.pilot['error_code'] = job.piloterrorcodes[0]
put_in_queue(job, queues.failed_data_in)
# do not set graceful stop if pilot has not finished sending the final job update
# i.e. wait until SERVER_UPDATE is DONE_FINAL
check_for_final_server_update(args.update_server)
args.graceful_stop.set()
except queue.Empty:
continue
# proceed to set the job_aborted flag?
if threads_aborted():
logger.debug('will proceed to set job_aborted')
args.job_aborted.set()
else:
logger.debug('will not set job_aborted yet')
logger.debug('[data] copytool_in thread has finished')
