def wait_graceful(args, proc, job): log = logger.getChild(str(job['PandaID'])) breaker = False exit_code = None while True: for i in xrange(100): if args.graceful_stop.is_set(): breaker = True log.debug('breaking -- sending SIGTERM pid=%s' % proc.pid) proc.terminate() break time.sleep(0.1) if breaker: log.debug('breaking -- sleep 3s before sending SIGKILL pid=%s' % proc.pid) time.sleep(3) proc.kill() break exit_code = proc.poll() log.info('running: pid=%s exit_code=%s' % (proc.pid, exit_code)) if exit_code is not None: break else: send_state(job, 'running') continue return exit_code
def _stage_out_all(job, args): outputs = {} for f in job['job_report']['files']['output']: outputs[f['subFiles'][0]['name']] = { 'scope': job['scopeOut'], 'name': f['subFiles'][0]['name'], 'guid': f['subFiles'][0]['file_guid'], 'bytes': f['subFiles'][0]['file_size'] } outputs['%s:%s' % (job['scopeLog'], job['logFile'])] = prepare_log( job, 'tarball_PandaJob_%s_%s' % (job['PandaID'], args.queue)) pfc = '''<?xml version="1.0" encoding="UTF-8" standalone="no" ?> <!DOCTYPE POOLFILECATALOG SYSTEM "InMemory"> <POOLFILECATALOG>''' pfc_file = ''' <File ID="{guid}"> <logical> <lfn name="{name}"/> </logical> <metadata att_name="surl" att_value="{pfn}"/> <metadata att_name="fsize" att_value="{bytes}"/> <metadata att_name="adler32" att_value="{adler32}"/> </File> ''' failed = False for outfile in outputs: summary = _stage_out(args, outputs[outfile], job) if summary is not None: outputs[outfile]['pfn'] = summary[ '%s:%s' % (outputs[outfile]['scope'], outputs[outfile]['name'])]['pfn'] outputs[outfile]['adler32'] = summary[ '%s:%s' % (outputs[outfile]['scope'], outputs[outfile]['name'])]['adler32'] pfc += pfc_file.format(**outputs[outfile]) else: failed = True pfc += '</POOLFILECATALOG>' if failed: send_state(job, 'failed') return False else: send_state(job, 'finished', xml=pfc) return True
def run(self): """ (add description) :return: """ log = get_logger(str(self.__job.jobid), logger) exit_code = 1 pilot_user = os.environ.get('PILOT_USER', 'generic').lower() if self.setup_payload(self.__job, self.__out, self.__err): log.debug('running payload') proc = self.run_payload(self.__job, self.__out, self.__err) if proc is not None: # the process is now running, update the server set_pilot_state(job=self.__job, state="running") send_state(self.__job, self.__args, self.__job.state) log.info('will wait for graceful exit') exit_code = self.wait_graceful(self.__args, proc, self.__job) state = 'finished' if exit_code == 0 else 'failed' set_pilot_state(job=self.__job, state=state) log.info('finished pid=%s exit_code=%s state=%s' % (proc.pid, exit_code, self.__job.state)) if exit_code is None: log.warning( 'detected unset exit_code from wait_graceful - reset to -1' ) exit_code = -1 self.utility_after_payload_finished(self.__job) self.post_payload(self.__job) # stop any running utilities if self.__job.utilities != {}: for utcmd in self.__job.utilities.keys(): utproc = self.__job.utilities[utcmd][0] if utproc: user = __import__( 'pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], -1) sig = user.get_utility_command_kill_signal(utcmd) log.info("stopping process \'%s\' with signal %d" % (utcmd, sig)) try: os.killpg(os.getpgid(utproc.pid), sig) except Exception as e: log.warning('exception caught: %s (ignoring)' % e) user.post_utility_command_action(utcmd, self.__job) return exit_code
def execute(queues, traces, args): while not args.graceful_stop.is_set(): try: job = queues.validated_payloads.get(block=True, timeout=1) log = logger.getChild(str(job['PandaID'])) q_snapshot = list(queues.finished_data_in.queue) peek = [ s_job for s_job in q_snapshot if job['PandaID'] == s_job['PandaID'] ] if len(peek) == 0: queues.validated_payloads.put(job) for i in xrange(10): if args.graceful_stop.is_set(): break time.sleep(0.1) continue log.debug('opening payload stdout/err logs') out = open(os.path.join(job['working_dir'], 'payload.stdout'), 'wb') err = open(os.path.join(job['working_dir'], 'payload.stderr'), 'wb') log.debug('setting up payload environment') send_state(job, 'starting') exit_code = 1 if setup_payload(job, out, err): log.debug('running payload') send_state(job, 'running') proc = run_payload(job, out, err) if proc is not None: exit_code = wait_graceful(args, proc, job) log.info('finished pid=%s exit_code=%s' % (proc.pid, exit_code)) log.debug('closing payload stdout/err logs') out.close() err.close() if exit_code == 0: queues.finished_payloads.put(job) else: queues.failed_payloads.put(job) except Queue.Empty: continue
def copytool_in(queues, traces, args): while not args.graceful_stop.is_set(): try: job = queues.data_in.get(block=True, timeout=1) send_state(job, 'transferring') if _stage_in(args, job): queues.finished_data_in.put(job) else: queues.failed_data_in.put(job) except Queue.Empty: continue
def copytool_out(queues, traces, args): while not args.graceful_stop.is_set(): try: job = queues.data_out.get(block=True, timeout=1) logger.info('dataset=%s rse=%s' % (job['destinationDblock'], job['ddmEndPointOut'].split(',')[0])) send_state(job, 'transferring') if _stage_out_all(job, args): queues.finished_data_out.put(job) else: queues.failed_data_out.put(job) except Queue.Empty: continue
def copytool_in(queues, traces, args): """ Call the stage-in function and put the job object in the proper queue. :param queues: internal queues for job handling. :param traces: tuple containing internal pilot states. :param args: Pilot arguments (e.g. containing queue name, queuedata dictionary, etc). :return: """ while not args.graceful_stop.is_set(): time.sleep(0.5) try: # abort if kill signal arrived too long time ago, ie loop is stuck current_time = int(time.time()) if args.kill_time and current_time - args.kill_time > MAX_KILL_WAIT_TIME: logger.warning( 'loop has run for too long time after first kill signal - will abort' ) break # extract a job to stage-in its input job = queues.data_in.get(block=True, timeout=1) # does the user want to execute any special commands before stage-in? pilot_user = os.environ.get('PILOT_USER', 'generic').lower() user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], 0) # Python 2/3 cmd = user.get_utility_commands(job=job, order=UTILITY_BEFORE_STAGEIN) if cmd: # xcache debug #_, _stdout, _stderr = execute('pgrep -x xrootd | awk \'{print \"ps -p \"$1\" -o args --no-headers --cols 300\"}\' | sh') #logger.debug('[before xcache start] stdout=%s', _stdout) #logger.debug('[before xcache start] stderr=%s', _stderr) _, stdout, stderr = execute(cmd.get('command')) logger.debug('stdout=%s', stdout) logger.debug('stderr=%s', stderr) # xcache debug #_, _stdout, _stderr = execute('pgrep -x xrootd | awk \'{print \"ps -p \"$1\" -o args --no-headers --cols 300\"}\' | sh') #logger.debug('[after xcache start] stdout=%s', _stdout) #logger.debug('[after xcache start] stderr=%s', _stderr) # perform any action necessary after command execution (e.g. stdout processing) kwargs = { 'label': cmd.get('label', 'utility'), 'output': stdout } user.post_prestagein_utility_command(**kwargs) # write output to log files write_utility_output(job.workdir, cmd.get('label', 'utility'), stdout, stderr) # place it in the current stage-in queue (used by the jobs' queue monitoring) put_in_queue(job, queues.current_data_in) # ready to set the job in running state send_state(job, args, 'running') # note: when sending a state change to the server, the server might respond with 'tobekilled' if job.state == 'failed': logger.warning( 'job state is \'failed\' - order log transfer and abort copytool_in()' ) job.stageout = 'log' # only stage-out log file put_in_queue(job, queues.data_out) break os.environ['SERVER_UPDATE'] = SERVER_UPDATE_RUNNING if args.abort_job.is_set(): traces.pilot['command'] = 'abort' logger.warning( 'copytool_in detected a set abort_job pre stage-in (due to a kill signal)' ) declare_failed_by_kill(job, queues.failed_data_in, args.signal) break if _stage_in(args, job): if args.abort_job.is_set(): traces.pilot['command'] = 'abort' logger.warning( 'copytool_in detected a set abort_job post stage-in (due to a kill signal)' ) declare_failed_by_kill(job, queues.failed_data_in, args.signal) break put_in_queue(job, queues.finished_data_in) # remove the job from the current stage-in queue _job = queues.current_data_in.get(block=True, timeout=1) if _job: logger.debug( 'job %s has been removed from the current_data_in queue', _job.jobid) # now create input file metadata if required by the payload if os.environ.get('PILOT_ES_EXECUTOR_TYPE', 'generic') == 'generic': pilot_user = os.environ.get('PILOT_USER', 'generic').lower() user = __import__('pilot.user.%s.metadata' % pilot_user, globals(), locals(), [pilot_user], 0) # Python 2/3 file_dictionary = get_input_file_dictionary(job.indata) xml = user.create_input_file_metadata( file_dictionary, job.workdir) logger.info('created input file metadata:\n%s', xml) else: # remove the job from the current stage-in queue _job = queues.current_data_in.get(block=True, timeout=1) if _job: logger.debug( 'job %s has been removed from the current_data_in queue', _job.jobid) logger.warning( 'stage-in failed, adding job object to failed_data_in queue' ) job.piloterrorcodes, job.piloterrordiags = errors.add_error_code( errors.STAGEINFAILED) set_pilot_state(job=job, state="failed") traces.pilot['error_code'] = job.piloterrorcodes[0] put_in_queue(job, queues.failed_data_in) # do not set graceful stop if pilot has not finished sending the final job update # i.e. wait until SERVER_UPDATE is DONE_FINAL check_for_final_server_update(args.update_server) args.graceful_stop.set() except queue.Empty: continue # proceed to set the job_aborted flag? if threads_aborted(): logger.debug('will proceed to set job_aborted') args.job_aborted.set() else: logger.debug('will not set job_aborted yet') logger.debug('[data] copytool_in thread has finished')
def execute_payloads(queues, traces, args): # noqa: C901 """ Execute queued payloads. Extract a Job object from the "validated_payloads" queue and put it in the "monitored_jobs" queue. The payload stdout/err streams are opened and the pilot state is changed to "starting". A payload executor is selected (for executing a normal job, an event service job or event service merge job). After the payload (or rather its executor) is started, the thread will wait for it to finish and then check for any failures. A successfully completed job is placed in the "finished_payloads" queue, and a failed job will be placed in the "failed_payloads" queue. :param queues: internal queues for job handling. :param traces: tuple containing internal pilot states. :param args: Pilot arguments (e.g. containing queue name, queuedata dictionary, etc). :return: """ job = None while not args.graceful_stop.is_set(): time.sleep(0.5) try: job = queues.validated_payloads.get(block=True, timeout=1) q_snapshot = list(queues.finished_data_in.queue) peek = [s_job for s_job in q_snapshot if job.jobid == s_job.jobid] if len(peek) == 0: #queues.validated_payloads.put(job) put_in_queue(job, queues.validated_payloads) for i in range(10): # Python 3 if args.graceful_stop.is_set(): break time.sleep(1) continue # this job is now to be monitored, so add it to the monitored_payloads queue #queues.monitored_payloads.put(job) put_in_queue(job, queues.monitored_payloads) logger.info('job %s added to monitored payloads queue' % job.jobid) try: out = open( os.path.join(job.workdir, config.Payload.payloadstdout), 'wb') err = open( os.path.join(job.workdir, config.Payload.payloadstderr), 'wb') except Exception as e: logger.warning('failed to open payload stdout/err: %s' % e) out = None err = None send_state(job, args, 'starting') # note: when sending a state change to the server, the server might respond with 'tobekilled' if job.state == 'failed': logger.warning( 'job state is \'failed\' - abort execute_payloads()') break payload_executor = get_payload_executor(args, job, out, err, traces) logger.info("Got payload executor: %s" % payload_executor) show_memory_usage() # run the payload and measure the execution time job.t0 = os.times() exit_code = payload_executor.run() set_cpu_consumption_time(job) job.transexitcode = exit_code % 255 out.close() err.close() pilot_user = os.environ.get('PILOT_USER', 'generic').lower() # some HPO jobs will produce new output files (following lfn name pattern), discover those and replace the job.outdata list if job.is_hpo: user = __import__('pilot.user.%s.common' % pilot_user, globals(), locals(), [pilot_user], 0) # Python 2/3 try: user.update_output_for_hpo(job) except Exception as e: logger.warning( 'exception caught by update_output_for_hpo(): %s' % e) else: for dat in job.outdata: if not dat.guid: dat.guid = get_guid() logger.warning( 'guid not set: generated guid=%s for lfn=%s' % (dat.guid, dat.lfn)) #if traces.pilot['nr_jobs'] == 1: # logger.debug('faking job failure in first multi-job') # job.transexitcode = 1 # exit_code = 1 # analyze and interpret the payload execution output perform_initial_payload_error_analysis(job, exit_code) # was an error already found? #if job.piloterrorcodes: # exit_code_interpret = 1 #else: user = __import__('pilot.user.%s.diagnose' % pilot_user, globals(), locals(), [pilot_user], 0) # Python 2/3 try: exit_code_interpret = user.interpret(job) except Exception as e: logger.warning('exception caught: %s' % e) #exit_code_interpret = -1 job.piloterrorcodes, job.piloterrordiags = errors.add_error_code( errors.INTERNALPILOTPROBLEM) if job.piloterrorcodes: exit_code_interpret = 1 if exit_code_interpret == 0 and exit_code == 0: logger.info( 'main payload error analysis completed - did not find any errors' ) # update output lists if zipmaps were used #job.add_archives_to_output_lists() # queues.finished_payloads.put(job) put_in_queue(job, queues.finished_payloads) else: logger.debug( 'main payload error analysis completed - adding job to failed_payloads queue' ) #queues.failed_payloads.put(job) put_in_queue(job, queues.failed_payloads) except queue.Empty: continue except Exception as e: logger.fatal( 'execute payloads caught an exception (cannot recover): %s, %s' % (e, traceback.format_exc())) if job: job.piloterrorcodes, job.piloterrordiags = errors.add_error_code( errors.PAYLOADEXECUTIONEXCEPTION) #queues.failed_payloads.put(job) put_in_queue(job, queues.failed_payloads) while not args.graceful_stop.is_set(): # let stage-out of log finish, but stop running payloads as there should be a problem with the pilot time.sleep(5) # proceed to set the job_aborted flag? if threads_aborted(): logger.debug('will proceed to set job_aborted') args.job_aborted.set() else: logger.debug('will not set job_aborted yet') logger.info('[payload] execute_payloads thread has finished')
def copytool_in(queues, traces, args): """ Call the stage-in function and put the job object in the proper queue. :param queues: :param traces: :param args: :return: """ while not args.graceful_stop.is_set(): try: # extract a job to stage-in its input job = queues.data_in.get(block=True, timeout=1) # place it in the current stage-in queue (used by the jobs' queue monitoring) if job: put_in_queue(job, queues.current_data_in) # ready to set the job in running state send_state(job, args, 'running') os.environ['SERVER_UPDATE'] = SERVER_UPDATE_RUNNING log = get_logger(job.jobid) if args.abort_job.is_set(): traces.pilot['command'] = 'abort' log.warning('copytool_in detected a set abort_job pre stage-in (due to a kill signal)') declare_failed_by_kill(job, queues.failed_data_in, args.signal) break if _stage_in(args, job): if args.abort_job.is_set(): traces.pilot['command'] = 'abort' log.warning('copytool_in detected a set abort_job post stage-in (due to a kill signal)') declare_failed_by_kill(job, queues.failed_data_in, args.signal) break #queues.finished_data_in.put(job) put_in_queue(job, queues.finished_data_in) # remove the job from the current stage-in queue _job = queues.current_data_in.get(block=True, timeout=1) if _job: log.debug('job %s has been removed from the current_data_in queue' % _job.jobid) # now create input file metadata if required by the payload try: pilot_user = os.environ.get('PILOT_USER', 'generic').lower() user = __import__('pilot.user.%s.metadata' % pilot_user, globals(), locals(), [pilot_user], -1) _dir = '/srv' if job.usecontainer else job.workdir file_dictionary = get_input_file_dictionary(job.indata, _dir) #file_dictionary = get_input_file_dictionary(job.indata, job.workdir) log.debug('file_dictionary=%s' % str(file_dictionary)) xml = user.create_input_file_metadata(file_dictionary, job.workdir) log.info('created input file metadata:\n%s' % xml) except Exception as e: pass else: log.warning('stage-in failed, adding job object to failed_data_in queue') job.piloterrorcodes, job.piloterrordiags = errors.add_error_code(errors.STAGEINFAILED) set_pilot_state(job=job, state="failed") traces.pilot['error_code'] = job.piloterrorcodes[0] #queues.failed_data_in.put(job) put_in_queue(job, queues.failed_data_in) # do not set graceful stop if pilot has not finished sending the final job update # i.e. wait until SERVER_UPDATE is DONE_FINAL check_for_final_server_update(args.update_server) args.graceful_stop.set() # send_state(job, args, 'failed') except queue.Empty: continue logger.debug('[data] copytool_in thread has finished')
def run(self): # noqa: C901 """ Run all payload processes (including pre- and post-processes, and utilities). In the case of HPO jobs, this function will loop over all processes until the preprocess returns a special exit code. :return: """ # get the payload command from the user specific code self.pre_setup(self.__job) cmd = self.get_payload_command(self.__job) # extract the setup in case the preprocess command needs it self.__job.setup = self.extract_setup(cmd) self.post_setup(self.__job) # a loop is needed for HPO jobs # abort when nothing more to run, or when the preprocess returns a special exit code iteration = 0 while True: logger.info('payload iteration loop #%d', iteration + 1) os.environ['PILOT_EXEC_ITERATION_COUNT'] = '%s' % iteration show_memory_usage() # first run the preprocess (if necessary) - note: this might update jobparams -> must update cmd jobparams_pre = self.__job.jobparams exit_code = self.run_preprocess(self.__job) jobparams_post = self.__job.jobparams if exit_code: if exit_code >= 160 and exit_code <= 162: exit_code = 0 # wipe the output file list since there won't be any new files # any output files from previous iterations, should have been transferred already logger.debug( 'reset outdata since further output should not be expected after preprocess exit' ) self.__job.outdata = [] break if jobparams_pre != jobparams_post: logger.debug( 'jobparams were updated by utility_before_payload()') # must update cmd cmd = cmd.replace(jobparams_pre, jobparams_post) # now run the main payload, when it finishes, run the postprocess (if necessary) # note: no need to run any main payload in HPO Horovod jobs on Kubernetes if os.environ.get('HARVESTER_HOROVOD', '') == '': #exit_code, _stdout, _stderr = execute('pgrep -x xrootd | awk \'{print \"ps -p \"$1\" -o args --no-headers --cols 300\"}\' | sh') #logger.debug('[before payload start] stdout=%s', _stdout) #logger.debug('[before payload start] stderr=%s', _stderr) proc = self.run_payload(self.__job, cmd, self.__out, self.__err) else: proc = None proc_co = None if proc is None: # run the post-process command even if there was no main payload if os.environ.get('HARVESTER_HOROVOD', '') != '': logger.info('No need to execute any main payload') exit_code = self.run_utility_after_payload_finished( exit_code, True, UTILITY_AFTER_PAYLOAD_FINISHED2) self.post_payload(self.__job) else: break else: # the process is now running, update the server # test 'tobekilled' from here to try payload kill send_state(self.__job, self.__args, self.__job.state) # note: when sending a state change to the server, the server might respond with 'tobekilled' if self.__job.state == 'failed': logger.warning( 'job state is \'failed\' - abort payload and run()') kill_processes(proc.pid) break # allow for a secondary command to be started after the payload (e.g. a coprocess) utility_cmd = self.get_utility_command( order=UTILITY_AFTER_PAYLOAD_STARTED2) if utility_cmd: logger.debug('starting utility command: %s', utility_cmd) label = 'coprocess' if 'coprocess' in utility_cmd else None proc_co = self.run_command(utility_cmd, label=label) logger.info('will wait for graceful exit') exit_code = self.wait_graceful(self.__args, proc) # reset error if Raythena decided to kill payload (no error) if errors.KILLPAYLOAD in self.__job.piloterrorcodes: logger.debug('ignoring KILLPAYLOAD error') self.__job.piloterrorcodes, self.__job.piloterrordiags = errors.remove_error_code( errors.KILLPAYLOAD, pilot_error_codes=self.__job.piloterrorcodes, pilot_error_diags=self.__job.piloterrordiags) exit_code = 0 state = 'finished' else: state = 'finished' if exit_code == 0 else 'failed' set_pilot_state(job=self.__job, state=state) logger.info('\n\nfinished pid=%s exit_code=%s state=%s\n', proc.pid, exit_code, self.__job.state) #exit_code, _stdout, _stderr = execute('pgrep -x xrootd | awk \'{print \"ps -p \"$1\" -o args --no-headers --cols 300\"}\' | sh') #logger.debug('[after payload finish] stdout=%s', _stdout) #logger.debug('[after payload finish] stderr=%s', _stderr) # stop the utility command (e.g. a coprocess if necessary if proc_co: logger.debug('stopping utility command: %s', utility_cmd) kill_processes(proc_co.pid) if exit_code is None: logger.warning( 'detected unset exit_code from wait_graceful - reset to -1' ) exit_code = -1 for order in [ UTILITY_AFTER_PAYLOAD_FINISHED, UTILITY_AFTER_PAYLOAD_FINISHED2 ]: exit_code = self.run_utility_after_payload_finished( exit_code, state, order) self.post_payload(self.__job) # stop any running utilities if self.__job.utilities != {}: self.stop_utilities() if self.__job.is_hpo and state != 'failed': # in case there are more hyper-parameter points, move away the previous log files #self.rename_log_files(iteration) iteration += 1 else: break return exit_code
def execute_payloads(queues, traces, args): """ Execute queued payloads. :param queues: :param traces: :param args: :return: """ job = None while not args.graceful_stop.is_set(): time.sleep(0.5) try: job = queues.validated_payloads.get(block=True, timeout=1) log = get_logger(job.jobid, logger) q_snapshot = list(queues.finished_data_in.queue) peek = [s_job for s_job in q_snapshot if job.jobid == s_job.jobid] if len(peek) == 0: #queues.validated_payloads.put(job) put_in_queue(job, queues.validated_payloads) for i in range(10): # Python 3 if args.graceful_stop.is_set(): break time.sleep(1) continue # this job is now to be monitored, so add it to the monitored_payloads queue #queues.monitored_payloads.put(job) put_in_queue(job, queues.monitored_payloads) log.info('job %s added to monitored payloads queue' % job.jobid) out = open(os.path.join(job.workdir, config.Payload.payloadstdout), 'wb') err = open(os.path.join(job.workdir, config.Payload.payloadstderr), 'wb') send_state(job, args, 'starting') payload_executor = get_payload_executor(args, job, out, err, traces) log.info("Got payload executor: %s" % payload_executor) # run the payload and measure the execution time job.t0 = os.times() exit_code = payload_executor.run() set_cpu_consumption_time(job) job.transexitcode = exit_code % 255 out.close() err.close() # analyze and interpret the payload execution output perform_initial_payload_error_analysis(job, exit_code) # was an error already found? if job.piloterrorcodes: exit_code_interpret = 1 else: pilot_user = os.environ.get('PILOT_USER', 'generic').lower() user = __import__('pilot.user.%s.diagnose' % pilot_user, globals(), locals(), [pilot_user], 0) # Python 2/3 try: exit_code_interpret = user.interpret(job) except Exception as e: log.warning('exception caught: %s' % e) exit_code_interpret = -1 job.piloterrorcodes, job.piloterrordiags = errors.add_error_code( errors.INTERNALPILOTPROBLEM) if exit_code_interpret == 0 and exit_code == 0: log.info( 'main payload error analysis completed - did not find any errors' ) # update output lists if zipmaps were used #job.add_archives_to_output_lists() # queues.finished_payloads.put(job) put_in_queue(job, queues.finished_payloads) else: log.debug( 'main payload error analysis completed - adding job to failed_payloads queue' ) #queues.failed_payloads.put(job) put_in_queue(job, queues.failed_payloads) except queue.Empty: continue except Exception as e: logger.fatal( 'execute payloads caught an exception (cannot recover): %s, %s' % (e, traceback.format_exc())) if job: job.piloterrorcodes, job.piloterrordiags = errors.add_error_code( errors.PAYLOADEXECUTIONEXCEPTION) #queues.failed_payloads.put(job) put_in_queue(job, queues.failed_payloads) while not args.graceful_stop.is_set(): # let stage-out of log finish, but stop running payloads as there should be a problem with the pilot time.sleep(5) logger.info('[payload] execute_payloads thread has finished')
def copytool_in(queues, traces, args): """ Call the stage-in function and put the job object in the proper queue. :param queues: internal queues for job handling. :param traces: tuple containing internal pilot states. :param args: Pilot arguments (e.g. containing queue name, queuedata dictionary, etc). :return: """ while not args.graceful_stop.is_set(): time.sleep(0.5) try: # abort if kill signal arrived too long time ago, ie loop is stuck current_time = int(time.time()) if args.kill_time and current_time - args.kill_time > MAX_KILL_WAIT_TIME: logger.warning( 'loop has run for too long time after first kill signal - will abort' ) break # extract a job to stage-in its input job = queues.data_in.get(block=True, timeout=1) # place it in the current stage-in queue (used by the jobs' queue monitoring) if job: put_in_queue(job, queues.current_data_in) # ready to set the job in running state send_state(job, args, 'running') # note: when sending a state change to the server, the server might respond with 'tobekilled' if job.state == 'failed': logger.warning( 'job state is \'failed\' - order log transfer and abort copytool_in()' ) job.stageout = 'log' # only stage-out log file put_in_queue(job, queues.data_out) break os.environ['SERVER_UPDATE'] = SERVER_UPDATE_RUNNING if args.abort_job.is_set(): traces.pilot['command'] = 'abort' logger.warning( 'copytool_in detected a set abort_job pre stage-in (due to a kill signal)' ) declare_failed_by_kill(job, queues.failed_data_in, args.signal) break if _stage_in(args, job): if args.abort_job.is_set(): traces.pilot['command'] = 'abort' logger.warning( 'copytool_in detected a set abort_job post stage-in (due to a kill signal)' ) declare_failed_by_kill(job, queues.failed_data_in, args.signal) break put_in_queue(job, queues.finished_data_in) # remove the job from the current stage-in queue _job = queues.current_data_in.get(block=True, timeout=1) if _job: logger.debug( 'job %s has been removed from the current_data_in queue' % _job.jobid) # now create input file metadata if required by the payload if config.Payload.executor_type.lower() != 'raythena': pilot_user = os.environ.get('PILOT_USER', 'generic').lower() user = __import__('pilot.user.%s.metadata' % pilot_user, globals(), locals(), [pilot_user], 0) # Python 2/3 file_dictionary = get_input_file_dictionary(job.indata) xml = user.create_input_file_metadata( file_dictionary, job.workdir) logger.info('created input file metadata:\n%s' % xml) else: logger.warning( 'stage-in failed, adding job object to failed_data_in queue' ) job.piloterrorcodes, job.piloterrordiags = errors.add_error_code( errors.STAGEINFAILED) set_pilot_state(job=job, state="failed") traces.pilot['error_code'] = job.piloterrorcodes[0] put_in_queue(job, queues.failed_data_in) # do not set graceful stop if pilot has not finished sending the final job update # i.e. wait until SERVER_UPDATE is DONE_FINAL check_for_final_server_update(args.update_server) args.graceful_stop.set() except queue.Empty: continue # proceed to set the job_aborted flag? if threads_aborted(): logger.debug('will proceed to set job_aborted') args.job_aborted.set() else: logger.debug('will not set job_aborted yet') logger.debug('[data] copytool_in thread has finished')