def getNodeStructureFromFile(self, workDir, jobId): """ get the node structure from the Job State file """ JS = JobState() _node = None # open the job state file tolog("workDir: %s" % (workDir)) tolog("jobId: %s" % (jobId)) filename = JS.getFilename(workDir, jobId) tolog("filename: %s" % (filename)) if os.path.exists(filename): # load the objects if JS.get(filename): # decode the job state info _job, _site, _node, _recoveryAttempt = JS.decode() else: tolog("JS.decode() failed to load objects") else: tolog("%s does not exist" % (filename)) return _node
def getNodeStructureFromFile(self, workDir, jobId): """ get the node structure from the Job State file """ JS = JobState() _node = None # open the job state file tolog("workDir: %s" % (workDir)) tolog("jobId: %s" % (jobId)) filename = JS.getFilename(workDir, jobId) tolog("filename: %s" % (filename)) if os.path.exists(filename): # load the objects if JS.get(filename): # decode the job state info _job, _site, _node, _recoveryAttempt = JS.decode() else: tolog("JS.decode() failed to load objects") else: tolog("%s does not exist" % (filename)) return _node
def DeferredStageoutJob(job_dir, job_state_file="", deferred_stageout_logfile=False, **kwargs): """ Performs stageing out preparation and stages out the job in specified directory. :param job_dir: (string) directory with a job. mandatory parameter :param job_state_file: (string) path to job state file or other file containing job state. If empty, job state file is located as job_dir+'/jobState-*.*'. defaults to "" :param deferred_stageout_logfile: (string|False) template name for deferred log stageout Replaces "{job_id}" with current job id like "log-{job_id}.txt" -> "log-124124.txt" Default False Other parameters are passed into other functions :return: (bool) the fact of stageout being performed """ log('Deferred stageout from job directory "%s"' % job_dir) job_state = JobState() if job_state_file == "": try: job_state_file = glob(job_dir + "/" + jobState_file_wildcart)[0] except: log("There is no job state file in the provided directory, exiting") return False log("Job state file is %s" % job_state_file) # lockfd, lockfn = createAtomicLockFile(job_dir) with LockFileWrapper(job_dir): if not TestJobDirForDeferredStageoutNecessity(job_dir, job_state_file, **kwargs): log('Job "%s" does not need deferred stageout procedure (yet)' % job_dir) # releaseAtomicLockFile(lockfd, lockfn) return False if not job_state.get(job_state_file): log("Job state file reading failed, exiting") # releaseAtomicLockFile(lockfd, lockfn) return False log('Working with job in "%s"' % job_dir) _job, _site, _node, _recoveryAttempt = job_state.decode() if not (_job and _site and _node): log("Can not decode jobState file, exiting") # releaseAtomicLockFile(lockfd, lockfn) return False with LogWrapper(deferred_stageout_logfile, _job.jobId) as logger: rc = PrepareJobForDeferredStageout(job_state, **kwargs) if rc == ReturnCode.PostJobOnly: pUtil.postJobTask( job_state.job, job_state.site, DorE(kwargs, "workerNode"), DorE(kwargs, "experiment"), jr=True, ra=job_state.recoveryAttempt, ) # releaseAtomicLockFile(lockfd, lockfn) return True if rc > 0: log("Job is not prepared for stageout, exiting") if rc == ReturnCode.Cleanup: cleanup(job_state) # releaseAtomicLockFile(lockfd, lockfn) return False rc, logfile, datadir, filelist = CreateTransferFileList(job_state, **kwargs) XMLStr = "" if datadir == "": try: XMLStr = job_state.node["xml"] except: pass if XMLStr == "": XMLStr = pUtil.getMetadata(job_state.site.workdir, job_state.job.jobId) currentdir = os.getcwd() pUtil.chdir(job_state.site.workdir) if len(filelist): log("Stageout will now transfer the files") rc = TransferFiles(job_state, datadir, filelist, **kwargs) if rc == ReturnCode.Holding: job_state.job.result[0] = "holding" if rc == ReturnCode.FailedJob: job_state.job.result[0] = "failed" job_state.job.setState(job_state.job.result) pUtil.chdir(job_state.site.workdir) ret = True if logfile != "" and not pUtil.isLogfileCopied(job_state.site.workdir): log("Stageout will now transfer the log") _log = JobLog() ret, _ = _log.transferLogFile( job_state.job, job_state.site, DorE(kwargs, "experiment"), dest=None, jr=True ) if not ret: rc = ReturnCode.Holding # We need to transfer log file regardless the files if rc == ReturnCode.OK: if pUtil.verifyTransfer(job_state.site.workdir): job_state.job.result[0] = "finished" else: job_state.job.result[0] = "failed" job_state.job.setState(job_state.job.result) if job_state.job.result[0] in finalJobStates: job_state.job.final_state = job_state.job.result[0] log("Stageout will now update the server with new status") rt, retNode = updatePandaServer(job_state, xmlstr=XMLStr, **kwargs) if rt == 0: log("Job %s updated (exit code %d)" % (job_state.job.jobId, job_state.job.result[2])) # did the server send back a command? if "tobekilled" in job_state.job.action: log("!!WARNING!!1120!! Panda server returned a 'tobekilled' command") job_state.job.result[0] = "failed" # further recovery attempt unnecessary, but keep the work dir for debugging if job_state.job.result[0] == "failed": log("Further recovery attempts will be prevented for failed job (will leave work dir)") if not job_state.rename(job_state.site, job_state.job): log("(Fate of job state file left for next pilot)") else: log("!!WARNING!!1120!! Panda server returned a %d" % (rt)) # store the final state so that the next pilot will know # store the metadata xml retNode["xml"] = XMLStr # update the job state file with the new state information _retjs = pUtil.updateJobState(job_state.job, job_state.site, retNode, job_state.recoveryAttempt) log("Stageout will now proceed to post-job actions") if job_state.job.result[0] in finalJobStates: pUtil.postJobTask( job_state.job, job_state.site, DorE(kwargs, "workerNode"), DorE(kwargs, "experiment"), jr=True, ra=job_state.recoveryAttempt, ) pUtil.chdir(currentdir) # releaseAtomicLockFile(lockfd, lockfn) if job_state.job.result[0] == "finished": log("Stageout will now remove the job, it is in finished state and can be removed") cleanup(job_state) return True
def DeferredStageoutHPCJob(job_dir, deferred_stageout_logfile=False, **kwargs): """ Performs stageing out preparation for the HPC job in specified directory. :param job_dir: (string) directory with a job. mandatory parameter :param deferred_stageout_logfile: (string|False) template name for deferred log stageout Replaces "{job_id}" with current job id like "log-{job_id}.txt" -> "log-124124.txt" Default False Other parameters are passed into other functions :return: (bool) the fact of stageout being performed """ log('Deferred stageout from HPC job directory "%s"' % job_dir) file_path = job_dir + "/" + hpc_jobState_file_wildcart current_dir = os.getcwd() log("Working on %s" % file_path) log("Chdir from current dir %s to %s" % (current_dir, job_dir)) pUtil.chdir(job_dir) try: with LockFileWrapper(file_path) as is_locked: if not is_locked: return False from json import load with open(file_path) as data_file: HPC_state = load(data_file) job_state_file = HPC_state["JobStateFile"] job_command = HPC_state["JobCommand"] # global_work_dir = HPC_state['GlobalWorkingDir'] JS = JobState() JS.get(job_state_file) _job, _site, _node, _recoveryAttempt = JS.decode() with LogWrapper(deferred_stageout_logfile, _job.jobId) as logger: jobStatus, jobAttemptNr, jobStatusCode = pUtil.getJobStatus( _job.jobId, DorE(kwargs, "pshttpurl"), DorE(kwargs, "psport"), DorE(kwargs, "pilot_initdir") ) # recover this job? if jobStatusCode == 20: log("Received general error code from dispatcher call (leave job for later pilot)") # release the atomic lockfile and go to the next directory # releaseAtomicLockFile(fd, lockfile_name) return False elif jobStatus in finalJobStates or "tobekilled" in _job.action: log( "Job %s is currently in state '%s' with attemptNr = %d (according to server - will not" " perform staging out)" % (_job.jobId, jobStatus, jobAttemptNr) ) # releaseAtomicLockFile(fd, lockfile_name) return False # update job state file at this point to prevent a parallel pilot from doing a simultaneous recovery _retjs = pUtil.updateJobState(_job, _site, _node, _recoveryAttempt) # releaseAtomicLockFile(fd, lockfile_name) monitor = Monitor(env) monitor.monitor_recovery_job(_job, _site, _node, job_command, job_state_file, recover_dir=job_dir) log("Chdir back to %s" % current_dir) pUtil.chdir(current_dir) panda_jobs = glob(job_dir + "/PandaJob_*_*") panda_logs = glob(job_dir + "/*.log.tgz.*") if panda_jobs or panda_logs: log( "Number of founded panda jobs: %d, number of panda log tar file %d, will not remove job dir" % (len(panda_jobs), len(panda_logs)) ) else: log( "Number of founded panda jobs: %d, number of panda log tar file %d, will remove job dir" % (len(panda_jobs), len(panda_logs)) ) log("Remove job dir %s" % job_dir) os.system("rm -rf %s" % job_dir) return True except: log("Failed to start deferred stage out for HPC job: %s" % traceback.format_exc()) return False
def cleanup(self): """ execute the clean-up """ status = True number_of_cleanups = 0 if self.clean: tolog("Executing empty dirs clean-up, stage 1/5") Cleaner.purgeEmptyDirs(self.path) tolog("Executing work dir clean-up, stage 2/5") Cleaner.purgeWorkDirs(self.path) tolog("Executing maxed-out dirs clean-up, stage 3/5") Cleaner.purgeMaxedoutDirs(self.path) tolog("Executing AthenaMP clean-up, stage 4/5 <SKIPPED>") #files = ['AthenaMP_*', 'fifo_*', 'TokenExtractorChannel*', 'zmq_EventService*', 'asetup*', 'tmp*.pkl'] #for f in files: # Cleaner.purgeFiles(self.path, f, limit=48*3600) tolog("Executing PanDA Pilot dir clean-up, stage 5/5") JS = JobState() # grab all job state files in all work directories job_state_files = glob(self.path + "/Panda_Pilot_*/jobState-*.pickle") number_of_files = len(job_state_files) file_number = 0 max_cleanups = 30 tolog("Number of found job state files: %d" % (number_of_files)) if job_state_files: # loop over all found job state files for file_path in job_state_files: file_number += 1 if file_number > max_cleanups: tolog("Maximum number of job recoveries exceeded for this pilot: %d" % (max_cleanups)) break tolog("Processing job state file %d/%d: %s" % (file_number, number_of_files, file_path)) current_time = int(time.time()) # when was file last modified? try: file_modification_time = os.path.getmtime(file_path) except: # skip this file since it was not possible to read the modification time pass else: # was the job state file updated longer than the time limit? (convert to seconds) mod_time = current_time - file_modification_time if mod_time > self.limit*3600: tolog("File was last modified %d seconds ago (proceed)" % (mod_time)) cmd = "whoami; ls -lF %s; ls -lF %s" % (file_path, os.path.dirname(file_path)) tolog("Executing command: %s" % (cmd)) ec, rs = commands.getstatusoutput(cmd) if ec == 0: tolog("%s" % (rs)) else: tolog("!!WARNING!!2999!! %d, %s" % (ec, rs)) # open the job state file if JS.get(file_path): # decode the job state info _job, _site, _node, _recoveryAttempt = JS.decode() # add member if it doesn't exist (new Job version) try: _tmp = _job.prodSourceLabel except: _job.prodSourceLabel = '' if _job and _site and _node: # query the job state file for job information if _job.result[0] == 'running' or _job.result[0] == 'starting' or (_job.result[0] == 'holding' and mod_time > 7*24*3600): if _job.result[0] == 'holding': tolog("Job %s was found in %s state but has not been modified for a long time - will be cleaned up" % (_job.jobId, _job.result[0])) else: tolog("Job %s was found in %s state - will be cleaned up" % (_job.jobId, _job.result[0])) tolog("Erasing directory: %s" % (_site.workdir)) cmd = "rm -rf %s" % (_site.workdir) try: ec, rs = commands.getstatusoutput(cmd) except: tolog("!!WARNING!!5500!! Could not erase lost job workdir: %d, %s" % (ec, rs)) status = False break else: tolog("Lost job workdir removed") else: tolog("Job found in state: %s" % (_job.result[0])) else: tolog("File was last modified %d seconds ago (skip)" % (mod_time)) else: tolog("No job state files were found, aborting clean-up") else: tolog("Clean-up turned off") status = False return status
def cleanup(self): """ execute the clean-up """ status = True number_of_cleanups = 0 if self.clean: tolog("Executing empty dirs clean-up, stage 1/5") Cleaner.purgeEmptyDirs(self.path) tolog("Executing work dir clean-up, stage 2/5") Cleaner.purgeWorkDirs(self.path) tolog("Executing maxed-out dirs clean-up, stage 3/5") Cleaner.purgeMaxedoutDirs(self.path) tolog("Executing AthenaMP clean-up, stage 4/5 <SKIPPED>") #files = ['AthenaMP_*', 'fifo_*', 'TokenExtractorChannel*', 'zmq_EventService*', 'asetup*', 'tmp*.pkl'] #for f in files: # Cleaner.purgeFiles(self.path, f, limit=48*3600) tolog("Executing PanDA Pilot dir clean-up, stage 5/5") JS = JobState() # grab all job state files in all work directories job_state_files = glob(self.path + "/Panda_Pilot_*/jobState-*.pickle") number_of_files = len(job_state_files) file_number = 0 max_cleanups = 30 tolog("Number of found job state files: %d" % (number_of_files)) if job_state_files: # loop over all found job state files for file_path in job_state_files: file_number += 1 if file_number > max_cleanups: tolog( "Maximum number of job recoveries exceeded for this pilot: %d" % (max_cleanups)) break tolog("Processing job state file %d/%d: %s" % (file_number, number_of_files, file_path)) current_time = int(time.time()) # when was file last modified? try: file_modification_time = os.path.getmtime(file_path) except: # skip this file since it was not possible to read the modification time pass else: # was the job state file updated longer than the time limit? (convert to seconds) mod_time = current_time - file_modification_time if mod_time > self.limit * 3600: tolog( "File was last modified %d seconds ago (proceed)" % (mod_time)) cmd = "whoami; ls -lF %s; ls -lF %s" % ( file_path, os.path.dirname(file_path)) tolog("Executing command: %s" % (cmd)) ec, rs = commands.getstatusoutput(cmd) if ec == 0: tolog("%s" % (rs)) else: tolog("!!WARNING!!2999!! %d, %s" % (ec, rs)) # open the job state file if JS.get(file_path): # decode the job state info _job, _site, _node, _recoveryAttempt = JS.decode( ) # add member if it doesn't exist (new Job version) try: _tmp = _job.prodSourceLabel except: _job.prodSourceLabel = '' if _job and _site and _node: # query the job state file for job information if _job.result[ 0] == 'running' or _job.result[ 0] == 'starting' or ( _job.result[0] == 'holding' and mod_time > 7 * 24 * 3600): if _job.result[0] == 'holding': tolog( "Job %s was found in %s state but has not been modified for a long time - will be cleaned up" % (_job.jobId, _job.result[0])) else: tolog( "Job %s was found in %s state - will be cleaned up" % (_job.jobId, _job.result[0])) tolog("Erasing directory: %s" % (_site.workdir)) cmd = "rm -rf %s" % (_site.workdir) try: ec, rs = commands.getstatusoutput( cmd) except: tolog( "!!WARNING!!5500!! Could not erase lost job workdir: %d, %s" % (ec, rs)) status = False break else: tolog("Lost job workdir removed") else: tolog("Job found in state: %s" % (_job.result[0])) else: tolog( "File was last modified %d seconds ago (skip)" % (mod_time)) else: tolog("No job state files were found, aborting clean-up") else: tolog("Clean-up turned off") status = False return status
def DeferredStageoutJob(job_dir, job_state_file="", deferred_stageout_logfile=False, **kwargs): """ Performs stageing out preparation and stages out the job in specified directory. :param job_dir: (string) directory with a job. mandatory parameter :param job_state_file: (string) path to job state file or other file containing job state. If empty, job state file is located as job_dir+'/jobState-*.*'. defaults to "" :param deferred_stageout_logfile: (string|False) template name for deferred log stageout Replaces "{job_id}" with current job id like "log-{job_id}.txt" -> "log-124124.txt" Default False Other parameters are passed into other functions :return: (bool) the fact of stageout being performed """ log("Deferred stageout from job directory \"%s\"" % job_dir) job_state = JobState() if job_state_file == "": try: job_state_file = glob(job_dir + "/" + jobState_file_wildcart)[0] except: log("There is no job state file in the provided directory, exiting") return False log("Job state file is %s"%job_state_file) # lockfd, lockfn = createAtomicLockFile(job_dir) with LockFileWrapper(job_dir): if not TestJobDirForDeferredStageoutNecessity(job_dir, job_state_file, **kwargs): log("Job \"%s\" does not need deferred stageout procedure (yet)" % job_dir) # releaseAtomicLockFile(lockfd, lockfn) return False if not job_state.get(job_state_file): log("Job state file reading failed, exiting") # releaseAtomicLockFile(lockfd, lockfn) return False log("Working with job in \"%s\"" % job_dir) _job, _site, _node, _recoveryAttempt = job_state.decode() if not (_job and _site and _node): log("Can not decode jobState file, exiting") # releaseAtomicLockFile(lockfd, lockfn) return False with LogWrapper(deferred_stageout_logfile, _job.jobId) as logger: rc = PrepareJobForDeferredStageout(job_state, **kwargs) if rc == ReturnCode.PostJobOnly: pUtil.postJobTask(job_state.job, job_state.site, DorE(kwargs, 'workerNode'), DorE(kwargs, 'experiment'), jr=True, ra=job_state.recoveryAttempt) # releaseAtomicLockFile(lockfd, lockfn) return True if rc > 0: log("Job is not prepared for stageout, exiting") if rc == ReturnCode.Cleanup: cleanup(job_state) # releaseAtomicLockFile(lockfd, lockfn) return False rc, logfile, datadir, filelist = CreateTransferFileList(job_state, **kwargs) XMLStr = '' if datadir == "": try: XMLStr = job_state.node['xml'] except: pass if XMLStr == '': XMLStr = pUtil.getMetadata(job_state.site.workdir, job_state.job.jobId) currentdir = os.getcwd() pUtil.chdir(job_state.site.workdir) if len(filelist): log("Stageout will now transfer the files") rc = TransferFiles(job_state, datadir, filelist, **kwargs) if rc == ReturnCode.Holding: job_state.job.result[0] = "holding" if rc == ReturnCode.FailedJob: job_state.job.result[0] = "failed" job_state.job.setState(job_state.job.result) pUtil.chdir(job_state.site.workdir) ret = True if logfile != "" and not pUtil.isLogfileCopied(job_state.site.workdir): log("Stageout will now transfer the log") _log = JobLog() ret, _ = _log.transferLogFile(job_state.job, job_state.site, DorE(kwargs, 'experiment'), dest=None, jr=True) if not ret: rc = ReturnCode.Holding # We need to transfer log file regardless the files if rc == ReturnCode.OK: if pUtil.verifyTransfer(job_state.site.workdir): job_state.job.result[0] = "finished" else: job_state.job.result[0] = "failed" job_state.job.setState(job_state.job.result) if job_state.job.result[0] in finalJobStates: job_state.job.final_state = job_state.job.result[0] log("Stageout will now update the server with new status") rt, retNode = updatePandaServer(job_state, xmlstr=XMLStr, **kwargs) if rt == 0: log("Job %s updated (exit code %d)" % (job_state.job.jobId, job_state.job.result[2])) # did the server send back a command? if "tobekilled" in job_state.job.action: log("!!WARNING!!1120!! Panda server returned a \'tobekilled\' command") job_state.job.result[0] = "failed" # further recovery attempt unnecessary, but keep the work dir for debugging if job_state.job.result[0] == "failed": log("Further recovery attempts will be prevented for failed job (will leave work dir)") if not job_state.rename(job_state.site, job_state.job): log("(Fate of job state file left for next pilot)") else: log("!!WARNING!!1120!! Panda server returned a %d" % (rt)) # store the final state so that the next pilot will know # store the metadata xml retNode['xml'] = XMLStr # update the job state file with the new state information _retjs = pUtil.updateJobState(job_state.job, job_state.site, retNode, job_state.recoveryAttempt) log("Stageout will now proceed to post-job actions") if job_state.job.result[0] in finalJobStates: pUtil.postJobTask(job_state.job, job_state.site, DorE(kwargs, 'workerNode'), DorE(kwargs, 'experiment'), jr=True, ra=job_state.recoveryAttempt) pUtil.chdir(currentdir) # releaseAtomicLockFile(lockfd, lockfn) if job_state.job.result[0] == "finished": log("Stageout will now remove the job, it is in finished state and can be removed") cleanup(job_state) return True
def DeferredStageoutHPCJob(job_dir, deferred_stageout_logfile=False, **kwargs): """ Performs stageing out preparation for the HPC job in specified directory. :param job_dir: (string) directory with a job. mandatory parameter :param deferred_stageout_logfile: (string|False) template name for deferred log stageout Replaces "{job_id}" with current job id like "log-{job_id}.txt" -> "log-124124.txt" Default False Other parameters are passed into other functions :return: (bool) the fact of stageout being performed """ log("Deferred stageout from HPC job directory \"%s\"" % job_dir) file_path = job_dir+"/"+hpc_jobState_file_wildcart current_dir = os.getcwd() log("Working on %s" % file_path) log("Chdir from current dir %s to %s" % (current_dir, job_dir)) pUtil.chdir(job_dir) try: with LockFileWrapper(file_path) as is_locked: if not is_locked: return False from json import load with open(file_path) as data_file: HPC_state = load(data_file) job_state_file = HPC_state['JobStateFile'] job_command = HPC_state['JobCommand'] # global_work_dir = HPC_state['GlobalWorkingDir'] JS = JobState() JS.get(job_state_file) _job, _site, _node, _recoveryAttempt = JS.decode() with LogWrapper(deferred_stageout_logfile, _job.jobId) as logger: jobStatus, jobAttemptNr, jobStatusCode = pUtil.getJobStatus(_job.jobId, DorE(kwargs,'pshttpurl'), DorE(kwargs,'psport'), DorE(kwargs,'pilot_initdir')) # recover this job? if jobStatusCode == 20: log("Received general error code from dispatcher call (leave job for later pilot)") # release the atomic lockfile and go to the next directory # releaseAtomicLockFile(fd, lockfile_name) return False elif jobStatus in finalJobStates or "tobekilled" in _job.action: log("Job %s is currently in state \'%s\' with attemptNr = %d (according to server - will not" " perform staging out)" % (_job.jobId, jobStatus, jobAttemptNr)) # releaseAtomicLockFile(fd, lockfile_name) return False # update job state file at this point to prevent a parallel pilot from doing a simultaneous recovery _retjs = pUtil.updateJobState(_job, _site, _node, _recoveryAttempt) # releaseAtomicLockFile(fd, lockfile_name) monitor = Monitor(env) monitor.monitor_recovery_job(_job, _site, _node, job_command, job_state_file, recover_dir=job_dir) log("Chdir back to %s" % current_dir) pUtil.chdir(current_dir) panda_jobs = glob(job_dir + "/PandaJob_*_*") panda_logs = glob(job_dir + "/*.log.tgz.*") if panda_jobs or panda_logs: log("Number of founded panda jobs: %d, number of panda log tar file %d, will not remove job dir" % (len(panda_jobs), len(panda_logs))) else: log("Number of founded panda jobs: %d, number of panda log tar file %d, will remove job dir" % (len(panda_jobs), len(panda_logs))) log("Remove job dir %s" % job_dir) os.system("rm -rf %s" % job_dir) return True except: log("Failed to start deferred stage out for HPC job: %s" % traceback.format_exc()) return False