def TransferFiles(job_state, datadir, files, **kwargs): """ Transfers files from list 'files' May change CWD with pUtil.chdir (several times) :param job_state: :param datadir: job data dir :param files: list of filenames :param kwargs: specific arguments for other purposes :return: """ job = job_state.job pUtil.chdir(datadir) XMLMetadata = pUtil.getMetadata(job_state.site.workdir, job.jobId) thisSite = DorE(kwargs, 'thisSite') if not setGuids(job_state, files, **kwargs): job.result[2] = PilotErrors().ERR_LOSTJOBPFC return ReturnCode.FailedJob outPFC = updateOutPFC(job, **kwargs) if not outPFC: return ReturnCode.FailedJob dsname = defaultDSname(job.destinationDblock) datasetDict = pUtil.getDatasetDict(job.outFiles, job.destinationDblock, job.logFile, job.logDblock) if not datasetDict: log("Output files will go to default dataset: %s" % (dsname)) # the cmtconfig is needed by at least the xrdcp site mover cmtconfig = pUtil.getCmtconfig(job.cmtconfig) tin_0 = os.times() rf = None _state = ReturnCode.OK _msg = "" ec = -1 try: # Note: alt stage-out numbers are not saved in recovery mode (job object not returned from this function) rc, pilotErrorDiag, rf, rs, job.filesNormalStageOut, job.filesAltStageOut, os_bucket_id = Mover.mover_put_data( "xmlcatalog_file:%s" % outPFC, dsname, thisSite.sitename, thisSite.computingElement, analysisJob=pUtil.isAnalysisJob(job.trf.split(",")[0]), proxycheck=DorE(kwargs, 'proxycheckFlag'), pinitdir=DorE(kwargs, 'pilot_initdir'), datasetDict=datasetDict, stageoutTries=DorE(kwargs, 'stageoutretry'), cmtconfig=cmtconfig, recoveryWorkDir=thisSite.workdir, job=job) except Exception, e: pilotErrorDiag = "Put function can not be called for staging out: %s" % str(e) log("!!%s!!1105!! %s" % (env['errorLabel'], pilotErrorDiag)) ec = PilotErrors().ERR_PUTFUNCNOCALL _state = ReturnCode.Holding _msg = env['errorLabel']
def CreateTransferFileList(job_state, **kwargs): """ Function prepares job for staging out :param job_state: (JobState) decoded job state file mandatory Other params can be passed into functions: 1. To overwrite environment variables: Unused parameters are ommited. :return: 1. (integer) return code 2. (string) logfile path 3. (string) dir with files 2. (list) list of files to be transferred """ currentDir = os.getcwd() pUtil.chdir(job_state.site.workdir) logfile = "%s/%s" % (job_state.site.workdir, job_state.job.logFile) logfile = ( os.path.abspath(logfile) if os.path.isfile(logfile) and not pUtil.isLogfileCopied(job_state.site.workdir) else "" ) remaining_files = [] filesDir = "" if os.path.isdir(job_state.job.datadir): pUtil.chdir(job_state.job.datadir) remaining_files = pUtil.getRemainingOutputFiles(job_state.job.outFiles) if not remaining_files: remaining_files = [] filesDir = os.path.abspath(job_state.job.datadir) pUtil.chdir(currentDir) return ReturnCode.OK, logfile, filesDir, remaining_files
def CreateTransferFileList(job_state, **kwargs): """ Function prepares job for staging out :param job_state: (JobState) decoded job state file mandatory Other params can be passed into functions: 1. To overwrite environment variables: Unused parameters are ommited. :return: 1. (integer) return code 2. (string) logfile path 3. (string) dir with files 2. (list) list of files to be transferred """ currentDir = os.getcwd() pUtil.chdir(job_state.site.workdir) logfile = "%s/%s" % (job_state.site.workdir, job_state.job.logFile) logfile = os.path.abspath(logfile)\ if os.path.isfile(logfile) and not pUtil.isLogfileCopied(job_state.site.workdir) else "" remaining_files = [] filesDir = '' if os.path.isdir(job_state.job.datadir): pUtil.chdir(job_state.job.datadir) remaining_files = pUtil.getRemainingOutputFiles(job_state.job.outFiles) if not remaining_files: remaining_files = [] filesDir = os.path.abspath(job_state.job.datadir) pUtil.chdir(currentDir) return ReturnCode.OK, logfile, filesDir, remaining_files
def DeferredStageoutJob(job_dir, job_state_file="", deferred_stageout_logfile=False, **kwargs): """ Performs stageing out preparation and stages out the job in specified directory. :param job_dir: (string) directory with a job. mandatory parameter :param job_state_file: (string) path to job state file or other file containing job state. If empty, job state file is located as job_dir+'/jobState-*.*'. defaults to "" :param deferred_stageout_logfile: (string|False) template name for deferred log stageout Replaces "{job_id}" with current job id like "log-{job_id}.txt" -> "log-124124.txt" Default False Other parameters are passed into other functions :return: (bool) the fact of stageout being performed """ log('Deferred stageout from job directory "%s"' % job_dir) job_state = JobState() if job_state_file == "": try: job_state_file = glob(job_dir + "/" + jobState_file_wildcart)[0] except: log("There is no job state file in the provided directory, exiting") return False log("Job state file is %s" % job_state_file) # lockfd, lockfn = createAtomicLockFile(job_dir) with LockFileWrapper(job_dir): if not TestJobDirForDeferredStageoutNecessity(job_dir, job_state_file, **kwargs): log('Job "%s" does not need deferred stageout procedure (yet)' % job_dir) # releaseAtomicLockFile(lockfd, lockfn) return False if not job_state.get(job_state_file): log("Job state file reading failed, exiting") # releaseAtomicLockFile(lockfd, lockfn) return False log('Working with job in "%s"' % job_dir) _job, _site, _node, _recoveryAttempt = job_state.decode() if not (_job and _site and _node): log("Can not decode jobState file, exiting") # releaseAtomicLockFile(lockfd, lockfn) return False with LogWrapper(deferred_stageout_logfile, _job.jobId) as logger: rc = PrepareJobForDeferredStageout(job_state, **kwargs) if rc == ReturnCode.PostJobOnly: pUtil.postJobTask( job_state.job, job_state.site, DorE(kwargs, "workerNode"), DorE(kwargs, "experiment"), jr=True, ra=job_state.recoveryAttempt, ) # releaseAtomicLockFile(lockfd, lockfn) return True if rc > 0: log("Job is not prepared for stageout, exiting") if rc == ReturnCode.Cleanup: cleanup(job_state) # releaseAtomicLockFile(lockfd, lockfn) return False rc, logfile, datadir, filelist = CreateTransferFileList(job_state, **kwargs) XMLStr = "" if datadir == "": try: XMLStr = job_state.node["xml"] except: pass if XMLStr == "": XMLStr = pUtil.getMetadata(job_state.site.workdir, job_state.job.jobId) currentdir = os.getcwd() pUtil.chdir(job_state.site.workdir) if len(filelist): log("Stageout will now transfer the files") rc = TransferFiles(job_state, datadir, filelist, **kwargs) if rc == ReturnCode.Holding: job_state.job.result[0] = "holding" if rc == ReturnCode.FailedJob: job_state.job.result[0] = "failed" job_state.job.setState(job_state.job.result) pUtil.chdir(job_state.site.workdir) ret = True if logfile != "" and not pUtil.isLogfileCopied(job_state.site.workdir): log("Stageout will now transfer the log") _log = JobLog() ret, _ = _log.transferLogFile( job_state.job, job_state.site, DorE(kwargs, "experiment"), dest=None, jr=True ) if not ret: rc = ReturnCode.Holding # We need to transfer log file regardless the files if rc == ReturnCode.OK: if pUtil.verifyTransfer(job_state.site.workdir): job_state.job.result[0] = "finished" else: job_state.job.result[0] = "failed" job_state.job.setState(job_state.job.result) if job_state.job.result[0] in finalJobStates: job_state.job.final_state = job_state.job.result[0] log("Stageout will now update the server with new status") rt, retNode = updatePandaServer(job_state, xmlstr=XMLStr, **kwargs) if rt == 0: log("Job %s updated (exit code %d)" % (job_state.job.jobId, job_state.job.result[2])) # did the server send back a command? if "tobekilled" in job_state.job.action: log("!!WARNING!!1120!! Panda server returned a 'tobekilled' command") job_state.job.result[0] = "failed" # further recovery attempt unnecessary, but keep the work dir for debugging if job_state.job.result[0] == "failed": log("Further recovery attempts will be prevented for failed job (will leave work dir)") if not job_state.rename(job_state.site, job_state.job): log("(Fate of job state file left for next pilot)") else: log("!!WARNING!!1120!! Panda server returned a %d" % (rt)) # store the final state so that the next pilot will know # store the metadata xml retNode["xml"] = XMLStr # update the job state file with the new state information _retjs = pUtil.updateJobState(job_state.job, job_state.site, retNode, job_state.recoveryAttempt) log("Stageout will now proceed to post-job actions") if job_state.job.result[0] in finalJobStates: pUtil.postJobTask( job_state.job, job_state.site, DorE(kwargs, "workerNode"), DorE(kwargs, "experiment"), jr=True, ra=job_state.recoveryAttempt, ) pUtil.chdir(currentdir) # releaseAtomicLockFile(lockfd, lockfn) if job_state.job.result[0] == "finished": log("Stageout will now remove the job, it is in finished state and can be removed") cleanup(job_state) return True
def DeferredStageoutHPCJob(job_dir, deferred_stageout_logfile=False, **kwargs): """ Performs stageing out preparation for the HPC job in specified directory. :param job_dir: (string) directory with a job. mandatory parameter :param deferred_stageout_logfile: (string|False) template name for deferred log stageout Replaces "{job_id}" with current job id like "log-{job_id}.txt" -> "log-124124.txt" Default False Other parameters are passed into other functions :return: (bool) the fact of stageout being performed """ log('Deferred stageout from HPC job directory "%s"' % job_dir) file_path = job_dir + "/" + hpc_jobState_file_wildcart current_dir = os.getcwd() log("Working on %s" % file_path) log("Chdir from current dir %s to %s" % (current_dir, job_dir)) pUtil.chdir(job_dir) try: with LockFileWrapper(file_path) as is_locked: if not is_locked: return False from json import load with open(file_path) as data_file: HPC_state = load(data_file) job_state_file = HPC_state["JobStateFile"] job_command = HPC_state["JobCommand"] # global_work_dir = HPC_state['GlobalWorkingDir'] JS = JobState() JS.get(job_state_file) _job, _site, _node, _recoveryAttempt = JS.decode() with LogWrapper(deferred_stageout_logfile, _job.jobId) as logger: jobStatus, jobAttemptNr, jobStatusCode = pUtil.getJobStatus( _job.jobId, DorE(kwargs, "pshttpurl"), DorE(kwargs, "psport"), DorE(kwargs, "pilot_initdir") ) # recover this job? if jobStatusCode == 20: log("Received general error code from dispatcher call (leave job for later pilot)") # release the atomic lockfile and go to the next directory # releaseAtomicLockFile(fd, lockfile_name) return False elif jobStatus in finalJobStates or "tobekilled" in _job.action: log( "Job %s is currently in state '%s' with attemptNr = %d (according to server - will not" " perform staging out)" % (_job.jobId, jobStatus, jobAttemptNr) ) # releaseAtomicLockFile(fd, lockfile_name) return False # update job state file at this point to prevent a parallel pilot from doing a simultaneous recovery _retjs = pUtil.updateJobState(_job, _site, _node, _recoveryAttempt) # releaseAtomicLockFile(fd, lockfile_name) monitor = Monitor(env) monitor.monitor_recovery_job(_job, _site, _node, job_command, job_state_file, recover_dir=job_dir) log("Chdir back to %s" % current_dir) pUtil.chdir(current_dir) panda_jobs = glob(job_dir + "/PandaJob_*_*") panda_logs = glob(job_dir + "/*.log.tgz.*") if panda_jobs or panda_logs: log( "Number of founded panda jobs: %d, number of panda log tar file %d, will not remove job dir" % (len(panda_jobs), len(panda_logs)) ) else: log( "Number of founded panda jobs: %d, number of panda log tar file %d, will remove job dir" % (len(panda_jobs), len(panda_logs)) ) log("Remove job dir %s" % job_dir) os.system("rm -rf %s" % job_dir) return True except: log("Failed to start deferred stage out for HPC job: %s" % traceback.format_exc()) return False
log( "!!WARNING!!1100!! Attempt number mismatch for job %s (according to server - will not be" " recovered)" % job_state.job.jobId ) log("....Initial attempt number: %d" % attemptNr) log("....Current attempt number: %d" % jobAttemptNr) log("....Job status (server) : %s" % jobStatus) log("....Job status (state) : %s" % job_state.job.result[0]) return lognret(ReturnCode.Cleanup, "Further recovery attempts will be prevented for this job") else: log("Attempt numbers from server and job state file agree: %d" % attemptNr) job_state.recoveryAttempt += 1 currentDir = os.getcwd() pUtil.chdir(job_state.site.workdir) # Update state to prevent another pilot to try to recover this job # This was at the main logic, but I don't see, why should we do that while lockfile is present pUtil.updateJobState(job_state.job, job_state.site, job_state.node, job_state.recoveryAttempt) pUtil.chdir(currentDir) ec = job_state.job.result[2] logfile = "%s/%s" % (job_state.site.workdir, job_state.job.logFile) if not ec: if ( (not os.path.isfile(logfile)) and os.path.isdir(job_state.job.newDirNM) and os.path.isfile(os.path.join(job_state.job.newDirNM, pilotLogFileInNewWD)) ): ec = pUtil.getExitCode(job_state.job.newDirNM, "pilotlog.txt")
def DeferredStageoutJob(job_dir, job_state_file="", deferred_stageout_logfile=False, **kwargs): """ Performs stageing out preparation and stages out the job in specified directory. :param job_dir: (string) directory with a job. mandatory parameter :param job_state_file: (string) path to job state file or other file containing job state. If empty, job state file is located as job_dir+'/jobState-*.*'. defaults to "" :param deferred_stageout_logfile: (string|False) template name for deferred log stageout Replaces "{job_id}" with current job id like "log-{job_id}.txt" -> "log-124124.txt" Default False Other parameters are passed into other functions :return: (bool) the fact of stageout being performed """ log("Deferred stageout from job directory \"%s\"" % job_dir) job_state = JobState() if job_state_file == "": try: job_state_file = glob(job_dir + "/" + jobState_file_wildcart)[0] except: log("There is no job state file in the provided directory, exiting") return False log("Job state file is %s"%job_state_file) # lockfd, lockfn = createAtomicLockFile(job_dir) with LockFileWrapper(job_dir): if not TestJobDirForDeferredStageoutNecessity(job_dir, job_state_file, **kwargs): log("Job \"%s\" does not need deferred stageout procedure (yet)" % job_dir) # releaseAtomicLockFile(lockfd, lockfn) return False if not job_state.get(job_state_file): log("Job state file reading failed, exiting") # releaseAtomicLockFile(lockfd, lockfn) return False log("Working with job in \"%s\"" % job_dir) _job, _site, _node, _recoveryAttempt = job_state.decode() if not (_job and _site and _node): log("Can not decode jobState file, exiting") # releaseAtomicLockFile(lockfd, lockfn) return False with LogWrapper(deferred_stageout_logfile, _job.jobId) as logger: rc = PrepareJobForDeferredStageout(job_state, **kwargs) if rc == ReturnCode.PostJobOnly: pUtil.postJobTask(job_state.job, job_state.site, DorE(kwargs, 'workerNode'), DorE(kwargs, 'experiment'), jr=True, ra=job_state.recoveryAttempt) # releaseAtomicLockFile(lockfd, lockfn) return True if rc > 0: log("Job is not prepared for stageout, exiting") if rc == ReturnCode.Cleanup: cleanup(job_state) # releaseAtomicLockFile(lockfd, lockfn) return False rc, logfile, datadir, filelist = CreateTransferFileList(job_state, **kwargs) XMLStr = '' if datadir == "": try: XMLStr = job_state.node['xml'] except: pass if XMLStr == '': XMLStr = pUtil.getMetadata(job_state.site.workdir, job_state.job.jobId) currentdir = os.getcwd() pUtil.chdir(job_state.site.workdir) if len(filelist): log("Stageout will now transfer the files") rc = TransferFiles(job_state, datadir, filelist, **kwargs) if rc == ReturnCode.Holding: job_state.job.result[0] = "holding" if rc == ReturnCode.FailedJob: job_state.job.result[0] = "failed" job_state.job.setState(job_state.job.result) pUtil.chdir(job_state.site.workdir) ret = True if logfile != "" and not pUtil.isLogfileCopied(job_state.site.workdir): log("Stageout will now transfer the log") _log = JobLog() ret, _ = _log.transferLogFile(job_state.job, job_state.site, DorE(kwargs, 'experiment'), dest=None, jr=True) if not ret: rc = ReturnCode.Holding # We need to transfer log file regardless the files if rc == ReturnCode.OK: if pUtil.verifyTransfer(job_state.site.workdir): job_state.job.result[0] = "finished" else: job_state.job.result[0] = "failed" job_state.job.setState(job_state.job.result) if job_state.job.result[0] in finalJobStates: job_state.job.final_state = job_state.job.result[0] log("Stageout will now update the server with new status") rt, retNode = updatePandaServer(job_state, xmlstr=XMLStr, **kwargs) if rt == 0: log("Job %s updated (exit code %d)" % (job_state.job.jobId, job_state.job.result[2])) # did the server send back a command? if "tobekilled" in job_state.job.action: log("!!WARNING!!1120!! Panda server returned a \'tobekilled\' command") job_state.job.result[0] = "failed" # further recovery attempt unnecessary, but keep the work dir for debugging if job_state.job.result[0] == "failed": log("Further recovery attempts will be prevented for failed job (will leave work dir)") if not job_state.rename(job_state.site, job_state.job): log("(Fate of job state file left for next pilot)") else: log("!!WARNING!!1120!! Panda server returned a %d" % (rt)) # store the final state so that the next pilot will know # store the metadata xml retNode['xml'] = XMLStr # update the job state file with the new state information _retjs = pUtil.updateJobState(job_state.job, job_state.site, retNode, job_state.recoveryAttempt) log("Stageout will now proceed to post-job actions") if job_state.job.result[0] in finalJobStates: pUtil.postJobTask(job_state.job, job_state.site, DorE(kwargs, 'workerNode'), DorE(kwargs, 'experiment'), jr=True, ra=job_state.recoveryAttempt) pUtil.chdir(currentdir) # releaseAtomicLockFile(lockfd, lockfn) if job_state.job.result[0] == "finished": log("Stageout will now remove the job, it is in finished state and can be removed") cleanup(job_state) return True
def DeferredStageoutHPCJob(job_dir, deferred_stageout_logfile=False, **kwargs): """ Performs stageing out preparation for the HPC job in specified directory. :param job_dir: (string) directory with a job. mandatory parameter :param deferred_stageout_logfile: (string|False) template name for deferred log stageout Replaces "{job_id}" with current job id like "log-{job_id}.txt" -> "log-124124.txt" Default False Other parameters are passed into other functions :return: (bool) the fact of stageout being performed """ log("Deferred stageout from HPC job directory \"%s\"" % job_dir) file_path = job_dir+"/"+hpc_jobState_file_wildcart current_dir = os.getcwd() log("Working on %s" % file_path) log("Chdir from current dir %s to %s" % (current_dir, job_dir)) pUtil.chdir(job_dir) try: with LockFileWrapper(file_path) as is_locked: if not is_locked: return False from json import load with open(file_path) as data_file: HPC_state = load(data_file) job_state_file = HPC_state['JobStateFile'] job_command = HPC_state['JobCommand'] # global_work_dir = HPC_state['GlobalWorkingDir'] JS = JobState() JS.get(job_state_file) _job, _site, _node, _recoveryAttempt = JS.decode() with LogWrapper(deferred_stageout_logfile, _job.jobId) as logger: jobStatus, jobAttemptNr, jobStatusCode = pUtil.getJobStatus(_job.jobId, DorE(kwargs,'pshttpurl'), DorE(kwargs,'psport'), DorE(kwargs,'pilot_initdir')) # recover this job? if jobStatusCode == 20: log("Received general error code from dispatcher call (leave job for later pilot)") # release the atomic lockfile and go to the next directory # releaseAtomicLockFile(fd, lockfile_name) return False elif jobStatus in finalJobStates or "tobekilled" in _job.action: log("Job %s is currently in state \'%s\' with attemptNr = %d (according to server - will not" " perform staging out)" % (_job.jobId, jobStatus, jobAttemptNr)) # releaseAtomicLockFile(fd, lockfile_name) return False # update job state file at this point to prevent a parallel pilot from doing a simultaneous recovery _retjs = pUtil.updateJobState(_job, _site, _node, _recoveryAttempt) # releaseAtomicLockFile(fd, lockfile_name) monitor = Monitor(env) monitor.monitor_recovery_job(_job, _site, _node, job_command, job_state_file, recover_dir=job_dir) log("Chdir back to %s" % current_dir) pUtil.chdir(current_dir) panda_jobs = glob(job_dir + "/PandaJob_*_*") panda_logs = glob(job_dir + "/*.log.tgz.*") if panda_jobs or panda_logs: log("Number of founded panda jobs: %d, number of panda log tar file %d, will not remove job dir" % (len(panda_jobs), len(panda_logs))) else: log("Number of founded panda jobs: %d, number of panda log tar file %d, will remove job dir" % (len(panda_jobs), len(panda_logs))) log("Remove job dir %s" % job_dir) os.system("rm -rf %s" % job_dir) return True except: log("Failed to start deferred stage out for HPC job: %s" % traceback.format_exc()) return False
# heartbeat jobs due to reassigned panda job id numbers) if attemptNr != jobAttemptNr: log("!!WARNING!!1100!! Attempt number mismatch for job %s (according to server - will not be" " recovered)" % job_state.job.jobId) log("....Initial attempt number: %d" % attemptNr) log("....Current attempt number: %d" % jobAttemptNr) log("....Job status (server) : %s" % jobStatus) log("....Job status (state) : %s" % job_state.job.result[0]) return lognret(ReturnCode.Cleanup, "Further recovery attempts will be prevented for this job") else: log("Attempt numbers from server and job state file agree: %d" % attemptNr) job_state.recoveryAttempt += 1 currentDir = os.getcwd() pUtil.chdir(job_state.site.workdir) # Update state to prevent another pilot to try to recover this job # This was at the main logic, but I don't see, why should we do that while lockfile is present pUtil.updateJobState(job_state.job, job_state.site, job_state.node, job_state.recoveryAttempt) pUtil.chdir(currentDir) ec = job_state.job.result[2] logfile = "%s/%s" % (job_state.site.workdir, job_state.job.logFile) if not ec: if (not os.path.isfile(logfile)) and os.path.isdir(job_state.job.newDirNM)\ and os.path.isfile(os.path.join(job_state.job.newDirNM, pilotLogFileInNewWD)): ec = pUtil.getExitCode(job_state.job.newDirNM, "pilotlog.txt") # Too big nesting, but I don't know how to do better. # We search an error code, if not found, we get it from the server if ec == -1: