def getJobExecutionCommand(self, job, jobSite, pilot_initdir): """ Define and test the command(s) that will be used to execute the payload """ # Input tuple: (method is called from RunJob*) # job: Job object # jobSite: Site object # pilot_initdir: launch directory of pilot.py # # Return tuple: # pilot_error_code, pilot_error_diagnostics, job_execution_command, special_setup_command, JEM, cmtconfig # where # pilot_error_code : self.__error.<PILOT ERROR CODE as defined in PilotErrors class> (value should be 0 for successful setup) # pilot_error_diagnostics: any output from problematic command or explanatory error diagnostics # job_execution_command : command to execute payload, e.g. cmd = "source <path>/setup.sh; <path>/python trf.py [options]" # special_setup_command : any special setup command that can be insterted into job_execution_command and is sent to stage-in/out methods # JEM : Job Execution Monitor activation state (default value "NO", meaning JEM is not to be used. See JEMstub.py) # cmtconfig : cmtconfig symbol from the job def or schedconfig, e.g. "x86_64-slc5-gcc43-opt" pilotErrorDiag = "" cmd = "" JEM = "NO" # Is it's an analysis job or not? isCMSRunJob = self.isCMSRunJob(job.trf) tolog("isCMSRunJob = %s " % isCMSRunJob) # Command used to download trf wgetCommand = 'wget' # Get the cmtconfig value cmtconfig = getCmtconfig(job.cmtconfig) if cmtconfig != "": tolog("cmtconfig: %s" % (cmtconfig)) # Set python executable ec, pilotErrorDiag, pybin = self.setPython() if ec == self.__error.ERR_MISSINGINSTALLATION: return ec, pilotErrorDiag, "", special_setup_cmd, JEM, cmtconfig # Define the job execution command if isCMSRunJob: # Try to download the analysis trf status, pilotErrorDiag, trfName = self.getAnalysisTrf(wgetCommand, job.trf, pilot_initdir) if status != 0: return status, pilotErrorDiag, "", special_setup_cmd, JEM, cmtconfig scramArchSetup = self.getScramArchSetupCommand(job) ec, pilotErrorDiag, cmdtrf = self.getCMSRunCommand(job, jobSite, trfName) cmd = "%s %s" % (scramArchSetup, cmdtrf) # Set special_setup_cmd if necessary special_setup_cmd = self.getSpecialSetupCommand() if special_setup_cmd != "": tolog("Special setup command: %s" % (special_setup_cmd)) # Pipe stdout/err for payload to files cmd += " 1>%s 2>%s" % (job.stdout, job.stderr) tolog("\nCommand to run the job is: \n%s" % (cmd)) return 0, pilotErrorDiag, cmd, special_setup_cmd, JEM, cmtconfig
def TransferFiles(job_state, datadir, files, **kwargs): """ Transfers files from list 'files' May change CWD with pUtil.chdir (several times) :param job_state: :param datadir: job data dir :param files: list of filenames :param kwargs: specific arguments for other purposes :return: """ job = job_state.job pUtil.chdir(datadir) XMLMetadata = pUtil.getMetadata(job_state.site.workdir, job.jobId) thisSite = DorE(kwargs, 'thisSite') if not setGuids(job_state, files, **kwargs): job.result[2] = PilotErrors().ERR_LOSTJOBPFC return ReturnCode.FailedJob outPFC = updateOutPFC(job, **kwargs) if not outPFC: return ReturnCode.FailedJob dsname = defaultDSname(job.destinationDblock) datasetDict = pUtil.getDatasetDict(job.outFiles, job.destinationDblock, job.logFile, job.logDblock) if not datasetDict: log("Output files will go to default dataset: %s" % (dsname)) # the cmtconfig is needed by at least the xrdcp site mover cmtconfig = pUtil.getCmtconfig(job.cmtconfig) tin_0 = os.times() rf = None _state = ReturnCode.OK _msg = "" ec = -1 try: # Note: alt stage-out numbers are not saved in recovery mode (job object not returned from this function) rc, pilotErrorDiag, rf, rs, job.filesNormalStageOut, job.filesAltStageOut, os_bucket_id = Mover.mover_put_data( "xmlcatalog_file:%s" % outPFC, dsname, thisSite.sitename, thisSite.computingElement, analysisJob=pUtil.isAnalysisJob(job.trf.split(",")[0]), proxycheck=DorE(kwargs, 'proxycheckFlag'), pinitdir=DorE(kwargs, 'pilot_initdir'), datasetDict=datasetDict, stageoutTries=DorE(kwargs, 'stageoutretry'), cmtconfig=cmtconfig, recoveryWorkDir=thisSite.workdir, job=job) except Exception, e: pilotErrorDiag = "Put function can not be called for staging out: %s" % str(e) log("!!%s!!1105!! %s" % (env['errorLabel'], pilotErrorDiag)) ec = PilotErrors().ERR_PUTFUNCNOCALL _state = ReturnCode.Holding _msg = env['errorLabel']
msgdic["transecode"] = job.result[1] msgdic["pilotecode"] = job.result[2] msgdic["timeStageIn"] = job.timeStageIn msgdic["timeStageOut"] = job.timeStageOut msgdic["timeSetup"] = job.timeSetup msgdic["timeExe"] = job.timeExe msgdic["cpuTime"] = job.cpuConsumptionTime msgdic["cpuUnit"] = job.cpuConsumptionUnit msgdic["cpuConversionFactor"] = job.cpuConversionFactor msgdic["nEvents"] = job.nEvents msgdic["nEventsW"] = job.nEventsW msgdic["vmPeakMax"] = job.vmPeakMax msgdic["vmPeakMean"] = job.vmPeakMean msgdic["RSSMean"] = job.RSSMean msgdic["JEM"] = job.JEM msgdic["cmtconfig"] = getCmtconfig(job.cmtconfig) msgdic["dbTime"] = job.dbTime msgdic["dbData"] = job.dbData if job.external_stageout_time: msgdic['external_stageout_time'] = job.external_stageout_time if job.outputZipName and job.outputZipBucketID: msgdic['outputZipName'] = job.outputZipName msgdic['outputZipBucketID'] = job.outputZipBucketID # hpc job status if job.mode: msgdic["mode"] = job.mode if job.hpcStatus: msgdic['hpcStatus'] = job.hpcStatus