def getJobExecutionCommand(self, job, jobSite, pilot_initdir):
""" Define and test the command(s) that will be used to execute the payload """
# Input tuple: (method is called from RunJob*)
# job: Job object
# jobSite: Site object
# pilot_initdir: launch directory of pilot.py
#
# Return tuple:
# pilot_error_code, pilot_error_diagnostics, job_execution_command, special_setup_command, JEM, cmtconfig
# where
# pilot_error_code : self.__error.<PILOT ERROR CODE as defined in PilotErrors class> (value should be 0 for successful setup)
# pilot_error_diagnostics: any output from problematic command or explanatory error diagnostics
# job_execution_command : command to execute payload, e.g. cmd = "source <path>/setup.sh; <path>/python trf.py [options]"
# special_setup_command : any special setup command that can be insterted into job_execution_command and is sent to stage-in/out methods
# JEM : Job Execution Monitor activation state (default value "NO", meaning JEM is not to be used. See JEMstub.py)
# cmtconfig : cmtconfig symbol from the job def or schedconfig, e.g. "x86_64-slc5-gcc43-opt"
pilotErrorDiag = ""
cmd = ""
JEM = "NO"
# Is it's an analysis job or not?
isCMSRunJob = self.isCMSRunJob(job.trf)
tolog("isCMSRunJob = %s " % isCMSRunJob)
# Command used to download trf
wgetCommand = 'wget'
# Get the cmtconfig value
cmtconfig = getCmtconfig(job.cmtconfig)
if cmtconfig != "":
tolog("cmtconfig: %s" % (cmtconfig))
# Set python executable
ec, pilotErrorDiag, pybin = self.setPython()
if ec == self.__error.ERR_MISSINGINSTALLATION:
return ec, pilotErrorDiag, "", special_setup_cmd, JEM, cmtconfig
# Define the job execution command
if isCMSRunJob:
# Try to download the analysis trf
status, pilotErrorDiag, trfName = self.getAnalysisTrf(wgetCommand, job.trf, pilot_initdir)
if status != 0:
return status, pilotErrorDiag, "", special_setup_cmd, JEM, cmtconfig
scramArchSetup = self.getScramArchSetupCommand(job)
ec, pilotErrorDiag, cmdtrf = self.getCMSRunCommand(job, jobSite, trfName)
cmd = "%s %s" % (scramArchSetup, cmdtrf)
# Set special_setup_cmd if necessary
special_setup_cmd = self.getSpecialSetupCommand()
if special_setup_cmd != "":
tolog("Special setup command: %s" % (special_setup_cmd))
# Pipe stdout/err for payload to files
cmd += " 1>%s 2>%s" % (job.stdout, job.stderr)
tolog("\nCommand to run the job is: \n%s" % (cmd))
return 0, pilotErrorDiag, cmd, special_setup_cmd, JEM, cmtconfig
def TransferFiles(job_state, datadir, files, **kwargs):
"""
Transfers files from list 'files'
May change CWD with pUtil.chdir (several times)
:param job_state:
:param datadir: job data dir
:param files: list of filenames
:param kwargs: specific arguments for other purposes
:return:
"""
job = job_state.job
pUtil.chdir(datadir)
XMLMetadata = pUtil.getMetadata(job_state.site.workdir, job.jobId)
thisSite = DorE(kwargs, 'thisSite')
if not setGuids(job_state, files, **kwargs):
job.result[2] = PilotErrors().ERR_LOSTJOBPFC
return ReturnCode.FailedJob
outPFC = updateOutPFC(job, **kwargs)
if not outPFC:
return ReturnCode.FailedJob
dsname = defaultDSname(job.destinationDblock)
datasetDict = pUtil.getDatasetDict(job.outFiles, job.destinationDblock, job.logFile, job.logDblock)
if not datasetDict:
log("Output files will go to default dataset: %s" % (dsname))
# the cmtconfig is needed by at least the xrdcp site mover
cmtconfig = pUtil.getCmtconfig(job.cmtconfig)
tin_0 = os.times()
rf = None
_state = ReturnCode.OK
_msg = ""
ec = -1
try:
# Note: alt stage-out numbers are not saved in recovery mode (job object not returned from this function)
rc, pilotErrorDiag, rf, rs, job.filesNormalStageOut, job.filesAltStageOut, os_bucket_id = Mover.mover_put_data(
"xmlcatalog_file:%s" % outPFC, dsname,
thisSite.sitename, thisSite.computingElement, analysisJob=pUtil.isAnalysisJob(job.trf.split(",")[0]),
proxycheck=DorE(kwargs, 'proxycheckFlag'),
pinitdir=DorE(kwargs, 'pilot_initdir'),
datasetDict=datasetDict,
stageoutTries=DorE(kwargs, 'stageoutretry'),
cmtconfig=cmtconfig, recoveryWorkDir=thisSite.workdir,
job=job)
except Exception, e:
pilotErrorDiag = "Put function can not be called for staging out: %s" % str(e)
log("!!%s!!1105!! %s" % (env['errorLabel'], pilotErrorDiag))
ec = PilotErrors().ERR_PUTFUNCNOCALL
_state = ReturnCode.Holding
_msg = env['errorLabel']
msgdic["transecode"] = job.result[1]
msgdic["pilotecode"] = job.result[2]
msgdic["timeStageIn"] = job.timeStageIn
msgdic["timeStageOut"] = job.timeStageOut
msgdic["timeSetup"] = job.timeSetup
msgdic["timeExe"] = job.timeExe
msgdic["cpuTime"] = job.cpuConsumptionTime
msgdic["cpuUnit"] = job.cpuConsumptionUnit
msgdic["cpuConversionFactor"] = job.cpuConversionFactor
msgdic["nEvents"] = job.nEvents
msgdic["nEventsW"] = job.nEventsW
msgdic["vmPeakMax"] = job.vmPeakMax
msgdic["vmPeakMean"] = job.vmPeakMean
msgdic["RSSMean"] = job.RSSMean
msgdic["JEM"] = job.JEM
msgdic["cmtconfig"] = getCmtconfig(job.cmtconfig)
msgdic["dbTime"] = job.dbTime
msgdic["dbData"] = job.dbData
if job.external_stageout_time:
msgdic['external_stageout_time'] = job.external_stageout_time
if job.outputZipName and job.outputZipBucketID:
msgdic['outputZipName'] = job.outputZipName
msgdic['outputZipBucketID'] = job.outputZipBucketID
# hpc job status
if job.mode:
msgdic["mode"] = job.mode
if job.hpcStatus:
msgdic['hpcStatus'] = job.hpcStatus
