def createFileMetadata(outFiles, job, outsDict, dsname, datasetDict, sitename):
""" create the metadata for the output + log files """
ec = 0
# get/assign guids to the output files
if outFiles:
tolog("outFiles=%s"%str(outFiles))
if not pUtil.isBuildJob(outFiles):
ec, job.pilotErrorDiag, job.outFilesGuids = RunJobUtilities.getOutFilesGuids(job.outFiles, job.workdir)
if ec:
# missing PoolFileCatalog (only error code from getOutFilesGuids)
return ec, job, None
else:
tolog("Build job - do not use PoolFileCatalog to get guid (generated)")
else:
tolog("This job has no output files")
# get the file sizes and checksums for the local output files
# WARNING: any errors are lost if occur in getOutputFileInfo()
ec, pilotErrorDiag, fsize, checksum = pUtil.getOutputFileInfo(list(outFiles), getChecksumCommand(), skiplog=True, logFile=job.logFile)
if ec != 0:
tolog("!!FAILED!!2999!! %s" % (pilotErrorDiag))
failJob(job.result[1], ec, job, pilotserver, pilotport, pilotErrorDiag=pilotErrorDiag)
if logguid:
guid = logguid
else:
guid = job.tarFileGuid
# create preliminary metadata (no metadata yet about log file - added later in pilot.py)
_fname = "%s/metadata-%d.xml" % (job.workdir, job.jobId)
try:
_status = pUtil.PFCxml(job.experiment, _fname, list(job.outFiles), fguids=job.outFilesGuids, fntag="lfn", alog=job.logFile, alogguid=guid,\
fsize=fsize, checksum=checksum, analJob=analJob)
except Exception, e:
pilotErrorDiag = "PFCxml failed due to problematic XML: %s" % (e)
tolog("!!WARNING!!1113!! %s" % (pilotErrorDiag))
failJob(job.result[1], error.ERR_MISSINGGUID, job, pilotserver, pilotport, pilotErrorDiag=pilotErrorDiag)
def getAnalysisRunCommand(self, job, jobSite, trfName):
""" Get the run command for analysis jobs """
# The run command is used to setup up runAthena/runGen
from RunJobUtilities import updateCopysetups
ec = 0
pilotErrorDiag = ""
run_command = ""
# get the queuedata info
# (directAccess info is stored in the copysetup variable)
region = readpar('region')
# get relevant file transfer info
dInfo, useCopyTool, useDirectAccess, useFileStager, oldPrefix, newPrefix, copysetup, usePFCTurl, lfcHost =\
self.getFileTransferInfo(job.transferType, isBuildJob(job.outFiles))
# extract the setup file from copysetup (and verify that it exists)
_copysetup = self.getSetupFromCopysetup(copysetup)
if _copysetup != "" and os.path.exists(_copysetup):
run_command = 'source %s;' % (_copysetup)
# add the user proxy
if os.environ.has_key('X509_USER_PROXY'):
run_command += 'export X509_USER_PROXY=%s;' % os.environ['X509_USER_PROXY']
else:
tolog("Could not add user proxy to the run command (proxy does not exist)")
# set up analysis trf
run_command += './%s %s' % (trfName, job.jobPars)
#PN
# job.jobPars += ' --accessmode=filestager'
# add options for file stager if necessary
if dInfo:
# sort out when directIn and useFileStager options should be used
if useDirectAccess and '--directIn' not in job.jobPars and '--directIn' not in run_command:
run_command += ' --directIn'
if useFileStager and '--useFileStager' not in job.jobPars:
run_command += ' --useFileStager'
# old style copysetups will contain oldPrefix and newPrefix needed for the old style remote I/O
if oldPrefix != "" and newPrefix != "":
run_command += ' --oldPrefix "%s" --newPrefix %s' % (oldPrefix, newPrefix)
else:
# --directIn should be used in combination with --usePFCTurl, but not --old/newPrefix and --lfcHost
if usePFCTurl:
run_command += ' --usePFCTurl'
if job.transferType == 'direct':
# update the copysetup
# transferType is only needed if copysetup does not contain remote I/O info
updateCopysetups(run_command, transferType=job.transferType, useCT=False, directIn=useDirectAccess, useFileStager=useFileStager)
# add options for file stager if necessary (ignore if transferType = direct)
if "accessmode" in job.jobPars and job.transferType != 'direct':
accessmode_useCT = None
accessmode_useFileStager = None
accessmode_directIn = None
_accessmode_dic = { "--accessmode=copy":["copy-to-scratch mode", ""],
"--accessmode=direct":["direct access mode", " --directIn"],
"--accessmode=filestager":["direct access / file stager mode", " --directIn --useFileStager"]}
# update run_command according to jobPars
for _mode in _accessmode_dic.keys():
if _mode in job.jobPars:
# any accessmode set in jobPars should overrule schedconfig
tolog("Enforcing %s" % (_accessmode_dic[_mode][0]))
if _mode == "--accessmode=copy":
# make sure direct access and file stager get turned off
usePFCTurl = False
accessmode_useCT = True
accessmode_directIn = False
accessmode_useFileStager = False
elif _mode == "--accessmode=direct":
# make sure copy-to-scratch and file stager get turned off
usePFCTurl = True
accessmode_useCT = False
accessmode_directIn = True
accessmode_useFileStager = False
else:
# make sure file stager gets turned on
usePFCTurl = False
accessmode_useCT = False
accessmode_directIn = True
accessmode_useFileStager = True
# update run_command (do not send the accessmode switch to runAthena)
run_command += _accessmode_dic[_mode][1]
if _mode in run_command:
run_command = run_command.replace(_mode, "")
if "directIn" in run_command and not dInfo:
if not usePFCTurl:
usePFCTurl = True
tolog("WARNING: accessmode mode specified directIn but direct access mode is not specified in copysetup (will attempt to create TURL based PFC later)")
if not "usePFCTurl" in run_command:
run_command += ' --usePFCTurl'
# need to add proxy if not there already
if ("--directIn" in run_command or "--useFileStager" in run_command) and not "export X509_USER_PROXY" in run_command:
if os.environ.has_key('X509_USER_PROXY'):
run_command = run_command.replace("./%s" % (trfName), "export X509_USER_PROXY=%s;./%s" % (os.environ['X509_USER_PROXY'], trfName))
else:
tolog("Did not add user proxy to the run command (proxy does not exist)")
# add the lfcHost if not there already
if not "--lfcHost" in run_command and lfcHost != "":
run_command += " --lfcHost %s" % (lfcHost)
# update the copysetup
updateCopysetups(run_command, transferType=None, useCT=accessmode_useCT, directIn=accessmode_directIn, useFileStager=accessmode_useFileStager)
# add guids and lfc host when needed
if region == 'US' and jobSite.dq2url != "" and lfcHost == "": # old style
run_command += ' -u %s' % (jobSite.dq2url)
else:
if lfcHost != "":
# get the correct guids list (with only the direct access files)
if not isBuildJob(job.outFiles):
_guids = self.getGuidsFromJobPars(job.jobPars, job.inFiles, job.inFilesGuids)
# only add the lfcHost if --usePFCTurl is not specified
if usePFCTurl:
run_command += ' --inputGUIDs \"%s\"' % (str(_guids))
else:
if not "--lfcHost" in run_command:
run_command += ' --lfcHost %s' % (lfcHost)
run_command += ' --inputGUIDs \"%s\"' % (str(_guids))
else:
if not usePFCTurl and not "--lfcHost" in run_command:
run_command += ' --lfcHost %s' % (lfcHost)
# if both direct access and the accessmode loop added a directIn switch, remove the first one from the string
if run_command.count("directIn") > 1:
run_command = run_command.replace("--directIn", "", 1)
# make sure that the site supports direct access / file stager
# if ("--directIn" in run_command or "--useFileStager" in run_command) and readpar('allowdirectaccess').lower() != "true":
# ec = self.__error.ERR_DAFSNOTALLOWED
# pilotErrorDiag = "Site does not allow requested direct access / file stager (payload will fail)"
# tolog("!!WARNING!!1234!! %s" % (pilotErrorDiag))
return ec, pilotErrorDiag, run_command
# update the current file states
updateFileStates(outs, runJob.getParentWorkDir(), job.jobId, mode="file_state", state="created")
dumpFileStates(runJob.getParentWorkDir(), job.jobId)
# create xml string to pass to dispatcher for atlas jobs
outputFileInfo = {}
if outs or (job.logFile and job.logFile != ''):
# get the datasets for the output files
dsname, datasetDict = runJob.getDatasets(job)
# re-create the metadata.xml file, putting guids of ALL output files into it.
# output files that miss guids from the job itself will get guids in PFCxml function
# first rename and copy the trf metadata file for non-build jobs
if not pUtil.isBuildJob(outs):
runJob.moveTrfMetadata(job.workdir, job.jobId)
# create the metadata for the output + log files
ec, job, outputFileInfo = runJob.createFileMetadata(list(outs), job, outsDict, dsname, datasetDict, jobSite.sitename, analysisJob=analysisJob)
if ec:
runJob.failJob(0, ec, job, pilotErrorDiag=job.pilotErrorDiag)
# move output files from workdir to local DDM area
finalUpdateDone = False
if outs:
tolog("Setting stage-out state until all output files have been copied")
job.setState(["stageout", 0, 0])
rt = RunJobUtilities.updatePilotServer(job, runJob.getPilotServer(), runJob.getPilotPort())
# stage-out output files
job.jobId,
mode="file_state",
state="created")
dumpFileStates(runJob.getParentWorkDir(), job.jobId)
# create xml string to pass to dispatcher for atlas jobs
outputFileInfo = {}
if outs or (job.logFile and job.logFile != ''):
# get the datasets for the output files
dsname, datasetDict = runJob.getDatasets(job)
# re-create the metadata.xml file, putting guids of ALL output files into it.
# output files that miss guids from the job itself will get guids in PFCxml function
# first rename and copy the trf metadata file for non-build jobs
if not pUtil.isBuildJob(outs):
runJob.moveTrfMetadata(job.workdir, job.jobId)
# create the metadata for the output + log files
ec, job, outputFileInfo = runJob.createFileMetadata(
list(outs),
job,
outsDict,
dsname,
datasetDict,
jobSite.sitename,
analysisJob=analysisJob)
if ec:
runJob.failJob(0, ec, job, pilotErrorDiag=job.pilotErrorDiag)
# move output files from workdir to local DDM area
def getCMSRunCommand(self, job, jobSite, trfName):
from RunJobUtilities import updateCopysetups
ec = 0
pilotErrorDiag = ""
run_command = ""
# get relevant file transfer info
dInfo, useCopyTool, useDirectAccess, useFileStager, oldPrefix, newPrefix, copysetup, usePFCTurl, lfcHost =\
self.getFileTransferInfo(job.transferType, isBuildJob(job.outFiles))
# extract the setup file from copysetup (and verify that it exists)
_copysetup = self.getSetupFromCopysetup(copysetup)
tolog("copysetup = %s" % _copysetup)
if _copysetup != "" and os.path.exists(_copysetup):
run_command = 'source %s; ' % (_copysetup)
# add the user proxy
if os.environ.has_key('X509_USER_PROXY'):
run_command += 'export X509_USER_PROXY=%s; ' % os.environ['X509_USER_PROXY']
else:
tolog("Could not add user proxy to the run command (proxy does not exist)")
"""
strpars = job.jobPars
cmdopt = shlex.split(strpars)
parser = PassThroughOptionParser()
parser.add_option('-a',\
dest='a',\
type='string')
parser.add_option('-o',\
dest='o',\
type='string')
parser.add_option('--inputFile',\
dest='inputFile',\
type='string')
parser.add_option('--sourceURL',\
dest='sourceURL',\
type='string')
parser.add_option('--jobNumber',\
dest='jobNumber',\
type='string')
parser.add_option('--cmsswVersion',\
dest='cmsswVersion',\
type='string')
parser.add_option('--scramArch',\
dest='scramArch',\
type='string')
parser.add_option('--runAndLumis',\
dest='runAndLumis',\
type='string')
(options,args) = parser.parse_args(cmdopt)
paramsstring = '-a %s ' % options.a
paramsstring += '--sourceURL %s ' % options.sourceURL
paramsstring += '--jobNumber=%s ' % options.jobNumber
paramsstring += '--cmsswVersion=%s ' % options.cmsswVersion
paramsstring += '--scramArch=%s ' % options.scramArch
paramsstring += "--inputFile='%s' " % options.inputFile
paramsstring += "--runAndLumis='%s' " % options.runAndLumis
paramsstring += '-o "%s" ' % options.o
tolog("paramsstring = %s" % paramsstring)
"""
run_command += './%s %s' % (trfName, job.jobPars)
return ec, pilotErrorDiag, run_command
tolog("Moved %d/%d output file(s) to: %s" % (nr_moved, len(remaining_files), job.datadir))
# remove all successfully copied files from the local directory
nr_removed = 0
for _file in moved_files_list:
try:
os.system("rm %s" % (_file))
except OSError, e:
tolog("!!WARNING!!3000!! Failed to remove output file: %s, %s" % (_file, e))
else:
nr_removed += 1
tolog("Removed %d output file(s) from local dir" % (nr_removed))
# copy the PoolFileCatalog.xml for non build jobs
if not pUtil.isBuildJob(remaining_files):
_fname = os.path.join(job.workdir, "PoolFileCatalog.xml")
tolog("Copying %s to %s" % (_fname, job.datadir))
try:
copy2(_fname, job.datadir)
except Exception, e:
tolog("!!WARNING!!3000!! Could not copy PoolFileCatalog.xml to data dir - expect ddm Adder problems during job recovery")
# remove all remaining output files from the work directory
# (a successfully copied file should already have been removed by the Mover)
rem = False
for inf in job.outFiles:
if inf and inf != 'NULL' and os.path.isfile("%s/%s" % (job.workdir, inf)): # non-empty string and not NULL
try:
os.remove("%s/%s" % (job.workdir, inf))
except Exception,e:
def getAnalysisRunCommand(self, job, jobSite, trfName):
""" Get the run command for analysis jobs """
# The run command is used to setup up runAthena/runGen
from RunJobUtilities import updateCopysetups
ec = 0
pilotErrorDiag = ""
run_command = ""
tolog("getAnalysisRunCommand called from Experiment")
# get the queuedata info
# (directAccess info is stored in the copysetup variable)
# get relevant file transfer info
dInfo, useCopyTool, useDirectAccess, useFileStager, oldPrefix, newPrefix, copysetup, usePFCTurl, lfcHost =\
self.getFileTransferInfo(job.transferType, isBuildJob(job.outFiles))
# extract the setup file from copysetup (and verify that it exists)
_copysetup = self.getSetupFromCopysetup(copysetup)
if _copysetup != "" and os.path.exists(_copysetup):
run_command = 'source %s;' % (_copysetup)
# add the user proxy
if os.environ.has_key('X509_USER_PROXY'):
run_command += 'export X509_USER_PROXY=%s;' % os.environ[
'X509_USER_PROXY']
else:
tolog(
"Could not add user proxy to the run command (proxy does not exist)"
)
# set up analysis trf
run_command += './%s %s' % (trfName, job.jobPars)
# add options for file stager if necessary
if dInfo:
# in case of forced usePFCTurl
if usePFCTurl and not '--usePFCTurl' in run_command:
oldPrefix = ""
newPrefix = ""
run_command += ' --usePFCTurl'
tolog("reset old/newPrefix (forced TURL mode (1))")
# sort out when directIn and useFileStager options should be used
if useDirectAccess and '--directIn' not in job.jobPars and '--directIn' not in run_command:
run_command += ' --directIn'
if useFileStager and '--useFileStager' not in job.jobPars:
run_command += ' --useFileStager'
# old style copysetups will contain oldPrefix and newPrefix needed for the old style remote I/O
if oldPrefix != "" and newPrefix != "":
run_command += ' --oldPrefix "%s" --newPrefix %s' % (oldPrefix,
newPrefix)
else:
# --directIn should be used in combination with --usePFCTurl, but not --old/newPrefix and --lfcHost
if usePFCTurl and not '--usePFCTurl' in run_command:
run_command += ' --usePFCTurl'
if job.transferType == 'direct':
# update the copysetup
# transferType is only needed if copysetup does not contain remote I/O info
updateCopysetups(run_command,
transferType=job.transferType,
useCT=False,
directIn=useDirectAccess,
useFileStager=useFileStager)
# add options for file stager if necessary (ignore if transferType = direct)
if "accessmode" in job.jobPars and job.transferType != 'direct':
accessmode_useCT = None
accessmode_useFileStager = None
accessmode_directIn = None
_accessmode_dic = {
"--accessmode=copy": ["copy-to-scratch mode", ""],
"--accessmode=direct": ["direct access mode", " --directIn"],
"--accessmode=filestager": [
"direct access / file stager mode",
" --directIn --useFileStager"
]
}
# update run_command according to jobPars
for _mode in _accessmode_dic.keys():
if _mode in job.jobPars:
# any accessmode set in jobPars should overrule schedconfig
tolog("Enforcing %s" % (_accessmode_dic[_mode][0]))
if _mode == "--accessmode=copy":
# make sure direct access and file stager get turned off
usePFCTurl = False
accessmode_useCT = True
accessmode_directIn = False
accessmode_useFileStager = False
elif _mode == "--accessmode=direct":
# make sure copy-to-scratch and file stager get turned off
usePFCTurl = True
accessmode_useCT = False
accessmode_directIn = True
accessmode_useFileStager = False
else:
# make sure file stager gets turned on
usePFCTurl = False
accessmode_useCT = False
accessmode_directIn = True
accessmode_useFileStager = True
# update run_command (do not send the accessmode switch to runAthena)
run_command += _accessmode_dic[_mode][1]
if _mode in run_command:
run_command = run_command.replace(_mode, "")
if "directIn" in run_command and not dInfo:
if not usePFCTurl:
usePFCTurl = True
tolog(
"WARNING: accessmode mode specified directIn but direct access mode is not specified in copysetup (will attempt to create TURL based PFC later)"
)
if not "usePFCTurl" in run_command:
run_command += ' --usePFCTurl'
# need to add proxy if not there already
if ("--directIn" in run_command or "--useFileStager" in run_command
) and not "export X509_USER_PROXY" in run_command:
if os.environ.has_key('X509_USER_PROXY'):
run_command = run_command.replace(
"./%s" % (trfName), "export X509_USER_PROXY=%s;./%s" %
(os.environ['X509_USER_PROXY'], trfName))
else:
tolog(
"Did not add user proxy to the run command (proxy does not exist)"
)
# add the lfcHost if not there already
if not "--lfcHost" in run_command and lfcHost != "":
run_command += " --lfcHost %s" % (lfcHost)
# update the copysetup
updateCopysetups(run_command,
transferType=None,
useCT=accessmode_useCT,
directIn=accessmode_directIn,
useFileStager=accessmode_useFileStager)
# add guids and lfc host when needed
if lfcHost != "":
# get the correct guids list (with only the direct access files)
if not isBuildJob(job.outFiles):
_guids = self.getGuidsFromJobPars(job.jobPars, job.inFiles,
job.inFilesGuids)
# only add the lfcHost if --usePFCTurl is not specified
if usePFCTurl:
run_command += ' --inputGUIDs \"%s\"' % (str(_guids))
else:
if not "--lfcHost" in run_command:
run_command += ' --lfcHost %s' % (lfcHost)
run_command += ' --inputGUIDs \"%s\"' % (str(_guids))
else:
if not usePFCTurl and not "--lfcHost" in run_command:
run_command += ' --lfcHost %s' % (lfcHost)
# if both direct access and the accessmode loop added a directIn switch, remove the first one from the string
if run_command.count("directIn") > 1:
run_command = run_command.replace("--directIn", "", 1)
# make sure that the site supports direct access / file stager
# if ("--directIn" in run_command or "--useFileStager" in run_command) and readpar('allowdirectaccess').lower() != "true":
# ec = self.__error.ERR_DAFSNOTALLOWED
# pilotErrorDiag = "Site does not allow requested direct access / file stager (payload will fail)"
# tolog("!!WARNING!!1234!! %s" % (pilotErrorDiag))
return ec, pilotErrorDiag, run_command
def getCMSRunCommand(self, job, jobSite, trfName):
from RunJobUtilities import updateCopysetups
ec = 0
pilotErrorDiag = ""
run_command = ""
# get relevant file transfer info
dInfo, useCopyTool, useDirectAccess, useFileStager, oldPrefix, newPrefix, copysetup, usePFCTurl, lfcHost =\
self.getFileTransferInfo(job.transferType, isBuildJob(job.outFiles))
# extract the setup file from copysetup (and verify that it exists)
_copysetup = "" #self.getSetupFromCopysetup(copysetup)
tolog("copysetup = %s" % _copysetup)
if _copysetup != "" and os.path.exists(_copysetup):
run_command = 'source %s; ' % (_copysetup)
# add the user proxy
if os.environ.has_key('X509_USER_PROXY'):
run_command += 'export X509_USER_PROXY=%s; ' % os.environ['X509_USER_PROXY']
else:
tolog("Could not add user proxy to the run command (proxy does not exist)")
"""
strpars = job.jobPars
cmdopt = shlex.split(strpars)
parser = PassThroughOptionParser()
parser.add_option('-a',\
dest='a',\
type='string')
parser.add_option('-o',\
dest='o',\
type='string')
parser.add_option('--inputFile',\
dest='inputFile',\
type='string')
parser.add_option('--sourceURL',\
dest='sourceURL',\
type='string')
parser.add_option('--jobNumber',\
dest='jobNumber',\
type='string')
parser.add_option('--cmsswVersion',\
dest='cmsswVersion',\
type='string')
parser.add_option('--scramArch',\
dest='scramArch',\
type='string')
parser.add_option('--runAndLumis',\
dest='runAndLumis',\
type='string')
(options,args) = parser.parse_args(cmdopt)
paramsstring = '-a %s ' % options.a
paramsstring += '--sourceURL %s ' % options.sourceURL
paramsstring += '--jobNumber=%s ' % options.jobNumber
paramsstring += '--cmsswVersion=%s ' % options.cmsswVersion
paramsstring += '--scramArch=%s ' % options.scramArch
paramsstring += "--inputFile='%s' " % options.inputFile
paramsstring += "--runAndLumis='%s' " % options.runAndLumis
paramsstring += '-o "%s" ' % options.o
tolog("paramsstring = %s" % paramsstring)
"""
run_command += './%s %s' % (trfName, job.jobPars)
return ec, pilotErrorDiag, run_command
