def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig):
'''prepare the subjob specific configuration'''
# PandaTools
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('AthenaMCPandaRTHandler prepare called for %s', job.getFQID('.'))
try:
assert self.outsite
except:
logger.error("outsite not set. Aborting")
raise Exception()
job.backend.site = self.outsite
job.backend.actualCE = self.outsite
cloud = job._getRoot().backend.requirements.cloud
job.backend.requirements.cloud = cloud
# now just filling the job from AthenaMC data
jspec = JobSpec()
jspec.jobDefinitionID = job._getRoot().id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.AtlasRelease = 'Atlas-%s' % app.atlas_rel
if app.transform_archive:
jspec.homepackage = 'AnalysisTransforms'+app.transform_archive
elif app.prod_release:
jspec.homepackage = 'AnalysisTransforms-AtlasProduction_'+str(app.prod_release)
jspec.transformation = '%s/runAthena-00-00-11' % Client.baseURLSUB
#---->???? prodDBlock and destinationDBlock when facing several input / output datasets?
jspec.prodDBlock = 'NULL'
if job.inputdata and len(app.inputfiles)>0 and app.inputfiles[0] in app.dsetmap:
jspec.prodDBlock = app.dsetmap[app.inputfiles[0]]
# How to specify jspec.destinationDBlock when more than one type of output is available? Panda prod jobs seem to specify only the last output dataset
outdset=""
for type in ["EVNT","RDO","HITS","AOD","ESD","NTUP"]:
if type in app.outputpaths.keys():
outdset=string.replace(app.outputpaths[type],"/",".")
outdset=outdset[1:-1]
break
if not outdset:
try:
assert len(app.outputpaths.keys())>0
except:
logger.error("app.outputpaths is empty: check your output datasets")
raise
type=app.outputpaths.keys()[0]
outdset=string.replace(app.outputpaths[type],"/",".")
outdset=outdset[1:-1]
jspec.destinationDBlock = outdset
jspec.destinationSE = self.outsite
jspec.prodSourceLabel = 'user'
jspec.assignedPriority = 1000
jspec.cloud = cloud
# memory
if job.backend.requirements.memory != -1:
jspec.minRamCount = job.backend.requirements.memory
jspec.computingSite = self.outsite
jspec.cmtConfig = AthenaUtils.getCmtConfig(athenaVer=app.atlas_rel)
# library (source files)
flib = FileSpec()
flib.lfn = self.library
# flib.GUID =
flib.type = 'input'
# flib.status =
flib.dataset = self.libDataset
flib.dispatchDBlock = self.libDataset
jspec.addFile(flib)
# input files FIXME: many more input types
for lfn in app.inputfiles:
useguid=app.turls[lfn].replace("guid:","")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# add dbfiles if any:
for lfn in app.dbfiles:
useguid=app.dbturls[lfn].replace("guid:","")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# then minbias files
for lfn in app.mbfiles:
useguid=app.minbias_turls[lfn].replace("guid:","")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# then cavern files
for lfn in app.cavernfiles:
useguid=app.cavern_turls[lfn].replace("guid:","")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# output files( this includes the logfiles)
# Output files
jidtag=""
job = app._getParent() # Returns job or subjob object
if job._getRoot().subjobs:
jidtag = job._getRoot().id
else:
jidtag = "%d" % job.id
outfiles=app.subjobsOutfiles[job.id]
pandaOutfiles={}
for type in outfiles.keys():
pandaOutfiles[type]=outfiles[type]+"."+str(jidtag)
if type=="LOG":
pandaOutfiles[type]+=".tgz"
#print pandaOutfiles
for outtype in pandaOutfiles.keys():
fout = FileSpec()
dset=string.replace(app.outputpaths[outtype],"/",".")
dset=dset[1:-1]
fout.dataset=dset
fout.lfn=pandaOutfiles[outtype]
fout.type = 'output'
# fout.destinationDBlock = jspec.destinationDBlock
fout.destinationDBlock = fout.dataset
fout.destinationSE = jspec.destinationSE
if outtype=='LOG':
fout.type='log'
fout.destinationDBlock = fout.dataset
fout.destinationSE = job.backend.site
jspec.addFile(fout)
# job parameters
param = '-l %s ' % self.library # user tarball.
# use corruption checker
if job.backend.requirements.corCheck:
param += '--corCheck '
# disable to skip missing files
if job.backend.requirements.notSkipMissing:
param += '--notSkipMissing '
# transform parameters
# need to update arglist with final output file name...
newArgs=[]
if app.mode == "evgen":
app.args[3]=app.args[3]+" -t "
if app.verbosity:
app.args[3]=app.args[3]+" -l %s " % app.verbosity
for arg in app.args[3:]:
for type in outfiles.keys():
if arg.find(outfiles[type])>-1:
arg=arg.replace(outfiles[type],pandaOutfiles[type])
newArgs.append(arg)
arglist=string.join(newArgs," ")
# print "Arglist:",arglist
param += ' -r ./ '
param += ' -j "%s"' % urllib.quote(arglist)
allinfiles=app.inputfiles+app.dbfiles
# Input files.
param += ' -i "%s" ' % allinfiles
if len(app.mbfiles)>0:
param+= ' -m "%s" ' % app.mbfiles
if len(app.cavernfiles)>0:
param+= ' -n "%s" ' % app.cavernfiles
# param += '-m "[]" ' #%minList FIXME
# param += '-n "[]" ' #%cavList FIXME
del pandaOutfiles["LOG"] # logfiles do not appear in IROOT block, and this one is not needed anymore...
param += ' -o "{\'IROOT\':%s }"' % str(pandaOutfiles.items())
# source URL
matchURL = re.search("(http.*://[^/]+)/",Client.baseURLSSL)
if matchURL != None:
param += " --sourceURL %s " % matchURL.group(1)
param += " --trf"
jspec.jobParameters = param
jspec.metadata="--trf \"%s\"" % arglist
#print "SUBJOB DETAILS:",jspec.values()
if app.dryrun:
print "job.application.dryrun activated, printing out job parameters"
print jspec.values()
return
return jspec
def doBrokerage(self,inputList,vo,prodSourceLabel,workQueue):
# list with a lock
inputListWorld = ListWithLock([])
# variables for submission
maxBunchTask = 100
# make logger
tmpLog = MsgWrapper(logger)
tmpLog.debug('start doBrokerage')
# return for failure
retFatal = self.SC_FATAL
retTmpError = self.SC_FAILED
tmpLog.debug('vo={0} label={1} queue={2} nTasks={3}'.format(vo,prodSourceLabel,
workQueue.queue_name,
len(inputList)))
# loop over all tasks
allRwMap = {}
prioMap = {}
tt2Map = {}
expRWs = {}
jobSpecList = []
for tmpJediTaskID,tmpInputList in inputList:
for taskSpec,cloudName,inputChunk in tmpInputList:
# collect tasks for WORLD
if taskSpec.useWorldCloud():
inputListWorld.append((taskSpec,inputChunk))
continue
# make JobSpec to be submitted for TaskAssigner
jobSpec = JobSpec()
jobSpec.taskID = taskSpec.jediTaskID
jobSpec.jediTaskID = taskSpec.jediTaskID
# set managed to trigger TA
jobSpec.prodSourceLabel = 'managed'
jobSpec.processingType = taskSpec.processingType
jobSpec.workingGroup = taskSpec.workingGroup
jobSpec.metadata = taskSpec.processingType
jobSpec.assignedPriority = taskSpec.taskPriority
jobSpec.currentPriority = taskSpec.currentPriority
jobSpec.maxDiskCount = (taskSpec.getOutDiskSize() + taskSpec.getWorkDiskSize()) / 1024 / 1024
if taskSpec.useWorldCloud():
# use destinationSE to trigger task brokerage in WORLD cloud
jobSpec.destinationSE = taskSpec.cloud
prodDBlock = None
setProdDBlock = False
for datasetSpec in inputChunk.getDatasets():
prodDBlock = datasetSpec.datasetName
if datasetSpec.isMaster():
jobSpec.prodDBlock = datasetSpec.datasetName
setProdDBlock = True
for fileSpec in datasetSpec.Files:
tmpInFileSpec = fileSpec.convertToJobFileSpec(datasetSpec)
jobSpec.addFile(tmpInFileSpec)
# use secondary dataset name as prodDBlock
if setProdDBlock == False and prodDBlock != None:
jobSpec.prodDBlock = prodDBlock
# append
jobSpecList.append(jobSpec)
prioMap[jobSpec.taskID] = jobSpec.currentPriority
tt2Map[jobSpec.taskID] = jobSpec.processingType
# get RW for a priority
if not allRwMap.has_key(jobSpec.currentPriority):
tmpRW = self.taskBufferIF.calculateRWwithPrio_JEDI(vo,prodSourceLabel,workQueue,
jobSpec.currentPriority)
if tmpRW == None:
tmpLog.error('failed to calculate RW with prio={0}'.format(jobSpec.currentPriority))
return retTmpError
allRwMap[jobSpec.currentPriority] = tmpRW
# get expected RW
expRW = self.taskBufferIF.calculateTaskRW_JEDI(jobSpec.jediTaskID)
if expRW == None:
tmpLog.error('failed to calculate RW for jediTaskID={0}'.format(jobSpec.jediTaskID))
return retTmpError
expRWs[jobSpec.taskID] = expRW
# for old clouds
if jobSpecList != []:
# get fullRWs
fullRWs = self.taskBufferIF.calculateRWwithPrio_JEDI(vo,prodSourceLabel,None,None)
if fullRWs == None:
tmpLog.error('failed to calculate full RW')
return retTmpError
# set metadata
for jobSpec in jobSpecList:
rwValues = allRwMap[jobSpec.currentPriority]
jobSpec.metadata = "%s;%s;%s;%s;%s;%s" % (jobSpec.metadata,
str(rwValues),str(expRWs),
str(prioMap),str(fullRWs),
str(tt2Map))
tmpLog.debug('run task assigner for {0} tasks'.format(len(jobSpecList)))
nBunchTask = 0
while nBunchTask < len(jobSpecList):
# get a bunch
jobsBunch = jobSpecList[nBunchTask:nBunchTask+maxBunchTask]
strIDs = 'jediTaskID='
for tmpJobSpec in jobsBunch:
strIDs += '{0},'.format(tmpJobSpec.taskID)
strIDs = strIDs[:-1]
tmpLog.debug(strIDs)
# increment index
nBunchTask += maxBunchTask
# run task brokerge
stS,outSs = PandaClient.runTaskAssignment(jobsBunch)
tmpLog.debug('{0}:{1}'.format(stS,str(outSs)))
# for WORLD
if len(inputListWorld) > 0:
# thread pool
threadPool = ThreadPool()
# get full RW for WORLD
fullRWs = self.taskBufferIF.calculateWorldRWwithPrio_JEDI(vo,prodSourceLabel,None,None)
if fullRWs == None:
tmpLog.error('failed to calculate full WORLD RW')
return retTmpError
# get RW per priority
for taskSpec,inputChunk in inputListWorld:
if not taskSpec.currentPriority in allRwMap:
tmpRW = self.taskBufferIF.calculateWorldRWwithPrio_JEDI(vo,prodSourceLabel,workQueue,
taskSpec.currentPriority)
if tmpRW == None:
tmpLog.error('failed to calculate RW with prio={0}'.format(taskSpec.currentPriority))
return retTmpError
allRwMap[taskSpec.currentPriority] = tmpRW
# live counter for RWs
liveCounter = MapWithLock(allRwMap)
# make workers
ddmIF = self.ddmIF.getInterface(vo)
for iWorker in range(4):
thr = AtlasProdTaskBrokerThread(inputListWorld,threadPool,
self.taskBufferIF,ddmIF,
fullRWs,liveCounter)
thr.start()
threadPool.join(60*10)
# return
tmpLog.debug('doBrokerage done')
return self.SC_SUCCEEDED
def doBrokerage(self, inputList, vo, prodSourceLabel, workQueue):
# variables for submission
maxBunchTask = 100
# make logger
tmpLog = MsgWrapper(logger)
tmpLog.debug('start doBrokerage')
# return for failure
retFatal = self.SC_FATAL
retTmpError = self.SC_FAILED
tmpLog.debug('vo={0} label={1} queue={2}'.format(
vo, prodSourceLabel, workQueue.queue_name))
# loop over all tasks
allRwMap = {}
prioMap = {}
tt2Map = {}
expRWs = {}
jobSpecList = []
for tmpJediTaskID, tmpInputList in inputList:
for taskSpec, cloudName, inputChunk in tmpInputList:
# make JobSpec to be submitted for TaskAssigner
jobSpec = JobSpec()
jobSpec.taskID = taskSpec.jediTaskID
jobSpec.jediTaskID = taskSpec.jediTaskID
# set managed to trigger TA
jobSpec.prodSourceLabel = 'managed'
jobSpec.processingType = taskSpec.processingType
jobSpec.workingGroup = taskSpec.workingGroup
jobSpec.metadata = taskSpec.processingType
jobSpec.assignedPriority = taskSpec.taskPriority
jobSpec.currentPriority = taskSpec.currentPriority
jobSpec.maxDiskCount = (
taskSpec.getOutDiskSize() +
taskSpec.getWorkDiskSize()) / 1024 / 1024
if taskSpec.useWorldCloud():
# use destinationSE to trigger task brokerage in WORLD cloud
jobSpec.destinationSE = taskSpec.cloud
prodDBlock = None
setProdDBlock = False
for datasetSpec in inputChunk.getDatasets():
prodDBlock = datasetSpec.datasetName
if datasetSpec.isMaster():
jobSpec.prodDBlock = datasetSpec.datasetName
setProdDBlock = True
for fileSpec in datasetSpec.Files:
tmpInFileSpec = fileSpec.convertToJobFileSpec(
datasetSpec)
jobSpec.addFile(tmpInFileSpec)
# use secondary dataset name as prodDBlock
if setProdDBlock == False and prodDBlock != None:
jobSpec.prodDBlock = prodDBlock
# append
jobSpecList.append(jobSpec)
prioMap[jobSpec.taskID] = jobSpec.currentPriority
tt2Map[jobSpec.taskID] = jobSpec.processingType
# get RW for a priority
if not allRwMap.has_key(jobSpec.currentPriority):
tmpRW = self.taskBufferIF.calculateRWwithPrio_JEDI(
vo, prodSourceLabel, workQueue,
jobSpec.currentPriority)
if tmpRW == None:
tmpLog.error(
'failed to calculate RW with prio={0}'.format(
jobSpec.currentPriority))
return retTmpError
allRwMap[jobSpec.currentPriority] = tmpRW
# get expected RW
expRW = self.taskBufferIF.calculateTaskRW_JEDI(
jobSpec.jediTaskID)
if expRW == None:
tmpLog.error(
'failed to calculate RW for jediTaskID={0}'.format(
jobSpec.jediTaskID))
return retTmpError
expRWs[jobSpec.taskID] = expRW
# get fullRWs
fullRWs = self.taskBufferIF.calculateRWwithPrio_JEDI(
vo, prodSourceLabel, None, None)
if fullRWs == None:
tmpLog.error('failed to calculate full RW')
return retTmpError
# set metadata
for jobSpec in jobSpecList:
rwValues = allRwMap[jobSpec.currentPriority]
jobSpec.metadata = "%s;%s;%s;%s;%s;%s" % (
jobSpec.metadata, str(rwValues), str(expRWs), str(prioMap),
str(fullRWs), str(tt2Map))
tmpLog.debug('run task assigner for {0} tasks'.format(
len(jobSpecList)))
nBunchTask = 0
while nBunchTask < len(jobSpecList):
# get a bunch
jobsBunch = jobSpecList[nBunchTask:nBunchTask + maxBunchTask]
strIDs = 'jediTaskID='
for tmpJobSpec in jobsBunch:
strIDs += '{0},'.format(tmpJobSpec.taskID)
strIDs = strIDs[:-1]
tmpLog.debug(strIDs)
# increment index
nBunchTask += maxBunchTask
# run task brokerge
stS, outSs = PandaClient.runTaskAssignment(jobsBunch)
tmpLog.debug('{0}:{1}'.format(stS, str(outSs)))
# return
tmpLog.debug('done')
return self.SC_SUCCEEDED
def doBrokerage(self, inputList, vo, prodSourceLabel, workQueue):
# variables for submission
maxBunchTask = 100
# make logger
tmpLog = MsgWrapper(logger)
tmpLog.debug("start doBrokerage")
# return for failure
retFatal = self.SC_FATAL
retTmpError = self.SC_FAILED
tmpLog.debug("vo={0} label={1} queue={2}".format(vo, prodSourceLabel, workQueue.queue_name))
# loop over all tasks
allRwMap = {}
prioMap = {}
tt2Map = {}
expRWs = {}
jobSpecList = []
for tmpJediTaskID, tmpInputList in inputList:
for taskSpec, cloudName, inputChunk in tmpInputList:
# make JobSpec to be submitted for TaskAssigner
jobSpec = JobSpec()
jobSpec.taskID = taskSpec.jediTaskID
jobSpec.jediTaskID = taskSpec.jediTaskID
# set managed to trigger TA
jobSpec.prodSourceLabel = "managed"
jobSpec.processingType = taskSpec.processingType
jobSpec.workingGroup = taskSpec.workingGroup
jobSpec.metadata = taskSpec.processingType
jobSpec.assignedPriority = taskSpec.taskPriority
jobSpec.currentPriority = taskSpec.currentPriority
jobSpec.maxDiskCount = (taskSpec.getOutDiskSize() + taskSpec.getWorkDiskSize()) / 1024 / 1024
if taskSpec.useWorldCloud():
# use destinationSE to trigger task brokerage in WORLD cloud
jobSpec.destinationSE = taskSpec.cloud
prodDBlock = None
setProdDBlock = False
for datasetSpec in inputChunk.getDatasets():
prodDBlock = datasetSpec.datasetName
if datasetSpec.isMaster():
jobSpec.prodDBlock = datasetSpec.datasetName
setProdDBlock = True
for fileSpec in datasetSpec.Files:
tmpInFileSpec = fileSpec.convertToJobFileSpec(datasetSpec)
jobSpec.addFile(tmpInFileSpec)
# use secondary dataset name as prodDBlock
if setProdDBlock == False and prodDBlock != None:
jobSpec.prodDBlock = prodDBlock
# append
jobSpecList.append(jobSpec)
prioMap[jobSpec.taskID] = jobSpec.currentPriority
tt2Map[jobSpec.taskID] = jobSpec.processingType
# get RW for a priority
if not allRwMap.has_key(jobSpec.currentPriority):
tmpRW = self.taskBufferIF.calculateRWwithPrio_JEDI(
vo, prodSourceLabel, workQueue, jobSpec.currentPriority
)
if tmpRW == None:
tmpLog.error("failed to calculate RW with prio={0}".format(jobSpec.currentPriority))
return retTmpError
allRwMap[jobSpec.currentPriority] = tmpRW
# get expected RW
expRW = self.taskBufferIF.calculateTaskRW_JEDI(jobSpec.jediTaskID)
if expRW == None:
tmpLog.error("failed to calculate RW for jediTaskID={0}".format(jobSpec.jediTaskID))
return retTmpError
expRWs[jobSpec.taskID] = expRW
# get fullRWs
fullRWs = self.taskBufferIF.calculateRWwithPrio_JEDI(vo, prodSourceLabel, None, None)
if fullRWs == None:
tmpLog.error("failed to calculate full RW")
return retTmpError
# set metadata
for jobSpec in jobSpecList:
rwValues = allRwMap[jobSpec.currentPriority]
jobSpec.metadata = "%s;%s;%s;%s;%s;%s" % (
jobSpec.metadata,
str(rwValues),
str(expRWs),
str(prioMap),
str(fullRWs),
str(tt2Map),
)
tmpLog.debug("run task assigner for {0} tasks".format(len(jobSpecList)))
nBunchTask = 0
while nBunchTask < len(jobSpecList):
# get a bunch
jobsBunch = jobSpecList[nBunchTask : nBunchTask + maxBunchTask]
strIDs = "jediTaskID="
for tmpJobSpec in jobsBunch:
strIDs += "{0},".format(tmpJobSpec.taskID)
strIDs = strIDs[:-1]
tmpLog.debug(strIDs)
# increment index
nBunchTask += maxBunchTask
# run task brokerge
stS, outSs = PandaClient.runTaskAssignment(jobsBunch)
tmpLog.debug("{0}:{1}".format(stS, str(outSs)))
# return
tmpLog.debug("done")
return self.SC_SUCCEEDED
def createJobSpec(self, task, outdataset, job, jobset, jobdef, site, jobname, lfnhanger, allsites, jobid):
"""Create a spec for one job
:arg TaskWorker.DataObject.Task task: the task to work on
:arg str outdataset: the output dataset name where all the produced files will be placed
:arg WMCore.DataStructs.Job job: the abstract job
:arg int jobset: the PanDA jobset corresponding to the current task
:arg int jobdef: the PanDA jobdef where to append the current jobs --- not used
:arg str site: the borkered site where to run the jobs
:arg str jobname: the job name
:arg str lfnhanger: the random string to be added in the output file name
:arg list str allsites: all possible sites where the job can potentially run
:arg int jobid: incremental job number
:return: the sepc object."""
pandajob = JobSpec()
## always setting a job definition ID
pandajob.jobDefinitionID = jobdef if jobdef else -1
## always setting a job set ID
pandajob.jobsetID = jobset if jobset else -1
pandajob.jobName = jobname
pandajob.prodUserID = task['tm_user_dn']
pandajob.destinationDBlock = outdataset
pandajob.prodDBlock = task['tm_input_dataset']
pandajob.prodSourceLabel = 'user'
pandajob.computingSite = site
pandajob.cloud = getSite(pandajob.computingSite)
pandajob.destinationSE = 'local'
pandajob.transformation = task['tm_transformation']
## need to initialize this
pandajob.metadata = ''
def outFileSpec(of=None, log=False):
"""Local routine to create an FileSpec for the an job output/log file
:arg str of: output file base name
:return: FileSpec object for the output file."""
outfile = FileSpec()
if log:
outfile.lfn = "job.log_%d_%s.tgz" % (jobid, lfnhanger)
outfile.type = 'log'
else:
outfile.lfn = '%s_%d_%s%s' %(os.path.splitext(of)[0], jobid, lfnhanger, os.path.splitext(of)[1])
outfile.type = 'output'
outfile.destinationDBlock = pandajob.destinationDBlock
outfile.destinationSE = task['tm_asyncdest']
outfile.dataset = pandajob.destinationDBlock
return outfile
alloutfiles = []
outjobpar = {}
outfilestring = ''
for outputfile in task['tm_outfiles']:
outfilestring += '%s,' % outputfile
filespec = outFileSpec(outputfile)
alloutfiles.append(filespec)
#pandajob.addFile(filespec)
outjobpar[outputfile] = filespec.lfn
for outputfile in task['tm_tfile_outfiles']:
outfilestring += '%s,' % outputfile
filespec = outFileSpec(outputfile)
alloutfiles.append(filespec)
#pandajob.addFile(filespec)
outjobpar[outputfile] = filespec.lfn
for outputfile in task['tm_edm_outfiles']:
outfilestring += '%s,' % outputfile
filespec = outFileSpec(outputfile)
alloutfiles.append(filespec)
#pandajob.addFile(filespec)
outjobpar[outputfile] = filespec.lfn
outfilestring = outfilestring[:-1]
infiles = []
for inputfile in job['input_files']:
infiles.append( inputfile['lfn'] )
pandajob.jobParameters = '-a %s ' % task['tm_user_sandbox']
pandajob.jobParameters += '--sourceURL %s ' % task['tm_cache_url']
pandajob.jobParameters += '--jobNumber=%s ' % jobid
pandajob.jobParameters += '--cmsswVersion=%s ' % task['tm_job_sw']
pandajob.jobParameters += '--scramArch=%s ' % task['tm_job_arch']
pandajob.jobParameters += '--inputFile=\'%s\' ' % json.dumps(infiles)
self.jobParametersSetting(pandajob, job, self.jobtypeMapper[task['tm_job_type']])
pandajob.jobParameters += '-o "%s" ' % str(outjobpar)
pandajob.jobParameters += '--dbs_url=%s ' % task['tm_dbs_url']
pandajob.jobParameters += '--publish_dbs_url=%s ' % task['tm_publish_dbs_url']
pandajob.jobParameters += '--publishFiles=%s ' % ('True' if task['tm_publication'] == 'T' else 'False')
pandajob.jobParameters += '--saveLogs=%s ' % ('True' if task['tm_save_logs'] == 'T' else 'False')
pandajob.jobParameters += '--availableSites=\'%s\' ' %json.dumps(allsites)
pandajob.jobParameters += '--group=%s ' % (task['tm_user_group'] if task['tm_user_group'] else '')
pandajob.jobParameters += '--role=%s ' % (task['tm_user_role'] if task['tm_user_role'] else '')
self.logger.info(type(task['tm_user_infiles']))
self.logger.info(task['tm_user_infiles'])
if task['tm_user_infiles']:
addinfilestring = ''
for addinfile in task['tm_user_infiles']:
addinfilestring += '%s,' % addinfile
pandajob.jobParameters += '--userFiles=%s ' % ( addinfilestring[:-1] )
pandajob.jobName = '%s' % task['tm_taskname'] #Needed by ASO and Dashboard
if 'panda_oldjobid' in job and job['panda_oldjobid']:
pandajob.parentID = job['panda_oldjobid']
pandajob.addFile(outFileSpec(log=True))
for filetoadd in alloutfiles:
pandajob.addFile(filetoadd)
return pandajob
def send_job(jobid, siteid):
_logger.debug('Jobid: ' + str(jobid))
site = sites_.get(siteid)
job = jobs_.get(int(jobid))
cont = job.container
files_catalog = cont.files
fscope = getScope(job.owner.username)
datasetName = '{}:{}'.format(fscope, cont.guid)
distributive = job.distr.name
release = job.distr.release
# Prepare runScript
parameters = job.distr.command
parameters = parameters.replace("$COMMAND$", job.params)
parameters = parameters.replace("$USERNAME$", job.owner.username)
parameters = parameters.replace("$WORKINGGROUP$", job.owner.working_group)
# Prepare metadata
metadata = dict(user=job.owner.username)
# Prepare PanDA Object
pandajob = JobSpec()
pandajob.jobDefinitionID = int(time.time()) % 10000
pandajob.jobName = cont.guid
pandajob.transformation = client_config.DEFAULT_TRF
pandajob.destinationDBlock = datasetName
pandajob.destinationSE = site.se
pandajob.currentPriority = 1000
pandajob.prodSourceLabel = 'user'
pandajob.computingSite = site.ce
pandajob.cloud = 'RU'
pandajob.VO = 'atlas'
pandajob.prodDBlock = "%s:%s" % (fscope, pandajob.jobName)
pandajob.coreCount = job.corecount
pandajob.metadata = json.dumps(metadata)
#pandajob.workingGroup = job.owner.working_group
if site.encode_commands:
# It requires script wrapper on cluster side
pandajob.jobParameters = '%s %s %s "%s"' % (cont.guid, release,
distributive, parameters)
else:
pandajob.jobParameters = parameters
has_input = False
for fcc in files_catalog:
if fcc.type == 'input':
f = fcc.file
guid = f.guid
fileIT = FileSpec()
fileIT.lfn = f.lfn
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = guid
pandajob.addFile(fileIT)
has_input = True
if fcc.type == 'output':
f = fcc.file
fileOT = FileSpec()
fileOT.lfn = f.lfn
fileOT.destinationDBlock = pandajob.prodDBlock
fileOT.destinationSE = pandajob.destinationSE
fileOT.dataset = pandajob.prodDBlock
fileOT.type = 'output'
fileOT.scope = fscope
fileOT.GUID = f.guid
pandajob.addFile(fileOT)
# Save replica meta
fc.new_replica(f, site)
if not has_input:
# Add fake input
fileIT = FileSpec()
fileIT.lfn = "fake.input"
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = "fake.guid"
pandajob.addFile(fileIT)
# Prepare lof file
fileOL = FileSpec()
fileOL.lfn = "%s.log.tgz" % pandajob.jobName
fileOL.destinationDBlock = pandajob.destinationDBlock
fileOL.destinationSE = pandajob.destinationSE
fileOL.dataset = '{}:logs'.format(fscope)
fileOL.type = 'log'
fileOL.scope = 'panda'
pandajob.addFile(fileOL)
# Save log meta
log = File()
log.scope = fscope
log.lfn = fileOL.lfn
log.guid = getGUID(log.scope, log.lfn)
log.type = 'log'
log.status = 'defined'
files_.save(log)
# Save replica meta
fc.new_replica(log, site)
# Register file in container
fc.reg_file_in_cont(log, cont, 'log')
# Submit job
o = submitJobs([pandajob])
x = o[0]
try:
#update PandaID
PandaID = int(x[0])
job.pandaid = PandaID
job.ce = site.ce
except:
job.status = 'submit_error'
jobs_.save(job)
return 0
def doBrokerage(self, inputList, vo, prodSourceLabel, workQueue,
resource_name):
# list with a lock
inputListWorld = ListWithLock([])
# variables for submission
maxBunchTask = 100
# make logger
tmpLog = MsgWrapper(logger)
tmpLog.debug('start doBrokerage')
# return for failure
retFatal = self.SC_FATAL
retTmpError = self.SC_FAILED
tmpLog.debug(
'vo={0} label={1} queue={2} resource_name={3} nTasks={4}'.format(
vo, prodSourceLabel, workQueue.queue_name, resource_name,
len(inputList)))
# loop over all tasks
allRwMap = {}
prioMap = {}
tt2Map = {}
expRWs = {}
jobSpecList = []
for tmpJediTaskID, tmpInputList in inputList:
for taskSpec, cloudName, inputChunk in tmpInputList:
# collect tasks for WORLD
if taskSpec.useWorldCloud():
inputListWorld.append((taskSpec, inputChunk))
continue
# make JobSpec to be submitted for TaskAssigner
jobSpec = JobSpec()
jobSpec.taskID = taskSpec.jediTaskID
jobSpec.jediTaskID = taskSpec.jediTaskID
# set managed to trigger TA
jobSpec.prodSourceLabel = 'managed'
jobSpec.processingType = taskSpec.processingType
jobSpec.workingGroup = taskSpec.workingGroup
jobSpec.metadata = taskSpec.processingType
jobSpec.assignedPriority = taskSpec.taskPriority
jobSpec.currentPriority = taskSpec.currentPriority
jobSpec.maxDiskCount = (
taskSpec.getOutDiskSize() +
taskSpec.getWorkDiskSize()) // 1024 // 1024
if taskSpec.useWorldCloud():
# use destinationSE to trigger task brokerage in WORLD cloud
jobSpec.destinationSE = taskSpec.cloud
prodDBlock = None
setProdDBlock = False
for datasetSpec in inputChunk.getDatasets():
prodDBlock = datasetSpec.datasetName
if datasetSpec.isMaster():
jobSpec.prodDBlock = datasetSpec.datasetName
setProdDBlock = True
for fileSpec in datasetSpec.Files:
tmpInFileSpec = fileSpec.convertToJobFileSpec(
datasetSpec)
jobSpec.addFile(tmpInFileSpec)
# use secondary dataset name as prodDBlock
if setProdDBlock is False and prodDBlock is not None:
jobSpec.prodDBlock = prodDBlock
# append
jobSpecList.append(jobSpec)
prioMap[jobSpec.taskID] = jobSpec.currentPriority
tt2Map[jobSpec.taskID] = jobSpec.processingType
# get RW for a priority
if jobSpec.currentPriority not in allRwMap:
tmpRW = self.taskBufferIF.calculateRWwithPrio_JEDI(
vo, prodSourceLabel, workQueue,
jobSpec.currentPriority)
if tmpRW is None:
tmpLog.error(
'failed to calculate RW with prio={0}'.format(
jobSpec.currentPriority))
return retTmpError
allRwMap[jobSpec.currentPriority] = tmpRW
# get expected RW
expRW = self.taskBufferIF.calculateTaskRW_JEDI(
jobSpec.jediTaskID)
if expRW is None:
tmpLog.error(
'failed to calculate RW for jediTaskID={0}'.format(
jobSpec.jediTaskID))
return retTmpError
expRWs[jobSpec.taskID] = expRW
# for old clouds
if jobSpecList != []:
# get fullRWs
fullRWs = self.taskBufferIF.calculateRWwithPrio_JEDI(
vo, prodSourceLabel, None, None)
if fullRWs is None:
tmpLog.error('failed to calculate full RW')
return retTmpError
# set metadata
for jobSpec in jobSpecList:
rwValues = allRwMap[jobSpec.currentPriority]
jobSpec.metadata = "%s;%s;%s;%s;%s;%s" % (
jobSpec.metadata, str(rwValues), str(expRWs), str(prioMap),
str(fullRWs), str(tt2Map))
tmpLog.debug('run task assigner for {0} tasks'.format(
len(jobSpecList)))
nBunchTask = 0
while nBunchTask < len(jobSpecList):
# get a bunch
jobsBunch = jobSpecList[nBunchTask:nBunchTask + maxBunchTask]
strIDs = 'jediTaskID='
for tmpJobSpec in jobsBunch:
strIDs += '{0},'.format(tmpJobSpec.taskID)
strIDs = strIDs[:-1]
tmpLog.debug(strIDs)
# increment index
nBunchTask += maxBunchTask
# run task brokerge
stS, outSs = PandaClient.runTaskAssignment(jobsBunch)
tmpLog.debug('{0}:{1}'.format(stS, str(outSs)))
# for WORLD
if len(inputListWorld) > 0:
# thread pool
threadPool = ThreadPool()
# get full RW for WORLD
fullRWs = self.taskBufferIF.calculateWorldRWwithPrio_JEDI(
vo, prodSourceLabel, None, None)
if fullRWs is None:
tmpLog.error('failed to calculate full WORLD RW')
return retTmpError
# get RW per priority
for taskSpec, inputChunk in inputListWorld:
if taskSpec.currentPriority not in allRwMap:
tmpRW = self.taskBufferIF.calculateWorldRWwithPrio_JEDI(
vo, prodSourceLabel, workQueue,
taskSpec.currentPriority)
if tmpRW is None:
tmpLog.error(
'failed to calculate RW with prio={0}'.format(
taskSpec.currentPriority))
return retTmpError
allRwMap[taskSpec.currentPriority] = tmpRW
# live counter for RWs
liveCounter = MapWithLock(allRwMap)
# make workers
ddmIF = self.ddmIF.getInterface(vo)
for iWorker in range(4):
thr = AtlasProdTaskBrokerThread(inputListWorld, threadPool,
self.taskBufferIF, ddmIF,
fullRWs, liveCounter,
workQueue)
thr.start()
threadPool.join(60 * 10)
# return
tmpLog.debug('doBrokerage done')
return self.SC_SUCCEEDED
for i in [999905,999906,999907]:
job = JobSpec()
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (commands.getoutput('uuidgen'),i)
job.AtlasRelease = 'Atlas-14.1.0'
job.homepackage = 'AtlasProduction/12.0.6.2'
job.transformation = 'csc_evgen_trf.py'
job.destinationDBlock = datasetName
#job.destinationSE = destName
job.currentPriority = 1000
job.prodSourceLabel = 'managed'
#job.prodSourceLabel = 'test'
#job.computingSite = site
job.cmtConfig = 'i686-slc4-gcc34-opt'
job.metadata = 'evgen;%s;%s;%s' % (str({'FR': 46, 'NL': 45, 'NDGF': 300, 'CERN': 19, 'TW': 44110, 'CA': 2922, 'DE': 9903, 'IT': 1168, 'US': 6226, 'UK': 1026, 'ES': 26619}),str({999907:100,999906:200,999905:300}),str({999905:100,999906:910,999907:500}))
#job.metadata = 'evgen;%s' % str({'FR': 46, 'NL': 45, 'NDGF': 300, 'CERN': 19, 'TW': 44110, 'CA': 2922, 'DE': 9903, 'IT': 1168, 'US': 6226, 'UK': 1026, 'ES': 26619})
#job.cloud = "UK"
job.taskID = i
file = FileSpec()
file.lfn = "%s.evgen.pool.root" % job.jobName
file.destinationDBlock = job.destinationDBlock
file.destinationSE = job.destinationSE
file.dataset = job.destinationDBlock
#file.destinationDBlockToken = 'ATLASDATADISK'
file.type = 'output'
job.addFile(file)
fileOL = FileSpec()
def createJobSpec(self, task, outdataset, job, jobset, jobdef, site,
jobname, lfnhanger, allsites, jobid):
"""Create a spec for one job
:arg TaskWorker.DataObject.Task task: the task to work on
:arg str outdataset: the output dataset name where all the produced files will be placed
:arg WMCore.DataStructs.Job job: the abstract job
:arg int jobset: the PanDA jobset corresponding to the current task
:arg int jobdef: the PanDA jobdef where to append the current jobs --- not used
:arg str site: the borkered site where to run the jobs
:arg str jobname: the job name
:arg str lfnhanger: the random string to be added in the output file name
:arg list str allsites: all possible sites where the job can potentially run
:arg int jobid: incremental job number
:return: the sepc object."""
pandajob = JobSpec()
## always setting a job definition ID
pandajob.jobDefinitionID = jobdef if jobdef else -1
## always setting a job set ID
pandajob.jobsetID = jobset if jobset else -1
pandajob.jobName = jobname
pandajob.prodUserID = task['tm_user_dn']
pandajob.destinationDBlock = outdataset
pandajob.prodDBlock = task['tm_input_dataset']
pandajob.prodSourceLabel = 'user'
pandajob.computingSite = site
pandajob.cloud = getSite(pandajob.computingSite)
pandajob.destinationSE = 'local'
pandajob.transformation = task['tm_transformation']
## need to initialize this
pandajob.metadata = ''
def outFileSpec(of=None, log=False):
"""Local routine to create an FileSpec for the an job output/log file
:arg str of: output file base name
:return: FileSpec object for the output file."""
outfile = FileSpec()
if log:
outfile.lfn = "job.log_%d_%s.tgz" % (jobid, lfnhanger)
outfile.type = 'log'
else:
outfile.lfn = '%s_%d_%s%s' % (os.path.splitext(of)[0],
jobid, lfnhanger,
os.path.splitext(of)[1])
outfile.type = 'output'
outfile.destinationDBlock = pandajob.destinationDBlock
outfile.destinationSE = task['tm_asyncdest']
outfile.dataset = pandajob.destinationDBlock
return outfile
alloutfiles = []
outjobpar = {}
outfilestring = ''
for outputfile in task['tm_outfiles']:
outfilestring += '%s,' % outputfile
filespec = outFileSpec(outputfile)
alloutfiles.append(filespec)
#pandajob.addFile(filespec)
outjobpar[outputfile] = filespec.lfn
for outputfile in task['tm_tfile_outfiles']:
outfilestring += '%s,' % outputfile
filespec = outFileSpec(outputfile)
alloutfiles.append(filespec)
#pandajob.addFile(filespec)
outjobpar[outputfile] = filespec.lfn
for outputfile in task['tm_edm_outfiles']:
outfilestring += '%s,' % outputfile
filespec = outFileSpec(outputfile)
alloutfiles.append(filespec)
#pandajob.addFile(filespec)
outjobpar[outputfile] = filespec.lfn
outfilestring = outfilestring[:-1]
infiles = []
for inputfile in job['input_files']:
infiles.append(inputfile['lfn'])
pandajob.jobParameters = '-a %s ' % task['tm_user_sandbox']
pandajob.jobParameters += '--sourceURL %s ' % task['tm_cache_url']
pandajob.jobParameters += '--jobNumber=%s ' % jobid
pandajob.jobParameters += '--cmsswVersion=%s ' % task['tm_job_sw']
pandajob.jobParameters += '--scramArch=%s ' % task['tm_job_arch']
pandajob.jobParameters += '--inputFile=\'%s\' ' % json.dumps(infiles)
self.jobParametersSetting(pandajob, job,
self.jobtypeMapper[task['tm_job_type']])
pandajob.jobParameters += '-o "%s" ' % str(outjobpar)
pandajob.jobParameters += '--dbs_url=%s ' % task['tm_dbs_url']
pandajob.jobParameters += '--publish_dbs_url=%s ' % task[
'tm_publish_dbs_url']
pandajob.jobParameters += '--publishFiles=%s ' % (
'True' if task['tm_publication'] == 'T' else 'False')
pandajob.jobParameters += '--saveLogs=%s ' % (
'True' if task['tm_save_logs'] == 'T' else 'False')
pandajob.jobParameters += '--availableSites=\'%s\' ' % json.dumps(
allsites)
pandajob.jobParameters += '--group=%s ' % (
task['tm_user_group'] if task['tm_user_group'] else '')
pandajob.jobParameters += '--role=%s ' % (task['tm_user_role'] if
task['tm_user_role'] else '')
self.logger.info(type(task['tm_user_infiles']))
self.logger.info(task['tm_user_infiles'])
if task['tm_user_infiles']:
addinfilestring = ''
for addinfile in task['tm_user_infiles']:
addinfilestring += '%s,' % addinfile
pandajob.jobParameters += '--userFiles=%s ' % (
addinfilestring[:-1])
pandajob.jobName = '%s' % task[
'tm_taskname'] #Needed by ASO and Dashboard
if 'panda_oldjobid' in job and job['panda_oldjobid']:
pandajob.parentID = job['panda_oldjobid']
pandajob.addFile(outFileSpec(log=True))
for filetoadd in alloutfiles:
pandajob.addFile(filetoadd)
return pandajob
def send_job(jobid, siteid):
_logger.debug('Jobid: ' + str(jobid))
site = sites_.get(siteid)
job = jobs_.get(int(jobid))
cont = job.container
files_catalog = cont.files
fscope = getScope(job.owner.username)
datasetName = '{}:{}'.format(fscope, cont.guid)
distributive = job.distr.name
release = job.distr.release
# Prepare runScript
parameters = job.distr.command
parameters = parameters.replace("$COMMAND$", job.params)
parameters = parameters.replace("$USERNAME$", job.owner.username)
parameters = parameters.replace("$WORKINGGROUP$", job.owner.working_group)
# Prepare metadata
metadata = dict(user=job.owner.username)
# Prepare PanDA Object
pandajob = JobSpec()
pandajob.jobDefinitionID = int(time.time()) % 10000
pandajob.jobName = cont.guid
pandajob.transformation = client_config.DEFAULT_TRF
pandajob.destinationDBlock = datasetName
pandajob.destinationSE = site.se
pandajob.currentPriority = 1000
pandajob.prodSourceLabel = 'user'
pandajob.computingSite = site.ce
pandajob.cloud = 'RU'
pandajob.VO = 'atlas'
pandajob.prodDBlock = "%s:%s" % (fscope, pandajob.jobName)
pandajob.coreCount = job.corecount
pandajob.metadata = json.dumps(metadata)
#pandajob.workingGroup = job.owner.working_group
if site.encode_commands:
# It requires script wrapper on cluster side
pandajob.jobParameters = '%s %s %s "%s"' % (cont.guid, release, distributive, parameters)
else:
pandajob.jobParameters = parameters
has_input = False
for fcc in files_catalog:
if fcc.type == 'input':
f = fcc.file
guid = f.guid
fileIT = FileSpec()
fileIT.lfn = f.lfn
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = guid
pandajob.addFile(fileIT)
has_input = True
if fcc.type == 'output':
f = fcc.file
fileOT = FileSpec()
fileOT.lfn = f.lfn
fileOT.destinationDBlock = pandajob.prodDBlock
fileOT.destinationSE = pandajob.destinationSE
fileOT.dataset = pandajob.prodDBlock
fileOT.type = 'output'
fileOT.scope = fscope
fileOT.GUID = f.guid
pandajob.addFile(fileOT)
# Save replica meta
fc.new_replica(f, site)
if not has_input:
# Add fake input
fileIT = FileSpec()
fileIT.lfn = "fake.input"
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = "fake.guid"
pandajob.addFile(fileIT)
# Prepare lof file
fileOL = FileSpec()
fileOL.lfn = "%s.log.tgz" % pandajob.jobName
fileOL.destinationDBlock = pandajob.destinationDBlock
fileOL.destinationSE = pandajob.destinationSE
fileOL.dataset = '{}:logs'.format(fscope)
fileOL.type = 'log'
fileOL.scope = 'panda'
pandajob.addFile(fileOL)
# Save log meta
log = File()
log.scope = fscope
log.lfn = fileOL.lfn
log.guid = getGUID(log.scope, log.lfn)
log.type = 'log'
log.status = 'defined'
files_.save(log)
# Save replica meta
fc.new_replica(log, site)
# Register file in container
fc.reg_file_in_cont(log, cont, 'log')
# Submit job
o = submitJobs([pandajob])
x = o[0]
try:
#update PandaID
PandaID = int(x[0])
job.pandaid = PandaID
job.ce = site.ce
except:
job.status = 'submit_error'
jobs_.save(job)
return 0
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
'''prepare the subjob specific configuration'''
# PandaTools
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('AthenaMCPandaRTHandler prepare called for %s',
job.getFQID('.'))
try:
assert self.outsite
except:
logger.error("outsite not set. Aborting")
raise Exception()
job.backend.site = self.outsite
job.backend.actualCE = self.outsite
cloud = job._getRoot().backend.requirements.cloud
job.backend.requirements.cloud = cloud
# now just filling the job from AthenaMC data
jspec = JobSpec()
jspec.jobDefinitionID = job._getRoot().id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.AtlasRelease = 'Atlas-%s' % app.atlas_rel
if app.transform_archive:
jspec.homepackage = 'AnalysisTransforms' + app.transform_archive
elif app.prod_release:
jspec.homepackage = 'AnalysisTransforms-AtlasProduction_' + str(
app.prod_release)
jspec.transformation = '%s/runAthena-00-00-11' % Client.baseURLSUB
#---->???? prodDBlock and destinationDBlock when facing several input / output datasets?
jspec.prodDBlock = 'NULL'
if job.inputdata and len(
app.inputfiles) > 0 and app.inputfiles[0] in app.dsetmap:
jspec.prodDBlock = app.dsetmap[app.inputfiles[0]]
# How to specify jspec.destinationDBlock when more than one type of output is available? Panda prod jobs seem to specify only the last output dataset
outdset = ""
for type in ["EVNT", "RDO", "HITS", "AOD", "ESD", "NTUP"]:
if type in app.outputpaths.keys():
outdset = string.replace(app.outputpaths[type], "/", ".")
outdset = outdset[1:-1]
break
if not outdset:
try:
assert len(app.outputpaths.keys()) > 0
except:
logger.error(
"app.outputpaths is empty: check your output datasets")
raise
type = app.outputpaths.keys()[0]
outdset = string.replace(app.outputpaths[type], "/", ".")
outdset = outdset[1:-1]
jspec.destinationDBlock = outdset
jspec.destinationSE = self.outsite
jspec.prodSourceLabel = 'user'
jspec.assignedPriority = 1000
jspec.cloud = cloud
# memory
if job.backend.requirements.memory != -1:
jspec.minRamCount = job.backend.requirements.memory
jspec.computingSite = self.outsite
jspec.cmtConfig = AthenaUtils.getCmtConfig(athenaVer=app.atlas_rel)
# library (source files)
flib = FileSpec()
flib.lfn = self.library
# flib.GUID =
flib.type = 'input'
# flib.status =
flib.dataset = self.libDataset
flib.dispatchDBlock = self.libDataset
jspec.addFile(flib)
# input files FIXME: many more input types
for lfn in app.inputfiles:
useguid = app.turls[lfn].replace("guid:", "")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# add dbfiles if any:
for lfn in app.dbfiles:
useguid = app.dbturls[lfn].replace("guid:", "")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# then minbias files
for lfn in app.mbfiles:
useguid = app.minbias_turls[lfn].replace("guid:", "")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# then cavern files
for lfn in app.cavernfiles:
useguid = app.cavern_turls[lfn].replace("guid:", "")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# output files( this includes the logfiles)
# Output files
jidtag = ""
job = app._getParent() # Returns job or subjob object
if job._getRoot().subjobs:
jidtag = job._getRoot().id
else:
jidtag = "%d" % job.id
outfiles = app.subjobsOutfiles[job.id]
pandaOutfiles = {}
for type in outfiles.keys():
pandaOutfiles[type] = outfiles[type] + "." + str(jidtag)
if type == "LOG":
pandaOutfiles[type] += ".tgz"
#print pandaOutfiles
for outtype in pandaOutfiles.keys():
fout = FileSpec()
dset = string.replace(app.outputpaths[outtype], "/", ".")
dset = dset[1:-1]
fout.dataset = dset
fout.lfn = pandaOutfiles[outtype]
fout.type = 'output'
# fout.destinationDBlock = jspec.destinationDBlock
fout.destinationDBlock = fout.dataset
fout.destinationSE = jspec.destinationSE
if outtype == 'LOG':
fout.type = 'log'
fout.destinationDBlock = fout.dataset
fout.destinationSE = job.backend.site
jspec.addFile(fout)
# job parameters
param = '-l %s ' % self.library # user tarball.
# use corruption checker
if job.backend.requirements.corCheck:
param += '--corCheck '
# disable to skip missing files
if job.backend.requirements.notSkipMissing:
param += '--notSkipMissing '
# transform parameters
# need to update arglist with final output file name...
newArgs = []
if app.mode == "evgen":
app.args[3] = app.args[3] + " -t "
if app.verbosity:
app.args[3] = app.args[3] + " -l %s " % app.verbosity
for arg in app.args[3:]:
for type in outfiles.keys():
if arg.find(outfiles[type]) > -1:
arg = arg.replace(outfiles[type], pandaOutfiles[type])
newArgs.append(arg)
arglist = string.join(newArgs, " ")
# print "Arglist:",arglist
param += ' -r ./ '
param += ' -j "%s"' % urllib.quote(arglist)
allinfiles = app.inputfiles + app.dbfiles
# Input files.
param += ' -i "%s" ' % allinfiles
if len(app.mbfiles) > 0:
param += ' -m "%s" ' % app.mbfiles
if len(app.cavernfiles) > 0:
param += ' -n "%s" ' % app.cavernfiles
# param += '-m "[]" ' #%minList FIXME
# param += '-n "[]" ' #%cavList FIXME
del pandaOutfiles[
"LOG"] # logfiles do not appear in IROOT block, and this one is not needed anymore...
param += ' -o "{\'IROOT\':%s }"' % str(pandaOutfiles.items())
# source URL
matchURL = re.search("(http.*://[^/]+)/", Client.baseURLSSL)
if matchURL != None:
param += " --sourceURL %s " % matchURL.group(1)
param += " --trf"
jspec.jobParameters = param
jspec.metadata = "--trf \"%s\"" % arglist
#print "SUBJOB DETAILS:",jspec.values()
if app.dryrun:
print "job.application.dryrun activated, printing out job parameters"
print jspec.values()
return
return jspec
