def prepare(self,app,appsubconfig,appmasterconfig,jobmasterconfig):
'''prepare the subjob specific configuration'''
from pandatools import Client
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('AthenaPandaRTHandler prepare called for %s', job.getFQID('.'))
# in case of a simple job get the dataset content, otherwise subjobs are filled by the splitter
if job.inputdata and not job._getRoot().subjobs:
if not job.inputdata.names:
contents = job.inputdata.get_contents(overlap=False, size=True)
for ds in contents.keys():
for f in contents[ds]:
job.inputdata.guids.append( f[0] )
job.inputdata.names.append( f[1][0] )
job.inputdata.sizes.append( f[1][1] )
job.inputdata.checksums.append( f[1][2] )
job.inputdata.scopes.append( f[1][3] )
site = job._getRoot().backend.site
job.backend.site = site
job.backend.actualCE = site
cloud = job._getRoot().backend.requirements.cloud
job.backend.requirements.cloud = cloud
# if no outputdata are given
if not job.outputdata:
job.outputdata = DQ2OutputDataset()
job.outputdata.datasetname = job._getRoot().outputdata.datasetname
#if not job.outputdata.datasetname:
else:
job.outputdata.datasetname = job._getRoot().outputdata.datasetname
if not job.outputdata.datasetname:
raise ApplicationConfigurationError('DQ2OutputDataset has no datasetname')
jspec = JobSpec()
jspec.jobDefinitionID = job._getRoot().id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.transformation = '%s/runGen-00-00-02' % Client.baseURLSUB
if job.inputdata:
jspec.prodDBlock = job.inputdata.dataset[0]
else:
jspec.prodDBlock = 'NULL'
jspec.destinationDBlock = job.outputdata.datasetname
if job.outputdata.location:
if not job._getRoot().subjobs or job.id == 0:
logger.warning('You have specified outputdata.location. Note that Panda may not support writing to a user-defined output location.')
jspec.destinationSE = job.outputdata.location
else:
jspec.destinationSE = site
jspec.prodSourceLabel = configPanda['prodSourceLabelRun']
jspec.processingType = configPanda['processingType']
jspec.assignedPriority = configPanda['assignedPriorityRun']
jspec.cloud = cloud
# memory
if job.backend.requirements.memory != -1:
jspec.minRamCount = job.backend.requirements.memory
# cputime
if job.backend.requirements.cputime != -1:
jspec.maxCpuCount = job.backend.requirements.cputime
jspec.computingSite = site
# library (source files)
if job.backend.libds:
flib = FileSpec()
flib.lfn = self.fileBO.lfn
flib.GUID = self.fileBO.GUID
flib.type = 'input'
flib.status = self.fileBO.status
flib.dataset = self.fileBO.destinationDBlock
flib.dispatchDBlock = self.fileBO.destinationDBlock
jspec.addFile(flib)
elif job.backend.bexec:
flib = FileSpec()
flib.lfn = self.library
flib.type = 'input'
flib.dataset = self.libDataset
flib.dispatchDBlock = self.libDataset
jspec.addFile(flib)
# input files FIXME: many more input types
if job.inputdata:
for guid, lfn, size, checksum, scope in zip(job.inputdata.guids,job.inputdata.names,job.inputdata.sizes, job.inputdata.checksums, job.inputdata.scopes):
finp = FileSpec()
finp.lfn = lfn
finp.GUID = guid
finp.scope = scope
# finp.fsize =
# finp.md5sum =
finp.dataset = job.inputdata.dataset[0]
finp.prodDBlock = job.inputdata.dataset[0]
finp.dispatchDBlock = job.inputdata.dataset[0]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# output files
# outMap = {}
#FIXME: if options.outMeta != []:
self.rundirectory = "."
# log files
flog = FileSpec()
flog.lfn = '%s._$PANDAID.log.tgz' % job.outputdata.datasetname
flog.type = 'log'
flog.dataset = job.outputdata.datasetname
flog.destinationDBlock = job.outputdata.datasetname
flog.destinationSE = job.backend.site
jspec.addFile(flog)
# job parameters
param = ''
# source URL
matchURL = re.search("(http.*://[^/]+)/",Client.baseURLCSRVSSL)
srcURL = ""
if matchURL != None:
srcURL = matchURL.group(1)
param += " --sourceURL %s " % srcURL
param += '-r "%s" ' % self.rundirectory
exe_name = job.application.exe
if job.backend.bexec == '':
if hasattr(job.application.exe, "name"):
exe_name = os.path.basename(job.application.exe.name)
# set jobO parameter
if job.application.args:
param += ' -j "" -p "%s %s" '%(exe_name,urllib.quote(" ".join(job.application.args)))
else:
param += ' -j "" -p "%s" '%exe_name
if self.inputsandbox:
param += ' -a %s '%self.inputsandbox
else:
param += '-l %s ' % self.library
param += '-j "" -p "%s %s" ' % ( exe_name,urllib.quote(" ".join(job.application.args)))
if job.inputdata:
param += '-i "%s" ' % job.inputdata.names
# fill outfiles
outfiles = {}
for f in self.extOutFile:
tarnum = 1
if f.find('*') != -1:
# archive *
outfiles[f] = "outputbox%i.%s.%s.tar.gz" % (tarnum, job.getFQID('.'), time.strftime("%Y%m%d%H%M%S") )
tarnum += 1
else:
outfiles[f] = "%s.%s.%s" %(f, job.getFQID('.'), time.strftime("%Y%m%d%H%M%S"))
fout = FileSpec()
fout.lfn = outfiles[f]
fout.type = 'output'
fout.dataset = job.outputdata.datasetname
fout.destinationDBlock = job.outputdata.datasetname
fout.destinationSE = job.backend.site
jspec.addFile(fout)
param += '-o "%s" ' % (outfiles) # must be double quotes, because python prints strings in 'single quotes'
for file in jspec.Files:
if file.type in [ 'output', 'log'] and configPanda['chirpconfig']:
file.dispatchDBlockToken = configPanda['chirpconfig']
logger.debug('chirp file %s',file)
jspec.jobParameters = param
return jspec
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
'''prepare the subjob specific configuration'''
from pandatools import Client
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('AthenaPandaRTHandler prepare called for %s',
job.getFQID('.'))
# in case of a simple job get the dataset content, otherwise subjobs are filled by the splitter
if job.inputdata and not job._getRoot().subjobs:
if not job.inputdata.names:
contents = job.inputdata.get_contents(overlap=False, size=True)
for ds in contents.keys():
for f in contents[ds]:
job.inputdata.guids.append(f[0])
job.inputdata.names.append(f[1][0])
job.inputdata.sizes.append(f[1][1])
job.inputdata.checksums.append(f[1][2])
job.inputdata.scopes.append(f[1][3])
site = job._getRoot().backend.site
job.backend.site = site
job.backend.actualCE = site
cloud = job._getRoot().backend.requirements.cloud
job.backend.requirements.cloud = cloud
# if no outputdata are given
if not job.outputdata:
job.outputdata = DQ2OutputDataset()
job.outputdata.datasetname = job._getRoot().outputdata.datasetname
#if not job.outputdata.datasetname:
else:
job.outputdata.datasetname = job._getRoot().outputdata.datasetname
if not job.outputdata.datasetname:
raise ApplicationConfigurationError(
None, 'DQ2OutputDataset has no datasetname')
jspec = JobSpec()
jspec.jobDefinitionID = job._getRoot().id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.transformation = '%s/runGen-00-00-02' % Client.baseURLSUB
if job.inputdata:
jspec.prodDBlock = job.inputdata.dataset[0]
else:
jspec.prodDBlock = 'NULL'
jspec.destinationDBlock = job.outputdata.datasetname
if job.outputdata.location:
if not job._getRoot().subjobs or job.id == 0:
logger.warning(
'You have specified outputdata.location. Note that Panda may not support writing to a user-defined output location.'
)
jspec.destinationSE = job.outputdata.location
else:
jspec.destinationSE = site
jspec.prodSourceLabel = configPanda['prodSourceLabelRun']
jspec.processingType = configPanda['processingType']
jspec.assignedPriority = configPanda['assignedPriorityRun']
jspec.cloud = cloud
# memory
if job.backend.requirements.memory != -1:
jspec.minRamCount = job.backend.requirements.memory
# cputime
if job.backend.requirements.cputime != -1:
jspec.maxCpuCount = job.backend.requirements.cputime
jspec.computingSite = site
# library (source files)
if job.backend.libds:
flib = FileSpec()
flib.lfn = self.fileBO.lfn
flib.GUID = self.fileBO.GUID
flib.type = 'input'
flib.status = self.fileBO.status
flib.dataset = self.fileBO.destinationDBlock
flib.dispatchDBlock = self.fileBO.destinationDBlock
jspec.addFile(flib)
elif job.backend.bexec:
flib = FileSpec()
flib.lfn = self.library
flib.type = 'input'
flib.dataset = self.libDataset
flib.dispatchDBlock = self.libDataset
jspec.addFile(flib)
# input files FIXME: many more input types
if job.inputdata:
for guid, lfn, size, checksum, scope in zip(
job.inputdata.guids, job.inputdata.names,
job.inputdata.sizes, job.inputdata.checksums,
job.inputdata.scopes):
finp = FileSpec()
finp.lfn = lfn
finp.GUID = guid
finp.scope = scope
# finp.fsize =
# finp.md5sum =
finp.dataset = job.inputdata.dataset[0]
finp.prodDBlock = job.inputdata.dataset[0]
finp.dispatchDBlock = job.inputdata.dataset[0]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# output files
# outMap = {}
#FIXME: if options.outMeta != []:
self.rundirectory = "."
# log files
flog = FileSpec()
flog.lfn = '%s._$PANDAID.log.tgz' % job.outputdata.datasetname
flog.type = 'log'
flog.dataset = job.outputdata.datasetname
flog.destinationDBlock = job.outputdata.datasetname
flog.destinationSE = job.backend.site
jspec.addFile(flog)
# job parameters
param = ''
# source URL
matchURL = re.search("(http.*://[^/]+)/", Client.baseURLCSRVSSL)
srcURL = ""
if matchURL != None:
srcURL = matchURL.group(1)
param += " --sourceURL %s " % srcURL
param += '-r "%s" ' % self.rundirectory
exe_name = job.application.exe
if job.backend.bexec == '':
if hasattr(job.application.exe, "name"):
exe_name = os.path.basename(job.application.exe.name)
# set jobO parameter
if job.application.args:
param += ' -j "" -p "%s %s" ' % (
exe_name, urllib.quote(" ".join(job.application.args)))
else:
param += ' -j "" -p "%s" ' % exe_name
if self.inputsandbox:
param += ' -a %s ' % self.inputsandbox
else:
param += '-l %s ' % self.library
param += '-j "" -p "%s %s" ' % (
exe_name, urllib.quote(" ".join(job.application.args)))
if job.inputdata:
param += '-i "%s" ' % job.inputdata.names
# fill outfiles
outfiles = {}
for f in self.extOutFile:
tarnum = 1
if f.find('*') != -1:
# archive *
outfiles[f] = "outputbox%i.%s.%s.tar.gz" % (
tarnum, job.getFQID('.'), time.strftime("%Y%m%d%H%M%S"))
tarnum += 1
else:
outfiles[f] = "%s.%s.%s" % (f, job.getFQID('.'),
time.strftime("%Y%m%d%H%M%S"))
fout = FileSpec()
fout.lfn = outfiles[f]
fout.type = 'output'
fout.dataset = job.outputdata.datasetname
fout.destinationDBlock = job.outputdata.datasetname
fout.destinationSE = job.backend.site
jspec.addFile(fout)
param += '-o "%s" ' % (
outfiles
) # must be double quotes, because python prints strings in 'single quotes'
for file in jspec.Files:
if file.type in ['output', 'log'] and configPanda['chirpconfig']:
file.dispatchDBlockToken = configPanda['chirpconfig']
logger.debug('chirp file %s', file)
jspec.jobParameters = param
return jspec
def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig):
'''prepare the subjob specific configuration'''
# PandaTools
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('AthenaMCPandaRTHandler prepare called for %s', job.getFQID('.'))
try:
assert self.outsite
except:
logger.error("outsite not set. Aborting")
raise Exception()
job.backend.site = self.outsite
job.backend.actualCE = self.outsite
cloud = job._getRoot().backend.requirements.cloud
job.backend.requirements.cloud = cloud
# now just filling the job from AthenaMC data
jspec = JobSpec()
jspec.jobDefinitionID = job._getRoot().id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.AtlasRelease = 'Atlas-%s' % app.atlas_rel
if app.transform_archive:
jspec.homepackage = 'AnalysisTransforms'+app.transform_archive
elif app.prod_release:
jspec.homepackage = 'AnalysisTransforms-AtlasProduction_'+str(app.prod_release)
jspec.transformation = '%s/runAthena-00-00-11' % Client.baseURLSUB
#---->???? prodDBlock and destinationDBlock when facing several input / output datasets?
jspec.prodDBlock = 'NULL'
if job.inputdata and len(app.inputfiles)>0 and app.inputfiles[0] in app.dsetmap:
jspec.prodDBlock = app.dsetmap[app.inputfiles[0]]
# How to specify jspec.destinationDBlock when more than one type of output is available? Panda prod jobs seem to specify only the last output dataset
outdset=""
for type in ["EVNT","RDO","HITS","AOD","ESD","NTUP"]:
if type in app.outputpaths.keys():
outdset=string.replace(app.outputpaths[type],"/",".")
outdset=outdset[1:-1]
break
if not outdset:
try:
assert len(app.outputpaths.keys())>0
except:
logger.error("app.outputpaths is empty: check your output datasets")
raise
type=app.outputpaths.keys()[0]
outdset=string.replace(app.outputpaths[type],"/",".")
outdset=outdset[1:-1]
jspec.destinationDBlock = outdset
jspec.destinationSE = self.outsite
jspec.prodSourceLabel = 'user'
jspec.assignedPriority = 1000
jspec.cloud = cloud
# memory
if job.backend.requirements.memory != -1:
jspec.minRamCount = job.backend.requirements.memory
jspec.computingSite = self.outsite
jspec.cmtConfig = AthenaUtils.getCmtConfig(athenaVer=app.atlas_rel)
# library (source files)
flib = FileSpec()
flib.lfn = self.library
# flib.GUID =
flib.type = 'input'
# flib.status =
flib.dataset = self.libDataset
flib.dispatchDBlock = self.libDataset
jspec.addFile(flib)
# input files FIXME: many more input types
for lfn in app.inputfiles:
useguid=app.turls[lfn].replace("guid:","")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# add dbfiles if any:
for lfn in app.dbfiles:
useguid=app.dbturls[lfn].replace("guid:","")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# then minbias files
for lfn in app.mbfiles:
useguid=app.minbias_turls[lfn].replace("guid:","")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# then cavern files
for lfn in app.cavernfiles:
useguid=app.cavern_turls[lfn].replace("guid:","")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# output files( this includes the logfiles)
# Output files
jidtag=""
job = app._getParent() # Returns job or subjob object
if job._getRoot().subjobs:
jidtag = job._getRoot().id
else:
jidtag = "%d" % job.id
outfiles=app.subjobsOutfiles[job.id]
pandaOutfiles={}
for type in outfiles.keys():
pandaOutfiles[type]=outfiles[type]+"."+str(jidtag)
if type=="LOG":
pandaOutfiles[type]+=".tgz"
#print pandaOutfiles
for outtype in pandaOutfiles.keys():
fout = FileSpec()
dset=string.replace(app.outputpaths[outtype],"/",".")
dset=dset[1:-1]
fout.dataset=dset
fout.lfn=pandaOutfiles[outtype]
fout.type = 'output'
# fout.destinationDBlock = jspec.destinationDBlock
fout.destinationDBlock = fout.dataset
fout.destinationSE = jspec.destinationSE
if outtype=='LOG':
fout.type='log'
fout.destinationDBlock = fout.dataset
fout.destinationSE = job.backend.site
jspec.addFile(fout)
# job parameters
param = '-l %s ' % self.library # user tarball.
# use corruption checker
if job.backend.requirements.corCheck:
param += '--corCheck '
# disable to skip missing files
if job.backend.requirements.notSkipMissing:
param += '--notSkipMissing '
# transform parameters
# need to update arglist with final output file name...
newArgs=[]
if app.mode == "evgen":
app.args[3]=app.args[3]+" -t "
if app.verbosity:
app.args[3]=app.args[3]+" -l %s " % app.verbosity
for arg in app.args[3:]:
for type in outfiles.keys():
if arg.find(outfiles[type])>-1:
arg=arg.replace(outfiles[type],pandaOutfiles[type])
newArgs.append(arg)
arglist=string.join(newArgs," ")
# print "Arglist:",arglist
param += ' -r ./ '
param += ' -j "%s"' % urllib.quote(arglist)
allinfiles=app.inputfiles+app.dbfiles
# Input files.
param += ' -i "%s" ' % allinfiles
if len(app.mbfiles)>0:
param+= ' -m "%s" ' % app.mbfiles
if len(app.cavernfiles)>0:
param+= ' -n "%s" ' % app.cavernfiles
# param += '-m "[]" ' #%minList FIXME
# param += '-n "[]" ' #%cavList FIXME
del pandaOutfiles["LOG"] # logfiles do not appear in IROOT block, and this one is not needed anymore...
param += ' -o "{\'IROOT\':%s }"' % str(pandaOutfiles.items())
# source URL
matchURL = re.search("(http.*://[^/]+)/",Client.baseURLSSL)
if matchURL != None:
param += " --sourceURL %s " % matchURL.group(1)
param += " --trf"
jspec.jobParameters = param
jspec.metadata="--trf \"%s\"" % arglist
#print "SUBJOB DETAILS:",jspec.values()
if app.dryrun:
print "job.application.dryrun activated, printing out job parameters"
print jspec.values()
return
return jspec
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
'''prepare the subjob specific configuration'''
# PandaTools
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('AthenaMCPandaRTHandler prepare called for %s',
job.getFQID('.'))
try:
assert self.outsite
except:
logger.error("outsite not set. Aborting")
raise Exception()
job.backend.site = self.outsite
job.backend.actualCE = self.outsite
cloud = job._getRoot().backend.requirements.cloud
job.backend.requirements.cloud = cloud
# now just filling the job from AthenaMC data
jspec = JobSpec()
jspec.jobDefinitionID = job._getRoot().id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.AtlasRelease = 'Atlas-%s' % app.atlas_rel
if app.transform_archive:
jspec.homepackage = 'AnalysisTransforms' + app.transform_archive
elif app.prod_release:
jspec.homepackage = 'AnalysisTransforms-AtlasProduction_' + str(
app.prod_release)
jspec.transformation = '%s/runAthena-00-00-11' % Client.baseURLSUB
#---->???? prodDBlock and destinationDBlock when facing several input / output datasets?
jspec.prodDBlock = 'NULL'
if job.inputdata and len(
app.inputfiles) > 0 and app.inputfiles[0] in app.dsetmap:
jspec.prodDBlock = app.dsetmap[app.inputfiles[0]]
# How to specify jspec.destinationDBlock when more than one type of output is available? Panda prod jobs seem to specify only the last output dataset
outdset = ""
for type in ["EVNT", "RDO", "HITS", "AOD", "ESD", "NTUP"]:
if type in app.outputpaths.keys():
outdset = string.replace(app.outputpaths[type], "/", ".")
outdset = outdset[1:-1]
break
if not outdset:
try:
assert len(app.outputpaths.keys()) > 0
except:
logger.error(
"app.outputpaths is empty: check your output datasets")
raise
type = app.outputpaths.keys()[0]
outdset = string.replace(app.outputpaths[type], "/", ".")
outdset = outdset[1:-1]
jspec.destinationDBlock = outdset
jspec.destinationSE = self.outsite
jspec.prodSourceLabel = 'user'
jspec.assignedPriority = 1000
jspec.cloud = cloud
# memory
if job.backend.requirements.memory != -1:
jspec.minRamCount = job.backend.requirements.memory
jspec.computingSite = self.outsite
jspec.cmtConfig = AthenaUtils.getCmtConfig(athenaVer=app.atlas_rel)
# library (source files)
flib = FileSpec()
flib.lfn = self.library
# flib.GUID =
flib.type = 'input'
# flib.status =
flib.dataset = self.libDataset
flib.dispatchDBlock = self.libDataset
jspec.addFile(flib)
# input files FIXME: many more input types
for lfn in app.inputfiles:
useguid = app.turls[lfn].replace("guid:", "")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# add dbfiles if any:
for lfn in app.dbfiles:
useguid = app.dbturls[lfn].replace("guid:", "")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# then minbias files
for lfn in app.mbfiles:
useguid = app.minbias_turls[lfn].replace("guid:", "")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# then cavern files
for lfn in app.cavernfiles:
useguid = app.cavern_turls[lfn].replace("guid:", "")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# output files( this includes the logfiles)
# Output files
jidtag = ""
job = app._getParent() # Returns job or subjob object
if job._getRoot().subjobs:
jidtag = job._getRoot().id
else:
jidtag = "%d" % job.id
outfiles = app.subjobsOutfiles[job.id]
pandaOutfiles = {}
for type in outfiles.keys():
pandaOutfiles[type] = outfiles[type] + "." + str(jidtag)
if type == "LOG":
pandaOutfiles[type] += ".tgz"
#print pandaOutfiles
for outtype in pandaOutfiles.keys():
fout = FileSpec()
dset = string.replace(app.outputpaths[outtype], "/", ".")
dset = dset[1:-1]
fout.dataset = dset
fout.lfn = pandaOutfiles[outtype]
fout.type = 'output'
# fout.destinationDBlock = jspec.destinationDBlock
fout.destinationDBlock = fout.dataset
fout.destinationSE = jspec.destinationSE
if outtype == 'LOG':
fout.type = 'log'
fout.destinationDBlock = fout.dataset
fout.destinationSE = job.backend.site
jspec.addFile(fout)
# job parameters
param = '-l %s ' % self.library # user tarball.
# use corruption checker
if job.backend.requirements.corCheck:
param += '--corCheck '
# disable to skip missing files
if job.backend.requirements.notSkipMissing:
param += '--notSkipMissing '
# transform parameters
# need to update arglist with final output file name...
newArgs = []
if app.mode == "evgen":
app.args[3] = app.args[3] + " -t "
if app.verbosity:
app.args[3] = app.args[3] + " -l %s " % app.verbosity
for arg in app.args[3:]:
for type in outfiles.keys():
if arg.find(outfiles[type]) > -1:
arg = arg.replace(outfiles[type], pandaOutfiles[type])
newArgs.append(arg)
arglist = string.join(newArgs, " ")
# print "Arglist:",arglist
param += ' -r ./ '
param += ' -j "%s"' % urllib.quote(arglist)
allinfiles = app.inputfiles + app.dbfiles
# Input files.
param += ' -i "%s" ' % allinfiles
if len(app.mbfiles) > 0:
param += ' -m "%s" ' % app.mbfiles
if len(app.cavernfiles) > 0:
param += ' -n "%s" ' % app.cavernfiles
# param += '-m "[]" ' #%minList FIXME
# param += '-n "[]" ' #%cavList FIXME
del pandaOutfiles[
"LOG"] # logfiles do not appear in IROOT block, and this one is not needed anymore...
param += ' -o "{\'IROOT\':%s }"' % str(pandaOutfiles.items())
# source URL
matchURL = re.search("(http.*://[^/]+)/", Client.baseURLSSL)
if matchURL != None:
param += " --sourceURL %s " % matchURL.group(1)
param += " --trf"
jspec.jobParameters = param
jspec.metadata = "--trf \"%s\"" % arglist
#print "SUBJOB DETAILS:",jspec.values()
if app.dryrun:
print "job.application.dryrun activated, printing out job parameters"
print jspec.values()
return
return jspec
