def defineEvgen16Job(self, i):
"""Define an Evgen16 job based on predefined values and randomly generated names
"""
job = JobSpec()
job.computingSite = self.__site
job.cloud = self.__cloud
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (uuid.uuid1(), i)
job.AtlasRelease = 'Atlas-16.6.2'
job.homepackage = 'AtlasProduction/16.6.2.1'
job.transformation = 'Evgen_trf.py'
job.destinationDBlock = self.__datasetName
job.destinationSE = self.__destName
job.currentPriority = 10000
job.prodSourceLabel = 'test'
job.cmtConfig = 'i686-slc5-gcc43-opt'
#Output file
fileO = FileSpec()
fileO.lfn = "%s.evgen.pool.root" % job.jobName
fileO.destinationDBlock = job.destinationDBlock
fileO.destinationSE = job.destinationSE
fileO.dataset = job.destinationDBlock
fileO.destinationDBlockToken = 'ATLASDATADISK'
fileO.type = 'output'
job.addFile(fileO)
#Log file
fileL = FileSpec()
fileL.lfn = "%s.job.log.tgz" % job.jobName
fileL.destinationDBlock = job.destinationDBlock
fileL.destinationSE = job.destinationSE
fileL.dataset = job.destinationDBlock
fileL.destinationDBlockToken = 'ATLASDATADISK'
fileL.type = 'log'
job.addFile(fileL)
job.jobParameters = "2760 105048 19901 101 200 MC10.105048.PythiaB_ccmu3mu1X.py %s NONE NONE NONE MC10JobOpts-latest-test.tar.gz" % fileO.lfn
return job
def defineEvgen16Job(self, i):
"""Define an Evgen16 job based on predefined values and randomly generated names
"""
job = JobSpec()
job.computingSite = self.__site
job.cloud = self.__cloud
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (uuid.uuid1(), i)
job.AtlasRelease = 'Atlas-16.6.2'
job.homepackage = 'AtlasProduction/16.6.2.1'
job.transformation = 'Evgen_trf.py'
job.destinationDBlock = self.__datasetName
job.destinationSE = self.__destName
job.currentPriority = 10000
job.prodSourceLabel = 'test'
job.cmtConfig = 'i686-slc5-gcc43-opt'
#Output file
fileO = FileSpec()
fileO.lfn = "%s.evgen.pool.root" % job.jobName
fileO.destinationDBlock = job.destinationDBlock
fileO.destinationSE = job.destinationSE
fileO.dataset = job.destinationDBlock
fileO.destinationDBlockToken = 'ATLASDATADISK'
fileO.type = 'output'
job.addFile(fileO)
#Log file
fileL = FileSpec()
fileL.lfn = "%s.job.log.tgz" % job.jobName
fileL.destinationDBlock = job.destinationDBlock
fileL.destinationSE = job.destinationSE
fileL.dataset = job.destinationDBlock
fileL.destinationDBlockToken = 'ATLASDATADISK'
fileL.type = 'log'
job.addFile(fileL)
job.jobParameters = "2760 105048 19901 101 200 MC10.105048.PythiaB_ccmu3mu1X.py %s NONE NONE NONE MC10JobOpts-latest-test.tar.gz" % fileO.lfn
return job
def run(self, data):
datasetName = 'panda:panda.destDB.%s' % commands.getoutput('uuidgen')
destName = 'ANALY_RRC-KI-HPC'
site = 'ANALY_RRC-KI-HPC'
scope = config['DEFAULT_SCOPE']
distributive = data['distributive']
release = data['release']
parameters = data['parameters']
input_type = data['input_type']
input_params = data['input_params']
input_files = data['input_files']
output_type = data['output_type']
output_params = data['output_params']
output_files = data['output_files']
jobid = data['jobid']
_logger.debug('Jobid: ' + str(jobid))
job = JobSpec()
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = commands.getoutput('uuidgen')
job.transformation = config['DEFAULT_TRF']
job.destinationDBlock = datasetName
job.destinationSE = destName
job.currentPriority = 1000
job.prodSourceLabel = 'user'
job.computingSite = site
job.cloud = 'RU'
job.prodDBlock = "%s:%s.%s" % (scope, scope, job.jobName)
job.jobParameters = '%s %s "%s"' % (release, distributive, parameters)
params = {}
_logger.debug('MoveData')
ec = 0
ec, uploaded_input_files = movedata(
params=params,
fileList=input_files,
fromType=input_type,
fromParams=input_params,
toType='hpc',
toParams={'dest': '/' + re.sub(':', '/', job.prodDBlock)})
if ec != 0:
_logger.error('Move data error: ' + ec[1])
return
for file in uploaded_input_files:
fileIT = FileSpec()
fileIT.lfn = file
fileIT.dataset = job.prodDBlock
fileIT.prodDBlock = job.prodDBlock
fileIT.type = 'input'
fileIT.scope = scope
fileIT.status = 'ready'
fileIT.GUID = commands.getoutput('uuidgen')
job.addFile(fileIT)
for file in output_files:
fileOT = FileSpec()
fileOT.lfn = file
fileOT.destinationDBlock = job.prodDBlock
fileOT.destinationSE = job.destinationSE
fileOT.dataset = job.prodDBlock
fileOT.type = 'output'
fileOT.scope = scope
fileOT.GUID = commands.getoutput('uuidgen')
job.addFile(fileOT)
fileOL = FileSpec()
fileOL.lfn = "%s.log.tgz" % job.jobName
fileOL.destinationDBlock = job.destinationDBlock
fileOL.destinationSE = job.destinationSE
fileOL.dataset = job.destinationDBlock
fileOL.type = 'log'
fileOL.scope = 'panda'
job.addFile(fileOL)
self.jobList.append(job)
#submitJob
o = self.submitJobs(self.jobList)
x = o[0]
#update PandaID
conn = MySQLdb.connect(
host=self.dbhost,
db=self.dbname,
# port=self.dbport, connect_timeout=self.dbtimeout,
user=self.dbuser,
passwd=self.dbpasswd)
cur = conn.cursor()
try:
varDict = {}
PandaID = int(x[0])
varDict['id'] = jobid
varDict['pandaId'] = PandaID
sql = "UPDATE %s SET %s.pandaId=%s WHERE %s.id=%s" % (
self.table_jobs, self.table_jobs, varDict['pandaId'],
self.table_jobs, varDict['id'])
cur.execute(sql, varDict)
except:
_logger.error('SENDJOB: Incorrect server response')
try:
conn.commit()
return True
except:
_logger.error("commit error")
return False
def master_prepare(self, app, appconfig):
'''Prepare the master job'''
from pandatools import Client
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('ExecutablePandaRTHandler master_prepare called for %s',
job.getFQID('.'))
# set chirp variables
if configPanda['chirpconfig'] or configPanda['chirpserver']:
setChirpVariables()
# Pack inputsandbox
inputsandbox = 'sources.%s.tar' % commands.getoutput(
'uuidgen 2> /dev/null')
inpw = job.getInputWorkspace()
# add user script to inputsandbox
if hasattr(job.application.exe, "name"):
if not job.application.exe in job.inputsandbox:
job.inputsandbox.append(job.application.exe)
for fname in [f.name for f in job.inputsandbox]:
fname.rstrip(os.sep)
path = fname[:fname.rfind(os.sep)]
f = fname[fname.rfind(os.sep) + 1:]
rc, output = commands.getstatusoutput(
'tar rf %s -C %s %s' % (inpw.getPath(inputsandbox), path, f))
if rc:
logger.error('Packing inputsandbox failed with status %d', rc)
logger.error(output)
raise ApplicationConfigurationError(
None, 'Packing inputsandbox failed.')
if len(job.inputsandbox) > 0:
rc, output = commands.getstatusoutput('gzip %s' %
(inpw.getPath(inputsandbox)))
if rc:
logger.error('Packing inputsandbox failed with status %d', rc)
logger.error(output)
raise ApplicationConfigurationError(
None, 'Packing inputsandbox failed.')
inputsandbox += ".gz"
else:
inputsandbox = None
# Upload Inputsandbox
if inputsandbox:
logger.debug('Uploading source tarball ...')
uploadSources(inpw.getPath(), os.path.basename(inputsandbox))
self.inputsandbox = inputsandbox
else:
self.inputsandbox = None
# input dataset
if job.inputdata:
if job.inputdata._name != 'DQ2Dataset':
raise ApplicationConfigurationError(
None, 'PANDA application supports only DQ2Datasets')
# run brokerage here if not splitting
if not job.splitter:
from GangaPanda.Lib.Panda.Panda import runPandaBrokerage
runPandaBrokerage(job)
elif job.splitter._name not in [
'DQ2JobSplitter', 'ArgSplitter', 'ArgSplitterTask'
]:
raise ApplicationConfigurationError(
None, 'Panda splitter must be DQ2JobSplitter or ArgSplitter')
if job.backend.site == 'AUTO':
raise ApplicationConfigurationError(
None, 'site is still AUTO after brokerage!')
# output dataset
if job.outputdata:
if job.outputdata._name != 'DQ2OutputDataset':
raise ApplicationConfigurationError(
None, 'Panda backend supports only DQ2OutputDataset')
else:
logger.info('Adding missing DQ2OutputDataset')
job.outputdata = DQ2OutputDataset()
job.outputdata.datasetname, outlfn = dq2outputdatasetname(
job.outputdata.datasetname, job.id, job.outputdata.isGroupDS,
job.outputdata.groupname)
self.outDsLocation = Client.PandaSites[job.backend.site]['ddm']
try:
Client.addDataset(job.outputdata.datasetname,
False,
location=self.outDsLocation)
logger.info('Output dataset %s registered at %s' %
(job.outputdata.datasetname, self.outDsLocation))
dq2_set_dataset_lifetime(job.outputdata.datasetname,
location=self.outDsLocation)
except exceptions.SystemExit:
raise BackendError(
'Panda', 'Exception in Client.addDataset %s: %s %s' %
(job.outputdata.datasetname, sys.exc_info()[0],
sys.exc_info()[1]))
# handle the libds
if job.backend.libds:
self.libDataset = job.backend.libds
self.fileBO = getLibFileSpecFromLibDS(self.libDataset)
self.library = self.fileBO.lfn
elif job.backend.bexec:
self.libDataset = job.outputdata.datasetname + '.lib'
self.library = '%s.tgz' % self.libDataset
try:
Client.addDataset(self.libDataset,
False,
location=self.outDsLocation)
dq2_set_dataset_lifetime(self.libDataset,
location=self.outDsLocation)
logger.info('Lib dataset %s registered at %s' %
(self.libDataset, self.outDsLocation))
except exceptions.SystemExit:
raise BackendError(
'Panda', 'Exception in Client.addDataset %s: %s %s' %
(self.libDataset, sys.exc_info()[0], sys.exc_info()[1]))
# collect extOutFiles
self.extOutFile = []
for tmpName in job.outputdata.outputdata:
if tmpName != '':
self.extOutFile.append(tmpName)
for tmpName in job.outputsandbox:
if tmpName != '':
self.extOutFile.append(tmpName)
for tmpName in job.backend.extOutFile:
if tmpName != '':
self.extOutFile.append(tmpName)
# create build job
if job.backend.bexec != '':
jspec = JobSpec()
jspec.jobDefinitionID = job.id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.transformation = '%s/buildGen-00-00-01' % Client.baseURLSUB
if Client.isDQ2free(job.backend.site):
jspec.destinationDBlock = '%s/%s' % (
job.outputdata.datasetname, self.libDataset)
jspec.destinationSE = 'local'
else:
jspec.destinationDBlock = self.libDataset
jspec.destinationSE = job.backend.site
jspec.prodSourceLabel = configPanda['prodSourceLabelBuild']
jspec.processingType = configPanda['processingType']
jspec.assignedPriority = configPanda['assignedPriorityBuild']
jspec.computingSite = job.backend.site
jspec.cloud = job.backend.requirements.cloud
jspec.jobParameters = '-o %s' % (self.library)
if self.inputsandbox:
jspec.jobParameters += ' -i %s' % (self.inputsandbox)
else:
raise ApplicationConfigurationError(
None,
'Executable on Panda with build job defined, but inputsandbox is emtpy !'
)
matchURL = re.search('(http.*://[^/]+)/', Client.baseURLCSRVSSL)
if matchURL:
jspec.jobParameters += ' --sourceURL %s ' % matchURL.group(1)
if job.backend.bexec != '':
jspec.jobParameters += ' --bexec "%s" ' % urllib.quote(
job.backend.bexec)
jspec.jobParameters += ' -r %s ' % '.'
fout = FileSpec()
fout.lfn = self.library
fout.type = 'output'
fout.dataset = self.libDataset
fout.destinationDBlock = self.libDataset
jspec.addFile(fout)
flog = FileSpec()
flog.lfn = '%s.log.tgz' % self.libDataset
flog.type = 'log'
flog.dataset = self.libDataset
flog.destinationDBlock = self.libDataset
jspec.addFile(flog)
return jspec
else:
return None
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
'''prepare the subjob specific configuration'''
from pandatools import Client
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('AthenaPandaRTHandler prepare called for %s',
job.getFQID('.'))
# in case of a simple job get the dataset content, otherwise subjobs are filled by the splitter
if job.inputdata and not job._getRoot().subjobs:
if not job.inputdata.names:
contents = job.inputdata.get_contents(overlap=False, size=True)
for ds in contents.keys():
for f in contents[ds]:
job.inputdata.guids.append(f[0])
job.inputdata.names.append(f[1][0])
job.inputdata.sizes.append(f[1][1])
job.inputdata.checksums.append(f[1][2])
job.inputdata.scopes.append(f[1][3])
site = job._getRoot().backend.site
job.backend.site = site
job.backend.actualCE = site
cloud = job._getRoot().backend.requirements.cloud
job.backend.requirements.cloud = cloud
# if no outputdata are given
if not job.outputdata:
job.outputdata = DQ2OutputDataset()
job.outputdata.datasetname = job._getRoot().outputdata.datasetname
#if not job.outputdata.datasetname:
else:
job.outputdata.datasetname = job._getRoot().outputdata.datasetname
if not job.outputdata.datasetname:
raise ApplicationConfigurationError(
None, 'DQ2OutputDataset has no datasetname')
jspec = JobSpec()
jspec.jobDefinitionID = job._getRoot().id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.transformation = '%s/runGen-00-00-02' % Client.baseURLSUB
if job.inputdata:
jspec.prodDBlock = job.inputdata.dataset[0]
else:
jspec.prodDBlock = 'NULL'
jspec.destinationDBlock = job.outputdata.datasetname
if job.outputdata.location:
if not job._getRoot().subjobs or job.id == 0:
logger.warning(
'You have specified outputdata.location. Note that Panda may not support writing to a user-defined output location.'
)
jspec.destinationSE = job.outputdata.location
else:
jspec.destinationSE = site
jspec.prodSourceLabel = configPanda['prodSourceLabelRun']
jspec.processingType = configPanda['processingType']
jspec.assignedPriority = configPanda['assignedPriorityRun']
jspec.cloud = cloud
# memory
if job.backend.requirements.memory != -1:
jspec.minRamCount = job.backend.requirements.memory
# cputime
if job.backend.requirements.cputime != -1:
jspec.maxCpuCount = job.backend.requirements.cputime
jspec.computingSite = site
# library (source files)
if job.backend.libds:
flib = FileSpec()
flib.lfn = self.fileBO.lfn
flib.GUID = self.fileBO.GUID
flib.type = 'input'
flib.status = self.fileBO.status
flib.dataset = self.fileBO.destinationDBlock
flib.dispatchDBlock = self.fileBO.destinationDBlock
jspec.addFile(flib)
elif job.backend.bexec:
flib = FileSpec()
flib.lfn = self.library
flib.type = 'input'
flib.dataset = self.libDataset
flib.dispatchDBlock = self.libDataset
jspec.addFile(flib)
# input files FIXME: many more input types
if job.inputdata:
for guid, lfn, size, checksum, scope in zip(
job.inputdata.guids, job.inputdata.names,
job.inputdata.sizes, job.inputdata.checksums,
job.inputdata.scopes):
finp = FileSpec()
finp.lfn = lfn
finp.GUID = guid
finp.scope = scope
# finp.fsize =
# finp.md5sum =
finp.dataset = job.inputdata.dataset[0]
finp.prodDBlock = job.inputdata.dataset[0]
finp.dispatchDBlock = job.inputdata.dataset[0]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# output files
# outMap = {}
#FIXME: if options.outMeta != []:
self.rundirectory = "."
# log files
flog = FileSpec()
flog.lfn = '%s._$PANDAID.log.tgz' % job.outputdata.datasetname
flog.type = 'log'
flog.dataset = job.outputdata.datasetname
flog.destinationDBlock = job.outputdata.datasetname
flog.destinationSE = job.backend.site
jspec.addFile(flog)
# job parameters
param = ''
# source URL
matchURL = re.search("(http.*://[^/]+)/", Client.baseURLCSRVSSL)
srcURL = ""
if matchURL != None:
srcURL = matchURL.group(1)
param += " --sourceURL %s " % srcURL
param += '-r "%s" ' % self.rundirectory
exe_name = job.application.exe
if job.backend.bexec == '':
if hasattr(job.application.exe, "name"):
exe_name = os.path.basename(job.application.exe.name)
# set jobO parameter
if job.application.args:
param += ' -j "" -p "%s %s" ' % (
exe_name, urllib.quote(" ".join(job.application.args)))
else:
param += ' -j "" -p "%s" ' % exe_name
if self.inputsandbox:
param += ' -a %s ' % self.inputsandbox
else:
param += '-l %s ' % self.library
param += '-j "" -p "%s %s" ' % (
exe_name, urllib.quote(" ".join(job.application.args)))
if job.inputdata:
param += '-i "%s" ' % job.inputdata.names
# fill outfiles
outfiles = {}
for f in self.extOutFile:
tarnum = 1
if f.find('*') != -1:
# archive *
outfiles[f] = "outputbox%i.%s.%s.tar.gz" % (
tarnum, job.getFQID('.'), time.strftime("%Y%m%d%H%M%S"))
tarnum += 1
else:
outfiles[f] = "%s.%s.%s" % (f, job.getFQID('.'),
time.strftime("%Y%m%d%H%M%S"))
fout = FileSpec()
fout.lfn = outfiles[f]
fout.type = 'output'
fout.dataset = job.outputdata.datasetname
fout.destinationDBlock = job.outputdata.datasetname
fout.destinationSE = job.backend.site
jspec.addFile(fout)
param += '-o "%s" ' % (
outfiles
) # must be double quotes, because python prints strings in 'single quotes'
for file in jspec.Files:
if file.type in ['output', 'log'] and configPanda['chirpconfig']:
file.dispatchDBlockToken = configPanda['chirpconfig']
logger.debug('chirp file %s', file)
jspec.jobParameters = param
return jspec
def master_prepare(self, app, appmasterconfig):
# PandaTools
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('AthenaMCPandaRTHandler master_prepare called for %s',
job.getFQID('.'))
usertag = configDQ2['usertag']
#usertag='user09'
nickname = getNickname(allowMissingNickname=True)
self.libDataset = '%s.%s.ganga.%s_%d.lib._%06d' % (
usertag, nickname, commands.getoutput('hostname').split('.')[0],
int(time.time()), job.id)
# self.userprefix='%s.%s.ganga' % (usertag,gridProxy.identity())
sources = 'sources.%s.tar.gz' % commands.getoutput(
'uuidgen 2> /dev/null')
self.library = '%s.lib.tgz' % self.libDataset
# check DBRelease
# if job.backend.dbRelease != '' and job.backend.dbRelease.find(':') == -1:
# raise ApplicationConfigurationError(None,"ERROR : invalid argument for backend.dbRelease. Must be 'DatasetName:FileName'")
# unpack library
logger.debug('Creating source tarball ...')
tmpdir = '/tmp/%s' % commands.getoutput('uuidgen 2> /dev/null')
os.mkdir(tmpdir)
inputbox = []
if os.path.exists(app.transform_archive):
# must add a condition on size.
inputbox += [File(app.transform_archive)]
if app.evgen_job_option:
self.evgen_job_option = app.evgen_job_option
if os.path.exists(app.evgen_job_option):
# locally modified job option file to add to the input sand box
inputbox += [File(app.evgen_job_option)]
self.evgen_job_option = app.evgen_job_option.split("/")[-1]
# add input sandbox files
if (job.inputsandbox):
for file in job.inputsandbox:
inputbox += [file]
# add option files
for extFile in job.backend.extOutFile:
try:
shutil.copy(extFile, tmpdir)
except IOError:
os.makedirs(tmpdir)
shutil.copy(extFile, tmpdir)
# fill the archive
for opt_file in inputbox:
try:
shutil.copy(opt_file.name, tmpdir)
except IOError:
os.makedirs(tmpdir)
shutil.copy(opt_file.name, tmpdir)
# now tar it up again
inpw = job.getInputWorkspace()
rc, output = commands.getstatusoutput('tar czf %s -C %s .' %
(inpw.getPath(sources), tmpdir))
if rc:
logger.error('Packing sources failed with status %d', rc)
logger.error(output)
raise ApplicationConfigurationError(None,
'Packing sources failed.')
shutil.rmtree(tmpdir)
# upload sources
logger.debug('Uploading source tarball ...')
try:
cwd = os.getcwd()
os.chdir(inpw.getPath())
rc, output = Client.putFile(sources)
if output != 'True':
logger.error('Uploading sources %s failed. Status = %d',
sources, rc)
logger.error(output)
raise ApplicationConfigurationError(
None, 'Uploading archive failed')
finally:
os.chdir(cwd)
# Use Panda's brokerage
## if job.inputdata and len(app.sites)>0:
## # update cloud, use inputdata's
## from dq2.info.TiersOfATLAS import whichCloud,ToACache
## inclouds=[]
## for site in app.sites:
## cloudSite=whichCloud(app.sites[0])
## if cloudSite not in inclouds:
## inclouds.append(cloudSite)
## # now converting inclouds content into proper brokering stuff.
## outclouds=[]
## for cloudSite in inclouds:
## for cloudID, eachCloud in ToACache.dbcloud.iteritems():
## if cloudSite==eachCloud:
## cloud=cloudID
## outclouds.append(cloud)
## break
## print outclouds
## # finally, matching with user's wishes
## if len(outclouds)>0:
## if not job.backend.requirements.cloud: # no user wish, update
## job.backend.requirements.cloud=outclouds[0]
## else:
## try:
## assert job.backend.requirements.cloud in outclouds
## except:
## raise ApplicationConfigurationError(None,'Input dataset not available in target cloud %s. Please try any of the following %s' % (job.backend.requirements.cloud, str(outclouds)))
from GangaPanda.Lib.Panda.Panda import runPandaBrokerage
runPandaBrokerage(job)
if job.backend.site == 'AUTO':
raise ApplicationConfigurationError(
None, 'site is still AUTO after brokerage!')
# output dataset preparation and registration
try:
outDsLocation = Client.PandaSites[job.backend.site]['ddm']
except:
raise ApplicationConfigurationError(
None,
"Could not extract output dataset location from job.backend.site value: %s. Aborting"
% job.backend.site)
if not app.dryrun:
for outtype in app.outputpaths.keys():
dset = string.replace(app.outputpaths[outtype], "/", ".")
dset = dset[1:]
# dataset registration must be done only once.
print "registering output dataset %s at %s" % (dset,
outDsLocation)
try:
Client.addDataset(dset, False, location=outDsLocation)
dq2_set_dataset_lifetime(dset, location=outDsLocation)
except:
raise ApplicationConfigurationError(
None,
"Fail to create output dataset %s. Aborting" % dset)
# extend registration to build job lib dataset:
print "registering output dataset %s at %s" % (self.libDataset,
outDsLocation)
try:
Client.addDataset(self.libDataset,
False,
location=outDsLocation)
dq2_set_dataset_lifetime(self.libDataset, outDsLocation)
except:
raise ApplicationConfigurationError(
None, "Fail to create output dataset %s. Aborting" %
self.libDataset)
###
cacheVer = "-AtlasProduction_" + str(app.prod_release)
logger.debug("master job submit?")
self.outsite = job.backend.site
if app.se_name and app.se_name != "none" and not self.outsite:
self.outsite = app.se_name
# create build job
jspec = JobSpec()
jspec.jobDefinitionID = job.id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.AtlasRelease = 'Atlas-%s' % app.atlas_rel
jspec.homepackage = 'AnalysisTransforms' + cacheVer #+nightVer
jspec.transformation = '%s/buildJob-00-00-03' % Client.baseURLSUB # common base to Athena and AthenaMC jobs: buildJob is a pilot job which takes care of all inputs for the real jobs (in prepare()
jspec.destinationDBlock = self.libDataset
jspec.destinationSE = job.backend.site
jspec.prodSourceLabel = 'panda'
jspec.assignedPriority = 2000
jspec.computingSite = job.backend.site
jspec.cloud = job.backend.requirements.cloud
# jspec.jobParameters = self.args not known yet
jspec.jobParameters = '-o %s' % (self.library)
if app.userarea:
print app.userarea
jspec.jobParameters += ' -i %s' % (os.path.basename(app.userarea))
else:
jspec.jobParameters += ' -i %s' % (sources)
jspec.cmtConfig = AthenaUtils.getCmtConfig(athenaVer=app.atlas_rel)
matchURL = re.search('(http.*://[^/]+)/', Client.baseURLSSL)
if matchURL:
jspec.jobParameters += ' --sourceURL %s' % matchURL.group(1)
fout = FileSpec()
fout.lfn = self.library
fout.type = 'output'
fout.dataset = self.libDataset
fout.destinationDBlock = self.libDataset
jspec.addFile(fout)
flog = FileSpec()
flog.lfn = '%s.log.tgz' % self.libDataset
flog.type = 'log'
flog.dataset = self.libDataset
flog.destinationDBlock = self.libDataset
jspec.addFile(flog)
#print "MASTER JOB DETAILS:",jspec.jobParameters
return jspec
def master_prepare(self,app,appconfig):
'''Prepare the master job'''
from pandatools import Client
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('ExecutablePandaRTHandler master_prepare called for %s', job.getFQID('.'))
# set chirp variables
if configPanda['chirpconfig'] or configPanda['chirpserver']:
setChirpVariables()
# Pack inputsandbox
inputsandbox = 'sources.%s.tar' % commands.getoutput('uuidgen 2> /dev/null')
inpw = job.getInputWorkspace()
# add user script to inputsandbox
if hasattr(job.application.exe, "name"):
if not job.application.exe in job.inputsandbox:
job.inputsandbox.append(job.application.exe)
for fname in [f.name for f in job.inputsandbox]:
fname.rstrip(os.sep)
path = fname[:fname.rfind(os.sep)]
f = fname[fname.rfind(os.sep)+1:]
rc, output = commands.getstatusoutput('tar rf %s -C %s %s' % (inpw.getPath(inputsandbox), path, f))
if rc:
logger.error('Packing inputsandbox failed with status %d',rc)
logger.error(output)
raise ApplicationConfigurationError('Packing inputsandbox failed.')
if len(job.inputsandbox) > 0:
rc, output = commands.getstatusoutput('gzip %s' % (inpw.getPath(inputsandbox)))
if rc:
logger.error('Packing inputsandbox failed with status %d',rc)
logger.error(output)
raise ApplicationConfigurationError('Packing inputsandbox failed.')
inputsandbox += ".gz"
else:
inputsandbox = None
# Upload Inputsandbox
if inputsandbox:
logger.debug('Uploading source tarball ...')
uploadSources(inpw.getPath(),os.path.basename(inputsandbox))
self.inputsandbox = inputsandbox
else:
self.inputsandbox = None
# input dataset
if job.inputdata:
if job.inputdata._name != 'DQ2Dataset':
raise ApplicationConfigurationError('PANDA application supports only DQ2Datasets')
# run brokerage here if not splitting
if not job.splitter:
from GangaPanda.Lib.Panda.Panda import runPandaBrokerage
runPandaBrokerage(job)
elif job.splitter._name not in ['DQ2JobSplitter', 'ArgSplitter', 'ArgSplitterTask']:
raise ApplicationConfigurationError('Panda splitter must be DQ2JobSplitter or ArgSplitter')
if job.backend.site == 'AUTO':
raise ApplicationConfigurationError('site is still AUTO after brokerage!')
# output dataset
if job.outputdata:
if job.outputdata._name != 'DQ2OutputDataset':
raise ApplicationConfigurationError('Panda backend supports only DQ2OutputDataset')
else:
logger.info('Adding missing DQ2OutputDataset')
job.outputdata = DQ2OutputDataset()
job.outputdata.datasetname,outlfn = dq2outputdatasetname(job.outputdata.datasetname, job.id, job.outputdata.isGroupDS, job.outputdata.groupname)
self.outDsLocation = Client.PandaSites[job.backend.site]['ddm']
try:
Client.addDataset(job.outputdata.datasetname,False,location=self.outDsLocation)
logger.info('Output dataset %s registered at %s'%(job.outputdata.datasetname,self.outDsLocation))
dq2_set_dataset_lifetime(job.outputdata.datasetname, location=self.outDsLocation)
except exceptions.SystemExit:
raise BackendError('Panda','Exception in Client.addDataset %s: %s %s'%(job.outputdata.datasetname,sys.exc_info()[0],sys.exc_info()[1]))
# handle the libds
if job.backend.libds:
self.libDataset = job.backend.libds
self.fileBO = getLibFileSpecFromLibDS(self.libDataset)
self.library = self.fileBO.lfn
elif job.backend.bexec:
self.libDataset = job.outputdata.datasetname+'.lib'
self.library = '%s.tgz' % self.libDataset
try:
Client.addDataset(self.libDataset,False,location=self.outDsLocation)
dq2_set_dataset_lifetime(self.libDataset, location=self.outDsLocation)
logger.info('Lib dataset %s registered at %s'%(self.libDataset,self.outDsLocation))
except exceptions.SystemExit:
raise BackendError('Panda','Exception in Client.addDataset %s: %s %s'%(self.libDataset,sys.exc_info()[0],sys.exc_info()[1]))
# collect extOutFiles
self.extOutFile = []
for tmpName in job.outputdata.outputdata:
if tmpName != '':
self.extOutFile.append(tmpName)
for tmpName in job.outputsandbox:
if tmpName != '':
self.extOutFile.append(tmpName)
for tmpName in job.backend.extOutFile:
if tmpName != '':
self.extOutFile.append(tmpName)
# create build job
if job.backend.bexec != '':
jspec = JobSpec()
jspec.jobDefinitionID = job.id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.transformation = '%s/buildGen-00-00-01' % Client.baseURLSUB
if Client.isDQ2free(job.backend.site):
jspec.destinationDBlock = '%s/%s' % (job.outputdata.datasetname,self.libDataset)
jspec.destinationSE = 'local'
else:
jspec.destinationDBlock = self.libDataset
jspec.destinationSE = job.backend.site
jspec.prodSourceLabel = configPanda['prodSourceLabelBuild']
jspec.processingType = configPanda['processingType']
jspec.assignedPriority = configPanda['assignedPriorityBuild']
jspec.computingSite = job.backend.site
jspec.cloud = job.backend.requirements.cloud
jspec.jobParameters = '-o %s' % (self.library)
if self.inputsandbox:
jspec.jobParameters += ' -i %s' % (self.inputsandbox)
else:
raise ApplicationConfigurationError('Executable on Panda with build job defined, but inputsandbox is emtpy !')
matchURL = re.search('(http.*://[^/]+)/',Client.baseURLCSRVSSL)
if matchURL:
jspec.jobParameters += ' --sourceURL %s ' % matchURL.group(1)
if job.backend.bexec != '':
jspec.jobParameters += ' --bexec "%s" ' % urllib.quote(job.backend.bexec)
jspec.jobParameters += ' -r %s ' % '.'
fout = FileSpec()
fout.lfn = self.library
fout.type = 'output'
fout.dataset = self.libDataset
fout.destinationDBlock = self.libDataset
jspec.addFile(fout)
flog = FileSpec()
flog.lfn = '%s.log.tgz' % self.libDataset
flog.type = 'log'
flog.dataset = self.libDataset
flog.destinationDBlock = self.libDataset
jspec.addFile(flog)
return jspec
else:
return None
for lfn in files.keys():
index += 1
job = JobSpec()
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (commands.getoutput('uuidgen'), index)
job.AtlasRelease = 'Atlas-13.0.40'
job.homepackage = 'AtlasProduction/13.0.40.3'
job.transformation = 'csc_simul_trf.py'
job.destinationDBlock = datasetName
job.destinationSE = destName
job.computingSite = site
job.prodDBlock = 'valid1.005001.pythia_minbias.evgen.EVNT.e306_tid019128'
job.prodSourceLabel = 'test'
job.currentPriority = 10000
job.cloud = 'IT'
fileI = FileSpec()
fileI.dataset = job.prodDBlock
fileI.prodDBlock = job.prodDBlock
fileI.lfn = lfn
fileI.type = 'input'
job.addFile(fileI)
fileD = FileSpec()
fileD.dataset = 'ddo.000001.Atlas.Ideal.DBRelease.v040701'
fileD.prodDBlock = 'ddo.000001.Atlas.Ideal.DBRelease.v030101'
fileD.lfn = 'DBRelease-4.7.1.tar.gz'
fileD.type = 'input'
job.addFile(fileD)
def master_prepare(self,app,appmasterconfig):
# PandaTools
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('AthenaMCPandaRTHandler master_prepare called for %s', job.getFQID('.'))
usertag = configDQ2['usertag']
#usertag='user09'
nickname = getNickname(allowMissingNickname=True)
self.libDataset = '%s.%s.ganga.%s_%d.lib._%06d' % (usertag,nickname,commands.getoutput('hostname').split('.')[0],int(time.time()),job.id)
# self.userprefix='%s.%s.ganga' % (usertag,gridProxy.identity())
sources = 'sources.%s.tar.gz' % commands.getoutput('uuidgen 2> /dev/null')
self.library = '%s.lib.tgz' % self.libDataset
# check DBRelease
# if job.backend.dbRelease != '' and job.backend.dbRelease.find(':') == -1:
# raise ApplicationConfigurationError(None,"ERROR : invalid argument for backend.dbRelease. Must be 'DatasetName:FileName'")
# unpack library
logger.debug('Creating source tarball ...')
tmpdir = '/tmp/%s' % commands.getoutput('uuidgen 2> /dev/null')
os.mkdir(tmpdir)
inputbox=[]
if os.path.exists(app.transform_archive):
# must add a condition on size.
inputbox += [ File(app.transform_archive) ]
if app.evgen_job_option:
self.evgen_job_option=app.evgen_job_option
if os.path.exists(app.evgen_job_option):
# locally modified job option file to add to the input sand box
inputbox += [ File(app.evgen_job_option) ]
self.evgen_job_option=app.evgen_job_option.split("/")[-1]
# add input sandbox files
if (job.inputsandbox):
for file in job.inputsandbox:
inputbox += [ file ]
# add option files
for extFile in job.backend.extOutFile:
try:
shutil.copy(extFile,tmpdir)
except IOError:
os.makedirs(tmpdir)
shutil.copy(extFile,tmpdir)
# fill the archive
for opt_file in inputbox:
try:
shutil.copy(opt_file.name,tmpdir)
except IOError:
os.makedirs(tmpdir)
shutil.copy(opt_file.name,tmpdir)
# now tar it up again
inpw = job.getInputWorkspace()
rc, output = commands.getstatusoutput('tar czf %s -C %s .' % (inpw.getPath(sources),tmpdir))
if rc:
logger.error('Packing sources failed with status %d',rc)
logger.error(output)
raise ApplicationConfigurationError(None,'Packing sources failed.')
shutil.rmtree(tmpdir)
# upload sources
logger.debug('Uploading source tarball ...')
try:
cwd = os.getcwd()
os.chdir(inpw.getPath())
rc, output = Client.putFile(sources)
if output != 'True':
logger.error('Uploading sources %s failed. Status = %d', sources, rc)
logger.error(output)
raise ApplicationConfigurationError(None,'Uploading archive failed')
finally:
os.chdir(cwd)
# Use Panda's brokerage
## if job.inputdata and len(app.sites)>0:
## # update cloud, use inputdata's
## from dq2.info.TiersOfATLAS import whichCloud,ToACache
## inclouds=[]
## for site in app.sites:
## cloudSite=whichCloud(app.sites[0])
## if cloudSite not in inclouds:
## inclouds.append(cloudSite)
## # now converting inclouds content into proper brokering stuff.
## outclouds=[]
## for cloudSite in inclouds:
## for cloudID, eachCloud in ToACache.dbcloud.iteritems():
## if cloudSite==eachCloud:
## cloud=cloudID
## outclouds.append(cloud)
## break
## print outclouds
## # finally, matching with user's wishes
## if len(outclouds)>0:
## if not job.backend.requirements.cloud: # no user wish, update
## job.backend.requirements.cloud=outclouds[0]
## else:
## try:
## assert job.backend.requirements.cloud in outclouds
## except:
## raise ApplicationConfigurationError(None,'Input dataset not available in target cloud %s. Please try any of the following %s' % (job.backend.requirements.cloud, str(outclouds)))
from GangaPanda.Lib.Panda.Panda import runPandaBrokerage
runPandaBrokerage(job)
if job.backend.site == 'AUTO':
raise ApplicationConfigurationError(None,'site is still AUTO after brokerage!')
# output dataset preparation and registration
try:
outDsLocation = Client.PandaSites[job.backend.site]['ddm']
except:
raise ApplicationConfigurationError(None,"Could not extract output dataset location from job.backend.site value: %s. Aborting" % job.backend.site)
if not app.dryrun:
for outtype in app.outputpaths.keys():
dset=string.replace(app.outputpaths[outtype],"/",".")
dset=dset[1:]
# dataset registration must be done only once.
print "registering output dataset %s at %s" % (dset,outDsLocation)
try:
Client.addDataset(dset,False,location=outDsLocation)
dq2_set_dataset_lifetime(dset, location=outDsLocation)
except:
raise ApplicationConfigurationError(None,"Fail to create output dataset %s. Aborting" % dset)
# extend registration to build job lib dataset:
print "registering output dataset %s at %s" % (self.libDataset,outDsLocation)
try:
Client.addDataset(self.libDataset,False,location=outDsLocation)
dq2_set_dataset_lifetime(self.libDataset, outDsLocation)
except:
raise ApplicationConfigurationError(None,"Fail to create output dataset %s. Aborting" % self.libDataset)
###
cacheVer = "-AtlasProduction_" + str(app.prod_release)
logger.debug("master job submit?")
self.outsite=job.backend.site
if app.se_name and app.se_name != "none" and not self.outsite:
self.outsite=app.se_name
# create build job
jspec = JobSpec()
jspec.jobDefinitionID = job.id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.AtlasRelease = 'Atlas-%s' % app.atlas_rel
jspec.homepackage = 'AnalysisTransforms'+cacheVer#+nightVer
jspec.transformation = '%s/buildJob-00-00-03' % Client.baseURLSUB # common base to Athena and AthenaMC jobs: buildJob is a pilot job which takes care of all inputs for the real jobs (in prepare()
jspec.destinationDBlock = self.libDataset
jspec.destinationSE = job.backend.site
jspec.prodSourceLabel = 'panda'
jspec.assignedPriority = 2000
jspec.computingSite = job.backend.site
jspec.cloud = job.backend.requirements.cloud
# jspec.jobParameters = self.args not known yet
jspec.jobParameters = '-o %s' % (self.library)
if app.userarea:
print app.userarea
jspec.jobParameters += ' -i %s' % (os.path.basename(app.userarea))
else:
jspec.jobParameters += ' -i %s' % (sources)
jspec.cmtConfig = AthenaUtils.getCmtConfig(athenaVer=app.atlas_rel)
matchURL = re.search('(http.*://[^/]+)/',Client.baseURLSSL)
if matchURL:
jspec.jobParameters += ' --sourceURL %s' % matchURL.group(1)
fout = FileSpec()
fout.lfn = self.library
fout.type = 'output'
fout.dataset = self.libDataset
fout.destinationDBlock = self.libDataset
jspec.addFile(fout)
flog = FileSpec()
flog.lfn = '%s.log.tgz' % self.libDataset
flog.type = 'log'
flog.dataset = self.libDataset
flog.destinationDBlock = self.libDataset
jspec.addFile(flog)
#print "MASTER JOB DETAILS:",jspec.jobParameters
return jspec
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
"""Prepare the specific aspec of each subjob.
Returns: subjobconfig list of objects understood by backends."""
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
from GangaAtlas.Lib.ATLASDataset.DQ2Dataset import dq2_set_dataset_lifetime
from GangaPanda.Lib.Panda.Panda import refreshPandaSpecs
# make sure we have the correct siteType
refreshPandaSpecs()
job = app._getParent()
masterjob = job._getRoot()
logger.debug('ProdTransPandaRTHandler prepare called for %s',
job.getFQID('.'))
job.backend.actualCE = job.backend.site
job.backend.requirements.cloud = Client.PandaSites[
job.backend.site]['cloud']
# check that the site is in a submit-able status
if not job.splitter or job.splitter._name != 'DQ2JobSplitter':
allowed_sites = job.backend.list_ddm_sites()
try:
outDsLocation = Client.PandaSites[job.backend.site]['ddm']
tmpDsExist = False
if (configPanda['processingType'].startswith('gangarobot') or
configPanda['processingType'].startswith('hammercloud')):
#if Client.getDatasets(job.outputdata.datasetname):
if getDatasets(job.outputdata.datasetname):
tmpDsExist = True
logger.info('Re-using output dataset %s' %
job.outputdata.datasetname)
if not configPanda[
'specialHandling'] == 'ddm:rucio' and not configPanda[
'processingType'].startswith(
'gangarobot'
) and not configPanda['processingType'].startswith(
'hammercloud') and not configPanda[
'processingType'].startswith('rucio_test'):
Client.addDataset(job.outputdata.datasetname,
False,
location=outDsLocation,
allowProdDisk=True,
dsExist=tmpDsExist)
logger.info('Output dataset %s registered at %s' %
(job.outputdata.datasetname, outDsLocation))
dq2_set_dataset_lifetime(job.outputdata.datasetname, outDsLocation)
except exceptions.SystemExit:
raise BackendError(
'Panda', 'Exception in adding dataset %s: %s %s' %
(job.outputdata.datasetname, sys.exc_info()[0],
sys.exc_info()[1]))
# JobSpec.
jspec = JobSpec()
jspec.currentPriority = app.priority
jspec.jobDefinitionID = masterjob.id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.coreCount = app.core_count
jspec.AtlasRelease = 'Atlas-%s' % app.atlas_release
jspec.homepackage = app.home_package
jspec.transformation = app.transformation
jspec.destinationDBlock = job.outputdata.datasetname
if job.outputdata.location:
jspec.destinationSE = job.outputdata.location
else:
jspec.destinationSE = job.backend.site
if job.inputdata:
jspec.prodDBlock = job.inputdata.dataset[0]
else:
jspec.prodDBlock = 'NULL'
if app.prod_source_label:
jspec.prodSourceLabel = app.prod_source_label
else:
jspec.prodSourceLabel = configPanda['prodSourceLabelRun']
jspec.processingType = configPanda['processingType']
jspec.specialHandling = configPanda['specialHandling']
jspec.computingSite = job.backend.site
jspec.cloud = job.backend.requirements.cloud
jspec.cmtConfig = app.atlas_cmtconfig
if app.dbrelease == 'LATEST':
try:
latest_dbrelease = getLatestDBReleaseCaching()
except:
from pandatools import Client
latest_dbrelease = Client.getLatestDBRelease()
m = re.search('(.*):DBRelease-(.*)\.tar\.gz', latest_dbrelease)
if m:
self.dbrelease_dataset = m.group(1)
self.dbrelease = m.group(2)
else:
raise ApplicationConfigurationError(
None,
"Error retrieving LATEST DBRelease. Try setting application.dbrelease manually."
)
else:
self.dbrelease_dataset = app.dbrelease_dataset
self.dbrelease = app.dbrelease
jspec.jobParameters = app.job_parameters
if self.dbrelease:
if self.dbrelease == 'current':
jspec.jobParameters += ' --DBRelease=current'
else:
if jspec.transformation.endswith(
"_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --DBRelease=DBRelease-%s.tar.gz' % (
self.dbrelease, )
else:
jspec.jobParameters += ' DBRelease=DBRelease-%s.tar.gz' % (
self.dbrelease, )
dbspec = FileSpec()
dbspec.lfn = 'DBRelease-%s.tar.gz' % self.dbrelease
dbspec.dataset = self.dbrelease_dataset
dbspec.prodDBlock = jspec.prodDBlock
dbspec.type = 'input'
jspec.addFile(dbspec)
if job.inputdata:
m = re.search('(.*)\.(.*)\.(.*)\.(.*)\.(.*)\.(.*)',
job.inputdata.dataset[0])
if not m:
logger.error("Error retrieving run number from dataset name")
#raise ApplicationConfigurationError(None, "Error retrieving run number from dataset name")
runnumber = 105200
else:
runnumber = int(m.group(2))
if jspec.transformation.endswith(
"_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --runNumber %d' % runnumber
else:
jspec.jobParameters += ' RunNumber=%d' % runnumber
# Output files.
randomized_lfns = []
ilfn = 0
for lfn, lfntype in zip(app.output_files, app.output_type):
ofspec = FileSpec()
if app.randomize_lfns:
randomized_lfn = lfn + (
'.%s.%d.%s' %
(job.backend.site, int(time.time()),
commands.getoutput('uuidgen 2> /dev/null')[:4]))
else:
randomized_lfn = lfn
ofspec.lfn = randomized_lfn
randomized_lfns.append(randomized_lfn)
ofspec.destinationDBlock = jspec.destinationDBlock
ofspec.destinationSE = jspec.destinationSE
ofspec.dataset = jspec.destinationDBlock
ofspec.type = 'output'
jspec.addFile(ofspec)
if jspec.transformation.endswith(
"_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --output%sFile %s' % (
lfntype, randomized_lfns[ilfn])
else:
jspec.jobParameters += ' output%sFile=%s' % (
lfntype, randomized_lfns[ilfn])
ilfn = ilfn + 1
# Input files.
if job.inputdata:
for guid, lfn, size, checksum, scope in zip(
job.inputdata.guids, job.inputdata.names,
job.inputdata.sizes, job.inputdata.checksums,
job.inputdata.scopes):
ifspec = FileSpec()
ifspec.lfn = lfn
ifspec.GUID = guid
ifspec.fsize = size
ifspec.md5sum = checksum
ifspec.scope = scope
ifspec.dataset = jspec.prodDBlock
ifspec.prodDBlock = jspec.prodDBlock
ifspec.type = 'input'
jspec.addFile(ifspec)
if app.input_type:
itype = app.input_type
else:
itype = m.group(5)
if jspec.transformation.endswith(
"_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --input%sFile %s' % (itype, ','.join(
job.inputdata.names))
else:
jspec.jobParameters += ' input%sFile=%s' % (itype, ','.join(
job.inputdata.names))
# Log files.
lfspec = FileSpec()
lfspec.lfn = '%s.job.log.tgz' % jspec.jobName
lfspec.destinationDBlock = jspec.destinationDBlock
lfspec.destinationSE = jspec.destinationSE
lfspec.dataset = jspec.destinationDBlock
lfspec.type = 'log'
jspec.addFile(lfspec)
return jspec
job = JobSpec()
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (commands.getoutput('uuidgen'), index)
job.AtlasRelease = 'Atlas-14.1.0\nAtlas-14.1.0'
job.homepackage = 'AtlasProduction/14.1.0.3\nAtlasProduction/14.1.0.3'
job.transformation = 'csc_digi_trf.py\ncsc_reco_trf.py'
job.destinationDBlock = datasetName
job.computingSite = site
job.prodDBlock = 'valid1.005200.T1_McAtNlo_Jimmy.simul.HITS.e322_s429_tid022081'
job.prodSourceLabel = 'test'
job.currentPriority = 10000
job.cloud = 'US'
for lfn in ['HITS.022081._00001.pool.root', 'HITS.022081._00002.pool.root']:
fileI = FileSpec()
fileI.dataset = job.prodDBlock
fileI.prodDBlock = job.prodDBlock
fileI.lfn = lfn
fileI.type = 'input'
job.addFile(fileI)
fileD1 = FileSpec()
fileD1.dataset = 'ddo.000001.Atlas.Ideal.DBRelease.v050001'
fileD1.prodDBlock = fileD1.dataset
fileD1.lfn = 'DBRelease-5.0.1.tar.gz'
fileD1.type = 'input'
job.addFile(fileD1)
def createJobSpec(self, task, outdataset, job, jobset, jobdef, site,
jobname, lfnhanger, allsites, jobid):
"""Create a spec for one job
:arg TaskWorker.DataObject.Task task: the task to work on
:arg str outdataset: the output dataset name where all the produced files will be placed
:arg WMCore.DataStructs.Job job: the abstract job
:arg int jobset: the PanDA jobset corresponding to the current task
:arg int jobdef: the PanDA jobdef where to append the current jobs --- not used
:arg str site: the borkered site where to run the jobs
:arg str jobname: the job name
:arg str lfnhanger: the random string to be added in the output file name
:arg list str allsites: all possible sites where the job can potentially run
:arg int jobid: incremental job number
:return: the sepc object."""
pandajob = JobSpec()
## always setting a job definition ID
pandajob.jobDefinitionID = jobdef if jobdef else -1
## always setting a job set ID
pandajob.jobsetID = jobset if jobset else -1
pandajob.jobName = jobname
pandajob.prodUserID = task['tm_user_dn']
pandajob.destinationDBlock = outdataset
pandajob.prodDBlock = task['tm_input_dataset']
pandajob.prodSourceLabel = 'user'
pandajob.computingSite = site
pandajob.cloud = getSite(pandajob.computingSite)
pandajob.destinationSE = 'local'
pandajob.transformation = task['tm_transformation']
## need to initialize this
pandajob.metadata = ''
def outFileSpec(of=None, log=False):
"""Local routine to create an FileSpec for the an job output/log file
:arg str of: output file base name
:return: FileSpec object for the output file."""
outfile = FileSpec()
if log:
outfile.lfn = "job.log_%d_%s.tgz" % (jobid, lfnhanger)
outfile.type = 'log'
else:
outfile.lfn = '%s_%d_%s%s' % (os.path.splitext(of)[0],
jobid, lfnhanger,
os.path.splitext(of)[1])
outfile.type = 'output'
outfile.destinationDBlock = pandajob.destinationDBlock
outfile.destinationSE = task['tm_asyncdest']
outfile.dataset = pandajob.destinationDBlock
return outfile
alloutfiles = []
outjobpar = {}
outfilestring = ''
for outputfile in task['tm_outfiles']:
outfilestring += '%s,' % outputfile
filespec = outFileSpec(outputfile)
alloutfiles.append(filespec)
#pandajob.addFile(filespec)
outjobpar[outputfile] = filespec.lfn
for outputfile in task['tm_tfile_outfiles']:
outfilestring += '%s,' % outputfile
filespec = outFileSpec(outputfile)
alloutfiles.append(filespec)
#pandajob.addFile(filespec)
outjobpar[outputfile] = filespec.lfn
for outputfile in task['tm_edm_outfiles']:
outfilestring += '%s,' % outputfile
filespec = outFileSpec(outputfile)
alloutfiles.append(filespec)
#pandajob.addFile(filespec)
outjobpar[outputfile] = filespec.lfn
outfilestring = outfilestring[:-1]
infiles = []
for inputfile in job['input_files']:
infiles.append(inputfile['lfn'])
pandajob.jobParameters = '-a %s ' % task['tm_user_sandbox']
pandajob.jobParameters += '--sourceURL %s ' % task['tm_cache_url']
pandajob.jobParameters += '--jobNumber=%s ' % jobid
pandajob.jobParameters += '--cmsswVersion=%s ' % task['tm_job_sw']
pandajob.jobParameters += '--scramArch=%s ' % task['tm_job_arch']
pandajob.jobParameters += '--inputFile=\'%s\' ' % json.dumps(infiles)
self.jobParametersSetting(pandajob, job,
self.jobtypeMapper[task['tm_job_type']])
pandajob.jobParameters += '-o "%s" ' % str(outjobpar)
pandajob.jobParameters += '--dbs_url=%s ' % task['tm_dbs_url']
pandajob.jobParameters += '--publish_dbs_url=%s ' % task[
'tm_publish_dbs_url']
pandajob.jobParameters += '--publishFiles=%s ' % (
'True' if task['tm_publication'] == 'T' else 'False')
pandajob.jobParameters += '--saveLogs=%s ' % (
'True' if task['tm_save_logs'] == 'T' else 'False')
pandajob.jobParameters += '--availableSites=\'%s\' ' % json.dumps(
allsites)
pandajob.jobParameters += '--group=%s ' % (
task['tm_user_group'] if task['tm_user_group'] else '')
pandajob.jobParameters += '--role=%s ' % (task['tm_user_role'] if
task['tm_user_role'] else '')
self.logger.info(type(task['tm_user_infiles']))
self.logger.info(task['tm_user_infiles'])
if task['tm_user_infiles']:
addinfilestring = ''
for addinfile in task['tm_user_infiles']:
addinfilestring += '%s,' % addinfile
pandajob.jobParameters += '--userFiles=%s ' % (
addinfilestring[:-1])
pandajob.jobName = '%s' % task[
'tm_taskname'] #Needed by ASO and Dashboard
if 'panda_oldjobid' in job and job['panda_oldjobid']:
pandajob.parentID = job['panda_oldjobid']
pandajob.addFile(outFileSpec(log=True))
for filetoadd in alloutfiles:
pandajob.addFile(filetoadd)
return pandajob
for lfn in files.keys():
index += 1
job = JobSpec()
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (commands.getoutput('uuidgen'),index)
job.AtlasRelease = 'Atlas-13.0.40'
job.homepackage = 'AtlasProduction/13.0.40.3'
job.transformation = 'csc_simul_trf.py'
job.destinationDBlock = datasetName
job.destinationSE = destName
job.computingSite = site
job.prodDBlock = 'valid1.005001.pythia_minbias.evgen.EVNT.e306_tid019128'
job.prodSourceLabel = 'test'
job.currentPriority = 10000
job.cloud = 'IT'
fileI = FileSpec()
fileI.dataset = job.prodDBlock
fileI.prodDBlock = job.prodDBlock
fileI.lfn = lfn
fileI.type = 'input'
job.addFile(fileI)
fileD = FileSpec()
fileD.dataset = 'ddo.000001.Atlas.Ideal.DBRelease.v040701'
fileD.prodDBlock = 'ddo.000001.Atlas.Ideal.DBRelease.v030101'
fileD.lfn = 'DBRelease-4.7.1.tar.gz'
fileD.type = 'input'
job.addFile(fileD)
def send_job(jobid, siteid):
_logger.debug('Jobid: ' + str(jobid))
site = sites_.get(siteid)
job = jobs_.get(int(jobid))
cont = job.container
files_catalog = cont.files
fscope = getScope(job.owner.username)
datasetName = '{}:{}'.format(fscope, cont.guid)
distributive = job.distr.name
release = job.distr.release
# Prepare runScript
parameters = job.distr.command
parameters = parameters.replace("$COMMAND$", job.params)
parameters = parameters.replace("$USERNAME$", job.owner.username)
parameters = parameters.replace("$WORKINGGROUP$", job.owner.working_group)
# Prepare metadata
metadata = dict(user=job.owner.username)
# Prepare PanDA Object
pandajob = JobSpec()
pandajob.jobDefinitionID = int(time.time()) % 10000
pandajob.jobName = cont.guid
pandajob.transformation = client_config.DEFAULT_TRF
pandajob.destinationDBlock = datasetName
pandajob.destinationSE = site.se
pandajob.currentPriority = 1000
pandajob.prodSourceLabel = 'user'
pandajob.computingSite = site.ce
pandajob.cloud = 'RU'
pandajob.VO = 'atlas'
pandajob.prodDBlock = "%s:%s" % (fscope, pandajob.jobName)
pandajob.coreCount = job.corecount
pandajob.metadata = json.dumps(metadata)
#pandajob.workingGroup = job.owner.working_group
if site.encode_commands:
# It requires script wrapper on cluster side
pandajob.jobParameters = '%s %s %s "%s"' % (cont.guid, release, distributive, parameters)
else:
pandajob.jobParameters = parameters
has_input = False
for fcc in files_catalog:
if fcc.type == 'input':
f = fcc.file
guid = f.guid
fileIT = FileSpec()
fileIT.lfn = f.lfn
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = guid
pandajob.addFile(fileIT)
has_input = True
if fcc.type == 'output':
f = fcc.file
fileOT = FileSpec()
fileOT.lfn = f.lfn
fileOT.destinationDBlock = pandajob.prodDBlock
fileOT.destinationSE = pandajob.destinationSE
fileOT.dataset = pandajob.prodDBlock
fileOT.type = 'output'
fileOT.scope = fscope
fileOT.GUID = f.guid
pandajob.addFile(fileOT)
# Save replica meta
fc.new_replica(f, site)
if not has_input:
# Add fake input
fileIT = FileSpec()
fileIT.lfn = "fake.input"
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = "fake.guid"
pandajob.addFile(fileIT)
# Prepare lof file
fileOL = FileSpec()
fileOL.lfn = "%s.log.tgz" % pandajob.jobName
fileOL.destinationDBlock = pandajob.destinationDBlock
fileOL.destinationSE = pandajob.destinationSE
fileOL.dataset = '{}:logs'.format(fscope)
fileOL.type = 'log'
fileOL.scope = 'panda'
pandajob.addFile(fileOL)
# Save log meta
log = File()
log.scope = fscope
log.lfn = fileOL.lfn
log.guid = getGUID(log.scope, log.lfn)
log.type = 'log'
log.status = 'defined'
files_.save(log)
# Save replica meta
fc.new_replica(log, site)
# Register file in container
fc.reg_file_in_cont(log, cont, 'log')
# Submit job
o = submitJobs([pandajob])
x = o[0]
try:
#update PandaID
PandaID = int(x[0])
job.pandaid = PandaID
job.ce = site.ce
except:
job.status = 'submit_error'
jobs_.save(job)
return 0
index = 0
for lfn in files.keys():
index += 1
job = JobSpec()
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (commands.getoutput('uuidgen'),index)
job.AtlasRelease = 'Atlas-12.0.6'
job.homepackage = 'AtlasProduction/12.0.6.4'
job.transformation = 'csc_reco_trf.py'
job.destinationDBlock = datasetName
job.destinationSE = destName
job.computingSite = site
#job.prodDBlock = 'misal1_mc12.005200.T1_McAtNlo_Jimmy.digit.RDO.v12000601_tid007554'
job.prodDBlock = 'misal1_mc12.005802.JF17_pythia_jet_filter.digit.RDO.v12000601_tid008610'
job.cloud = 'US'
job.prodSourceLabel = 'test'
job.currentPriority = 10000
job.cmtConfig = 'i686-slc4-gcc34-opt'
fileI = FileSpec()
fileI.dataset = job.prodDBlock
fileI.prodDBlock = job.prodDBlock
fileI.lfn = lfn
fileI.type = 'input'
job.addFile(fileI)
fileD = FileSpec()
fileD.dataset = 'ddo.000001.Atlas.Ideal.DBRelease.v030101'
fileD.prodDBlock = 'ddo.000001.Atlas.Ideal.DBRelease.v030101'
def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig):
'''prepare the subjob specific configuration'''
# PandaTools
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('AthenaMCPandaRTHandler prepare called for %s', job.getFQID('.'))
try:
assert self.outsite
except:
logger.error("outsite not set. Aborting")
raise Exception()
job.backend.site = self.outsite
job.backend.actualCE = self.outsite
cloud = job._getRoot().backend.requirements.cloud
job.backend.requirements.cloud = cloud
# now just filling the job from AthenaMC data
jspec = JobSpec()
jspec.jobDefinitionID = job._getRoot().id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.AtlasRelease = 'Atlas-%s' % app.atlas_rel
if app.transform_archive:
jspec.homepackage = 'AnalysisTransforms'+app.transform_archive
elif app.prod_release:
jspec.homepackage = 'AnalysisTransforms-AtlasProduction_'+str(app.prod_release)
jspec.transformation = '%s/runAthena-00-00-11' % Client.baseURLSUB
#---->???? prodDBlock and destinationDBlock when facing several input / output datasets?
jspec.prodDBlock = 'NULL'
if job.inputdata and len(app.inputfiles)>0 and app.inputfiles[0] in app.dsetmap:
jspec.prodDBlock = app.dsetmap[app.inputfiles[0]]
# How to specify jspec.destinationDBlock when more than one type of output is available? Panda prod jobs seem to specify only the last output dataset
outdset=""
for type in ["EVNT","RDO","HITS","AOD","ESD","NTUP"]:
if type in app.outputpaths.keys():
outdset=string.replace(app.outputpaths[type],"/",".")
outdset=outdset[1:-1]
break
if not outdset:
try:
assert len(app.outputpaths.keys())>0
except:
logger.error("app.outputpaths is empty: check your output datasets")
raise
type=app.outputpaths.keys()[0]
outdset=string.replace(app.outputpaths[type],"/",".")
outdset=outdset[1:-1]
jspec.destinationDBlock = outdset
jspec.destinationSE = self.outsite
jspec.prodSourceLabel = 'user'
jspec.assignedPriority = 1000
jspec.cloud = cloud
# memory
if job.backend.requirements.memory != -1:
jspec.minRamCount = job.backend.requirements.memory
jspec.computingSite = self.outsite
jspec.cmtConfig = AthenaUtils.getCmtConfig(athenaVer=app.atlas_rel)
# library (source files)
flib = FileSpec()
flib.lfn = self.library
# flib.GUID =
flib.type = 'input'
# flib.status =
flib.dataset = self.libDataset
flib.dispatchDBlock = self.libDataset
jspec.addFile(flib)
# input files FIXME: many more input types
for lfn in app.inputfiles:
useguid=app.turls[lfn].replace("guid:","")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# add dbfiles if any:
for lfn in app.dbfiles:
useguid=app.dbturls[lfn].replace("guid:","")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# then minbias files
for lfn in app.mbfiles:
useguid=app.minbias_turls[lfn].replace("guid:","")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# then cavern files
for lfn in app.cavernfiles:
useguid=app.cavern_turls[lfn].replace("guid:","")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# output files( this includes the logfiles)
# Output files
jidtag=""
job = app._getParent() # Returns job or subjob object
if job._getRoot().subjobs:
jidtag = job._getRoot().id
else:
jidtag = "%d" % job.id
outfiles=app.subjobsOutfiles[job.id]
pandaOutfiles={}
for type in outfiles.keys():
pandaOutfiles[type]=outfiles[type]+"."+str(jidtag)
if type=="LOG":
pandaOutfiles[type]+=".tgz"
#print pandaOutfiles
for outtype in pandaOutfiles.keys():
fout = FileSpec()
dset=string.replace(app.outputpaths[outtype],"/",".")
dset=dset[1:-1]
fout.dataset=dset
fout.lfn=pandaOutfiles[outtype]
fout.type = 'output'
# fout.destinationDBlock = jspec.destinationDBlock
fout.destinationDBlock = fout.dataset
fout.destinationSE = jspec.destinationSE
if outtype=='LOG':
fout.type='log'
fout.destinationDBlock = fout.dataset
fout.destinationSE = job.backend.site
jspec.addFile(fout)
# job parameters
param = '-l %s ' % self.library # user tarball.
# use corruption checker
if job.backend.requirements.corCheck:
param += '--corCheck '
# disable to skip missing files
if job.backend.requirements.notSkipMissing:
param += '--notSkipMissing '
# transform parameters
# need to update arglist with final output file name...
newArgs=[]
if app.mode == "evgen":
app.args[3]=app.args[3]+" -t "
if app.verbosity:
app.args[3]=app.args[3]+" -l %s " % app.verbosity
for arg in app.args[3:]:
for type in outfiles.keys():
if arg.find(outfiles[type])>-1:
arg=arg.replace(outfiles[type],pandaOutfiles[type])
newArgs.append(arg)
arglist=string.join(newArgs," ")
# print "Arglist:",arglist
param += ' -r ./ '
param += ' -j "%s"' % urllib.quote(arglist)
allinfiles=app.inputfiles+app.dbfiles
# Input files.
param += ' -i "%s" ' % allinfiles
if len(app.mbfiles)>0:
param+= ' -m "%s" ' % app.mbfiles
if len(app.cavernfiles)>0:
param+= ' -n "%s" ' % app.cavernfiles
# param += '-m "[]" ' #%minList FIXME
# param += '-n "[]" ' #%cavList FIXME
del pandaOutfiles["LOG"] # logfiles do not appear in IROOT block, and this one is not needed anymore...
param += ' -o "{\'IROOT\':%s }"' % str(pandaOutfiles.items())
# source URL
matchURL = re.search("(http.*://[^/]+)/",Client.baseURLSSL)
if matchURL != None:
param += " --sourceURL %s " % matchURL.group(1)
param += " --trf"
jspec.jobParameters = param
jspec.metadata="--trf \"%s\"" % arglist
#print "SUBJOB DETAILS:",jspec.values()
if app.dryrun:
print "job.application.dryrun activated, printing out job parameters"
print jspec.values()
return
return jspec
def createJobSpec(self, task, outdataset, job, jobset, jobdef, site, jobname, lfnhanger, allsites, jobid):
"""Create a spec for one job
:arg TaskWorker.DataObject.Task task: the task to work on
:arg str outdataset: the output dataset name where all the produced files will be placed
:arg WMCore.DataStructs.Job job: the abstract job
:arg int jobset: the PanDA jobset corresponding to the current task
:arg int jobdef: the PanDA jobdef where to append the current jobs --- not used
:arg str site: the borkered site where to run the jobs
:arg str jobname: the job name
:arg str lfnhanger: the random string to be added in the output file name
:arg list str allsites: all possible sites where the job can potentially run
:arg int jobid: incremental job number
:return: the sepc object."""
pandajob = JobSpec()
## always setting a job definition ID
pandajob.jobDefinitionID = jobdef if jobdef else -1
## always setting a job set ID
pandajob.jobsetID = jobset if jobset else -1
pandajob.jobName = jobname
pandajob.prodUserID = task['tm_user_dn']
pandajob.destinationDBlock = outdataset
pandajob.prodDBlock = task['tm_input_dataset']
pandajob.prodSourceLabel = 'user'
pandajob.computingSite = site
pandajob.cloud = getSite(pandajob.computingSite)
pandajob.destinationSE = 'local'
pandajob.transformation = task['tm_transformation']
## need to initialize this
pandajob.metadata = ''
def outFileSpec(of=None, log=False):
"""Local routine to create an FileSpec for the an job output/log file
:arg str of: output file base name
:return: FileSpec object for the output file."""
outfile = FileSpec()
if log:
outfile.lfn = "job.log_%d_%s.tgz" % (jobid, lfnhanger)
outfile.type = 'log'
else:
outfile.lfn = '%s_%d_%s%s' %(os.path.splitext(of)[0], jobid, lfnhanger, os.path.splitext(of)[1])
outfile.type = 'output'
outfile.destinationDBlock = pandajob.destinationDBlock
outfile.destinationSE = task['tm_asyncdest']
outfile.dataset = pandajob.destinationDBlock
return outfile
alloutfiles = []
outjobpar = {}
outfilestring = ''
for outputfile in task['tm_outfiles']:
outfilestring += '%s,' % outputfile
filespec = outFileSpec(outputfile)
alloutfiles.append(filespec)
#pandajob.addFile(filespec)
outjobpar[outputfile] = filespec.lfn
for outputfile in task['tm_tfile_outfiles']:
outfilestring += '%s,' % outputfile
filespec = outFileSpec(outputfile)
alloutfiles.append(filespec)
#pandajob.addFile(filespec)
outjobpar[outputfile] = filespec.lfn
for outputfile in task['tm_edm_outfiles']:
outfilestring += '%s,' % outputfile
filespec = outFileSpec(outputfile)
alloutfiles.append(filespec)
#pandajob.addFile(filespec)
outjobpar[outputfile] = filespec.lfn
outfilestring = outfilestring[:-1]
infiles = []
for inputfile in job['input_files']:
infiles.append( inputfile['lfn'] )
pandajob.jobParameters = '-a %s ' % task['tm_user_sandbox']
pandajob.jobParameters += '--sourceURL %s ' % task['tm_cache_url']
pandajob.jobParameters += '--jobNumber=%s ' % jobid
pandajob.jobParameters += '--cmsswVersion=%s ' % task['tm_job_sw']
pandajob.jobParameters += '--scramArch=%s ' % task['tm_job_arch']
pandajob.jobParameters += '--inputFile=\'%s\' ' % json.dumps(infiles)
self.jobParametersSetting(pandajob, job, self.jobtypeMapper[task['tm_job_type']])
pandajob.jobParameters += '-o "%s" ' % str(outjobpar)
pandajob.jobParameters += '--dbs_url=%s ' % task['tm_dbs_url']
pandajob.jobParameters += '--publish_dbs_url=%s ' % task['tm_publish_dbs_url']
pandajob.jobParameters += '--publishFiles=%s ' % ('True' if task['tm_publication'] == 'T' else 'False')
pandajob.jobParameters += '--saveLogs=%s ' % ('True' if task['tm_save_logs'] == 'T' else 'False')
pandajob.jobParameters += '--availableSites=\'%s\' ' %json.dumps(allsites)
pandajob.jobParameters += '--group=%s ' % (task['tm_user_group'] if task['tm_user_group'] else '')
pandajob.jobParameters += '--role=%s ' % (task['tm_user_role'] if task['tm_user_role'] else '')
self.logger.info(type(task['tm_user_infiles']))
self.logger.info(task['tm_user_infiles'])
if task['tm_user_infiles']:
addinfilestring = ''
for addinfile in task['tm_user_infiles']:
addinfilestring += '%s,' % addinfile
pandajob.jobParameters += '--userFiles=%s ' % ( addinfilestring[:-1] )
pandajob.jobName = '%s' % task['tm_taskname'] #Needed by ASO and Dashboard
if 'panda_oldjobid' in job and job['panda_oldjobid']:
pandajob.parentID = job['panda_oldjobid']
pandajob.addFile(outFileSpec(log=True))
for filetoadd in alloutfiles:
pandajob.addFile(filetoadd)
return pandajob
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
"""Prepare the specific aspec of each subjob.
Returns: subjobconfig list of objects understood by backends."""
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
from GangaAtlas.Lib.ATLASDataset.DQ2Dataset import dq2_set_dataset_lifetime
from GangaPanda.Lib.Panda.Panda import refreshPandaSpecs
# make sure we have the correct siteType
refreshPandaSpecs()
job = app._getParent()
masterjob = job._getRoot()
logger.debug('ProdTransPandaRTHandler prepare called for %s',
job.getFQID('.'))
job.backend.actualCE = job.backend.site
job.backend.requirements.cloud = Client.PandaSites[job.backend.site]['cloud']
# check that the site is in a submit-able status
if not job.splitter or job.splitter._name != 'DQ2JobSplitter':
allowed_sites = job.backend.list_ddm_sites()
try:
outDsLocation = Client.PandaSites[job.backend.site]['ddm']
tmpDsExist = False
if (configPanda['processingType'].startswith('gangarobot') or configPanda['processingType'].startswith('hammercloud')):
#if Client.getDatasets(job.outputdata.datasetname):
if getDatasets(job.outputdata.datasetname):
tmpDsExist = True
logger.info('Re-using output dataset %s'%job.outputdata.datasetname)
if not configPanda['specialHandling']=='ddm:rucio' and not configPanda['processingType'].startswith('gangarobot') and not configPanda['processingType'].startswith('hammercloud') and not configPanda['processingType'].startswith('rucio_test'):
Client.addDataset(job.outputdata.datasetname,False,location=outDsLocation,allowProdDisk=True,dsExist=tmpDsExist)
logger.info('Output dataset %s registered at %s'%(job.outputdata.datasetname,outDsLocation))
dq2_set_dataset_lifetime(job.outputdata.datasetname, outDsLocation)
except exceptions.SystemExit:
raise BackendError('Panda','Exception in adding dataset %s: %s %s'%(job.outputdata.datasetname,sys.exc_info()[0],sys.exc_info()[1]))
# JobSpec.
jspec = JobSpec()
jspec.currentPriority = app.priority
jspec.jobDefinitionID = masterjob.id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.coreCount = app.core_count
jspec.AtlasRelease = 'Atlas-%s' % app.atlas_release
jspec.homepackage = app.home_package
jspec.transformation = app.transformation
jspec.destinationDBlock = job.outputdata.datasetname
if job.outputdata.location:
jspec.destinationSE = job.outputdata.location
else:
jspec.destinationSE = job.backend.site
if job.inputdata:
jspec.prodDBlock = job.inputdata.dataset[0]
else:
jspec.prodDBlock = 'NULL'
if app.prod_source_label:
jspec.prodSourceLabel = app.prod_source_label
else:
jspec.prodSourceLabel = configPanda['prodSourceLabelRun']
jspec.processingType = configPanda['processingType']
jspec.specialHandling = configPanda['specialHandling']
jspec.computingSite = job.backend.site
jspec.cloud = job.backend.requirements.cloud
jspec.cmtConfig = app.atlas_cmtconfig
if app.dbrelease == 'LATEST':
try:
latest_dbrelease = getLatestDBReleaseCaching()
except:
from pandatools import Client
latest_dbrelease = Client.getLatestDBRelease()
m = re.search('(.*):DBRelease-(.*)\.tar\.gz', latest_dbrelease)
if m:
self.dbrelease_dataset = m.group(1)
self.dbrelease = m.group(2)
else:
raise ApplicationConfigurationError(None, "Error retrieving LATEST DBRelease. Try setting application.dbrelease manually.")
else:
self.dbrelease_dataset = app.dbrelease_dataset
self.dbrelease = app.dbrelease
jspec.jobParameters = app.job_parameters
if self.dbrelease:
if self.dbrelease == 'current':
jspec.jobParameters += ' --DBRelease=current'
else:
if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --DBRelease=DBRelease-%s.tar.gz' % (self.dbrelease,)
else:
jspec.jobParameters += ' DBRelease=DBRelease-%s.tar.gz' % (self.dbrelease,)
dbspec = FileSpec()
dbspec.lfn = 'DBRelease-%s.tar.gz' % self.dbrelease
dbspec.dataset = self.dbrelease_dataset
dbspec.prodDBlock = jspec.prodDBlock
dbspec.type = 'input'
jspec.addFile(dbspec)
if job.inputdata:
m = re.search('(.*)\.(.*)\.(.*)\.(.*)\.(.*)\.(.*)',
job.inputdata.dataset[0])
if not m:
logger.error("Error retrieving run number from dataset name")
#raise ApplicationConfigurationError(None, "Error retrieving run number from dataset name")
runnumber = 105200
else:
runnumber = int(m.group(2))
if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --runNumber %d' % runnumber
else:
jspec.jobParameters += ' RunNumber=%d' % runnumber
# Output files.
randomized_lfns = []
ilfn = 0
for lfn, lfntype in zip(app.output_files,app.output_type):
ofspec = FileSpec()
if app.randomize_lfns:
randomized_lfn = lfn + ('.%s.%d.%s' % (job.backend.site, int(time.time()), commands.getoutput('uuidgen 2> /dev/null')[:4] ) )
else:
randomized_lfn = lfn
ofspec.lfn = randomized_lfn
randomized_lfns.append(randomized_lfn)
ofspec.destinationDBlock = jspec.destinationDBlock
ofspec.destinationSE = jspec.destinationSE
ofspec.dataset = jspec.destinationDBlock
ofspec.type = 'output'
jspec.addFile(ofspec)
if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --output%sFile %s' % (lfntype, randomized_lfns[ilfn])
else:
jspec.jobParameters += ' output%sFile=%s' % (lfntype, randomized_lfns[ilfn])
ilfn=ilfn+1
# Input files.
if job.inputdata:
for guid, lfn, size, checksum, scope in zip(job.inputdata.guids, job.inputdata.names, job.inputdata.sizes, job.inputdata.checksums, job.inputdata.scopes):
ifspec = FileSpec()
ifspec.lfn = lfn
ifspec.GUID = guid
ifspec.fsize = size
ifspec.md5sum = checksum
ifspec.scope = scope
ifspec.dataset = jspec.prodDBlock
ifspec.prodDBlock = jspec.prodDBlock
ifspec.type = 'input'
jspec.addFile(ifspec)
if app.input_type:
itype = app.input_type
else:
itype = m.group(5)
if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --input%sFile %s' % (itype, ','.join(job.inputdata.names))
else:
jspec.jobParameters += ' input%sFile=%s' % (itype, ','.join(job.inputdata.names))
# Log files.
lfspec = FileSpec()
lfspec.lfn = '%s.job.log.tgz' % jspec.jobName
lfspec.destinationDBlock = jspec.destinationDBlock
lfspec.destinationSE = jspec.destinationSE
lfspec.dataset = jspec.destinationDBlock
lfspec.type = 'log'
jspec.addFile(lfspec)
return jspec
jobList = []
for i in range(1):
job = JobSpec()
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (commands.getoutput('uuidgen'), i)
job.AtlasRelease = 'Atlas-17.0.5'
job.homepackage = 'AtlasProduction/17.0.5.6'
job.transformation = 'Evgen_trf.py'
job.destinationDBlock = datasetName
job.destinationSE = destName
job.currentPriority = 10000
job.prodSourceLabel = 'test'
job.computingSite = site
job.cloud = cloud
job.cmtConfig = 'i686-slc5-gcc43-opt'
file = FileSpec()
file.lfn = "%s.evgen.pool.root" % job.jobName
file.destinationDBlock = job.destinationDBlock
file.destinationSE = job.destinationSE
file.dataset = job.destinationDBlock
file.destinationDBlockToken = 'ATLASDATADISK'
file.type = 'output'
job.addFile(file)
fileOL = FileSpec()
fileOL.lfn = "%s.job.log.tgz" % job.jobName
fileOL.destinationDBlock = job.destinationDBlock
fileOL.destinationSE = job.destinationSE
def prepare(self,app,appsubconfig,appmasterconfig,jobmasterconfig):
'''prepare the subjob specific configuration'''
from pandatools import Client
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('AthenaPandaRTHandler prepare called for %s', job.getFQID('.'))
# in case of a simple job get the dataset content, otherwise subjobs are filled by the splitter
if job.inputdata and not job._getRoot().subjobs:
if not job.inputdata.names:
contents = job.inputdata.get_contents(overlap=False, size=True)
for ds in contents.keys():
for f in contents[ds]:
job.inputdata.guids.append( f[0] )
job.inputdata.names.append( f[1][0] )
job.inputdata.sizes.append( f[1][1] )
job.inputdata.checksums.append( f[1][2] )
job.inputdata.scopes.append( f[1][3] )
site = job._getRoot().backend.site
job.backend.site = site
job.backend.actualCE = site
cloud = job._getRoot().backend.requirements.cloud
job.backend.requirements.cloud = cloud
# if no outputdata are given
if not job.outputdata:
job.outputdata = DQ2OutputDataset()
job.outputdata.datasetname = job._getRoot().outputdata.datasetname
#if not job.outputdata.datasetname:
else:
job.outputdata.datasetname = job._getRoot().outputdata.datasetname
if not job.outputdata.datasetname:
raise ApplicationConfigurationError('DQ2OutputDataset has no datasetname')
jspec = JobSpec()
jspec.jobDefinitionID = job._getRoot().id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.transformation = '%s/runGen-00-00-02' % Client.baseURLSUB
if job.inputdata:
jspec.prodDBlock = job.inputdata.dataset[0]
else:
jspec.prodDBlock = 'NULL'
jspec.destinationDBlock = job.outputdata.datasetname
if job.outputdata.location:
if not job._getRoot().subjobs or job.id == 0:
logger.warning('You have specified outputdata.location. Note that Panda may not support writing to a user-defined output location.')
jspec.destinationSE = job.outputdata.location
else:
jspec.destinationSE = site
jspec.prodSourceLabel = configPanda['prodSourceLabelRun']
jspec.processingType = configPanda['processingType']
jspec.assignedPriority = configPanda['assignedPriorityRun']
jspec.cloud = cloud
# memory
if job.backend.requirements.memory != -1:
jspec.minRamCount = job.backend.requirements.memory
# cputime
if job.backend.requirements.cputime != -1:
jspec.maxCpuCount = job.backend.requirements.cputime
jspec.computingSite = site
# library (source files)
if job.backend.libds:
flib = FileSpec()
flib.lfn = self.fileBO.lfn
flib.GUID = self.fileBO.GUID
flib.type = 'input'
flib.status = self.fileBO.status
flib.dataset = self.fileBO.destinationDBlock
flib.dispatchDBlock = self.fileBO.destinationDBlock
jspec.addFile(flib)
elif job.backend.bexec:
flib = FileSpec()
flib.lfn = self.library
flib.type = 'input'
flib.dataset = self.libDataset
flib.dispatchDBlock = self.libDataset
jspec.addFile(flib)
# input files FIXME: many more input types
if job.inputdata:
for guid, lfn, size, checksum, scope in zip(job.inputdata.guids,job.inputdata.names,job.inputdata.sizes, job.inputdata.checksums, job.inputdata.scopes):
finp = FileSpec()
finp.lfn = lfn
finp.GUID = guid
finp.scope = scope
# finp.fsize =
# finp.md5sum =
finp.dataset = job.inputdata.dataset[0]
finp.prodDBlock = job.inputdata.dataset[0]
finp.dispatchDBlock = job.inputdata.dataset[0]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# output files
# outMap = {}
#FIXME: if options.outMeta != []:
self.rundirectory = "."
# log files
flog = FileSpec()
flog.lfn = '%s._$PANDAID.log.tgz' % job.outputdata.datasetname
flog.type = 'log'
flog.dataset = job.outputdata.datasetname
flog.destinationDBlock = job.outputdata.datasetname
flog.destinationSE = job.backend.site
jspec.addFile(flog)
# job parameters
param = ''
# source URL
matchURL = re.search("(http.*://[^/]+)/",Client.baseURLCSRVSSL)
srcURL = ""
if matchURL != None:
srcURL = matchURL.group(1)
param += " --sourceURL %s " % srcURL
param += '-r "%s" ' % self.rundirectory
exe_name = job.application.exe
if job.backend.bexec == '':
if hasattr(job.application.exe, "name"):
exe_name = os.path.basename(job.application.exe.name)
# set jobO parameter
if job.application.args:
param += ' -j "" -p "%s %s" '%(exe_name,urllib.quote(" ".join(job.application.args)))
else:
param += ' -j "" -p "%s" '%exe_name
if self.inputsandbox:
param += ' -a %s '%self.inputsandbox
else:
param += '-l %s ' % self.library
param += '-j "" -p "%s %s" ' % ( exe_name,urllib.quote(" ".join(job.application.args)))
if job.inputdata:
param += '-i "%s" ' % job.inputdata.names
# fill outfiles
outfiles = {}
for f in self.extOutFile:
tarnum = 1
if f.find('*') != -1:
# archive *
outfiles[f] = "outputbox%i.%s.%s.tar.gz" % (tarnum, job.getFQID('.'), time.strftime("%Y%m%d%H%M%S") )
tarnum += 1
else:
outfiles[f] = "%s.%s.%s" %(f, job.getFQID('.'), time.strftime("%Y%m%d%H%M%S"))
fout = FileSpec()
fout.lfn = outfiles[f]
fout.type = 'output'
fout.dataset = job.outputdata.datasetname
fout.destinationDBlock = job.outputdata.datasetname
fout.destinationSE = job.backend.site
jspec.addFile(fout)
param += '-o "%s" ' % (outfiles) # must be double quotes, because python prints strings in 'single quotes'
for file in jspec.Files:
if file.type in [ 'output', 'log'] and configPanda['chirpconfig']:
file.dispatchDBlockToken = configPanda['chirpconfig']
logger.debug('chirp file %s',file)
jspec.jobParameters = param
return jspec
def send_job(jobid, siteid):
_logger.debug('Jobid: ' + str(jobid))
site = sites_.get(siteid)
job = jobs_.get(int(jobid))
cont = job.container
files_catalog = cont.files
fscope = getScope(job.owner.username)
datasetName = '{}:{}'.format(fscope, cont.guid)
distributive = job.distr.name
release = job.distr.release
# Prepare runScript
parameters = job.distr.command
parameters = parameters.replace("$COMMAND$", job.params)
parameters = parameters.replace("$USERNAME$", job.owner.username)
parameters = parameters.replace("$WORKINGGROUP$", job.owner.working_group)
# Prepare metadata
metadata = dict(user=job.owner.username)
# Prepare PanDA Object
pandajob = JobSpec()
pandajob.jobDefinitionID = int(time.time()) % 10000
pandajob.jobName = cont.guid
pandajob.transformation = client_config.DEFAULT_TRF
pandajob.destinationDBlock = datasetName
pandajob.destinationSE = site.se
pandajob.currentPriority = 1000
pandajob.prodSourceLabel = 'user'
pandajob.computingSite = site.ce
pandajob.cloud = 'RU'
pandajob.VO = 'atlas'
pandajob.prodDBlock = "%s:%s" % (fscope, pandajob.jobName)
pandajob.coreCount = job.corecount
pandajob.metadata = json.dumps(metadata)
#pandajob.workingGroup = job.owner.working_group
if site.encode_commands:
# It requires script wrapper on cluster side
pandajob.jobParameters = '%s %s %s "%s"' % (cont.guid, release,
distributive, parameters)
else:
pandajob.jobParameters = parameters
has_input = False
for fcc in files_catalog:
if fcc.type == 'input':
f = fcc.file
guid = f.guid
fileIT = FileSpec()
fileIT.lfn = f.lfn
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = guid
pandajob.addFile(fileIT)
has_input = True
if fcc.type == 'output':
f = fcc.file
fileOT = FileSpec()
fileOT.lfn = f.lfn
fileOT.destinationDBlock = pandajob.prodDBlock
fileOT.destinationSE = pandajob.destinationSE
fileOT.dataset = pandajob.prodDBlock
fileOT.type = 'output'
fileOT.scope = fscope
fileOT.GUID = f.guid
pandajob.addFile(fileOT)
# Save replica meta
fc.new_replica(f, site)
if not has_input:
# Add fake input
fileIT = FileSpec()
fileIT.lfn = "fake.input"
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = "fake.guid"
pandajob.addFile(fileIT)
# Prepare lof file
fileOL = FileSpec()
fileOL.lfn = "%s.log.tgz" % pandajob.jobName
fileOL.destinationDBlock = pandajob.destinationDBlock
fileOL.destinationSE = pandajob.destinationSE
fileOL.dataset = '{}:logs'.format(fscope)
fileOL.type = 'log'
fileOL.scope = 'panda'
pandajob.addFile(fileOL)
# Save log meta
log = File()
log.scope = fscope
log.lfn = fileOL.lfn
log.guid = getGUID(log.scope, log.lfn)
log.type = 'log'
log.status = 'defined'
files_.save(log)
# Save replica meta
fc.new_replica(log, site)
# Register file in container
fc.reg_file_in_cont(log, cont, 'log')
# Submit job
o = submitJobs([pandajob])
x = o[0]
try:
#update PandaID
PandaID = int(x[0])
job.pandaid = PandaID
job.ce = site.ce
except:
job.status = 'submit_error'
jobs_.save(job)
return 0
jobList = []
for i in range(1):
job = JobSpec()
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (commands.getoutput('uuidgen'),i)
# job.AtlasRelease = 'Atlas-12.0.6'
# job.homepackage = 'AtlasProduction/12.0.6.5'
job.AtlasRelease = 'Atlas-12.0.7'
job.homepackage = 'AtlasProduction/12.0.7.1'
job.transformation = 'csc_evgen_trf.py'
job.destinationDBlock = datasetName
# job.destinationSE = destName
# job.cloud = 'CA'
job.cloud = cloud
job.taskID = taskid
job.currentPriority = 1000
job.prodSourceLabel = 'test'
# job.prodSourceLabel = 'cloudtest'
job.computingSite = site
file = FileSpec()
file.lfn = "%s.evgen.pool.root" % job.jobName
file.destinationDBlock = job.destinationDBlock
file.destinationSE = job.destinationSE
file.dataset = job.destinationDBlock
file.type = 'output'
job.addFile(file)
fileOL = FileSpec()
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
'''prepare the subjob specific configuration'''
# PandaTools
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
logger.debug('AthenaMCPandaRTHandler prepare called for %s',
job.getFQID('.'))
try:
assert self.outsite
except:
logger.error("outsite not set. Aborting")
raise Exception()
job.backend.site = self.outsite
job.backend.actualCE = self.outsite
cloud = job._getRoot().backend.requirements.cloud
job.backend.requirements.cloud = cloud
# now just filling the job from AthenaMC data
jspec = JobSpec()
jspec.jobDefinitionID = job._getRoot().id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.AtlasRelease = 'Atlas-%s' % app.atlas_rel
if app.transform_archive:
jspec.homepackage = 'AnalysisTransforms' + app.transform_archive
elif app.prod_release:
jspec.homepackage = 'AnalysisTransforms-AtlasProduction_' + str(
app.prod_release)
jspec.transformation = '%s/runAthena-00-00-11' % Client.baseURLSUB
#---->???? prodDBlock and destinationDBlock when facing several input / output datasets?
jspec.prodDBlock = 'NULL'
if job.inputdata and len(
app.inputfiles) > 0 and app.inputfiles[0] in app.dsetmap:
jspec.prodDBlock = app.dsetmap[app.inputfiles[0]]
# How to specify jspec.destinationDBlock when more than one type of output is available? Panda prod jobs seem to specify only the last output dataset
outdset = ""
for type in ["EVNT", "RDO", "HITS", "AOD", "ESD", "NTUP"]:
if type in app.outputpaths.keys():
outdset = string.replace(app.outputpaths[type], "/", ".")
outdset = outdset[1:-1]
break
if not outdset:
try:
assert len(app.outputpaths.keys()) > 0
except:
logger.error(
"app.outputpaths is empty: check your output datasets")
raise
type = app.outputpaths.keys()[0]
outdset = string.replace(app.outputpaths[type], "/", ".")
outdset = outdset[1:-1]
jspec.destinationDBlock = outdset
jspec.destinationSE = self.outsite
jspec.prodSourceLabel = 'user'
jspec.assignedPriority = 1000
jspec.cloud = cloud
# memory
if job.backend.requirements.memory != -1:
jspec.minRamCount = job.backend.requirements.memory
jspec.computingSite = self.outsite
jspec.cmtConfig = AthenaUtils.getCmtConfig(athenaVer=app.atlas_rel)
# library (source files)
flib = FileSpec()
flib.lfn = self.library
# flib.GUID =
flib.type = 'input'
# flib.status =
flib.dataset = self.libDataset
flib.dispatchDBlock = self.libDataset
jspec.addFile(flib)
# input files FIXME: many more input types
for lfn in app.inputfiles:
useguid = app.turls[lfn].replace("guid:", "")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# add dbfiles if any:
for lfn in app.dbfiles:
useguid = app.dbturls[lfn].replace("guid:", "")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# then minbias files
for lfn in app.mbfiles:
useguid = app.minbias_turls[lfn].replace("guid:", "")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# then cavern files
for lfn in app.cavernfiles:
useguid = app.cavern_turls[lfn].replace("guid:", "")
finp = FileSpec()
finp.lfn = lfn
finp.GUID = useguid
finp.dataset = app.dsetmap[lfn]
finp.prodDBlock = app.dsetmap[lfn]
finp.prodDBlockToken = 'local'
finp.dispatchDBlock = app.dsetmap[lfn]
finp.type = 'input'
finp.status = 'ready'
jspec.addFile(finp)
# output files( this includes the logfiles)
# Output files
jidtag = ""
job = app._getParent() # Returns job or subjob object
if job._getRoot().subjobs:
jidtag = job._getRoot().id
else:
jidtag = "%d" % job.id
outfiles = app.subjobsOutfiles[job.id]
pandaOutfiles = {}
for type in outfiles.keys():
pandaOutfiles[type] = outfiles[type] + "." + str(jidtag)
if type == "LOG":
pandaOutfiles[type] += ".tgz"
#print pandaOutfiles
for outtype in pandaOutfiles.keys():
fout = FileSpec()
dset = string.replace(app.outputpaths[outtype], "/", ".")
dset = dset[1:-1]
fout.dataset = dset
fout.lfn = pandaOutfiles[outtype]
fout.type = 'output'
# fout.destinationDBlock = jspec.destinationDBlock
fout.destinationDBlock = fout.dataset
fout.destinationSE = jspec.destinationSE
if outtype == 'LOG':
fout.type = 'log'
fout.destinationDBlock = fout.dataset
fout.destinationSE = job.backend.site
jspec.addFile(fout)
# job parameters
param = '-l %s ' % self.library # user tarball.
# use corruption checker
if job.backend.requirements.corCheck:
param += '--corCheck '
# disable to skip missing files
if job.backend.requirements.notSkipMissing:
param += '--notSkipMissing '
# transform parameters
# need to update arglist with final output file name...
newArgs = []
if app.mode == "evgen":
app.args[3] = app.args[3] + " -t "
if app.verbosity:
app.args[3] = app.args[3] + " -l %s " % app.verbosity
for arg in app.args[3:]:
for type in outfiles.keys():
if arg.find(outfiles[type]) > -1:
arg = arg.replace(outfiles[type], pandaOutfiles[type])
newArgs.append(arg)
arglist = string.join(newArgs, " ")
# print "Arglist:",arglist
param += ' -r ./ '
param += ' -j "%s"' % urllib.quote(arglist)
allinfiles = app.inputfiles + app.dbfiles
# Input files.
param += ' -i "%s" ' % allinfiles
if len(app.mbfiles) > 0:
param += ' -m "%s" ' % app.mbfiles
if len(app.cavernfiles) > 0:
param += ' -n "%s" ' % app.cavernfiles
# param += '-m "[]" ' #%minList FIXME
# param += '-n "[]" ' #%cavList FIXME
del pandaOutfiles[
"LOG"] # logfiles do not appear in IROOT block, and this one is not needed anymore...
param += ' -o "{\'IROOT\':%s }"' % str(pandaOutfiles.items())
# source URL
matchURL = re.search("(http.*://[^/]+)/", Client.baseURLSSL)
if matchURL != None:
param += " --sourceURL %s " % matchURL.group(1)
param += " --trf"
jspec.jobParameters = param
jspec.metadata = "--trf \"%s\"" % arglist
#print "SUBJOB DETAILS:",jspec.values()
if app.dryrun:
print "job.application.dryrun activated, printing out job parameters"
print jspec.values()
return
return jspec