def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
"""Prepare the specific aspec of each subjob.
Returns: subjobconfig list of objects understood by backends."""
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
from GangaAtlas.Lib.ATLASDataset.DQ2Dataset import dq2_set_dataset_lifetime
from GangaPanda.Lib.Panda.Panda import refreshPandaSpecs
# make sure we have the correct siteType
refreshPandaSpecs()
job = app._getParent()
masterjob = job._getRoot()
logger.debug('ProdTransPandaRTHandler prepare called for %s',
job.getFQID('.'))
job.backend.actualCE = job.backend.site
job.backend.requirements.cloud = Client.PandaSites[job.backend.site]['cloud']
# check that the site is in a submit-able status
if not job.splitter or job.splitter._name != 'DQ2JobSplitter':
allowed_sites = job.backend.list_ddm_sites()
try:
outDsLocation = Client.PandaSites[job.backend.site]['ddm']
tmpDsExist = False
if (configPanda['processingType'].startswith('gangarobot') or configPanda['processingType'].startswith('hammercloud')):
#if Client.getDatasets(job.outputdata.datasetname):
if getDatasets(job.outputdata.datasetname):
tmpDsExist = True
logger.info('Re-using output dataset %s'%job.outputdata.datasetname)
if not configPanda['specialHandling']=='ddm:rucio' and not configPanda['processingType'].startswith('gangarobot') and not configPanda['processingType'].startswith('hammercloud') and not configPanda['processingType'].startswith('rucio_test'):
Client.addDataset(job.outputdata.datasetname,False,location=outDsLocation,allowProdDisk=True,dsExist=tmpDsExist)
logger.info('Output dataset %s registered at %s'%(job.outputdata.datasetname,outDsLocation))
dq2_set_dataset_lifetime(job.outputdata.datasetname, outDsLocation)
except exceptions.SystemExit:
raise BackendError('Panda','Exception in adding dataset %s: %s %s'%(job.outputdata.datasetname,sys.exc_info()[0],sys.exc_info()[1]))
# JobSpec.
jspec = JobSpec()
jspec.currentPriority = app.priority
jspec.jobDefinitionID = masterjob.id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.coreCount = app.core_count
jspec.AtlasRelease = 'Atlas-%s' % app.atlas_release
jspec.homepackage = app.home_package
jspec.transformation = app.transformation
jspec.destinationDBlock = job.outputdata.datasetname
if job.outputdata.location:
jspec.destinationSE = job.outputdata.location
else:
jspec.destinationSE = job.backend.site
if job.inputdata:
jspec.prodDBlock = job.inputdata.dataset[0]
else:
jspec.prodDBlock = 'NULL'
if app.prod_source_label:
jspec.prodSourceLabel = app.prod_source_label
else:
jspec.prodSourceLabel = configPanda['prodSourceLabelRun']
jspec.processingType = configPanda['processingType']
jspec.specialHandling = configPanda['specialHandling']
jspec.computingSite = job.backend.site
jspec.cloud = job.backend.requirements.cloud
jspec.cmtConfig = app.atlas_cmtconfig
if app.dbrelease == 'LATEST':
try:
latest_dbrelease = getLatestDBReleaseCaching()
except:
from pandatools import Client
latest_dbrelease = Client.getLatestDBRelease()
m = re.search('(.*):DBRelease-(.*)\.tar\.gz', latest_dbrelease)
if m:
self.dbrelease_dataset = m.group(1)
self.dbrelease = m.group(2)
else:
raise ApplicationConfigurationError(None, "Error retrieving LATEST DBRelease. Try setting application.dbrelease manually.")
else:
self.dbrelease_dataset = app.dbrelease_dataset
self.dbrelease = app.dbrelease
jspec.jobParameters = app.job_parameters
if self.dbrelease:
if self.dbrelease == 'current':
jspec.jobParameters += ' --DBRelease=current'
else:
if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --DBRelease=DBRelease-%s.tar.gz' % (self.dbrelease,)
else:
jspec.jobParameters += ' DBRelease=DBRelease-%s.tar.gz' % (self.dbrelease,)
dbspec = FileSpec()
dbspec.lfn = 'DBRelease-%s.tar.gz' % self.dbrelease
dbspec.dataset = self.dbrelease_dataset
dbspec.prodDBlock = jspec.prodDBlock
dbspec.type = 'input'
jspec.addFile(dbspec)
if job.inputdata:
m = re.search('(.*)\.(.*)\.(.*)\.(.*)\.(.*)\.(.*)',
job.inputdata.dataset[0])
if not m:
logger.error("Error retrieving run number from dataset name")
#raise ApplicationConfigurationError(None, "Error retrieving run number from dataset name")
runnumber = 105200
else:
runnumber = int(m.group(2))
if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --runNumber %d' % runnumber
else:
jspec.jobParameters += ' RunNumber=%d' % runnumber
# Output files.
randomized_lfns = []
ilfn = 0
for lfn, lfntype in zip(app.output_files,app.output_type):
ofspec = FileSpec()
if app.randomize_lfns:
randomized_lfn = lfn + ('.%s.%d.%s' % (job.backend.site, int(time.time()), commands.getoutput('uuidgen 2> /dev/null')[:4] ) )
else:
randomized_lfn = lfn
ofspec.lfn = randomized_lfn
randomized_lfns.append(randomized_lfn)
ofspec.destinationDBlock = jspec.destinationDBlock
ofspec.destinationSE = jspec.destinationSE
ofspec.dataset = jspec.destinationDBlock
ofspec.type = 'output'
jspec.addFile(ofspec)
if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --output%sFile %s' % (lfntype, randomized_lfns[ilfn])
else:
jspec.jobParameters += ' output%sFile=%s' % (lfntype, randomized_lfns[ilfn])
ilfn=ilfn+1
# Input files.
if job.inputdata:
for guid, lfn, size, checksum, scope in zip(job.inputdata.guids, job.inputdata.names, job.inputdata.sizes, job.inputdata.checksums, job.inputdata.scopes):
ifspec = FileSpec()
ifspec.lfn = lfn
ifspec.GUID = guid
ifspec.fsize = size
ifspec.md5sum = checksum
ifspec.scope = scope
ifspec.dataset = jspec.prodDBlock
ifspec.prodDBlock = jspec.prodDBlock
ifspec.type = 'input'
jspec.addFile(ifspec)
if app.input_type:
itype = app.input_type
else:
itype = m.group(5)
if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --input%sFile %s' % (itype, ','.join(job.inputdata.names))
else:
jspec.jobParameters += ' input%sFile=%s' % (itype, ','.join(job.inputdata.names))
# Log files.
lfspec = FileSpec()
lfspec.lfn = '%s.job.log.tgz' % jspec.jobName
lfspec.destinationDBlock = jspec.destinationDBlock
lfspec.destinationSE = jspec.destinationSE
lfspec.dataset = jspec.destinationDBlock
lfspec.type = 'log'
jspec.addFile(lfspec)
return jspec
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
"""Prepare the specific aspec of each subjob.
Returns: subjobconfig list of objects understood by backends."""
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
from GangaAtlas.Lib.ATLASDataset.DQ2Dataset import dq2_set_dataset_lifetime
from GangaPanda.Lib.Panda.Panda import refreshPandaSpecs
# make sure we have the correct siteType
refreshPandaSpecs()
job = app._getParent()
masterjob = job._getRoot()
logger.debug('ProdTransPandaRTHandler prepare called for %s',
job.getFQID('.'))
job.backend.actualCE = job.backend.site
job.backend.requirements.cloud = Client.PandaSites[
job.backend.site]['cloud']
# check that the site is in a submit-able status
if not job.splitter or job.splitter._name != 'DQ2JobSplitter':
allowed_sites = job.backend.list_ddm_sites()
try:
outDsLocation = Client.PandaSites[job.backend.site]['ddm']
tmpDsExist = False
if (configPanda['processingType'].startswith('gangarobot') or
configPanda['processingType'].startswith('hammercloud')):
#if Client.getDatasets(job.outputdata.datasetname):
if getDatasets(job.outputdata.datasetname):
tmpDsExist = True
logger.info('Re-using output dataset %s' %
job.outputdata.datasetname)
if not configPanda[
'specialHandling'] == 'ddm:rucio' and not configPanda[
'processingType'].startswith(
'gangarobot'
) and not configPanda['processingType'].startswith(
'hammercloud') and not configPanda[
'processingType'].startswith('rucio_test'):
Client.addDataset(job.outputdata.datasetname,
False,
location=outDsLocation,
allowProdDisk=True,
dsExist=tmpDsExist)
logger.info('Output dataset %s registered at %s' %
(job.outputdata.datasetname, outDsLocation))
dq2_set_dataset_lifetime(job.outputdata.datasetname, outDsLocation)
except exceptions.SystemExit:
raise BackendError(
'Panda', 'Exception in adding dataset %s: %s %s' %
(job.outputdata.datasetname, sys.exc_info()[0],
sys.exc_info()[1]))
# JobSpec.
jspec = JobSpec()
jspec.currentPriority = app.priority
jspec.jobDefinitionID = masterjob.id
jspec.jobName = commands.getoutput('uuidgen 2> /dev/null')
jspec.coreCount = app.core_count
jspec.AtlasRelease = 'Atlas-%s' % app.atlas_release
jspec.homepackage = app.home_package
jspec.transformation = app.transformation
jspec.destinationDBlock = job.outputdata.datasetname
if job.outputdata.location:
jspec.destinationSE = job.outputdata.location
else:
jspec.destinationSE = job.backend.site
if job.inputdata:
jspec.prodDBlock = job.inputdata.dataset[0]
else:
jspec.prodDBlock = 'NULL'
if app.prod_source_label:
jspec.prodSourceLabel = app.prod_source_label
else:
jspec.prodSourceLabel = configPanda['prodSourceLabelRun']
jspec.processingType = configPanda['processingType']
jspec.specialHandling = configPanda['specialHandling']
jspec.computingSite = job.backend.site
jspec.cloud = job.backend.requirements.cloud
jspec.cmtConfig = app.atlas_cmtconfig
if app.dbrelease == 'LATEST':
try:
latest_dbrelease = getLatestDBReleaseCaching()
except:
from pandatools import Client
latest_dbrelease = Client.getLatestDBRelease()
m = re.search('(.*):DBRelease-(.*)\.tar\.gz', latest_dbrelease)
if m:
self.dbrelease_dataset = m.group(1)
self.dbrelease = m.group(2)
else:
raise ApplicationConfigurationError(
None,
"Error retrieving LATEST DBRelease. Try setting application.dbrelease manually."
)
else:
self.dbrelease_dataset = app.dbrelease_dataset
self.dbrelease = app.dbrelease
jspec.jobParameters = app.job_parameters
if self.dbrelease:
if self.dbrelease == 'current':
jspec.jobParameters += ' --DBRelease=current'
else:
if jspec.transformation.endswith(
"_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --DBRelease=DBRelease-%s.tar.gz' % (
self.dbrelease, )
else:
jspec.jobParameters += ' DBRelease=DBRelease-%s.tar.gz' % (
self.dbrelease, )
dbspec = FileSpec()
dbspec.lfn = 'DBRelease-%s.tar.gz' % self.dbrelease
dbspec.dataset = self.dbrelease_dataset
dbspec.prodDBlock = jspec.prodDBlock
dbspec.type = 'input'
jspec.addFile(dbspec)
if job.inputdata:
m = re.search('(.*)\.(.*)\.(.*)\.(.*)\.(.*)\.(.*)',
job.inputdata.dataset[0])
if not m:
logger.error("Error retrieving run number from dataset name")
#raise ApplicationConfigurationError(None, "Error retrieving run number from dataset name")
runnumber = 105200
else:
runnumber = int(m.group(2))
if jspec.transformation.endswith(
"_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --runNumber %d' % runnumber
else:
jspec.jobParameters += ' RunNumber=%d' % runnumber
# Output files.
randomized_lfns = []
ilfn = 0
for lfn, lfntype in zip(app.output_files, app.output_type):
ofspec = FileSpec()
if app.randomize_lfns:
randomized_lfn = lfn + (
'.%s.%d.%s' %
(job.backend.site, int(time.time()),
commands.getoutput('uuidgen 2> /dev/null')[:4]))
else:
randomized_lfn = lfn
ofspec.lfn = randomized_lfn
randomized_lfns.append(randomized_lfn)
ofspec.destinationDBlock = jspec.destinationDBlock
ofspec.destinationSE = jspec.destinationSE
ofspec.dataset = jspec.destinationDBlock
ofspec.type = 'output'
jspec.addFile(ofspec)
if jspec.transformation.endswith(
"_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --output%sFile %s' % (
lfntype, randomized_lfns[ilfn])
else:
jspec.jobParameters += ' output%sFile=%s' % (
lfntype, randomized_lfns[ilfn])
ilfn = ilfn + 1
# Input files.
if job.inputdata:
for guid, lfn, size, checksum, scope in zip(
job.inputdata.guids, job.inputdata.names,
job.inputdata.sizes, job.inputdata.checksums,
job.inputdata.scopes):
ifspec = FileSpec()
ifspec.lfn = lfn
ifspec.GUID = guid
ifspec.fsize = size
ifspec.md5sum = checksum
ifspec.scope = scope
ifspec.dataset = jspec.prodDBlock
ifspec.prodDBlock = jspec.prodDBlock
ifspec.type = 'input'
jspec.addFile(ifspec)
if app.input_type:
itype = app.input_type
else:
itype = m.group(5)
if jspec.transformation.endswith(
"_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --input%sFile %s' % (itype, ','.join(
job.inputdata.names))
else:
jspec.jobParameters += ' input%sFile=%s' % (itype, ','.join(
job.inputdata.names))
# Log files.
lfspec = FileSpec()
lfspec.lfn = '%s.job.log.tgz' % jspec.jobName
lfspec.destinationDBlock = jspec.destinationDBlock
lfspec.destinationSE = jspec.destinationSE
lfspec.dataset = jspec.destinationDBlock
lfspec.type = 'log'
jspec.addFile(lfspec)
return jspec
def send_job(jobid, siteid):
_logger.debug('Jobid: ' + str(jobid))
site = sites_.get(siteid)
job = jobs_.get(int(jobid))
cont = job.container
files_catalog = cont.files
fscope = getScope(job.owner.username)
datasetName = '{}:{}'.format(fscope, cont.guid)
distributive = job.distr.name
release = job.distr.release
# Prepare runScript
parameters = job.distr.command
parameters = parameters.replace("$COMMAND$", job.params)
parameters = parameters.replace("$USERNAME$", job.owner.username)
parameters = parameters.replace("$WORKINGGROUP$", job.owner.working_group)
# Prepare metadata
metadata = dict(user=job.owner.username)
# Prepare PanDA Object
pandajob = JobSpec()
pandajob.jobDefinitionID = int(time.time()) % 10000
pandajob.jobName = cont.guid
pandajob.transformation = client_config.DEFAULT_TRF
pandajob.destinationDBlock = datasetName
pandajob.destinationSE = site.se
pandajob.currentPriority = 1000
pandajob.prodSourceLabel = 'user'
pandajob.computingSite = site.ce
pandajob.cloud = 'RU'
pandajob.VO = 'atlas'
pandajob.prodDBlock = "%s:%s" % (fscope, pandajob.jobName)
pandajob.coreCount = job.corecount
pandajob.metadata = json.dumps(metadata)
#pandajob.workingGroup = job.owner.working_group
if site.encode_commands:
# It requires script wrapper on cluster side
pandajob.jobParameters = '%s %s %s "%s"' % (cont.guid, release,
distributive, parameters)
else:
pandajob.jobParameters = parameters
has_input = False
for fcc in files_catalog:
if fcc.type == 'input':
f = fcc.file
guid = f.guid
fileIT = FileSpec()
fileIT.lfn = f.lfn
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = guid
pandajob.addFile(fileIT)
has_input = True
if fcc.type == 'output':
f = fcc.file
fileOT = FileSpec()
fileOT.lfn = f.lfn
fileOT.destinationDBlock = pandajob.prodDBlock
fileOT.destinationSE = pandajob.destinationSE
fileOT.dataset = pandajob.prodDBlock
fileOT.type = 'output'
fileOT.scope = fscope
fileOT.GUID = f.guid
pandajob.addFile(fileOT)
# Save replica meta
fc.new_replica(f, site)
if not has_input:
# Add fake input
fileIT = FileSpec()
fileIT.lfn = "fake.input"
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = "fake.guid"
pandajob.addFile(fileIT)
# Prepare lof file
fileOL = FileSpec()
fileOL.lfn = "%s.log.tgz" % pandajob.jobName
fileOL.destinationDBlock = pandajob.destinationDBlock
fileOL.destinationSE = pandajob.destinationSE
fileOL.dataset = '{}:logs'.format(fscope)
fileOL.type = 'log'
fileOL.scope = 'panda'
pandajob.addFile(fileOL)
# Save log meta
log = File()
log.scope = fscope
log.lfn = fileOL.lfn
log.guid = getGUID(log.scope, log.lfn)
log.type = 'log'
log.status = 'defined'
files_.save(log)
# Save replica meta
fc.new_replica(log, site)
# Register file in container
fc.reg_file_in_cont(log, cont, 'log')
# Submit job
o = submitJobs([pandajob])
x = o[0]
try:
#update PandaID
PandaID = int(x[0])
job.pandaid = PandaID
job.ce = site.ce
except:
job.status = 'submit_error'
jobs_.save(job)
return 0
def send_job(jobid, siteid):
_logger.debug('Jobid: ' + str(jobid))
site = sites_.get(siteid)
job = jobs_.get(int(jobid))
cont = job.container
files_catalog = cont.files
fscope = getScope(job.owner.username)
datasetName = '{}:{}'.format(fscope, cont.guid)
distributive = job.distr.name
release = job.distr.release
# Prepare runScript
parameters = job.distr.command
parameters = parameters.replace("$COMMAND$", job.params)
parameters = parameters.replace("$USERNAME$", job.owner.username)
parameters = parameters.replace("$WORKINGGROUP$", job.owner.working_group)
# Prepare metadata
metadata = dict(user=job.owner.username)
# Prepare PanDA Object
pandajob = JobSpec()
pandajob.jobDefinitionID = int(time.time()) % 10000
pandajob.jobName = cont.guid
pandajob.transformation = client_config.DEFAULT_TRF
pandajob.destinationDBlock = datasetName
pandajob.destinationSE = site.se
pandajob.currentPriority = 1000
pandajob.prodSourceLabel = 'user'
pandajob.computingSite = site.ce
pandajob.cloud = 'RU'
pandajob.VO = 'atlas'
pandajob.prodDBlock = "%s:%s" % (fscope, pandajob.jobName)
pandajob.coreCount = job.corecount
pandajob.metadata = json.dumps(metadata)
#pandajob.workingGroup = job.owner.working_group
if site.encode_commands:
# It requires script wrapper on cluster side
pandajob.jobParameters = '%s %s %s "%s"' % (cont.guid, release, distributive, parameters)
else:
pandajob.jobParameters = parameters
has_input = False
for fcc in files_catalog:
if fcc.type == 'input':
f = fcc.file
guid = f.guid
fileIT = FileSpec()
fileIT.lfn = f.lfn
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = guid
pandajob.addFile(fileIT)
has_input = True
if fcc.type == 'output':
f = fcc.file
fileOT = FileSpec()
fileOT.lfn = f.lfn
fileOT.destinationDBlock = pandajob.prodDBlock
fileOT.destinationSE = pandajob.destinationSE
fileOT.dataset = pandajob.prodDBlock
fileOT.type = 'output'
fileOT.scope = fscope
fileOT.GUID = f.guid
pandajob.addFile(fileOT)
# Save replica meta
fc.new_replica(f, site)
if not has_input:
# Add fake input
fileIT = FileSpec()
fileIT.lfn = "fake.input"
fileIT.dataset = pandajob.prodDBlock
fileIT.prodDBlock = pandajob.prodDBlock
fileIT.type = 'input'
fileIT.scope = fscope
fileIT.status = 'ready'
fileIT.GUID = "fake.guid"
pandajob.addFile(fileIT)
# Prepare lof file
fileOL = FileSpec()
fileOL.lfn = "%s.log.tgz" % pandajob.jobName
fileOL.destinationDBlock = pandajob.destinationDBlock
fileOL.destinationSE = pandajob.destinationSE
fileOL.dataset = '{}:logs'.format(fscope)
fileOL.type = 'log'
fileOL.scope = 'panda'
pandajob.addFile(fileOL)
# Save log meta
log = File()
log.scope = fscope
log.lfn = fileOL.lfn
log.guid = getGUID(log.scope, log.lfn)
log.type = 'log'
log.status = 'defined'
files_.save(log)
# Save replica meta
fc.new_replica(log, site)
# Register file in container
fc.reg_file_in_cont(log, cont, 'log')
# Submit job
o = submitJobs([pandajob])
x = o[0]
try:
#update PandaID
PandaID = int(x[0])
job.pandaid = PandaID
job.ce = site.ce
except:
job.status = 'submit_error'
jobs_.save(job)
return 0