def defineEvgen16Job(self, i):
"""Define an Evgen16 job based on predefined values and randomly generated names
"""
job = JobSpec()
job.computingSite = self.__site
job.cloud = self.__cloud
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (uuid.uuid1(), i)
job.AtlasRelease = 'Atlas-16.6.2'
job.homepackage = 'AtlasProduction/16.6.2.1'
job.transformation = 'Evgen_trf.py'
job.destinationDBlock = self.__datasetName
job.destinationSE = self.__destName
job.currentPriority = 10000
job.prodSourceLabel = 'test'
job.cmtConfig = 'i686-slc5-gcc43-opt'
#Output file
fileO = FileSpec()
fileO.lfn = "%s.evgen.pool.root" % job.jobName
fileO.destinationDBlock = job.destinationDBlock
fileO.destinationSE = job.destinationSE
fileO.dataset = job.destinationDBlock
fileO.destinationDBlockToken = 'ATLASDATADISK'
fileO.type = 'output'
job.addFile(fileO)
#Log file
fileL = FileSpec()
fileL.lfn = "%s.job.log.tgz" % job.jobName
fileL.destinationDBlock = job.destinationDBlock
fileL.destinationSE = job.destinationSE
fileL.dataset = job.destinationDBlock
fileL.destinationDBlockToken = 'ATLASDATADISK'
fileL.type = 'log'
job.addFile(fileL)
job.jobParameters = "2760 105048 19901 101 200 MC10.105048.PythiaB_ccmu3mu1X.py %s NONE NONE NONE MC10JobOpts-latest-test.tar.gz" % fileO.lfn
return job
if len(sys.argv) > 1:
site = sys.argv[1]
else:
site = None
datasetName = 'panda.destDB.%s' % str(uuid.uuid4())
destName = None
jobList = []
for i in range(1):
job = JobSpec()
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (str(uuid.uuid4()), i)
job.AtlasRelease = 'Atlas-14.1.0'
job.homepackage = 'AtlasProduction/14.1.0.3'
job.transformation = 'csc_evgen_trf.py'
job.destinationDBlock = datasetName
job.destinationSE = destName
job.currentPriority = 100
job.prodSourceLabel = 'test'
job.computingSite = site
job.cloud = 'US'
job.cmtConfig = 'i686-slc4-gcc34-opt'
file = FileSpec()
file.lfn = "%s.evgen.pool.root" % job.jobName
file.destinationDBlock = job.destinationDBlock
file.destinationSE = job.destinationSE
file.dataset = job.destinationDBlock
def run(self):
try:
self.putLog('start %s' % self.evpFileName)
# lock evp file
self.evpFile = open(self.evpFileName)
try:
fcntl.flock(self.evpFile.fileno(),
fcntl.LOCK_EX | fcntl.LOCK_NB)
except Exception:
# relase
self.putLog("cannot lock %s" % self.evpFileName)
self.evpFile.close()
return True
# options
runEvtList = []
eventPickDataType = ''
eventPickStreamName = ''
eventPickDS = []
eventPickAmiTag = ''
eventPickNumSites = 1
inputFileList = []
tagDsList = []
tagQuery = ''
tagStreamRef = ''
skipDaTRI = False
runEvtGuidMap = {}
ei_api = ''
# read evp file
for tmpLine in self.evpFile:
tmpMatch = re.search('^([^=]+)=(.+)$', tmpLine)
# check format
if tmpMatch is None:
continue
tmpItems = tmpMatch.groups()
if tmpItems[0] == 'runEvent':
# get run and event number
tmpRunEvt = tmpItems[1].split(',')
if len(tmpRunEvt) == 2:
runEvtList.append(tmpRunEvt)
elif tmpItems[0] == 'eventPickDataType':
# data type
eventPickDataType = tmpItems[1]
elif tmpItems[0] == 'eventPickStreamName':
# stream name
eventPickStreamName = tmpItems[1]
elif tmpItems[0] == 'eventPickDS':
# dataset pattern
eventPickDS = tmpItems[1].split(',')
elif tmpItems[0] == 'eventPickAmiTag':
# AMI tag
eventPickAmiTag = tmpItems[1]
elif tmpItems[0] == 'eventPickNumSites':
# the number of sites where datasets are distributed
try:
eventPickNumSites = int(tmpItems[1])
except Exception:
pass
elif tmpItems[0] == 'userName':
# user name
self.userDN = tmpItems[1]
self.putLog("user=%s" % self.userDN)
elif tmpItems[0] == 'userTaskName':
# user task name
self.userTaskName = tmpItems[1]
elif tmpItems[0] == 'userDatasetName':
# user dataset name
self.userDatasetName = tmpItems[1]
elif tmpItems[0] == 'lockedBy':
# client name
self.lockedBy = tmpItems[1]
elif tmpItems[0] == 'creationTime':
# creation time
self.creationTime = tmpItems[1]
elif tmpItems[0] == 'params':
# parameters
self.params = tmpItems[1]
elif tmpItems[0] == 'ei_api':
# ei api parameter for MC
ei_api = tmpItems[1]
elif tmpItems[0] == 'inputFileList':
# input file list
inputFileList = tmpItems[1].split(',')
try:
inputFileList.remove('')
except Exception:
pass
elif tmpItems[0] == 'tagDS':
# TAG dataset
tagDsList = tmpItems[1].split(',')
elif tmpItems[0] == 'tagQuery':
# query for TAG
tagQuery = tmpItems[1]
elif tmpItems[0] == 'tagStreamRef':
# StreamRef for TAG
tagStreamRef = tmpItems[1]
if not tagStreamRef.endswith('_ref'):
tagStreamRef += '_ref'
elif tmpItems[0] == 'runEvtGuidMap':
# GUIDs
try:
runEvtGuidMap = eval(tmpItems[1])
except Exception:
pass
# extract task name
if self.userTaskName == '' and self.params != '':
try:
tmpMatch = re.search('--outDS(=| ) *([^ ]+)', self.params)
if tmpMatch is not None:
self.userTaskName = tmpMatch.group(2)
if not self.userTaskName.endswith('/'):
self.userTaskName += '/'
except Exception:
pass
# suppress DaTRI
if self.params != '':
if '--eventPickSkipDaTRI' in self.params:
skipDaTRI = True
# get compact user name
compactDN = self.taskBuffer.cleanUserID(self.userDN)
# get jediTaskID
self.jediTaskID = self.taskBuffer.getTaskIDwithTaskNameJEDI(
compactDN, self.userTaskName)
# get prodSourceLabel
self.prodSourceLabel, self.job_label = self.taskBuffer.getProdSourceLabelwithTaskID(
self.jediTaskID)
# convert run/event list to dataset/file list
tmpRet, locationMap, allFiles = self.pd2p.convertEvtRunToDatasets(
runEvtList, eventPickDataType, eventPickStreamName,
eventPickDS, eventPickAmiTag, self.userDN, runEvtGuidMap,
ei_api)
if not tmpRet:
if 'isFatal' in locationMap and locationMap['isFatal'] is True:
self.ignoreError = False
self.endWithError(
'Failed to convert the run/event list to a dataset/file list'
)
return False
# use only files in the list
if inputFileList != []:
tmpAllFiles = []
for tmpFile in allFiles:
if tmpFile['lfn'] in inputFileList:
tmpAllFiles.append(tmpFile)
allFiles = tmpAllFiles
# remove redundant CN from DN
tmpDN = self.userDN
tmpDN = re.sub('/CN=limited proxy', '', tmpDN)
tmpDN = re.sub('(/CN=proxy)+$', '', tmpDN)
# make dataset container
tmpRet = self.pd2p.registerDatasetContainerWithDatasets(
self.userDatasetName,
allFiles,
locationMap,
nSites=eventPickNumSites,
owner=tmpDN)
if not tmpRet:
self.endWithError('Failed to make a dataset container %s' %
self.userDatasetName)
return False
# skip DaTRI
if skipDaTRI:
# successfully terminated
self.putLog("skip DaTRI")
# update task
self.taskBuffer.updateTaskModTimeJEDI(self.jediTaskID)
else:
# get candidates
tmpRet, candidateMaps = self.pd2p.getCandidates(
self.userDatasetName,
self.prodSourceLabel,
self.job_label,
checkUsedFile=False,
useHidden=True)
if not tmpRet:
self.endWithError(
'Failed to find candidate for destination')
return False
# collect all candidates
allCandidates = []
for tmpDS in candidateMaps:
tmpDsVal = candidateMaps[tmpDS]
for tmpCloud in tmpDsVal:
tmpCloudVal = tmpDsVal[tmpCloud]
for tmpSiteName in tmpCloudVal[0]:
if tmpSiteName not in allCandidates:
allCandidates.append(tmpSiteName)
if allCandidates == []:
self.endWithError('No candidate for destination')
return False
# get list of dataset (container) names
if eventPickNumSites > 1:
# decompose container to transfer datasets separately
tmpRet, tmpOut = self.pd2p.getListDatasetReplicasInContainer(
self.userDatasetName)
if not tmpRet:
self.endWithError('Failed to get replicas in %s' %
self.userDatasetName)
return False
userDatasetNameList = list(tmpOut)
else:
# transfer container at once
userDatasetNameList = [self.userDatasetName]
# loop over all datasets
sitesUsed = []
for tmpUserDatasetName in userDatasetNameList:
# get size of dataset container
tmpRet, totalInputSize = rucioAPI.getDatasetSize(
tmpUserDatasetName)
if not tmpRet:
self.endWithError(
'Failed to get the size of {0} with {1}'.format(
tmpUserDatasetName, totalInputSize))
return False
# run brokerage
tmpJob = JobSpec()
tmpJob.AtlasRelease = ''
self.putLog("run brokerage for %s" % tmpDS)
pandaserver.brokerage.broker.schedule(
[tmpJob],
self.taskBuffer,
self.siteMapper,
True,
allCandidates,
True,
datasetSize=totalInputSize)
if tmpJob.computingSite.startswith('ERROR'):
self.endWithError('brokerage failed with %s' %
tmpJob.computingSite)
return False
self.putLog("site -> %s" % tmpJob.computingSite)
# send transfer request
try:
tmpDN = rucioAPI.parse_dn(tmpDN)
tmpStatus, userInfo = rucioAPI.finger(tmpDN)
if not tmpStatus:
raise RuntimeError(
'user info not found for {0} with {1}'.format(
tmpDN, userInfo))
tmpDN = userInfo['nickname']
tmpSiteSpec = self.siteMapper.getSite(
tmpJob.computingSite)
scope_input, scope_output = select_scope(
tmpSiteSpec, JobUtils.ANALY_PS, JobUtils.ANALY_PS)
tmpDQ2ID = tmpSiteSpec.ddm_input[scope_input]
tmpMsg = "%s ds=%s site=%s id=%s" % (
'registerDatasetLocation for DaTRI ',
tmpUserDatasetName, tmpDQ2ID, tmpDN)
self.putLog(tmpMsg)
rucioAPI.registerDatasetLocation(
tmpDS, [tmpDQ2ID],
lifetime=14,
owner=tmpDN,
activity="User Subscriptions")
self.putLog('OK')
except Exception:
errType, errValue = sys.exc_info()[:2]
tmpStr = 'Failed to send transfer request : %s %s' % (
errType, errValue)
tmpStr.strip()
tmpStr += traceback.format_exc()
self.endWithError(tmpStr)
return False
# list of sites already used
sitesUsed.append(tmpJob.computingSite)
self.putLog("used %s sites" % len(sitesUsed))
# set candidates
if len(sitesUsed) >= eventPickNumSites:
# reset candidates to limit the number of sites
allCandidates = sitesUsed
sitesUsed = []
else:
# remove site
allCandidates.remove(tmpJob.computingSite)
# send email notification for success
tmpMsg = 'A transfer request was successfully sent to Rucio.\n'
tmpMsg += 'Your task will get started once transfer is completed.'
self.sendEmail(True, tmpMsg)
try:
# unlock and delete evp file
fcntl.flock(self.evpFile.fileno(), fcntl.LOCK_UN)
self.evpFile.close()
os.remove(self.evpFileName)
except Exception:
pass
# successfully terminated
self.putLog("end %s" % self.evpFileName)
return True
except Exception:
errType, errValue = sys.exc_info()[:2]
self.endWithError('Got exception %s:%s %s' %
(errType, errValue, traceback.format_exc()))
return False
from pandaserver.taskbuffer.JobSpec import JobSpec
from pandaserver.taskbuffer.FileSpec import FileSpec
site = sys.argv[1]
cloud = sys.argv[2]
datasetName = 'panda.destDB.%s' % str(uuid.uuid4())
destName = None
jobList = []
for i in range(1):
job = JobSpec()
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (str(uuid.uuid4()), i)
job.AtlasRelease = 'Atlas-17.0.5'
job.homepackage = 'AtlasProduction/17.0.5.6'
job.transformation = 'Evgen_trf.py'
job.destinationDBlock = datasetName
job.destinationSE = destName
job.currentPriority = 10000
job.prodSourceLabel = 'test'
job.computingSite = site
job.cloud = cloud
job.cmtConfig = 'i686-slc5-gcc43-opt'
file = FileSpec()
file.lfn = "%s.evgen.pool.root" % job.jobName
file.destinationDBlock = job.destinationDBlock
file.destinationSE = job.destinationSE
file.dataset = job.destinationDBlock
None,
#'misal1_mc12.005802.JF17_pythia_jet_filter.digit.RDO.v12000601_tid008610._11639.pool.root.1':None,
#'misal1_mc12.005200.T1_McAtNlo_Jimmy.digit.RDO.v12000601_tid007554._03634.pool.root.1':None,
#'misal1_mc12.005200.T1_McAtNlo_Jimmy.digit.RDO.v12000601_tid007554._03248.pool.root.1':None,
#'misal1_mc12.005200.T1_McAtNlo_Jimmy.digit.RDO.v12000601_tid007554._03634.pool.root.1':None,
}
jobList = []
index = 0
for lfn in files:
index += 1
job = JobSpec()
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = "%s_%d" % (str(uuid.uuid4()), index)
job.AtlasRelease = 'Atlas-12.0.6'
job.homepackage = 'AtlasProduction/12.0.6.4'
job.transformation = 'csc_reco_trf.py'
job.destinationDBlock = datasetName
job.destinationSE = destName
job.computingSite = site
#job.prodDBlock = 'misal1_mc12.005200.T1_McAtNlo_Jimmy.digit.RDO.v12000601_tid007554'
job.prodDBlock = 'misal1_mc12.005802.JF17_pythia_jet_filter.digit.RDO.v12000601_tid008610'
job.cloud = 'US'
job.prodSourceLabel = 'test'
job.currentPriority = 10000
job.cmtConfig = 'i686-slc4-gcc34-opt'
fileI = FileSpec()
fileI.dataset = job.prodDBlock
destName = 'BNL_ATLAS_2'
files = {
'EVNT.023986._00001.pool.root.1': None,
#'EVNT.023989._00001.pool.root.1':None,
}
jobList = []
index = 0
for lfn in files:
index += 1
job = JobSpec()
job.jobDefinitionID = (time.time()) % 10000
job.jobName = "%s_%d" % (str(uuid.uuid4()), index)
job.AtlasRelease = 'Atlas-14.2.20'
job.homepackage = 'AtlasProduction/14.2.20.1'
job.transformation = 'csc_simul_reco_trf.py'
job.destinationDBlock = datasetName
job.destinationSE = destName
job.computingSite = site
job.prodDBlock = 'mc08.105031.Jimmy_jetsJ2.evgen.EVNT.e347_tid023986'
#job.prodDBlock = 'mc08.105034.Jimmy_jetsJ5.evgen.EVNT.e347_tid023989'
job.prodSourceLabel = 'test'
job.processingType = 'test'
job.currentPriority = 10000
job.cloud = cloud
fileI = FileSpec()
fileI.dataset = job.prodDBlock
datasetName = 'panda.destDB.%s' % str(uuid.uuid4())
destName = 'BNL_ATLAS_2'
#destName = 'BU_ATLAS_Tier2'
files = {
'mc11.007204.singlepart_mu4.evgen.EVNT.v11000302._00037.pool.root.1': None,
'mc11.007204.singlepart_mu4.evgen.EVNT.v11000302._00038.pool.root.1': None,
}
jobList = []
for lfn in files:
job = JobSpec()
job.jobDefinitionID = int(time.time()) % 10000
job.jobName = str(uuid.uuid4())
job.AtlasRelease = 'Atlas-11.0.3'
job.homepackage = 'JobTransforms-11-00-03-02'
job.transformation = 'share/csc.simul.trf'
job.destinationDBlock = datasetName
job.destinationSE = destName
job.computingSite = site
job.prodDBlock = 'mc11.007204.singlepart_mu4.evgen.EVNT.v11000302'
job.cmtConfig = 'i686-slc4-gcc34-opt'
job.prodSourceLabel = 'test'
job.currentPriority = 1000
fileI = FileSpec()
fileI.dataset = job.prodDBlock
fileI.prodDBlock = job.prodDBlock
fileI.lfn = lfn