def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule):
processingTypes = taskParamMap['processingType'].split('-')
# set ddmBackEnd
if not 'ddmBackEnd' in taskParamMap:
taskParamMap['ddmBackEnd'] = 'rucio'
# set sourceURL
try:
if taskParamMap.has_key('sourceURL'):
for tmpItem in taskParamMap['jobParameters']:
if tmpItem.has_key('value'):
tmpItem['value'] = re.sub('\$\{SURL\}',taskParamMap['sourceURL'],tmpItem['value'])
except:
pass
# set transPath
if not taskParamMap.has_key('transPath'):
if 'athena' in processingTypes:
# athena
taskParamMap['transPath'] = 'http://{0}:{1}/trf/user/runAthena-00-00-12'.format(panda_config.pserveralias,
panda_config.pserverportcache)
else:
# general executable
taskParamMap['transPath'] = 'http://{0}:{1}/trf/user/runGen-00-00-02'.format(panda_config.pserveralias,
panda_config.pserverportcache)
# set transPath for build
if taskParamMap.has_key('buildSpec') and not taskParamMap['buildSpec'].has_key('transPath'):
if 'athena' in processingTypes:
# athena
taskParamMap['buildSpec']['transPath'] = 'http://{0}:{1}/trf/user/buildJob-00-00-03'.format(panda_config.pserveralias,
panda_config.pserverportcache)
else:
# general executable
taskParamMap['buildSpec']['transPath'] = 'http://{0}:{1}/trf/user/buildGen-00-00-01'.format(panda_config.pserveralias,
panda_config.pserverportcache)
# set transPath for preprocessing
if taskParamMap.has_key('preproSpec') and not taskParamMap['preproSpec'].has_key('transPath'):
if 'evp' in processingTypes:
# event picking
taskParamMap['preproSpec']['transPath'] = 'http://{0}:{1}/trf/user/preEvtPick-00-00-01'.format(panda_config.pserveralias,
panda_config.pserverportcache)
elif 'grl' in processingTypes:
# good run list
taskParamMap['preproSpec']['transPath'] = 'http://{0}:{1}/trf/user/preGoodRunList-00-00-01'.format(panda_config.pserveralias,
panda_config.pserverportcache)
# set transPath for merge
if taskParamMap.has_key('mergeSpec') and not taskParamMap['mergeSpec'].has_key('transPath'):
taskParamMap['mergeSpec']['transPath'] = 'http://{0}:{1}/trf/user/runMerge-00-00-02'.format(panda_config.pserveralias,
panda_config.pserverportcache)
# min ram count
if not 'ramCount' in taskParamMap:
taskParamMap['ramCount'] = 2000
# update task parameters
self.updatedTaskParams = taskParamMap
# call base method
TaskRefinerBase.extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule)
def __init__(self, taskBufferIF, ddmIF):
TaskRefinerBase.__init__(self, taskBufferIF, ddmIF)
def extractCommon(self, jediTaskID, taskParamMap, workQueueMapper,
splitRule):
processingTypes = taskParamMap['processingType'].split('-')
# set ddmBackEnd
if not 'ddmBackEnd' in taskParamMap:
taskParamMap['ddmBackEnd'] = 'rucio'
# set sourceURL
try:
if taskParamMap.has_key('sourceURL'):
for tmpItem in taskParamMap['jobParameters']:
if tmpItem.has_key('value'):
tmpItem['value'] = re.sub('\$\{SURL\}',
taskParamMap['sourceURL'],
tmpItem['value'])
except:
pass
# set transPath
if not taskParamMap.has_key('transPath'):
if 'athena' in processingTypes:
# athena
taskParamMap[
'transPath'] = 'http://{0}:{1}/trf/user/runAthena-00-00-12'.format(
panda_config.pserveralias,
panda_config.pserverportcache)
else:
# general executable
taskParamMap[
'transPath'] = 'http://{0}:{1}/trf/user/runGen-00-00-02'.format(
panda_config.pserveralias,
panda_config.pserverportcache)
# set transPath for build
if taskParamMap.has_key('buildSpec') and not taskParamMap[
'buildSpec'].has_key('transPath'):
if 'athena' in processingTypes:
# athena
taskParamMap['buildSpec'][
'transPath'] = 'http://{0}:{1}/trf/user/buildJob-00-00-03'.format(
panda_config.pserveralias,
panda_config.pserverportcache)
else:
# general executable
taskParamMap['buildSpec'][
'transPath'] = 'http://{0}:{1}/trf/user/buildGen-00-00-01'.format(
panda_config.pserveralias,
panda_config.pserverportcache)
# set transPath for preprocessing
if taskParamMap.has_key('preproSpec') and not taskParamMap[
'preproSpec'].has_key('transPath'):
if 'evp' in processingTypes:
# event picking
taskParamMap['preproSpec'][
'transPath'] = 'http://{0}:{1}/trf/user/preEvtPick-00-00-01'.format(
panda_config.pserveralias,
panda_config.pserverportcache)
elif 'grl' in processingTypes:
# good run list
taskParamMap['preproSpec'][
'transPath'] = 'http://{0}:{1}/trf/user/preGoodRunList-00-00-01'.format(
panda_config.pserveralias,
panda_config.pserverportcache)
# set transPath for merge
if taskParamMap.has_key('mergeSpec') and not taskParamMap[
'mergeSpec'].has_key('transPath'):
taskParamMap['mergeSpec'][
'transPath'] = 'http://{0}:{1}/trf/user/runMerge-00-00-02'.format(
panda_config.pserveralias, panda_config.pserverportcache)
# min ram count
if not 'ramCount' in taskParamMap:
taskParamMap['ramCount'] = 2000
# update task parameters
self.updatedTaskParams = taskParamMap
# call base method
TaskRefinerBase.extractCommon(self, jediTaskID, taskParamMap,
workQueueMapper, splitRule)
def __init__(self, taskBufferIF, ddmIF):
TaskRefinerBase.__init__(self, taskBufferIF, ddmIF)
def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule):
processingTypes = taskParamMap['processingType'].split('-')
# set ddmBackEnd
if not 'ddmBackEnd' in taskParamMap:
taskParamMap['ddmBackEnd'] = 'rucio'
# set sourceURL
try:
if taskParamMap.has_key('sourceURL'):
for tmpItem in taskParamMap['jobParameters']:
if tmpItem.has_key('value'):
tmpItem['value'] = re.sub('\$\{SURL\}',taskParamMap['sourceURL'],tmpItem['value'])
except:
pass
# set transPath
if not taskParamMap.has_key('transPath'):
if 'athena' in processingTypes:
# athena
taskParamMap['transPath'] = 'http://{0}:{1}/trf/user/runAthena-00-00-12'.format(panda_config.pserveralias,
panda_config.pserverportcache)
elif 'cont' in processingTypes:
# container
taskParamMap['transPath'] = 'http://{0}:{1}/trf/user/runcontainer'.format(panda_config.pserveralias,
panda_config.pserverportcache)
else:
# general executable
taskParamMap['transPath'] = 'http://{0}:{1}/trf/user/runGen-00-00-02'.format(panda_config.pserveralias,
panda_config.pserverportcache)
# set transPath for build
if taskParamMap.has_key('buildSpec') and not taskParamMap['buildSpec'].has_key('transPath'):
if 'athena' in processingTypes:
# athena
taskParamMap['buildSpec']['transPath'] = 'http://{0}:{1}/trf/user/buildJob-00-00-03'.format(panda_config.pserveralias,
panda_config.pserverportcache)
else:
# general executable
taskParamMap['buildSpec']['transPath'] = 'http://{0}:{1}/trf/user/buildGen-00-00-01'.format(panda_config.pserveralias,
panda_config.pserverportcache)
# set transPath for preprocessing
if taskParamMap.has_key('preproSpec') and not taskParamMap['preproSpec'].has_key('transPath'):
if 'evp' in processingTypes:
# event picking
taskParamMap['preproSpec']['transPath'] = 'http://{0}:{1}/trf/user/preEvtPick-00-00-01'.format(panda_config.pserveralias,
panda_config.pserverportcache)
elif 'grl' in processingTypes:
# good run list
taskParamMap['preproSpec']['transPath'] = 'http://{0}:{1}/trf/user/preGoodRunList-00-00-01'.format(panda_config.pserveralias,
panda_config.pserverportcache)
# set transPath for merge
if taskParamMap.has_key('mergeSpec') and not taskParamMap['mergeSpec'].has_key('transPath'):
taskParamMap['mergeSpec']['transPath'] = 'http://{0}:{1}/trf/user/runMerge-00-00-02'.format(panda_config.pserveralias,
panda_config.pserverportcache)
# min ram count
if not 'ramCount' in taskParamMap:
taskParamMap['ramCount'] = 2000
# disk count
if 'outDiskCount' not in taskParamMap:
taskParamMap['outDiskCount'] = 500
taskParamMap['outDiskUnit'] = 'kB'
# directIO
if 'useLocalIO' not in taskParamMap and 'allowInputLAN' not in taskParamMap:
taskParamMap['allowInputLAN'] = 'use'
# current priority
if 'currentPriority' in taskParamMap and (taskParamMap['currentPriority'] < 900 or taskParamMap['currentPriority'] > 1100):
taskParamMap['currentPriority'] = 1000
isSU, isSG = self.taskBufferIF.isSuperUser(taskParamMap['userName'])
if isSU or (isSG and 'workingGroup' in taskParamMap):
# super high priority to jump over others
if 'currentPriority' not in taskParamMap or taskParamMap['currentPriority'] < JobUtils.priorityTasksToJumpOver:
taskParamMap['currentPriority'] = JobUtils.priorityTasksToJumpOver
# max attempts
if 'maxAttempt' not in taskParamMap:
taskParamMap['maxAttempt'] = 10
if 'maxFailure' not in taskParamMap:
taskParamMap['maxFailure'] = 3
# choose N % of tasks to enable input data motion
fracTaskWithDataMotion = self.taskBufferIF.getConfigValue('taskrefiner', 'USER_TASKS_MOVE_INPUT', 'jedi', 'atlas')
if fracTaskWithDataMotion is not None and fracTaskWithDataMotion > 0:
if random.randint(1, 100) <= fracTaskWithDataMotion:
if 'currentPriority' not in taskParamMap:
taskParamMap['currentPriority'] = taskParamMap['taskPriority']
taskParamMap['taskPriority'] = 1001
# update task parameters
self.updatedTaskParams = taskParamMap
# call base method
TaskRefinerBase.extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule)
def extractCommon(self, jediTaskID, taskParamMap, workQueueMapper, splitRule):
# set ddmBackEnd
if not "ddmBackEnd" in taskParamMap:
taskParamMap["ddmBackEnd"] = "rucio"
TaskRefinerBase.extractCommon(self, jediTaskID, taskParamMap, workQueueMapper, splitRule)
def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule):
tmpLog = self.tmpLog
# set ddmBackEnd
if not 'ddmBackEnd' in taskParamMap:
taskParamMap['ddmBackEnd'] = 'rucio'
# get number of unprocessed events for event service
autoEsConversion = False
if 'esConvertible' in taskParamMap and taskParamMap['esConvertible'] == True:
maxPrio = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_MAXTASKPRIORITY', 'jedi', 'atlas')
minPrio = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_MINTASKPRIORITY', 'jedi', 'atlas')
if maxPrio is not None and maxPrio < taskParamMap['taskPriority']:
pass
elif minPrio is not None and minPrio > taskParamMap['taskPriority']:
pass
else:
# get threshold
minNumEvents = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_EVENTPOOLSIZE', 'jedi', 'atlas')
maxPending = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_MAXPENDING', 'jedi', 'atlas')
nEvents, lastTaskTime, nPendingTasks = self.taskBufferIF.getNumUnprocessedEvents_JEDI(taskParamMap['vo'],
taskParamMap['prodSourceLabel'],
{'eventService': 1},
{'gshare': 'Validation'})
tmpStr = 'check for ES '
tmpStr += 'tot_num_unprocessed_events_AES={0} target_num_events_AES={1} last_AES_task_time={2} '.format(nEvents,
minNumEvents,
lastTaskTime)
tmpStr += 'num_pending_tasks_AES={0} max_pending_tasks_AES={1} '.format(nPendingTasks, maxPending)
tmpLog.info(tmpStr)
# not chane many tasks at once
if lastTaskTime is None or (lastTaskTime < datetime.datetime.utcnow() - datetime.timedelta(minutes=5)):
if minNumEvents is not None and nEvents < minNumEvents and \
maxPending is not None and (maxPending is None or maxPending > nPendingTasks):
autoEsConversion = True
tmpLog.info('will be converted to AES unless it goes to pending')
# add ES paramsters
if ('esFraction' in taskParamMap and taskParamMap['esFraction'] > 0) or \
('esConvertible' in taskParamMap and taskParamMap['esConvertible'] is True):
tmpStr = '<PANDA_ES_ONLY>--eventService=True</PANDA_ES_ONLY>'
taskParamMap['jobParameters'].append({'type':'constant',
'value':tmpStr})
if ('esFraction' in taskParamMap and taskParamMap['esFraction'] > 0) or autoEsConversion:
if 'nEventsPerWorker' not in taskParamMap and \
(('esFraction' in taskParamMap and taskParamMap['esFraction'] > random.random()) or autoEsConversion):
taskParamMap['nEventsPerWorker'] = 1
taskParamMap['registerEsFiles'] = True
if 'nEsConsumers' not in taskParamMap:
tmpVal = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_NESCONSUMERS', 'jedi', 'atlas')
if tmpVal is None:
tmpVal = 1
taskParamMap['nEsConsumers'] = tmpVal
if 'nSitesPerJob' not in taskParamMap:
tmpVal = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_NSITESPERJOB', 'jedi', 'atlas')
if tmpVal is not None:
taskParamMap['nSitesPerJob'] = tmpVal
if 'mergeEsOnOS' not in taskParamMap:
taskParamMap['mergeEsOnOS'] = True
if 'maxAttemptES' not in taskParamMap:
taskParamMap['maxAttemptES'] = 1
if 'maxAttemptEsJob' not in taskParamMap:
taskParamMap['maxAttemptEsJob'] = 0
if 'notDiscardEvents' not in taskParamMap:
taskParamMap['notDiscardEvents'] = True
if 'decAttOnFailedES' not in taskParamMap:
taskParamMap['decAttOnFailedES'] = True
taskParamMap['coreCount'] = 0
taskParamMap['resurrectConsumers'] = True
TaskRefinerBase.extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule)
def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule):
# set ddmBackEnd
if not 'ddmBackEnd' in taskParamMap:
taskParamMap['ddmBackEnd'] = 'rucio'
TaskRefinerBase.extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule)
def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule):
# set ddmBackEnd
if not 'ddmBackEnd' in taskParamMap:
taskParamMap['ddmBackEnd'] = 'rucio'
TaskRefinerBase.extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule)
