def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule): processingTypes = taskParamMap['processingType'].split('-') # set ddmBackEnd if not 'ddmBackEnd' in taskParamMap: taskParamMap['ddmBackEnd'] = 'rucio' # set sourceURL try: if taskParamMap.has_key('sourceURL'): for tmpItem in taskParamMap['jobParameters']: if tmpItem.has_key('value'): tmpItem['value'] = re.sub('\$\{SURL\}',taskParamMap['sourceURL'],tmpItem['value']) except: pass # set transPath if not taskParamMap.has_key('transPath'): if 'athena' in processingTypes: # athena taskParamMap['transPath'] = 'http://{0}:{1}/trf/user/runAthena-00-00-12'.format(panda_config.pserveralias, panda_config.pserverportcache) else: # general executable taskParamMap['transPath'] = 'http://{0}:{1}/trf/user/runGen-00-00-02'.format(panda_config.pserveralias, panda_config.pserverportcache) # set transPath for build if taskParamMap.has_key('buildSpec') and not taskParamMap['buildSpec'].has_key('transPath'): if 'athena' in processingTypes: # athena taskParamMap['buildSpec']['transPath'] = 'http://{0}:{1}/trf/user/buildJob-00-00-03'.format(panda_config.pserveralias, panda_config.pserverportcache) else: # general executable taskParamMap['buildSpec']['transPath'] = 'http://{0}:{1}/trf/user/buildGen-00-00-01'.format(panda_config.pserveralias, panda_config.pserverportcache) # set transPath for preprocessing if taskParamMap.has_key('preproSpec') and not taskParamMap['preproSpec'].has_key('transPath'): if 'evp' in processingTypes: # event picking taskParamMap['preproSpec']['transPath'] = 'http://{0}:{1}/trf/user/preEvtPick-00-00-01'.format(panda_config.pserveralias, panda_config.pserverportcache) elif 'grl' in processingTypes: # good run list taskParamMap['preproSpec']['transPath'] = 'http://{0}:{1}/trf/user/preGoodRunList-00-00-01'.format(panda_config.pserveralias, panda_config.pserverportcache) # set transPath for merge if taskParamMap.has_key('mergeSpec') and not taskParamMap['mergeSpec'].has_key('transPath'): taskParamMap['mergeSpec']['transPath'] = 'http://{0}:{1}/trf/user/runMerge-00-00-02'.format(panda_config.pserveralias, panda_config.pserverportcache) # min ram count if not 'ramCount' in taskParamMap: taskParamMap['ramCount'] = 2000 # update task parameters self.updatedTaskParams = taskParamMap # call base method TaskRefinerBase.extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule)
def __init__(self, taskBufferIF, ddmIF): TaskRefinerBase.__init__(self, taskBufferIF, ddmIF)
def extractCommon(self, jediTaskID, taskParamMap, workQueueMapper, splitRule): processingTypes = taskParamMap['processingType'].split('-') # set ddmBackEnd if not 'ddmBackEnd' in taskParamMap: taskParamMap['ddmBackEnd'] = 'rucio' # set sourceURL try: if taskParamMap.has_key('sourceURL'): for tmpItem in taskParamMap['jobParameters']: if tmpItem.has_key('value'): tmpItem['value'] = re.sub('\$\{SURL\}', taskParamMap['sourceURL'], tmpItem['value']) except: pass # set transPath if not taskParamMap.has_key('transPath'): if 'athena' in processingTypes: # athena taskParamMap[ 'transPath'] = 'http://{0}:{1}/trf/user/runAthena-00-00-12'.format( panda_config.pserveralias, panda_config.pserverportcache) else: # general executable taskParamMap[ 'transPath'] = 'http://{0}:{1}/trf/user/runGen-00-00-02'.format( panda_config.pserveralias, panda_config.pserverportcache) # set transPath for build if taskParamMap.has_key('buildSpec') and not taskParamMap[ 'buildSpec'].has_key('transPath'): if 'athena' in processingTypes: # athena taskParamMap['buildSpec'][ 'transPath'] = 'http://{0}:{1}/trf/user/buildJob-00-00-03'.format( panda_config.pserveralias, panda_config.pserverportcache) else: # general executable taskParamMap['buildSpec'][ 'transPath'] = 'http://{0}:{1}/trf/user/buildGen-00-00-01'.format( panda_config.pserveralias, panda_config.pserverportcache) # set transPath for preprocessing if taskParamMap.has_key('preproSpec') and not taskParamMap[ 'preproSpec'].has_key('transPath'): if 'evp' in processingTypes: # event picking taskParamMap['preproSpec'][ 'transPath'] = 'http://{0}:{1}/trf/user/preEvtPick-00-00-01'.format( panda_config.pserveralias, panda_config.pserverportcache) elif 'grl' in processingTypes: # good run list taskParamMap['preproSpec'][ 'transPath'] = 'http://{0}:{1}/trf/user/preGoodRunList-00-00-01'.format( panda_config.pserveralias, panda_config.pserverportcache) # set transPath for merge if taskParamMap.has_key('mergeSpec') and not taskParamMap[ 'mergeSpec'].has_key('transPath'): taskParamMap['mergeSpec'][ 'transPath'] = 'http://{0}:{1}/trf/user/runMerge-00-00-02'.format( panda_config.pserveralias, panda_config.pserverportcache) # min ram count if not 'ramCount' in taskParamMap: taskParamMap['ramCount'] = 2000 # update task parameters self.updatedTaskParams = taskParamMap # call base method TaskRefinerBase.extractCommon(self, jediTaskID, taskParamMap, workQueueMapper, splitRule)
def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule): processingTypes = taskParamMap['processingType'].split('-') # set ddmBackEnd if not 'ddmBackEnd' in taskParamMap: taskParamMap['ddmBackEnd'] = 'rucio' # set sourceURL try: if taskParamMap.has_key('sourceURL'): for tmpItem in taskParamMap['jobParameters']: if tmpItem.has_key('value'): tmpItem['value'] = re.sub('\$\{SURL\}',taskParamMap['sourceURL'],tmpItem['value']) except: pass # set transPath if not taskParamMap.has_key('transPath'): if 'athena' in processingTypes: # athena taskParamMap['transPath'] = 'http://{0}:{1}/trf/user/runAthena-00-00-12'.format(panda_config.pserveralias, panda_config.pserverportcache) elif 'cont' in processingTypes: # container taskParamMap['transPath'] = 'http://{0}:{1}/trf/user/runcontainer'.format(panda_config.pserveralias, panda_config.pserverportcache) else: # general executable taskParamMap['transPath'] = 'http://{0}:{1}/trf/user/runGen-00-00-02'.format(panda_config.pserveralias, panda_config.pserverportcache) # set transPath for build if taskParamMap.has_key('buildSpec') and not taskParamMap['buildSpec'].has_key('transPath'): if 'athena' in processingTypes: # athena taskParamMap['buildSpec']['transPath'] = 'http://{0}:{1}/trf/user/buildJob-00-00-03'.format(panda_config.pserveralias, panda_config.pserverportcache) else: # general executable taskParamMap['buildSpec']['transPath'] = 'http://{0}:{1}/trf/user/buildGen-00-00-01'.format(panda_config.pserveralias, panda_config.pserverportcache) # set transPath for preprocessing if taskParamMap.has_key('preproSpec') and not taskParamMap['preproSpec'].has_key('transPath'): if 'evp' in processingTypes: # event picking taskParamMap['preproSpec']['transPath'] = 'http://{0}:{1}/trf/user/preEvtPick-00-00-01'.format(panda_config.pserveralias, panda_config.pserverportcache) elif 'grl' in processingTypes: # good run list taskParamMap['preproSpec']['transPath'] = 'http://{0}:{1}/trf/user/preGoodRunList-00-00-01'.format(panda_config.pserveralias, panda_config.pserverportcache) # set transPath for merge if taskParamMap.has_key('mergeSpec') and not taskParamMap['mergeSpec'].has_key('transPath'): taskParamMap['mergeSpec']['transPath'] = 'http://{0}:{1}/trf/user/runMerge-00-00-02'.format(panda_config.pserveralias, panda_config.pserverportcache) # min ram count if not 'ramCount' in taskParamMap: taskParamMap['ramCount'] = 2000 # disk count if 'outDiskCount' not in taskParamMap: taskParamMap['outDiskCount'] = 500 taskParamMap['outDiskUnit'] = 'kB' # directIO if 'useLocalIO' not in taskParamMap and 'allowInputLAN' not in taskParamMap: taskParamMap['allowInputLAN'] = 'use' # current priority if 'currentPriority' in taskParamMap and (taskParamMap['currentPriority'] < 900 or taskParamMap['currentPriority'] > 1100): taskParamMap['currentPriority'] = 1000 isSU, isSG = self.taskBufferIF.isSuperUser(taskParamMap['userName']) if isSU or (isSG and 'workingGroup' in taskParamMap): # super high priority to jump over others if 'currentPriority' not in taskParamMap or taskParamMap['currentPriority'] < JobUtils.priorityTasksToJumpOver: taskParamMap['currentPriority'] = JobUtils.priorityTasksToJumpOver # max attempts if 'maxAttempt' not in taskParamMap: taskParamMap['maxAttempt'] = 10 if 'maxFailure' not in taskParamMap: taskParamMap['maxFailure'] = 3 # choose N % of tasks to enable input data motion fracTaskWithDataMotion = self.taskBufferIF.getConfigValue('taskrefiner', 'USER_TASKS_MOVE_INPUT', 'jedi', 'atlas') if fracTaskWithDataMotion is not None and fracTaskWithDataMotion > 0: if random.randint(1, 100) <= fracTaskWithDataMotion: if 'currentPriority' not in taskParamMap: taskParamMap['currentPriority'] = taskParamMap['taskPriority'] taskParamMap['taskPriority'] = 1001 # update task parameters self.updatedTaskParams = taskParamMap # call base method TaskRefinerBase.extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule)
def extractCommon(self, jediTaskID, taskParamMap, workQueueMapper, splitRule): # set ddmBackEnd if not "ddmBackEnd" in taskParamMap: taskParamMap["ddmBackEnd"] = "rucio" TaskRefinerBase.extractCommon(self, jediTaskID, taskParamMap, workQueueMapper, splitRule)
def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule): tmpLog = self.tmpLog # set ddmBackEnd if not 'ddmBackEnd' in taskParamMap: taskParamMap['ddmBackEnd'] = 'rucio' # get number of unprocessed events for event service autoEsConversion = False if 'esConvertible' in taskParamMap and taskParamMap['esConvertible'] == True: maxPrio = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_MAXTASKPRIORITY', 'jedi', 'atlas') minPrio = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_MINTASKPRIORITY', 'jedi', 'atlas') if maxPrio is not None and maxPrio < taskParamMap['taskPriority']: pass elif minPrio is not None and minPrio > taskParamMap['taskPriority']: pass else: # get threshold minNumEvents = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_EVENTPOOLSIZE', 'jedi', 'atlas') maxPending = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_MAXPENDING', 'jedi', 'atlas') nEvents, lastTaskTime, nPendingTasks = self.taskBufferIF.getNumUnprocessedEvents_JEDI(taskParamMap['vo'], taskParamMap['prodSourceLabel'], {'eventService': 1}, {'gshare': 'Validation'}) tmpStr = 'check for ES ' tmpStr += 'tot_num_unprocessed_events_AES={0} target_num_events_AES={1} last_AES_task_time={2} '.format(nEvents, minNumEvents, lastTaskTime) tmpStr += 'num_pending_tasks_AES={0} max_pending_tasks_AES={1} '.format(nPendingTasks, maxPending) tmpLog.info(tmpStr) # not chane many tasks at once if lastTaskTime is None or (lastTaskTime < datetime.datetime.utcnow() - datetime.timedelta(minutes=5)): if minNumEvents is not None and nEvents < minNumEvents and \ maxPending is not None and (maxPending is None or maxPending > nPendingTasks): autoEsConversion = True tmpLog.info('will be converted to AES unless it goes to pending') # add ES paramsters if ('esFraction' in taskParamMap and taskParamMap['esFraction'] > 0) or \ ('esConvertible' in taskParamMap and taskParamMap['esConvertible'] is True): tmpStr = '<PANDA_ES_ONLY>--eventService=True</PANDA_ES_ONLY>' taskParamMap['jobParameters'].append({'type':'constant', 'value':tmpStr}) if ('esFraction' in taskParamMap and taskParamMap['esFraction'] > 0) or autoEsConversion: if 'nEventsPerWorker' not in taskParamMap and \ (('esFraction' in taskParamMap and taskParamMap['esFraction'] > random.random()) or autoEsConversion): taskParamMap['nEventsPerWorker'] = 1 taskParamMap['registerEsFiles'] = True if 'nEsConsumers' not in taskParamMap: tmpVal = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_NESCONSUMERS', 'jedi', 'atlas') if tmpVal is None: tmpVal = 1 taskParamMap['nEsConsumers'] = tmpVal if 'nSitesPerJob' not in taskParamMap: tmpVal = self.taskBufferIF.getConfigValue('taskrefiner', 'AES_NSITESPERJOB', 'jedi', 'atlas') if tmpVal is not None: taskParamMap['nSitesPerJob'] = tmpVal if 'mergeEsOnOS' not in taskParamMap: taskParamMap['mergeEsOnOS'] = True if 'maxAttemptES' not in taskParamMap: taskParamMap['maxAttemptES'] = 1 if 'maxAttemptEsJob' not in taskParamMap: taskParamMap['maxAttemptEsJob'] = 0 if 'notDiscardEvents' not in taskParamMap: taskParamMap['notDiscardEvents'] = True if 'decAttOnFailedES' not in taskParamMap: taskParamMap['decAttOnFailedES'] = True taskParamMap['coreCount'] = 0 taskParamMap['resurrectConsumers'] = True TaskRefinerBase.extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule)
def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule): # set ddmBackEnd if not 'ddmBackEnd' in taskParamMap: taskParamMap['ddmBackEnd'] = 'rucio' TaskRefinerBase.extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule)