Beispiel #1
0
 def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule):
     # make task spec
     taskSpec = JediTaskSpec()
     taskSpec.jediTaskID = jediTaskID
     taskSpec.taskName = taskParamMap['taskName']
     taskSpec.userName = taskParamMap['userName']
     taskSpec.vo = taskParamMap['vo']     
     taskSpec.prodSourceLabel = taskParamMap['prodSourceLabel']
     taskSpec.taskPriority = taskParamMap['taskPriority']
     taskSpec.currentPriority = taskSpec.taskPriority
     taskSpec.architecture = taskParamMap['architecture']
     taskSpec.transUses = taskParamMap['transUses']
     taskSpec.transHome = taskParamMap['transHome']
     taskSpec.transPath = taskParamMap['transPath']
     taskSpec.processingType = taskParamMap['processingType']
     taskSpec.taskType = taskParamMap['taskType']
     taskSpec.splitRule = splitRule
     taskSpec.startTime = datetime.datetime.utcnow()
     if taskParamMap.has_key('workingGroup'):
         taskSpec.workingGroup = taskParamMap['workingGroup']
     if taskParamMap.has_key('countryGroup'):
         taskSpec.countryGroup = taskParamMap['countryGroup']
     if taskParamMap.has_key('ticketID'):
         taskSpec.ticketID = taskParamMap['ticketID']
     if taskParamMap.has_key('ticketSystemType'):
         taskSpec.ticketSystemType = taskParamMap['ticketSystemType']
     if taskParamMap.has_key('reqID'):
         taskSpec.reqID = taskParamMap['reqID']
     else:
         taskSpec.reqID = jediTaskID
     if taskParamMap.has_key('coreCount'):
         taskSpec.coreCount = taskParamMap['coreCount']
     else:
         taskSpec.coreCount = 1
     if taskParamMap.has_key('walltime'):
         taskSpec.walltime = taskParamMap['walltime']
     else:
         taskSpec.walltime = 0
     if taskParamMap.has_key('walltimeUnit'):
         taskSpec.walltimeUnit = taskParamMap['walltimeUnit']
     if taskParamMap.has_key('outDiskCount'):
         taskSpec.outDiskCount = taskParamMap['outDiskCount']
     else:
         taskSpec.outDiskCount = 0
     if 'outDiskUnit' in taskParamMap:
         taskSpec.outDiskUnit = taskParamMap['outDiskUnit']
     if taskParamMap.has_key('workDiskCount'):
         taskSpec.workDiskCount = taskParamMap['workDiskCount']
     else:
         taskSpec.workDiskCount = 0
     if taskParamMap.has_key('workDiskUnit'):
         taskSpec.workDiskUnit = taskParamMap['workDiskUnit']
     if taskParamMap.has_key('ramCount'):
         taskSpec.ramCount = taskParamMap['ramCount']
     else:
         taskSpec.ramCount = 0
     if taskParamMap.has_key('ramUnit'):
         taskSpec.ramUnit = taskParamMap['ramUnit']
     if taskParamMap.has_key('baseRamCount'):
         taskSpec.baseRamCount = taskParamMap['baseRamCount']
     else:
         taskSpec.baseRamCount = 0
     # HS06 stuff
     if 'cpuTimeUnit' in taskParamMap:
         taskSpec.cpuTimeUnit = taskParamMap['cpuTimeUnit']
     if 'cpuTime' in taskParamMap:
         taskSpec.cpuTime = taskParamMap['cpuTime']
     if 'cpuEfficiency' in taskParamMap:
         taskSpec.cpuEfficiency = taskParamMap['cpuEfficiency']
     else:
         # 90% of cpu efficiency by default
         taskSpec.cpuEfficiency = 90
     if 'baseWalltime' in taskParamMap:
         taskSpec.baseWalltime = taskParamMap['baseWalltime']
     else:
         # 10min of offset by default
         taskSpec.baseWalltime = 10*60
     # for merge
     if 'mergeRamCount' in taskParamMap:
         taskSpec.mergeRamCount = taskParamMap['mergeRamCount']
     if 'mergeCoreCount' in taskParamMap:
         taskSpec.mergeCoreCount = taskParamMap['mergeCoreCount']
     # scout
     if not taskParamMap.has_key('skipScout') and not taskSpec.isPostScout():
         taskSpec.setUseScout(True)
     # cloud
     if taskParamMap.has_key('cloud'):
         self.cloudName = taskParamMap['cloud']
         taskSpec.cloud = self.cloudName
     else:
         # set dummy to force update
         taskSpec.cloud = 'dummy'
         taskSpec.cloud = None
     # site
     if taskParamMap.has_key('site'):
         self.siteName = taskParamMap['site']
         taskSpec.site = self.siteName
     else:
         # set dummy to force update
         taskSpec.site = 'dummy'
         taskSpec.site = None
     # nucleus
     if 'nucleus' in taskParamMap:
         taskSpec.nucleus = taskParamMap['nucleus']
     # preset some parameters for job cloning
     if 'useJobCloning' in taskParamMap:
         # set implicit parameters
         if not 'nEventsPerWorker' in taskParamMap:
             taskParamMap['nEventsPerWorker'] = 1
         if not 'nSitesPerJob' in taskParamMap:
             taskParamMap['nSitesPerJob'] = 2
         if not 'nEsConsumers' in taskParamMap:
             taskParamMap['nEsConsumers'] = taskParamMap['nSitesPerJob']
     # event service
     if taskParamMap.has_key('nEventsPerWorker'):
         taskSpec.eventService = 1
     else:
         taskSpec.eventService = 0
     # ttcr: requested time to completion
     if taskParamMap.has_key('ttcrTimestamp'):
         try:
             # get rid of the +00:00 timezone string and parse the timestamp
             taskSpec.ttcRequested = datetime.datetime.strptime(taskParamMap['ttcrTimestamp'].split('+')[0], '%Y-%m-%d %H:%M:%S.%f')
         except (IndexError, ValueError):
             pass
     # goal
     if 'goal' in taskParamMap:
         try:
             taskSpec.goal = int(float(taskParamMap['goal'])*10)
             if taskSpec.goal >= 1000:
                 taskSpec.goal = None
         except:
             pass
     # campaign
     if taskParamMap.has_key('campaign'):
         taskSpec.campaign = taskParamMap['campaign']
     # work queue
     workQueue = None
     if 'workQueueName' in taskParamMap:
         # work queue is specified
         workQueue = workQueueMapper.getQueueWithName(taskSpec.vo,taskSpec.prodSourceLabel,taskParamMap['workQueueName'])
     if workQueue == None:
         # get work queue based on task attributes
         workQueue,tmpStr = workQueueMapper.getQueueWithSelParams(taskSpec.vo,
                                                                  taskSpec.prodSourceLabel,
                                                                  processingType=taskSpec.processingType,
                                                                  workingGroup=taskSpec.workingGroup,
                                                                  coreCount=taskSpec.coreCount,
                                                                  site=taskSpec.site)
     if workQueue == None:
         errStr  = 'workqueue is undefined for vo={0} labal={1} '.format(taskSpec.vo,taskSpec.prodSourceLabel)
         errStr += 'processingType={0} workingGroup={1} coreCount={2} '.format(taskSpec.processingType,
                                                                               taskSpec.workingGroup,
                                                                               taskSpec.coreCount)
         raise RuntimeError,errStr
     taskSpec.workQueue_ID = workQueue.queue_id
     self.taskSpec = taskSpec
     # set split rule    
     if 'tgtNumEventsPerJob' in taskParamMap:
         # set nEventsPerJob not respect file boundaries when nFilesPerJob is not used
         if not 'nFilesPerJob' in taskParamMap:
             self.setSplitRule(None,taskParamMap['tgtNumEventsPerJob'],JediTaskSpec.splitRuleToken['nEventsPerJob'])
     self.setSplitRule(taskParamMap,'nFilesPerJob',     JediTaskSpec.splitRuleToken['nFilesPerJob'])
     self.setSplitRule(taskParamMap,'nEventsPerJob',    JediTaskSpec.splitRuleToken['nEventsPerJob'])
     self.setSplitRule(taskParamMap,'nGBPerJob',        JediTaskSpec.splitRuleToken['nGBPerJob'])
     self.setSplitRule(taskParamMap,'nMaxFilesPerJob',  JediTaskSpec.splitRuleToken['nMaxFilesPerJob'])
     self.setSplitRule(taskParamMap,'nEventsPerWorker', JediTaskSpec.splitRuleToken['nEventsPerWorker'])
     self.setSplitRule(taskParamMap,'useLocalIO',       JediTaskSpec.splitRuleToken['useLocalIO'])
     self.setSplitRule(taskParamMap,'disableAutoRetry', JediTaskSpec.splitRuleToken['disableAutoRetry'])
     self.setSplitRule(taskParamMap,'nEsConsumers',     JediTaskSpec.splitRuleToken['nEsConsumers'])
     self.setSplitRule(taskParamMap,'waitInput',        JediTaskSpec.splitRuleToken['waitInput'])
     self.setSplitRule(taskParamMap,'addNthFieldToLFN', JediTaskSpec.splitRuleToken['addNthFieldToLFN'])
     self.setSplitRule(taskParamMap,'scoutSuccessRate', JediTaskSpec.splitRuleToken['scoutSuccessRate'])
     self.setSplitRule(taskParamMap,'t1Weight',         JediTaskSpec.splitRuleToken['t1Weight'])
     self.setSplitRule(taskParamMap,'maxAttemptES',     JediTaskSpec.splitRuleToken['maxAttemptES'])
     self.setSplitRule(taskParamMap,'nSitesPerJob',     JediTaskSpec.splitRuleToken['nSitesPerJob'])
     self.setSplitRule(taskParamMap,'nEventsPerMergeJob',   JediTaskSpec.splitRuleToken['nEventsPerMergeJob'])
     self.setSplitRule(taskParamMap,'nFilesPerMergeJob',    JediTaskSpec.splitRuleToken['nFilesPerMergeJob'])
     self.setSplitRule(taskParamMap,'nGBPerMergeJob',       JediTaskSpec.splitRuleToken['nGBPerMergeJob'])
     self.setSplitRule(taskParamMap,'nMaxFilesPerMergeJob', JediTaskSpec.splitRuleToken['nMaxFilesPerMergeJob'])
     if taskParamMap.has_key('loadXML'):
         self.setSplitRule(None,3,JediTaskSpec.splitRuleToken['loadXML'])
         self.setSplitRule(None,4,JediTaskSpec.splitRuleToken['groupBoundaryID'])
     if taskParamMap.has_key('pfnList'):
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['pfnList'])
     if taskParamMap.has_key('noWaitParent'):
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['noWaitParent'])
     if 'respectLB' in taskParamMap:
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['respectLB'])
     if taskParamMap.has_key('reuseSecOnDemand'):
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['reuseSecOnDemand'])
     if 'ddmBackEnd' in taskParamMap:
         self.taskSpec.setDdmBackEnd(taskParamMap['ddmBackEnd'])
     if 'disableReassign' in taskParamMap:
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['disableReassign'])
     if 'allowPartialFinish' in taskParamMap:
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['allowPartialFinish'])
     if 'useExhausted' in taskParamMap:
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['useExhausted'])
     if 'useRealNumEvents' in taskParamMap:
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['useRealNumEvents'])
     if 'ipConnectivity' in taskParamMap:
         self.taskSpec.setIpConnectivity(taskParamMap['ipConnectivity'])
     if 'altStageOut' in taskParamMap:
         self.taskSpec.setAltStageOut(taskParamMap['altStageOut'])
     if 'allowInputLAN' in taskParamMap:
         self.taskSpec.setAllowInputLAN(taskParamMap['allowInputLAN'])
     if 'runUntilClosed' in taskParamMap:
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['runUntilClosed'])
     if 'stayOutputOnSite' in taskParamMap:
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['stayOutputOnSite'])
     if 'useJobCloning' in taskParamMap:
         scValue = EventServiceUtils.getJobCloningValue(taskParamMap['useJobCloning'])
         self.setSplitRule(None,scValue,JediTaskSpec.splitRuleToken['useJobCloning'])
     if 'failWhenGoalUnreached' in taskParamMap and taskParamMap['failWhenGoalUnreached'] == True:
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['failGoalUnreached'])
     if 'switchEStoNormal' in taskParamMap:
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['switchEStoNormal'])
     if 'nEventsPerRange' in taskParamMap:
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['dynamicNumEvents'])
     if 'allowInputWAN' in taskParamMap and taskParamMap['allowInputWAN'] == True:
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['allowInputWAN'])
     if 'putLogToOS' in taskParamMap and taskParamMap['putLogToOS'] == True:
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['putLogToOS'])
     # return
     return
    def extractCommon(self, jediTaskID, taskParamMap, workQueueMapper,
                      splitRule):
        # make task spec
        taskSpec = JediTaskSpec()
        taskSpec.jediTaskID = jediTaskID
        taskSpec.taskName = taskParamMap['taskName']
        taskSpec.userName = taskParamMap['userName']
        taskSpec.vo = taskParamMap['vo']
        taskSpec.prodSourceLabel = taskParamMap['prodSourceLabel']
        taskSpec.taskPriority = taskParamMap['taskPriority']
        taskSpec.currentPriority = taskSpec.taskPriority
        taskSpec.architecture = taskParamMap['architecture']
        taskSpec.transUses = taskParamMap['transUses']
        taskSpec.transHome = taskParamMap['transHome']
        taskSpec.transPath = taskParamMap['transPath']
        taskSpec.processingType = taskParamMap['processingType']
        taskSpec.taskType = taskParamMap['taskType']
        taskSpec.splitRule = splitRule
        taskSpec.startTime = datetime.datetime.utcnow()
        if taskParamMap.has_key('workingGroup'):
            taskSpec.workingGroup = taskParamMap['workingGroup']
        if taskParamMap.has_key('countryGroup'):
            taskSpec.countryGroup = taskParamMap['countryGroup']
        if taskParamMap.has_key('ticketID'):
            taskSpec.ticketID = taskParamMap['ticketID']
        if taskParamMap.has_key('ticketSystemType'):
            taskSpec.ticketSystemType = taskParamMap['ticketSystemType']
        if taskParamMap.has_key('reqID'):
            taskSpec.reqID = taskParamMap['reqID']
        else:
            taskSpec.reqID = jediTaskID
        if taskParamMap.has_key('coreCount'):
            taskSpec.coreCount = taskParamMap['coreCount']
        else:
            taskSpec.coreCount = 1
        if taskParamMap.has_key('walltime'):
            taskSpec.walltime = taskParamMap['walltime']
        else:
            taskSpec.walltime = 0
        if not taskParamMap.has_key('walltimeUnit'):
            # force to set NULL so that retried tasks get data from scouts again
            taskSpec.forceUpdate('walltimeUnit')
        if taskParamMap.has_key('outDiskCount'):
            taskSpec.outDiskCount = taskParamMap['outDiskCount']
        else:
            taskSpec.outDiskCount = 0
        if 'outDiskUnit' in taskParamMap:
            taskSpec.outDiskUnit = taskParamMap['outDiskUnit']
        if taskParamMap.has_key('workDiskCount'):
            taskSpec.workDiskCount = taskParamMap['workDiskCount']
        else:
            taskSpec.workDiskCount = 0
        if taskParamMap.has_key('workDiskUnit'):
            taskSpec.workDiskUnit = taskParamMap['workDiskUnit']
        if taskParamMap.has_key('ramCount'):
            taskSpec.ramCount = taskParamMap['ramCount']
        else:
            taskSpec.ramCount = 0
        if taskParamMap.has_key('ramUnit'):
            taskSpec.ramUnit = taskParamMap['ramUnit']
        if taskParamMap.has_key('baseRamCount'):
            taskSpec.baseRamCount = taskParamMap['baseRamCount']
        else:
            taskSpec.baseRamCount = 0
        # HS06 stuff
        if 'cpuTimeUnit' in taskParamMap:
            taskSpec.cpuTimeUnit = taskParamMap['cpuTimeUnit']
        if 'cpuTime' in taskParamMap:
            taskSpec.cpuTime = taskParamMap['cpuTime']
        if 'cpuEfficiency' in taskParamMap:
            taskSpec.cpuEfficiency = taskParamMap['cpuEfficiency']
        else:
            # 90% of cpu efficiency by default
            taskSpec.cpuEfficiency = 90
        if 'baseWalltime' in taskParamMap:
            taskSpec.baseWalltime = taskParamMap['baseWalltime']
        else:
            # 10min of offset by default
            taskSpec.baseWalltime = 10 * 60
        # for merge
        if 'mergeRamCount' in taskParamMap:
            taskSpec.mergeRamCount = taskParamMap['mergeRamCount']
        if 'mergeCoreCount' in taskParamMap:
            taskSpec.mergeCoreCount = taskParamMap['mergeCoreCount']
        # scout
        if not taskParamMap.has_key(
                'skipScout') and not taskSpec.isPostScout():
            taskSpec.setUseScout(True)
        # cloud
        if taskParamMap.has_key('cloud'):
            self.cloudName = taskParamMap['cloud']
            taskSpec.cloud = self.cloudName
        else:
            # set dummy to force update
            taskSpec.cloud = 'dummy'
            taskSpec.cloud = None
        # site
        if taskParamMap.has_key('site'):
            self.siteName = taskParamMap['site']
            taskSpec.site = self.siteName
        else:
            # set dummy to force update
            taskSpec.site = 'dummy'
            taskSpec.site = None
        # nucleus
        if 'nucleus' in taskParamMap:
            taskSpec.nucleus = taskParamMap['nucleus']
        # preset some parameters for job cloning
        if 'useJobCloning' in taskParamMap:
            # set implicit parameters
            if not 'nEventsPerWorker' in taskParamMap:
                taskParamMap['nEventsPerWorker'] = 1
            if not 'nSitesPerJob' in taskParamMap:
                taskParamMap['nSitesPerJob'] = 2
            if not 'nEsConsumers' in taskParamMap:
                taskParamMap['nEsConsumers'] = taskParamMap['nSitesPerJob']
        # event service flag
        if 'useJobCloning' in taskParamMap:
            taskSpec.eventService = 2
        elif taskParamMap.has_key('nEventsPerWorker'):
            taskSpec.eventService = 1
        else:
            taskSpec.eventService = 0
        # ttcr: requested time to completion
        if taskParamMap.has_key('ttcrTimestamp'):
            try:
                # get rid of the +00:00 timezone string and parse the timestamp
                taskSpec.ttcRequested = datetime.datetime.strptime(
                    taskParamMap['ttcrTimestamp'].split('+')[0],
                    '%Y-%m-%d %H:%M:%S.%f')
            except (IndexError, ValueError):
                pass
        # goal
        if 'goal' in taskParamMap:
            try:
                taskSpec.goal = int(float(taskParamMap['goal']) * 10)
                if taskSpec.goal >= 1000:
                    taskSpec.goal = None
            except:
                pass
        # campaign
        if taskParamMap.has_key('campaign'):
            taskSpec.campaign = taskParamMap['campaign']
        # request type
        if 'requestType' in taskParamMap:
            taskSpec.requestType = taskParamMap['requestType']
        self.taskSpec = taskSpec
        # set split rule
        if 'tgtNumEventsPerJob' in taskParamMap:
            # set nEventsPerJob not respect file boundaries when nFilesPerJob is not used
            if not 'nFilesPerJob' in taskParamMap:
                self.setSplitRule(None, taskParamMap['tgtNumEventsPerJob'],
                                  JediTaskSpec.splitRuleToken['nEventsPerJob'])
        self.setSplitRule(taskParamMap, 'nFilesPerJob',
                          JediTaskSpec.splitRuleToken['nFilesPerJob'])
        self.setSplitRule(taskParamMap, 'nEventsPerJob',
                          JediTaskSpec.splitRuleToken['nEventsPerJob'])
        self.setSplitRule(taskParamMap, 'nGBPerJob',
                          JediTaskSpec.splitRuleToken['nGBPerJob'])
        self.setSplitRule(taskParamMap, 'nMaxFilesPerJob',
                          JediTaskSpec.splitRuleToken['nMaxFilesPerJob'])
        self.setSplitRule(taskParamMap, 'nEventsPerWorker',
                          JediTaskSpec.splitRuleToken['nEventsPerWorker'])
        self.setSplitRule(taskParamMap, 'useLocalIO',
                          JediTaskSpec.splitRuleToken['useLocalIO'])
        self.setSplitRule(taskParamMap, 'disableAutoRetry',
                          JediTaskSpec.splitRuleToken['disableAutoRetry'])
        self.setSplitRule(taskParamMap, 'nEsConsumers',
                          JediTaskSpec.splitRuleToken['nEsConsumers'])
        self.setSplitRule(taskParamMap, 'waitInput',
                          JediTaskSpec.splitRuleToken['waitInput'])
        self.setSplitRule(taskParamMap, 'addNthFieldToLFN',
                          JediTaskSpec.splitRuleToken['addNthFieldToLFN'])
        self.setSplitRule(taskParamMap, 'scoutSuccessRate',
                          JediTaskSpec.splitRuleToken['scoutSuccessRate'])
        self.setSplitRule(taskParamMap, 't1Weight',
                          JediTaskSpec.splitRuleToken['t1Weight'])
        self.setSplitRule(taskParamMap, 'maxAttemptES',
                          JediTaskSpec.splitRuleToken['maxAttemptES'])
        self.setSplitRule(taskParamMap, 'nSitesPerJob',
                          JediTaskSpec.splitRuleToken['nSitesPerJob'])
        self.setSplitRule(taskParamMap, 'nJumboJobs',
                          JediTaskSpec.splitRuleToken['nJumboJobs'])
        self.setSplitRule(taskParamMap, 'nEventsPerMergeJob',
                          JediTaskSpec.splitRuleToken['nEventsPerMergeJob'])
        self.setSplitRule(taskParamMap, 'nFilesPerMergeJob',
                          JediTaskSpec.splitRuleToken['nFilesPerMergeJob'])
        self.setSplitRule(taskParamMap, 'nGBPerMergeJob',
                          JediTaskSpec.splitRuleToken['nGBPerMergeJob'])
        self.setSplitRule(taskParamMap, 'nMaxFilesPerMergeJob',
                          JediTaskSpec.splitRuleToken['nMaxFilesPerMergeJob'])
        if taskParamMap.has_key('loadXML'):
            self.setSplitRule(None, 3, JediTaskSpec.splitRuleToken['loadXML'])
            self.setSplitRule(None, 4,
                              JediTaskSpec.splitRuleToken['groupBoundaryID'])
        if taskParamMap.has_key('pfnList'):
            self.setSplitRule(None, 1, JediTaskSpec.splitRuleToken['pfnList'])
        if taskParamMap.has_key(
                'noWaitParent') and taskParamMap['noWaitParent'] == True:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['noWaitParent'])
        if 'respectLB' in taskParamMap:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['respectLB'])
        if taskParamMap.has_key('reuseSecOnDemand'):
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['reuseSecOnDemand'])
        if 'ddmBackEnd' in taskParamMap:
            self.taskSpec.setDdmBackEnd(taskParamMap['ddmBackEnd'])
        if 'disableReassign' in taskParamMap:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['disableReassign'])
        if 'allowPartialFinish' in taskParamMap:
            self.setSplitRule(
                None, 1, JediTaskSpec.splitRuleToken['allowPartialFinish'])
        if 'useExhausted' in taskParamMap:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['useExhausted'])
        if 'useRealNumEvents' in taskParamMap:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['useRealNumEvents'])
        if 'ipConnectivity' in taskParamMap:
            self.taskSpec.setIpConnectivity(taskParamMap['ipConnectivity'])
        if 'altStageOut' in taskParamMap:
            self.taskSpec.setAltStageOut(taskParamMap['altStageOut'])
        if 'allowInputLAN' in taskParamMap:
            self.taskSpec.setAllowInputLAN(taskParamMap['allowInputLAN'])
        if 'runUntilClosed' in taskParamMap:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['runUntilClosed'])
        if 'stayOutputOnSite' in taskParamMap:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['stayOutputOnSite'])
        if 'useJobCloning' in taskParamMap:
            scValue = EventServiceUtils.getJobCloningValue(
                taskParamMap['useJobCloning'])
            self.setSplitRule(None, scValue,
                              JediTaskSpec.splitRuleToken['useJobCloning'])
        if 'failWhenGoalUnreached' in taskParamMap and taskParamMap[
                'failWhenGoalUnreached'] == True:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['failGoalUnreached'])
        if 'switchEStoNormal' in taskParamMap:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['switchEStoNormal'])
        if 'nEventsPerRange' in taskParamMap:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['dynamicNumEvents'])
        if 'allowInputWAN' in taskParamMap and taskParamMap[
                'allowInputWAN'] == True:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['allowInputWAN'])
        if 'putLogToOS' in taskParamMap and taskParamMap['putLogToOS'] == True:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['putLogToOS'])
        if 'mergeEsOnOS' in taskParamMap and taskParamMap[
                'mergeEsOnOS'] == True:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['mergeEsOnOS'])
        if 'writeInputToFile' in taskParamMap and taskParamMap[
                'writeInputToFile'] == True:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['writeInputToFile'])
        if 'useFileAsSourceLFN' in taskParamMap and taskParamMap[
                'useFileAsSourceLFN'] == True:
            self.setSplitRule(
                None, 1, JediTaskSpec.splitRuleToken['useFileAsSourceLFN'])
        if 'ignoreMissingInDS' in taskParamMap and taskParamMap[
                'ignoreMissingInDS'] == True:
            self.setSplitRule(None, 1,
                              JediTaskSpec.splitRuleToken['ignoreMissingInDS'])
        # work queue
        workQueue = None
        if 'workQueueName' in taskParamMap:
            # work queue is specified
            workQueue = workQueueMapper.getQueueWithName(
                taskSpec.vo, taskSpec.prodSourceLabel,
                taskParamMap['workQueueName'])
        if workQueue is None:
            # get work queue based on task attributes
            workQueue, tmpStr = workQueueMapper.getQueueWithSelParams(
                taskSpec.vo,
                taskSpec.prodSourceLabel,
                processingType=taskSpec.processingType,
                workingGroup=taskSpec.workingGroup,
                coreCount=taskSpec.coreCount,
                site=taskSpec.site,
                eventService=taskSpec.eventService,
                splitRule=taskSpec.splitRule,
                campaign=taskSpec.campaign)
        if workQueue is None:
            errStr = 'workqueue is undefined for vo={0} label={1} '.format(
                taskSpec.vo, taskSpec.prodSourceLabel)
            errStr += 'processingType={0} workingGroup={1} coreCount={2} eventService={3} '.format(
                taskSpec.processingType, taskSpec.workingGroup,
                taskSpec.coreCount, taskSpec.eventService)
            errStr += 'splitRule={0} campaign={1}'.format(
                taskSpec.splitRule, taskSpec.campaign)
            raise RuntimeError, errStr
        self.taskSpec.workQueue_ID = workQueue.queue_id

        # Initialize the global share
        gshare = None
        if 'gshare' in taskParamMap and self.taskBufferIF.is_valid_share(
                taskParamMap['gshare']):
            # work queue is specified
            gshare = taskParamMap['gshare']
        else:
            # get share based on definition
            gshare = self.taskBufferIF.get_share_for_task(self.taskSpec)
            if gshare is None:
                gshare = 'No match'
                # errStr  = 'share is undefined for vo={0} label={1} '.format(taskSpec.vo,taskSpec.prodSourceLabel)
                # errStr += 'workingGroup={0} campaign={1} '.format(taskSpec.workingGroup, taskSpec.campaign)
                # raise RuntimeError,errStr

            self.taskSpec.gshare = gshare

        # return
        return
Beispiel #3
0
 def extractCommon(self, jediTaskID, taskParamMap, workQueueMapper,
                   splitRule):
     # make task spec
     taskSpec = JediTaskSpec()
     taskSpec.jediTaskID = jediTaskID
     taskSpec.taskName = taskParamMap['taskName']
     taskSpec.userName = taskParamMap['userName']
     taskSpec.vo = taskParamMap['vo']
     taskSpec.prodSourceLabel = taskParamMap['prodSourceLabel']
     taskSpec.taskPriority = taskParamMap['taskPriority']
     taskSpec.currentPriority = taskSpec.taskPriority
     taskSpec.architecture = taskParamMap['architecture']
     taskSpec.transUses = taskParamMap['transUses']
     taskSpec.transHome = taskParamMap['transHome']
     taskSpec.transPath = taskParamMap['transPath']
     taskSpec.processingType = taskParamMap['processingType']
     taskSpec.taskType = taskParamMap['taskType']
     taskSpec.splitRule = splitRule
     taskSpec.startTime = datetime.datetime.utcnow()
     if taskParamMap.has_key('workingGroup'):
         taskSpec.workingGroup = taskParamMap['workingGroup']
     if taskParamMap.has_key('countryGroup'):
         taskSpec.countryGroup = taskParamMap['countryGroup']
     if taskParamMap.has_key('ticketID'):
         taskSpec.ticketID = taskParamMap['ticketID']
     if taskParamMap.has_key('ticketSystemType'):
         taskSpec.ticketSystemType = taskParamMap['ticketSystemType']
     if taskParamMap.has_key('reqID'):
         taskSpec.reqID = taskParamMap['reqID']
     else:
         taskSpec.reqID = jediTaskID
     if taskParamMap.has_key('coreCount'):
         taskSpec.coreCount = taskParamMap['coreCount']
     else:
         taskSpec.coreCount = 1
     if taskParamMap.has_key('walltime'):
         taskSpec.walltime = taskParamMap['walltime']
     else:
         taskSpec.walltime = 0
     if taskParamMap.has_key('walltimeUnit'):
         taskSpec.walltimeUnit = taskParamMap['walltimeUnit']
     if taskParamMap.has_key('outDiskCount'):
         taskSpec.outDiskCount = taskParamMap['outDiskCount']
     else:
         taskSpec.outDiskCount = 0
     if 'outDiskUnit' in taskParamMap:
         taskSpec.outDiskUnit = taskParamMap['outDiskUnit']
     if taskParamMap.has_key('workDiskCount'):
         taskSpec.workDiskCount = taskParamMap['workDiskCount']
     else:
         taskSpec.workDiskCount = 0
     if taskParamMap.has_key('workDiskUnit'):
         taskSpec.workDiskUnit = taskParamMap['workDiskUnit']
     if taskParamMap.has_key('ramCount'):
         taskSpec.ramCount = taskParamMap['ramCount']
     else:
         taskSpec.ramCount = 0
     # HS06 stuff
     if 'cpuTimeUnit' in taskParamMap:
         taskSpec.cpuTimeUnit = taskParamMap['cpuTimeUnit']
     if 'cpuTime' in taskParamMap:
         taskSpec.cpuTime = taskParamMap['cpuTime']
     if 'cpuEfficiency' in taskParamMap:
         taskSpec.cpuEfficiency = taskParamMap['cpuEfficiency']
     else:
         # 90% of cpu efficiency by default
         taskSpec.cpuEfficiency = 90
     if 'baseWalltime' in taskParamMap:
         taskSpec.baseWalltime = taskParamMap['baseWalltime']
     else:
         # 10min of offset by default
         taskSpec.baseWalltime = 10 * 60
     # for merge
     if 'mergeRamCount' in taskParamMap:
         taskSpec.mergeRamCount = taskParamMap['mergeRamCount']
     if 'mergeCoreCount' in taskParamMap:
         taskSpec.mergeCoreCount = taskParamMap['mergeCoreCount']
     # scout
     if not taskParamMap.has_key(
             'skipScout') and not taskSpec.isPostScout():
         taskSpec.setUseScout(True)
     # cloud
     if taskParamMap.has_key('cloud'):
         self.cloudName = taskParamMap['cloud']
         taskSpec.cloud = self.cloudName
     else:
         # set dummy to force update
         taskSpec.cloud = 'dummy'
         taskSpec.cloud = None
     # site
     if taskParamMap.has_key('site'):
         self.siteName = taskParamMap['site']
         taskSpec.site = self.siteName
     else:
         # set dummy to force update
         taskSpec.site = 'dummy'
         taskSpec.site = None
     # event service
     if taskParamMap.has_key('nEventsPerWorker'):
         taskSpec.eventService = 1
     else:
         taskSpec.eventService = 0
     # goal
     if 'goal' in taskParamMap:
         try:
             taskSpec.goal = int(float(taskParamMap['goal']) * 10)
             if taskSpec.goal >= 1000:
                 taskSpec.goal = None
         except:
             pass
     # campaign
     if taskParamMap.has_key('campaign'):
         taskSpec.campaign = taskParamMap['campaign']
     # work queue
     workQueue, tmpStr = workQueueMapper.getQueueWithSelParams(
         taskSpec.vo,
         taskSpec.prodSourceLabel,
         processingType=taskSpec.processingType,
         workingGroup=taskSpec.workingGroup,
         coreCount=taskSpec.coreCount,
         site=taskSpec.site)
     if workQueue == None:
         errStr = 'workqueue is undefined for vo={0} labal={1} '.format(
             taskSpec.vo, taskSpec.prodSourceLabel)
         errStr += 'processingType={0} workingGroup={1} coreCount={2} '.format(
             taskSpec.processingType, taskSpec.workingGroup,
             taskSpec.coreCount)
         raise RuntimeError, errStr
     taskSpec.workQueue_ID = workQueue.queue_id
     self.taskSpec = taskSpec
     # set split rule
     if 'tgtNumEventsPerJob' in taskParamMap:
         # set nEventsPerJob not respect file boundaries when nFilesPerJob is not used
         if not 'nFilesPerJob' in taskParamMap:
             self.setSplitRule(None, taskParamMap['tgtNumEventsPerJob'],
                               JediTaskSpec.splitRuleToken['nEventsPerJob'])
     self.setSplitRule(taskParamMap, 'nFilesPerJob',
                       JediTaskSpec.splitRuleToken['nFilesPerJob'])
     self.setSplitRule(taskParamMap, 'nEventsPerJob',
                       JediTaskSpec.splitRuleToken['nEventsPerJob'])
     self.setSplitRule(taskParamMap, 'nGBPerJob',
                       JediTaskSpec.splitRuleToken['nGBPerJob'])
     self.setSplitRule(taskParamMap, 'nMaxFilesPerJob',
                       JediTaskSpec.splitRuleToken['nMaxFilesPerJob'])
     self.setSplitRule(taskParamMap, 'nEventsPerWorker',
                       JediTaskSpec.splitRuleToken['nEventsPerWorker'])
     self.setSplitRule(taskParamMap, 'useLocalIO',
                       JediTaskSpec.splitRuleToken['useLocalIO'])
     self.setSplitRule(taskParamMap, 'disableAutoRetry',
                       JediTaskSpec.splitRuleToken['disableAutoRetry'])
     self.setSplitRule(taskParamMap, 'nEsConsumers',
                       JediTaskSpec.splitRuleToken['nEsConsumers'])
     self.setSplitRule(taskParamMap, 'waitInput',
                       JediTaskSpec.splitRuleToken['waitInput'])
     self.setSplitRule(taskParamMap, 'addNthFieldToLFN',
                       JediTaskSpec.splitRuleToken['addNthFieldToLFN'])
     self.setSplitRule(taskParamMap, 'scoutSuccessRate',
                       JediTaskSpec.splitRuleToken['scoutSuccessRate'])
     self.setSplitRule(taskParamMap, 't1Weight',
                       JediTaskSpec.splitRuleToken['t1Weight'])
     self.setSplitRule(taskParamMap, 'maxAttemptES',
                       JediTaskSpec.splitRuleToken['maxAttemptES'])
     self.setSplitRule(taskParamMap, 'nSitesPerJob',
                       JediTaskSpec.splitRuleToken['nSitesPerJob'])
     self.setSplitRule(taskParamMap, 'nEventsPerMergeJob',
                       JediTaskSpec.splitRuleToken['nEventsPerMergeJob'])
     self.setSplitRule(taskParamMap, 'nFilesPerMergeJob',
                       JediTaskSpec.splitRuleToken['nFilesPerMergeJob'])
     self.setSplitRule(taskParamMap, 'nGBPerMergeJob',
                       JediTaskSpec.splitRuleToken['nGBPerMergeJob'])
     self.setSplitRule(taskParamMap, 'nMaxFilesPerMergeJob',
                       JediTaskSpec.splitRuleToken['nMaxFilesPerMergeJob'])
     if taskParamMap.has_key('loadXML'):
         self.setSplitRule(None, 3, JediTaskSpec.splitRuleToken['loadXML'])
         self.setSplitRule(None, 4,
                           JediTaskSpec.splitRuleToken['groupBoundaryID'])
     if taskParamMap.has_key('pfnList'):
         self.setSplitRule(None, 1, JediTaskSpec.splitRuleToken['pfnList'])
     if taskParamMap.has_key('noWaitParent'):
         self.setSplitRule(None, 1,
                           JediTaskSpec.splitRuleToken['noWaitParent'])
     if 'respectLB' in taskParamMap:
         self.setSplitRule(None, 1,
                           JediTaskSpec.splitRuleToken['respectLB'])
     if taskParamMap.has_key('reuseSecOnDemand'):
         self.setSplitRule(None, 1,
                           JediTaskSpec.splitRuleToken['reuseSecOnDemand'])
     if 'ddmBackEnd' in taskParamMap:
         self.taskSpec.setDdmBackEnd(taskParamMap['ddmBackEnd'])
     if 'disableReassign' in taskParamMap:
         self.setSplitRule(None, 1,
                           JediTaskSpec.splitRuleToken['disableReassign'])
     if 'allowPartialFinish' in taskParamMap:
         self.setSplitRule(
             None, 1, JediTaskSpec.splitRuleToken['allowPartialFinish'])
     if 'useExhausted' in taskParamMap:
         self.setSplitRule(None, 1,
                           JediTaskSpec.splitRuleToken['useExhausted'])
     if 'useRealNumEvents' in taskParamMap:
         self.setSplitRule(None, 1,
                           JediTaskSpec.splitRuleToken['useRealNumEvents'])
     if 'ipConnectivity' in taskParamMap:
         self.taskSpec.setIpConnectivity(taskParamMap['ipConnectivity'])
     if 'runUntilClosed' in taskParamMap:
         self.setSplitRule(None, 1,
                           JediTaskSpec.splitRuleToken['runUntilClosed'])
     if 'stayOutputOnSite' in taskParamMap:
         self.setSplitRule(None, 1,
                           JediTaskSpec.splitRuleToken['stayOutputOnSite'])
     if 'useJobCloning' in taskParamMap:
         scValue = EventServiceUtils.getJobCloningValue(
             taskParamMap['useJobCloning'])
         self.setSplitRule(None, scValue,
                           JediTaskSpec.splitRuleToken['useJobCloning'])
     if 'failWhenGoalUnreached' in taskParamMap:
         self.setSplitRule(None, 1,
                           JediTaskSpec.splitRuleToken['failGoalUnreached'])
     if 'switchEStoNormal' in taskParamMap:
         self.setSplitRule(None, 1,
                           JediTaskSpec.splitRuleToken['switchEStoNormal'])
     # return
     return
Beispiel #4
0
 def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule):
     # make task spec
     taskSpec = JediTaskSpec()
     taskSpec.jediTaskID = jediTaskID
     taskSpec.taskName = taskParamMap['taskName']
     taskSpec.userName = taskParamMap['userName']
     taskSpec.vo = taskParamMap['vo']     
     taskSpec.prodSourceLabel = taskParamMap['prodSourceLabel']
     taskSpec.taskPriority = taskParamMap['taskPriority']
     taskSpec.currentPriority = taskSpec.taskPriority
     taskSpec.architecture = taskParamMap['architecture']
     taskSpec.transUses = taskParamMap['transUses']
     taskSpec.transHome = taskParamMap['transHome']
     taskSpec.transPath = taskParamMap['transPath']
     taskSpec.processingType = taskParamMap['processingType']
     taskSpec.taskType = taskParamMap['taskType']
     taskSpec.splitRule = splitRule
     if taskParamMap.has_key('workingGroup'):
         taskSpec.workingGroup = taskParamMap['workingGroup']
     if taskParamMap.has_key('countryGroup'):
         taskSpec.countryGroup = taskParamMap['countryGroup']
     if taskParamMap.has_key('ticketID'):
         taskSpec.ticketID = taskParamMap['ticketID']
     if taskParamMap.has_key('ticketSystemType'):
         taskSpec.ticketSystemType = taskParamMap['ticketSystemType']
     if taskParamMap.has_key('reqID'):
         taskSpec.reqID = taskParamMap['reqID']
     else:
         taskSpec.reqID = jediTaskID
     if taskParamMap.has_key('coreCount'):
         taskSpec.coreCount = taskParamMap['coreCount']
     else:
         taskSpec.coreCount = 1
     if taskParamMap.has_key('walltime'):
         taskSpec.walltime = taskParamMap['walltime']
     else:
         taskSpec.walltime = 0
     if taskParamMap.has_key('outDiskCount'):
         taskSpec.outDiskCount = taskParamMap['outDiskCount']
     else:
         taskSpec.outDiskCount = 0
     if taskParamMap.has_key('workDiskCount'):
         taskSpec.outDiskCount = taskParamMap['workDiskCount']
     else:
         taskSpec.outDiskCount = 0
     if taskParamMap.has_key('ramCount'):
         taskSpec.ramCount = taskParamMap['ramCount']
     else:
         taskSpec.ramCount = 0
     # scout
     if not taskParamMap.has_key('skipScout'):
         taskSpec.setUseScout(True)
     # cloud
     if taskParamMap.has_key('cloud'):
         self.cloudName = taskParamMap['cloud']
         taskSpec.cloud = self.cloudName
     # site
     if taskParamMap.has_key('site'):
         self.siteName = taskParamMap['site']
         taskSpec.site = self.siteName
     # event service
     if taskParamMap.has_key('nEventsPerWorker'):
         taskSpec.eventService = 1
     else:
         taskSpec.eventService = 0
     # work queue
     workQueue,tmpStr = workQueueMapper.getQueueWithSelParams(taskSpec.vo,
                                                              taskSpec.prodSourceLabel,
                                                              processingType=taskSpec.processingType,
                                                              workingGroup=taskSpec.workingGroup,
                                                              coreCount=taskSpec.coreCount)
     if workQueue == None:
         errStr  = 'workqueue is undefined for vo={0} labal={1} '.format(taskSpec.vo,taskSpec.prodSourceLabel)
         errStr += 'processingType={0} workingGroup={1} coreCount={2} '.format(taskSpec.processingType,
                                                                               taskSpec.workingGroup,
                                                                               taskSpec.coreCount)
         raise RuntimeError,errStr
     taskSpec.workQueue_ID = workQueue.queue_id
     self.taskSpec = taskSpec
     # set split rule    
     self.setSplitRule(taskParamMap,'nFilesPerJob',     JediTaskSpec.splitRuleToken['nFilesPerJob'])
     self.setSplitRule(taskParamMap,'nEventsPerJob',    JediTaskSpec.splitRuleToken['nEventsPerJob'])
     self.setSplitRule(taskParamMap,'nGBPerJob',        JediTaskSpec.splitRuleToken['nGBPerJob'])
     self.setSplitRule(taskParamMap,'nMaxFilesPerJob',  JediTaskSpec.splitRuleToken['nMaxFilesPerJob'])
     self.setSplitRule(taskParamMap,'nEventsPerWorker', JediTaskSpec.splitRuleToken['nEventsPerWorker'])
     self.setSplitRule(taskParamMap,'useLocalIO',       JediTaskSpec.splitRuleToken['useLocalIO'])
     self.setSplitRule(taskParamMap,'disableAutoRetry', JediTaskSpec.splitRuleToken['disableAutoRetry'])
     self.setSplitRule(taskParamMap,'nEsConsumers',     JediTaskSpec.splitRuleToken['nEsConsumers'])
     if taskParamMap.has_key('loadXML'):
         self.setSplitRule(None,3,JediTaskSpec.splitRuleToken['loadXML'])
         self.setSplitRule(None,4,JediTaskSpec.splitRuleToken['groupBoundaryID'])
     if taskParamMap.has_key('pfnList'):
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['pfnList'])
     if taskParamMap.has_key('noWaitParent'):
         self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['noWaitParent'])
     # return
     return
Beispiel #5
0
    def extractCommon(self,jediTaskID,taskParamMap,workQueueMapper,splitRule):
        # make task spec
        taskSpec = JediTaskSpec()
        taskSpec.jediTaskID = jediTaskID
        taskSpec.taskName = taskParamMap['taskName']
        taskSpec.userName = taskParamMap['userName']
        taskSpec.vo = taskParamMap['vo']     
        taskSpec.prodSourceLabel = taskParamMap['prodSourceLabel']
        taskSpec.taskPriority = taskParamMap['taskPriority']
        if 'currentPriority' in taskParamMap:
            taskSpec.currentPriority = taskParamMap['currentPriority']
        else:
            taskSpec.currentPriority = taskSpec.taskPriority
        taskSpec.architecture = taskParamMap['architecture']
        taskSpec.transUses = taskParamMap['transUses']
        taskSpec.transHome = taskParamMap['transHome']
        taskSpec.transPath = taskParamMap['transPath']
        taskSpec.processingType = taskParamMap['processingType']
        taskSpec.taskType = taskParamMap['taskType']
        taskSpec.splitRule = splitRule
        taskSpec.startTime = datetime.datetime.utcnow()
        if taskParamMap.has_key('workingGroup'):
            taskSpec.workingGroup = taskParamMap['workingGroup']
        if taskParamMap.has_key('countryGroup'):
            taskSpec.countryGroup = taskParamMap['countryGroup']
        if taskParamMap.has_key('ticketID'):
            taskSpec.ticketID = taskParamMap['ticketID']
        if taskParamMap.has_key('ticketSystemType'):
            taskSpec.ticketSystemType = taskParamMap['ticketSystemType']
        if taskParamMap.has_key('reqID'):
            taskSpec.reqID = taskParamMap['reqID']
        else:
            taskSpec.reqID = jediTaskID
        if taskParamMap.has_key('coreCount'):
            taskSpec.coreCount = taskParamMap['coreCount']
        else:
            taskSpec.coreCount = 1
        if taskParamMap.has_key('walltime'):
            taskSpec.walltime = taskParamMap['walltime']
        else:
            taskSpec.walltime = 0
        if not taskParamMap.has_key('walltimeUnit'):
            # force to set NULL so that retried tasks get data from scouts again
            taskSpec.forceUpdate('walltimeUnit')
        if taskParamMap.has_key('outDiskCount'):
            taskSpec.outDiskCount = taskParamMap['outDiskCount']
        else:
            taskSpec.outDiskCount = 0
        if 'outDiskUnit' in taskParamMap:
            taskSpec.outDiskUnit = taskParamMap['outDiskUnit']
        if taskParamMap.has_key('workDiskCount'):
            taskSpec.workDiskCount = taskParamMap['workDiskCount']
        else:
            taskSpec.workDiskCount = 0
        if taskParamMap.has_key('workDiskUnit'):
            taskSpec.workDiskUnit = taskParamMap['workDiskUnit']
        if taskParamMap.has_key('ramCount'):
            taskSpec.ramCount = taskParamMap['ramCount']
        else:
            taskSpec.ramCount = 0
        if taskParamMap.has_key('ramUnit'):
            taskSpec.ramUnit = taskParamMap['ramUnit']
        if taskParamMap.has_key('baseRamCount'):
            taskSpec.baseRamCount = taskParamMap['baseRamCount']
        else:
            taskSpec.baseRamCount = 0
        # IO
        if 'ioIntensity' in taskParamMap:
            taskSpec.ioIntensity = taskParamMap['ioIntensity']
        if 'ioIntensityUnit' in taskParamMap:
            taskSpec.ioIntensityUnit = taskParamMap['ioIntensityUnit']
        # HS06 stuff
        if 'cpuTimeUnit' in taskParamMap:
            taskSpec.cpuTimeUnit = taskParamMap['cpuTimeUnit']
        if 'cpuTime' in taskParamMap:
            taskSpec.cpuTime = taskParamMap['cpuTime']
        if 'cpuEfficiency' in taskParamMap:
            taskSpec.cpuEfficiency = taskParamMap['cpuEfficiency']
        else:
            # 90% of cpu efficiency by default
            taskSpec.cpuEfficiency = 90
        if 'baseWalltime' in taskParamMap:
            taskSpec.baseWalltime = taskParamMap['baseWalltime']
        else:
            # 10min of offset by default
            taskSpec.baseWalltime = 10*60
        # for merge
        if 'mergeRamCount' in taskParamMap:
            taskSpec.mergeRamCount = taskParamMap['mergeRamCount']
        if 'mergeCoreCount' in taskParamMap:
            taskSpec.mergeCoreCount = taskParamMap['mergeCoreCount']
        # scout
        if not taskParamMap.has_key('skipScout') and not taskSpec.isPostScout():
            taskSpec.setUseScout(True)
        # cloud
        if taskParamMap.has_key('cloud'):
            self.cloudName = taskParamMap['cloud']
            taskSpec.cloud = self.cloudName
        else:
            # set dummy to force update
            taskSpec.cloud = 'dummy'
            taskSpec.cloud = None
        # site
        if taskParamMap.has_key('site'):
            self.siteName = taskParamMap['site']
            taskSpec.site = self.siteName
        else:
            # set dummy to force update
            taskSpec.site = 'dummy'
            taskSpec.site = None
        # nucleus
        if 'nucleus' in taskParamMap:
            taskSpec.nucleus = taskParamMap['nucleus']
        # preset some parameters for job cloning
        if 'useJobCloning' in taskParamMap:
            # set implicit parameters
            if not 'nEventsPerWorker' in taskParamMap:
                taskParamMap['nEventsPerWorker'] = 1
            if not 'nSitesPerJob' in taskParamMap:
                taskParamMap['nSitesPerJob'] = 2
            if not 'nEsConsumers' in taskParamMap:
                taskParamMap['nEsConsumers'] = taskParamMap['nSitesPerJob']
        # minimum granularity
        if 'minGranularity' in taskParamMap:
            taskParamMap['nEventsPerRange'] = taskParamMap['minGranularity']
        # event service flag
        if 'useJobCloning' in taskParamMap:
            taskSpec.eventService = 2
        elif taskParamMap.has_key('nEventsPerWorker'):
            taskSpec.eventService = 1
        else:
            taskSpec.eventService = 0
        # OS
        if 'osInfo' in taskParamMap:
            taskSpec.termCondition = taskParamMap['osInfo']
        # ttcr: requested time to completion
        if taskParamMap.has_key('ttcrTimestamp'):
            try:
                # get rid of the +00:00 timezone string and parse the timestamp
                taskSpec.ttcRequested = datetime.datetime.strptime(taskParamMap['ttcrTimestamp'].split('+')[0], '%Y-%m-%d %H:%M:%S.%f')
            except (IndexError, ValueError):
                pass
        # goal
        if 'goal' in taskParamMap:
            try:
                taskSpec.goal = int(float(taskParamMap['goal'])*10)
                if taskSpec.goal > 1000:
                    taskSpec.goal = None
            except:
                pass
        # campaign
        if taskParamMap.has_key('campaign'):
            taskSpec.campaign = taskParamMap['campaign']
        # request type
        if 'requestType' in taskParamMap:
            taskSpec.requestType = taskParamMap['requestType']
        self.taskSpec = taskSpec
        # set split rule    
        if 'tgtNumEventsPerJob' in taskParamMap:
            # set nEventsPerJob not respect file boundaries when nFilesPerJob is not used
            if not 'nFilesPerJob' in taskParamMap:
                self.setSplitRule(None,taskParamMap['tgtNumEventsPerJob'],JediTaskSpec.splitRuleToken['nEventsPerJob'])
        self.setSplitRule(taskParamMap,'nFilesPerJob',     JediTaskSpec.splitRuleToken['nFilesPerJob'])
        self.setSplitRule(taskParamMap,'nEventsPerJob',    JediTaskSpec.splitRuleToken['nEventsPerJob'])
        self.setSplitRule(taskParamMap,'nGBPerJob',        JediTaskSpec.splitRuleToken['nGBPerJob'])
        self.setSplitRule(taskParamMap,'nMaxFilesPerJob',  JediTaskSpec.splitRuleToken['nMaxFilesPerJob'])
        self.setSplitRule(taskParamMap,'nEventsPerWorker', JediTaskSpec.splitRuleToken['nEventsPerWorker'])
        self.setSplitRule(taskParamMap,'useLocalIO',       JediTaskSpec.splitRuleToken['useLocalIO'])
        self.setSplitRule(taskParamMap,'disableAutoRetry', JediTaskSpec.splitRuleToken['disableAutoRetry'])
        self.setSplitRule(taskParamMap,'nEsConsumers',     JediTaskSpec.splitRuleToken['nEsConsumers'])
        self.setSplitRule(taskParamMap,'waitInput',        JediTaskSpec.splitRuleToken['waitInput'])
        self.setSplitRule(taskParamMap,'addNthFieldToLFN', JediTaskSpec.splitRuleToken['addNthFieldToLFN'])
        self.setSplitRule(taskParamMap,'scoutSuccessRate', JediTaskSpec.splitRuleToken['scoutSuccessRate'])
        self.setSplitRule(taskParamMap,'t1Weight',         JediTaskSpec.splitRuleToken['t1Weight'])
        self.setSplitRule(taskParamMap,'maxAttemptES',     JediTaskSpec.splitRuleToken['maxAttemptES'])
        self.setSplitRule(taskParamMap,'maxAttemptEsJob',  JediTaskSpec.splitRuleToken['maxAttemptEsJob'])
        self.setSplitRule(taskParamMap,'nSitesPerJob',     JediTaskSpec.splitRuleToken['nSitesPerJob'])
        self.setSplitRule(taskParamMap,'nEventsPerMergeJob',   JediTaskSpec.splitRuleToken['nEventsPerMergeJob'])
        self.setSplitRule(taskParamMap,'nFilesPerMergeJob',    JediTaskSpec.splitRuleToken['nFilesPerMergeJob'])
        self.setSplitRule(taskParamMap,'nGBPerMergeJob',       JediTaskSpec.splitRuleToken['nGBPerMergeJob'])
        self.setSplitRule(taskParamMap,'nMaxFilesPerMergeJob', JediTaskSpec.splitRuleToken['nMaxFilesPerMergeJob'])
        self.setSplitRule(taskParamMap,'maxWalltime', JediTaskSpec.splitRuleToken['maxWalltime'])
        self.setSplitRule(taskParamMap,'tgtMaxOutputForNG', JediTaskSpec.splitRuleToken['tgtMaxOutputForNG'])
        if 'nJumboJobs' in taskParamMap:
            self.setSplitRule(taskParamMap,'nJumboJobs',JediTaskSpec.splitRuleToken['nJumboJobs'])
            taskSpec.useJumbo = JediTaskSpec.enum_useJumbo['waiting']
            if 'maxJumboPerSite' in taskParamMap:
                self.setSplitRule(taskParamMap,'maxJumboPerSite',JediTaskSpec.splitRuleToken['maxJumboPerSite'])
        if 'minCpuEfficiency' in taskParamMap: 
            self.setSplitRule(taskParamMap,'minCpuEfficiency',JediTaskSpec.splitRuleToken['minCpuEfficiency'])
        if taskParamMap.has_key('loadXML'):
            self.setSplitRule(None,3,JediTaskSpec.splitRuleToken['loadXML'])
            self.setSplitRule(None,4,JediTaskSpec.splitRuleToken['groupBoundaryID'])
        if taskParamMap.has_key('pfnList'):
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['pfnList'])
        if taskParamMap.has_key('noWaitParent') and taskParamMap['noWaitParent'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['noWaitParent'])
        if 'respectLB' in taskParamMap:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['respectLB'])
        if 'orderByLB' in taskParamMap:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['orderByLB'])
        if 'respectSplitRule' in taskParamMap:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['respectSplitRule'])
        if taskParamMap.has_key('reuseSecOnDemand'):
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['reuseSecOnDemand'])
        if 'ddmBackEnd' in taskParamMap:
            self.taskSpec.setDdmBackEnd(taskParamMap['ddmBackEnd'])
        if 'disableReassign' in taskParamMap:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['disableReassign'])
        if 'allowPartialFinish' in taskParamMap:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['allowPartialFinish'])
        if 'useExhausted' in taskParamMap:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['useExhausted'])
        if 'useRealNumEvents' in taskParamMap:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['useRealNumEvents'])
        if 'ipConnectivity' in taskParamMap:
            self.taskSpec.setIpConnectivity(taskParamMap['ipConnectivity'])
        if 'altStageOut' in taskParamMap:
            self.taskSpec.setAltStageOut(taskParamMap['altStageOut'])
        if 'allowInputLAN' in taskParamMap:
            self.taskSpec.setAllowInputLAN(taskParamMap['allowInputLAN'])
        if 'runUntilClosed' in taskParamMap:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['runUntilClosed'])
        if 'stayOutputOnSite' in taskParamMap:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['stayOutputOnSite'])
        if 'useJobCloning' in taskParamMap:
            scValue = EventServiceUtils.getJobCloningValue(taskParamMap['useJobCloning'])
            self.setSplitRule(None,scValue,JediTaskSpec.splitRuleToken['useJobCloning'])
        if 'failWhenGoalUnreached' in taskParamMap and taskParamMap['failWhenGoalUnreached'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['failGoalUnreached'])
        if 'switchEStoNormal' in taskParamMap:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['switchEStoNormal'])
        if 'nEventsPerRange' in taskParamMap:
            self.setSplitRule(taskParamMap,'nEventsPerRange',JediTaskSpec.splitRuleToken['dynamicNumEvents'])
        if 'allowInputWAN' in taskParamMap and taskParamMap['allowInputWAN'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['allowInputWAN'])
        if 'putLogToOS' in taskParamMap and taskParamMap['putLogToOS'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['putLogToOS'])
        if 'mergeEsOnOS' in taskParamMap and taskParamMap['mergeEsOnOS'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['mergeEsOnOS'])
        if 'writeInputToFile' in taskParamMap and taskParamMap['writeInputToFile'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['writeInputToFile'])
        if 'useFileAsSourceLFN' in taskParamMap and taskParamMap['useFileAsSourceLFN'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['useFileAsSourceLFN'])
        if 'ignoreMissingInDS' in taskParamMap and taskParamMap['ignoreMissingInDS'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['ignoreMissingInDS'])
        if 'noExecStrCnv' in taskParamMap and taskParamMap['noExecStrCnv'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['noExecStrCnv'])
        if 'inFilePosEvtNum' in taskParamMap and taskParamMap['inFilePosEvtNum'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['inFilePosEvtNum'])
        if self.taskSpec.useEventService() and not taskSpec.useJobCloning():
            if 'registerEsFiles' in taskParamMap and taskParamMap['registerEsFiles'] == True:
                self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['registerEsFiles'])
        if 'disableAutoFinish' in taskParamMap and taskParamMap['disableAutoFinish'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['disableAutoFinish'])
        if 'resurrectConsumers' in taskParamMap and taskParamMap['resurrectConsumers'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['resurrectConsumers'])
        if 'usePrefetcher' in taskParamMap and taskParamMap['usePrefetcher'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['usePrefetcher'])
        if 'notDiscardEvents' in taskParamMap and taskParamMap['notDiscardEvents'] == True:
            self.setSplitRule(None,1,JediTaskSpec.splitRuleToken['notDiscardEvents'])
        if 'decAttOnFailedES' in taskParamMap and taskParamMap['decAttOnFailedES'] is True:
            self.setSplitRule(None, 1, JediTaskSpec.splitRuleToken['decAttOnFailedES'])
        if 'useZipToPin' in taskParamMap and taskParamMap['useZipToPin'] is True:
            self.setSplitRule(None, 1, JediTaskSpec.splitRuleToken['useZipToPin'])
        if 'osMatching' in taskParamMap and taskParamMap['osMatching'] is True:
            self.setSplitRule(None, 1, JediTaskSpec.splitRuleToken['osMatching'])
        # work queue
        workQueue = None
        if 'workQueueName' in taskParamMap:
            # work queue is specified
            workQueue = workQueueMapper.getQueueByName(taskSpec.vo, taskSpec.prodSourceLabel, taskParamMap['workQueueName'])
        if workQueue is None:
            # get work queue based on task attributes
            workQueue,tmpStr = workQueueMapper.getQueueWithSelParams(taskSpec.vo,
                                                                     taskSpec.prodSourceLabel,
                                                                     prodSourceLabel=taskSpec.prodSourceLabel,
                                                                     processingType=taskSpec.processingType,
                                                                     workingGroup=taskSpec.workingGroup,
                                                                     coreCount=taskSpec.coreCount,
                                                                     site=taskSpec.site,
                                                                     eventService=taskSpec.eventService,
                                                                     splitRule=taskSpec.splitRule,
                                                                     campaign=taskSpec.campaign)
        if workQueue is None:
            errStr  = 'workqueue is undefined for vo={0} label={1} '.format(taskSpec.vo,taskSpec.prodSourceLabel)
            errStr += 'processingType={0} workingGroup={1} coreCount={2} eventService={3} '.format(taskSpec.processingType,
                                                                                                   taskSpec.workingGroup,
                                                                                                   taskSpec.coreCount,
                                                                                                   taskSpec.eventService)
            errStr += 'splitRule={0} campaign={1}'.format(taskSpec.splitRule,taskSpec.campaign)
            raise RuntimeError,errStr
        self.taskSpec.workQueue_ID = workQueue.queue_id

        # Initialize the global share
        gshare = None
        if 'gshare' in taskParamMap and self.taskBufferIF.is_valid_share(taskParamMap['gshare']):
            # work queue is specified
            gshare = taskParamMap['gshare']
        else:
            # get share based on definition
            gshare = self.taskBufferIF.get_share_for_task(self.taskSpec)
            if gshare is None:
                gshare = 'Undefined' # Should not happen. Undefined is set when no share is found
                # errStr  = 'share is undefined for vo={0} label={1} '.format(taskSpec.vo,taskSpec.prodSourceLabel)
                # errStr += 'workingGroup={0} campaign={1} '.format(taskSpec.workingGroup, taskSpec.campaign)
                # raise RuntimeError,errStr

            self.taskSpec.gshare = gshare

        # Initialize the resource type
        try:
            self.taskSpec.resource_type = self.taskBufferIF.get_resource_type_task(self.taskSpec)
        except:
            self.taskSpec.resource_type = 'Undefined'

        # return
        return