def get_unified_sites(self, scan_site_list): unified_list = set() for tmpSiteName in scan_site_list: tmpSiteSpec = self.siteMapper.getSite(tmpSiteName) unifiedName = tmpSiteSpec.get_unified_name() unified_list.add(unifiedName) return tuple(unified_list) # get list of pseudo sites def get_pseudo_sites(self, unified_list, scan_site_list): unified_list = set(unified_list) pseudo_list = set() for tmpSiteName in scan_site_list: tmpSiteSpec = self.siteMapper.getSite(tmpSiteName) if tmpSiteSpec.get_unified_name() in unified_list: pseudo_list.add(tmpSiteName) return tuple(pseudo_list) # add pseudo sites to skip def add_pseudo_sites_to_skip(self, unified_dict, scan_site_list, skipped_dict): for tmpSiteName in scan_site_list: tmpSiteSpec = self.siteMapper.getSite(tmpSiteName) if tmpSiteSpec.get_unified_name() in unified_dict: skipped_dict[tmpSiteName] = unified_dict[ tmpSiteSpec.get_unified_name()] return skipped_dict Interaction.installSC(JobBrokerBase)
from pandajedi.jedicore import Interaction # base class for watchdog class WatchDogBase(object): # constructor def __init__(self, taskBufferIF, ddmIF): self.taskBufferIF = taskBufferIF self.ddmIF = ddmIF self.refresh() # refresh def refresh(self): self.siteMapper = self.taskBufferIF.getSiteMapper() # pre-action def pre_action(self, tmpLog, vo, prodSourceLabel, pid, *args, **kwargs): pass Interaction.installSC(WatchDogBase)
maxNumEventRanges=maxNumEventRanges, multiplicity=multiplicity, splitByFields=splitByFields, tmpLog=tmpLog, useDirectIO=useDirectIO, maxDiskSize=maxDiskSize) if subChunk == None: break if subChunk != []: # append subChunks.append(subChunk) iSubChunks += 1 # append to return map if remain if subChunks != []: # get site names for parallel execution if taskSpec.getNumSitesPerJob() > 1 and not inputChunk.isMerging: siteName = inputChunk.getParallelSites(taskSpec.getNumSitesPerJob(), nSubChunks,[siteName]) returnList.append({'siteName':siteName, 'subChunks':subChunks, 'siteCandidate':siteCandidate, }) tmpLog.debug('split to %s subchunks' % len(subChunks)) # return tmpLog.debug('done') return self.SC_SUCCEEDED,returnList Interaction.installSC(JobSplitter)
from pandajedi.jedicore import Interaction # base class for task setup class TaskSetupperBase(object): def __init__(self, taskBufferIF, ddmIF): self.ddmIF = ddmIF self.taskBufferIF = taskBufferIF self.refresh() def refresh(self): self.siteMapper = self.taskBufferIF.getSiteMapper() Interaction.installSC(TaskSetupperBase)
self.taskSpec.splitRule = tmpStr else: tmpMatch = re.search(valName + '=(-*\d+)', self.taskSpec.splitRule) if tmpMatch == None: # append self.taskSpec.splitRule += ',{0}'.format(tmpStr) else: # replace self.taskSpec.splitRule = re.sub(valName + '=(-*\d+)', tmpStr, self.taskSpec.splitRule) return # get parameters for event service merging def getParamsForEventServiceMerging(self, taskParamMap): # no event service if not self.taskSpec.useEventService(): return None # extract parameters transPath = 'UnDefined' jobParameters = 'UnDefined' if taskParamMap.has_key('esmergeSpec'): if taskParamMap['esmergeSpec'].has_key('transPath'): transPath = taskParamMap['esmergeSpec']['transPath'] if taskParamMap['esmergeSpec'].has_key('jobParameters'): jobParameters = taskParamMap['esmergeSpec']['jobParameters'] # return return '<PANDA_ESMERGE_TRF>' + transPath + '</PANDA_ESMERGE_TRF>' + '<PANDA_ESMERGE_JOBP>' + jobParameters + '</PANDA_ESMERGE_JOBP>' Interaction.installSC(TaskRefinerBase)
# append self.taskSpec.splitRule += ',{0}'.format(tmpStr) else: # replace self.taskSpec.splitRule = re.sub(valName+'=(-*\d+)', tmpStr, self.taskSpec.splitRule) return # get parameters for event service merging def getParamsForEventServiceMerging(self,taskParamMap): # no event service if not self.taskSpec.useEventService(): return None # extract parameters transPath = 'UnDefined' jobParameters = 'UnDefined' if taskParamMap.has_key('esmergeSpec'): if taskParamMap['esmergeSpec'].has_key('transPath'): transPath = taskParamMap['esmergeSpec']['transPath'] if taskParamMap['esmergeSpec'].has_key('jobParameters'): jobParameters = taskParamMap['esmergeSpec']['jobParameters'] # return return '<PANDA_ESMERGE_TRF>'+transPath+'</PANDA_ESMERGE_TRF>'+'<PANDA_ESMERGE_JOBP>'+jobParameters+'</PANDA_ESMERGE_JOBP>' Interaction.installSC(TaskRefinerBase)
from pandajedi.jedicore import Interaction # base class for task generator class TaskGeneratorBase (object): def __init__(self,taskBufferIF,ddmIF): self.ddmIF = ddmIF self.taskBufferIF = taskBufferIF self.refresh() def refresh(self): self.siteMapper = self.taskBufferIF.getSiteMapper() Interaction.installSC(TaskGeneratorBase)
if allow_chunk_size_limit and strict_chunkSize and len( subChunks) < nSubChunks: tmpLog.debug( 'skip splitting since chunk size {} is less than chunk size limit {} at {}' .format(len(subChunks), nSubChunks, siteName)) inputChunk.rollback_file_usage() isSkipped = True else: # get site names for parallel execution if taskSpec.getNumSitesPerJob( ) > 1 and not inputChunk.isMerging: siteName = inputChunk.getParallelSites( taskSpec.getNumSitesPerJob(), nSubChunks, [siteName]) returnList.append({ 'siteName': siteName, 'subChunks': subChunks, 'siteCandidate': siteCandidate, }) try: gshare = taskSpec.gshare.replace(' ', '_') except Exception: gshare = None tmpLog.info('split to nJobs=%s at site=%s gshare=%s' % (len(subChunks), siteName, gshare)) # return tmpLog.debug('done') return self.SC_SUCCEEDED, returnList, isSkipped Interaction.installSC(JobSplitter)
from pandajedi.jedicore import Interaction # base class for job throttle class JobThrottlerBase(object): def __init__(self, taskBufferIF): self.taskBufferIF = taskBufferIF # returns self.retTmpError = self.SC_FAILED, True self.retThrottled = self.SC_SUCCEEDED, True self.retUnThrottled = self.SC_SUCCEEDED, False # limit self.maxNumJobs = None self.minPriority = None self.refresh() # refresh def refresh(self): self.siteMapper = self.taskBufferIF.getSiteMapper() # set maximum number of jobs to be submitted def setMaxNumJobs(self, maxNumJobs): self.maxNumJobs = maxNumJobs # set min priority of jobs to be submitted def setMinPriority(self, minPriority): self.minPriority = minPriority Interaction.installSC(JobThrottlerBase)
self.maxNumJobs = None self.minPriority = None self.underNqLimit = False self.siteMapper = self.taskBufferIF.getSiteMapper() # set maximum number of jobs to be submitted def setMaxNumJobs(self, maxNumJobs): self.maxNumJobs = maxNumJobs # set min priority of jobs to be submitted def setMinPriority(self, minPriority): self.minPriority = minPriority # check throttle level def mergeThrottled(self, thrLevel): # un-leveled flag if thrLevel in [True, False]: return thrLevel return thrLevel > THR_LEVEL5 # check if lack of jobs def lackOfJobs(self): return self.underNqLimit # not enough jobs are queued def notEnoughJobsQueued(self): self.underNqLimit = True Interaction.installSC(JobThrottlerBase)
unified_list = set() for tmpSiteName in scan_site_list: tmpSiteSpec = self.siteMapper.getSite(tmpSiteName) unifiedName = tmpSiteSpec.get_unified_name() unified_list.add(unifiedName) return tuple(unified_list) # get list of pseudo sites def get_pseudo_sites(self, unified_list, scan_site_list): unified_list = set(unified_list) pseudo_list = set() for tmpSiteName in scan_site_list: tmpSiteSpec = self.siteMapper.getSite(tmpSiteName) if tmpSiteSpec.get_unified_name() in unified_list: pseudo_list.add(tmpSiteName) return tuple(pseudo_list) # add pseudo sites to skip def add_pseudo_sites_to_skip(self, unified_dict, scan_site_list, skipped_dict): for tmpSiteName in scan_site_list: tmpSiteSpec = self.siteMapper.getSite(tmpSiteName) if tmpSiteSpec.get_unified_name() in unified_dict: skipped_dict[tmpSiteName] = unified_dict[tmpSiteSpec.get_unified_name()] return skipped_dict Interaction.installSC(JobBrokerBase)
from pandajedi.jedicore import Interaction # base class for task brokerge class TaskBrokerBase (object): def __init__(self,taskBufferIF,ddmIF): self.ddmIF = ddmIF self.taskBufferIF = taskBufferIF self.refresh() def refresh(self): self.siteMapper = self.taskBufferIF.getSiteMapper() Interaction.installSC(TaskBrokerBase)
thr.start() # implementation of start() def startImpl(self): try: Interaction.CommandReceiveInterface.start(self) except: errtype,errvalue = sys.exc_info()[:2] self.logger.error('crashed in JediKnight.startImpl() with %s %s' % (errtype.__name__,errvalue)) # parse init params def parseInit(self,par): if isinstance(par,list): return par try: return par.split('|') except: return [par] # sleep to avoid synchronization of loop def randomSleep(self,minVal=0,maxVal=30): time.sleep(random.randint(minVal,maxVal)) # install SCs Interaction.installSC(JediKnight)
from pandajedi.jedicore import Interaction # base class for watchdog class WatchDogBase (object): # constructor def __init__(self,taskBufferIF,ddmIF): self.ddmIF = ddmIF self.taskBufferIF = taskBufferIF self.refresh() def refresh(self): self.siteMapper = self.taskBufferIF.getSiteMapper() Interaction.installSC(WatchDogBase)
# check goal only if checkGoal: # no goal if taskSpec.goal != None and taskCompleteness >= taskGoal: return True return False # return status return status # pre-check def doPreCheck(self, taskSpec, tmpLog): # send task to exhausted if taskSpec.useExhausted() and not taskSpec.status in ['passed'] \ and self.getFinalTaskStatus(taskSpec) in ['finished'] \ and not self.getFinalTaskStatus(taskSpec,checkParent=False) in ['done'] \ and not self.getFinalTaskStatus(taskSpec,checkGoal=True): taskSpec.status = 'exhausted' taskSpec.lockedBy = None taskSpec.lockedTime = None # update task tmpLog.info('set task.status={0}'.format(taskSpec.status)) self.taskBufferIF.updateTask_JEDI( taskSpec, {'jediTaskID': taskSpec.jediTaskID}, updateDEFT=True) # kick child tasks self.taskBufferIF.kickChildTasks_JEDI(taskSpec.jediTaskID) return True return False Interaction.installSC(PostProcessorBase)
from pandajedi.jedicore import Interaction # base class for task setup class TaskSetupperBase (object): def __init__(self,taskBufferIF,ddmIF): self.ddmIF = ddmIF self.taskBufferIF = taskBufferIF self.refresh() def refresh(self): self.siteMapper = self.taskBufferIF.getSiteMapper() Interaction.installSC(TaskSetupperBase)
from pandajedi.jedicore import Interaction # base class for task brokerge class TaskBrokerBase(object): def __init__(self, taskBufferIF, ddmIF): self.ddmIF = ddmIF self.taskBufferIF = taskBufferIF self.refresh() def refresh(self): self.siteMapper = self.taskBufferIF.getSiteMapper() Interaction.installSC(TaskBrokerBase)
# start communication channel import threading thr = threading.Thread(target=self.startImpl) thr.start() # implementation of start() def startImpl(self): try: Interaction.CommandReceiveInterface.start(self) except: errtype, errvalue = sys.exc_info()[:2] self.logger.error('crashed in JediKnight.startImpl() with %s %s' % (errtype.__name__, errvalue)) # parse init params def parseInit(self, par): if isinstance(par, list): return par try: return par.split('|') except: return [par] # sleep to avoid synchronization of loop def randomSleep(self, minVal=0, maxVal=30): time.sleep(random.randint(minVal, maxVal)) # install SCs Interaction.installSC(JediKnight)
except: errType,errValue = sys.exc_info()[:2] if iTry+1 < nTry: # sleep for retry tmpLog.debug("sleep {0} due to {1}:{2}".format(iTry,errType,errValue)) time.sleep(30) else: tmpLog.error("failed to send notification with {0}:{1}".format(errType,errValue)) if fileBackUp: # write to file which is processed in add.py mailFile = '{0}/jmail_{1}_{2}' % (panda_config.logdir,jediTaskID,commands.getoutput('uuidgen')) oMail = open(mailFile,"w") oMail.write(str(jediTaskID)+'\n'+toAdd+'\n'+msgBody) oMail.close() break try: smtplib.stderr = org_smtpstderr except: pass # return email sender def senderAddress(self): return panda_config.emailSender Interaction.installSC(PostProcessorBase)
# finish tasks when goal is reached tmpLog.info('finish achieved tasks for vo={0} label={1}'.format( vo, prodSourceLabel)) tmpRet = self.taskBufferIF.getAchievedTasks_JEDI( vo, prodSourceLabel, jedi_config.watchdog.waitForAchieved) if tmpRet is None: # failed tmpLog.error('failed to finish') else: for jediTaskID in tmpRet: self.taskBufferIF.sendCommandTaskPanda(jediTaskID, 'JEDI. Goal reached', True, 'finish', comQualifier='soft') tmpLog.info('finished {0} tasks'.format(tmpRet)) # rescue unlocked tasks with picked files tmpLog.info( 'rescue unlocked tasks with picked files for vo={0} label={1}'. format(vo, prodSourceLabel)) tmpRet = self.taskBufferIF.rescueUnLockedTasksWithPicked_JEDI( vo, prodSourceLabel, 60, pid) if tmpRet is None: # failed tmpLog.error('failed to rescue unlocked tasks') else: tmpLog.info('rescue unlocked {0} tasks'.format(tmpRet)) Interaction.installSC(TypicalWatchDogBase)