def get_unified_sites(self, scan_site_list):
unified_list = set()
for tmpSiteName in scan_site_list:
tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
unifiedName = tmpSiteSpec.get_unified_name()
unified_list.add(unifiedName)
return tuple(unified_list)
# get list of pseudo sites
def get_pseudo_sites(self, unified_list, scan_site_list):
unified_list = set(unified_list)
pseudo_list = set()
for tmpSiteName in scan_site_list:
tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
if tmpSiteSpec.get_unified_name() in unified_list:
pseudo_list.add(tmpSiteName)
return tuple(pseudo_list)
# add pseudo sites to skip
def add_pseudo_sites_to_skip(self, unified_dict, scan_site_list,
skipped_dict):
for tmpSiteName in scan_site_list:
tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
if tmpSiteSpec.get_unified_name() in unified_dict:
skipped_dict[tmpSiteName] = unified_dict[
tmpSiteSpec.get_unified_name()]
return skipped_dict
Interaction.installSC(JobBrokerBase)
from pandajedi.jedicore import Interaction
# base class for watchdog
class WatchDogBase(object):
# constructor
def __init__(self, taskBufferIF, ddmIF):
self.taskBufferIF = taskBufferIF
self.ddmIF = ddmIF
self.refresh()
# refresh
def refresh(self):
self.siteMapper = self.taskBufferIF.getSiteMapper()
# pre-action
def pre_action(self, tmpLog, vo, prodSourceLabel, pid, *args, **kwargs):
pass
Interaction.installSC(WatchDogBase)
maxNumEventRanges=maxNumEventRanges,
multiplicity=multiplicity,
splitByFields=splitByFields,
tmpLog=tmpLog,
useDirectIO=useDirectIO,
maxDiskSize=maxDiskSize)
if subChunk == None:
break
if subChunk != []:
# append
subChunks.append(subChunk)
iSubChunks += 1
# append to return map if remain
if subChunks != []:
# get site names for parallel execution
if taskSpec.getNumSitesPerJob() > 1 and not inputChunk.isMerging:
siteName = inputChunk.getParallelSites(taskSpec.getNumSitesPerJob(),
nSubChunks,[siteName])
returnList.append({'siteName':siteName,
'subChunks':subChunks,
'siteCandidate':siteCandidate,
})
tmpLog.debug('split to %s subchunks' % len(subChunks))
# return
tmpLog.debug('done')
return self.SC_SUCCEEDED,returnList
Interaction.installSC(JobSplitter)
from pandajedi.jedicore import Interaction
# base class for task setup
class TaskSetupperBase(object):
def __init__(self, taskBufferIF, ddmIF):
self.ddmIF = ddmIF
self.taskBufferIF = taskBufferIF
self.refresh()
def refresh(self):
self.siteMapper = self.taskBufferIF.getSiteMapper()
Interaction.installSC(TaskSetupperBase)
self.taskSpec.splitRule = tmpStr
else:
tmpMatch = re.search(valName + '=(-*\d+)', self.taskSpec.splitRule)
if tmpMatch == None:
# append
self.taskSpec.splitRule += ',{0}'.format(tmpStr)
else:
# replace
self.taskSpec.splitRule = re.sub(valName + '=(-*\d+)', tmpStr,
self.taskSpec.splitRule)
return
# get parameters for event service merging
def getParamsForEventServiceMerging(self, taskParamMap):
# no event service
if not self.taskSpec.useEventService():
return None
# extract parameters
transPath = 'UnDefined'
jobParameters = 'UnDefined'
if taskParamMap.has_key('esmergeSpec'):
if taskParamMap['esmergeSpec'].has_key('transPath'):
transPath = taskParamMap['esmergeSpec']['transPath']
if taskParamMap['esmergeSpec'].has_key('jobParameters'):
jobParameters = taskParamMap['esmergeSpec']['jobParameters']
# return
return '<PANDA_ESMERGE_TRF>' + transPath + '</PANDA_ESMERGE_TRF>' + '<PANDA_ESMERGE_JOBP>' + jobParameters + '</PANDA_ESMERGE_JOBP>'
Interaction.installSC(TaskRefinerBase)
# append
self.taskSpec.splitRule += ',{0}'.format(tmpStr)
else:
# replace
self.taskSpec.splitRule = re.sub(valName+'=(-*\d+)',
tmpStr,
self.taskSpec.splitRule)
return
# get parameters for event service merging
def getParamsForEventServiceMerging(self,taskParamMap):
# no event service
if not self.taskSpec.useEventService():
return None
# extract parameters
transPath = 'UnDefined'
jobParameters = 'UnDefined'
if taskParamMap.has_key('esmergeSpec'):
if taskParamMap['esmergeSpec'].has_key('transPath'):
transPath = taskParamMap['esmergeSpec']['transPath']
if taskParamMap['esmergeSpec'].has_key('jobParameters'):
jobParameters = taskParamMap['esmergeSpec']['jobParameters']
# return
return '<PANDA_ESMERGE_TRF>'+transPath+'</PANDA_ESMERGE_TRF>'+'<PANDA_ESMERGE_JOBP>'+jobParameters+'</PANDA_ESMERGE_JOBP>'
Interaction.installSC(TaskRefinerBase)
from pandajedi.jedicore import Interaction
# base class for task generator
class TaskGeneratorBase (object):
def __init__(self,taskBufferIF,ddmIF):
self.ddmIF = ddmIF
self.taskBufferIF = taskBufferIF
self.refresh()
def refresh(self):
self.siteMapper = self.taskBufferIF.getSiteMapper()
Interaction.installSC(TaskGeneratorBase)
if allow_chunk_size_limit and strict_chunkSize and len(
subChunks) < nSubChunks:
tmpLog.debug(
'skip splitting since chunk size {} is less than chunk size limit {} at {}'
.format(len(subChunks), nSubChunks, siteName))
inputChunk.rollback_file_usage()
isSkipped = True
else:
# get site names for parallel execution
if taskSpec.getNumSitesPerJob(
) > 1 and not inputChunk.isMerging:
siteName = inputChunk.getParallelSites(
taskSpec.getNumSitesPerJob(), nSubChunks, [siteName])
returnList.append({
'siteName': siteName,
'subChunks': subChunks,
'siteCandidate': siteCandidate,
})
try:
gshare = taskSpec.gshare.replace(' ', '_')
except Exception:
gshare = None
tmpLog.info('split to nJobs=%s at site=%s gshare=%s' %
(len(subChunks), siteName, gshare))
# return
tmpLog.debug('done')
return self.SC_SUCCEEDED, returnList, isSkipped
Interaction.installSC(JobSplitter)
from pandajedi.jedicore import Interaction
# base class for job throttle
class JobThrottlerBase(object):
def __init__(self, taskBufferIF):
self.taskBufferIF = taskBufferIF
# returns
self.retTmpError = self.SC_FAILED, True
self.retThrottled = self.SC_SUCCEEDED, True
self.retUnThrottled = self.SC_SUCCEEDED, False
# limit
self.maxNumJobs = None
self.minPriority = None
self.refresh()
# refresh
def refresh(self):
self.siteMapper = self.taskBufferIF.getSiteMapper()
# set maximum number of jobs to be submitted
def setMaxNumJobs(self, maxNumJobs):
self.maxNumJobs = maxNumJobs
# set min priority of jobs to be submitted
def setMinPriority(self, minPriority):
self.minPriority = minPriority
Interaction.installSC(JobThrottlerBase)
self.maxNumJobs = None
self.minPriority = None
self.underNqLimit = False
self.siteMapper = self.taskBufferIF.getSiteMapper()
# set maximum number of jobs to be submitted
def setMaxNumJobs(self, maxNumJobs):
self.maxNumJobs = maxNumJobs
# set min priority of jobs to be submitted
def setMinPriority(self, minPriority):
self.minPriority = minPriority
# check throttle level
def mergeThrottled(self, thrLevel):
# un-leveled flag
if thrLevel in [True, False]:
return thrLevel
return thrLevel > THR_LEVEL5
# check if lack of jobs
def lackOfJobs(self):
return self.underNqLimit
# not enough jobs are queued
def notEnoughJobsQueued(self):
self.underNqLimit = True
Interaction.installSC(JobThrottlerBase)
unified_list = set()
for tmpSiteName in scan_site_list:
tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
unifiedName = tmpSiteSpec.get_unified_name()
unified_list.add(unifiedName)
return tuple(unified_list)
# get list of pseudo sites
def get_pseudo_sites(self, unified_list, scan_site_list):
unified_list = set(unified_list)
pseudo_list = set()
for tmpSiteName in scan_site_list:
tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
if tmpSiteSpec.get_unified_name() in unified_list:
pseudo_list.add(tmpSiteName)
return tuple(pseudo_list)
# add pseudo sites to skip
def add_pseudo_sites_to_skip(self, unified_dict, scan_site_list, skipped_dict):
for tmpSiteName in scan_site_list:
tmpSiteSpec = self.siteMapper.getSite(tmpSiteName)
if tmpSiteSpec.get_unified_name() in unified_dict:
skipped_dict[tmpSiteName] = unified_dict[tmpSiteSpec.get_unified_name()]
return skipped_dict
Interaction.installSC(JobBrokerBase)
from pandajedi.jedicore import Interaction
# base class for task brokerge
class TaskBrokerBase (object):
def __init__(self,taskBufferIF,ddmIF):
self.ddmIF = ddmIF
self.taskBufferIF = taskBufferIF
self.refresh()
def refresh(self):
self.siteMapper = self.taskBufferIF.getSiteMapper()
Interaction.installSC(TaskBrokerBase)
thr.start()
# implementation of start()
def startImpl(self):
try:
Interaction.CommandReceiveInterface.start(self)
except:
errtype,errvalue = sys.exc_info()[:2]
self.logger.error('crashed in JediKnight.startImpl() with %s %s' % (errtype.__name__,errvalue))
# parse init params
def parseInit(self,par):
if isinstance(par,list):
return par
try:
return par.split('|')
except:
return [par]
# sleep to avoid synchronization of loop
def randomSleep(self,minVal=0,maxVal=30):
time.sleep(random.randint(minVal,maxVal))
# install SCs
Interaction.installSC(JediKnight)
from pandajedi.jedicore import Interaction
# base class for watchdog
class WatchDogBase (object):
# constructor
def __init__(self,taskBufferIF,ddmIF):
self.ddmIF = ddmIF
self.taskBufferIF = taskBufferIF
self.refresh()
def refresh(self):
self.siteMapper = self.taskBufferIF.getSiteMapper()
Interaction.installSC(WatchDogBase)
# check goal only
if checkGoal:
# no goal
if taskSpec.goal != None and taskCompleteness >= taskGoal:
return True
return False
# return status
return status
# pre-check
def doPreCheck(self, taskSpec, tmpLog):
# send task to exhausted
if taskSpec.useExhausted() and not taskSpec.status in ['passed'] \
and self.getFinalTaskStatus(taskSpec) in ['finished'] \
and not self.getFinalTaskStatus(taskSpec,checkParent=False) in ['done'] \
and not self.getFinalTaskStatus(taskSpec,checkGoal=True):
taskSpec.status = 'exhausted'
taskSpec.lockedBy = None
taskSpec.lockedTime = None
# update task
tmpLog.info('set task.status={0}'.format(taskSpec.status))
self.taskBufferIF.updateTask_JEDI(
taskSpec, {'jediTaskID': taskSpec.jediTaskID}, updateDEFT=True)
# kick child tasks
self.taskBufferIF.kickChildTasks_JEDI(taskSpec.jediTaskID)
return True
return False
Interaction.installSC(PostProcessorBase)
from pandajedi.jedicore import Interaction
# base class for task setup
class TaskSetupperBase (object):
def __init__(self,taskBufferIF,ddmIF):
self.ddmIF = ddmIF
self.taskBufferIF = taskBufferIF
self.refresh()
def refresh(self):
self.siteMapper = self.taskBufferIF.getSiteMapper()
Interaction.installSC(TaskSetupperBase)
from pandajedi.jedicore import Interaction
# base class for task brokerge
class TaskBrokerBase(object):
def __init__(self, taskBufferIF, ddmIF):
self.ddmIF = ddmIF
self.taskBufferIF = taskBufferIF
self.refresh()
def refresh(self):
self.siteMapper = self.taskBufferIF.getSiteMapper()
Interaction.installSC(TaskBrokerBase)
# start communication channel
import threading
thr = threading.Thread(target=self.startImpl)
thr.start()
# implementation of start()
def startImpl(self):
try:
Interaction.CommandReceiveInterface.start(self)
except:
errtype, errvalue = sys.exc_info()[:2]
self.logger.error('crashed in JediKnight.startImpl() with %s %s' %
(errtype.__name__, errvalue))
# parse init params
def parseInit(self, par):
if isinstance(par, list):
return par
try:
return par.split('|')
except:
return [par]
# sleep to avoid synchronization of loop
def randomSleep(self, minVal=0, maxVal=30):
time.sleep(random.randint(minVal, maxVal))
# install SCs
Interaction.installSC(JediKnight)
except:
errType,errValue = sys.exc_info()[:2]
if iTry+1 < nTry:
# sleep for retry
tmpLog.debug("sleep {0} due to {1}:{2}".format(iTry,errType,errValue))
time.sleep(30)
else:
tmpLog.error("failed to send notification with {0}:{1}".format(errType,errValue))
if fileBackUp:
# write to file which is processed in add.py
mailFile = '{0}/jmail_{1}_{2}' % (panda_config.logdir,jediTaskID,commands.getoutput('uuidgen'))
oMail = open(mailFile,"w")
oMail.write(str(jediTaskID)+'\n'+toAdd+'\n'+msgBody)
oMail.close()
break
try:
smtplib.stderr = org_smtpstderr
except:
pass
# return email sender
def senderAddress(self):
return panda_config.emailSender
Interaction.installSC(PostProcessorBase)
# finish tasks when goal is reached
tmpLog.info('finish achieved tasks for vo={0} label={1}'.format(
vo, prodSourceLabel))
tmpRet = self.taskBufferIF.getAchievedTasks_JEDI(
vo, prodSourceLabel, jedi_config.watchdog.waitForAchieved)
if tmpRet is None:
# failed
tmpLog.error('failed to finish')
else:
for jediTaskID in tmpRet:
self.taskBufferIF.sendCommandTaskPanda(jediTaskID,
'JEDI. Goal reached',
True,
'finish',
comQualifier='soft')
tmpLog.info('finished {0} tasks'.format(tmpRet))
# rescue unlocked tasks with picked files
tmpLog.info(
'rescue unlocked tasks with picked files for vo={0} label={1}'.
format(vo, prodSourceLabel))
tmpRet = self.taskBufferIF.rescueUnLockedTasksWithPicked_JEDI(
vo, prodSourceLabel, 60, pid)
if tmpRet is None:
# failed
tmpLog.error('failed to rescue unlocked tasks')
else:
tmpLog.info('rescue unlocked {0} tasks'.format(tmpRet))
Interaction.installSC(TypicalWatchDogBase)