def __init__(self, *args, **kwargs):
"""Constructor."""
AgentModule.__init__(self, *args, **kwargs)
self.rsClient = None
self.months = 36
def __init__(self, *args, **kwargs):
AgentModule.__init__(self, *args, **kwargs)
self.xmlPath = None
self.dryRun = False
def __init__( self, *args, **kwargs ):
""" c'tor
"""
AgentModule.__init__( self, *args, **kwargs )
self.queueDict = {}
self.queueCECache = {}
self.queueSlots = {}
self.failedQueues = defaultdict( int )
self.firstPass = True
self.maxJobsInFillMode = MAX_JOBS_IN_FILLMODE
self.maxPilotsToSubmit = MAX_PILOTS_TO_SUBMIT
self.gridEnv = ''
self.vo = ''
self.group = ''
# self.voGroups contain all the eligible user groups for pilots submitted by this SiteDirector
self.voGroups = []
self.pilotDN = ''
self.pilotGroup = ''
self.platforms = []
self.sites = []
self.proxy = None
self.updateStatus = True
self.getOutput = False
self.sendAccounting = True
def __init__(self, *args, **kwargs):
"""c'tor"""
AgentModule.__init__(self, *args, **kwargs)
self.jobDB = None
self.pilotDB = None
self.diracadmin = None
def __init__(self, *args, **kwargs):
AgentModule.__init__(self, *args, **kwargs)
self.diracAdmin = None
self.default_value = None
self.rsClient = ResourceStatusClient()
def __init__(self, *args, **kwargs):
""" c'tor
Always call this in the extension agent
"""
AgentModule.__init__(self, *args, **kwargs)
TransformationAgentsUtilities.__init__(self)
self.transClient = None
self.jobManagerClient = None
self.transType = []
self.tasksPerLoop = 50
self.maxParametricJobs = 20 # will be updated in execute()
# credentials
self.shifterProxy = None
self.credentials = None
self.credTuple = (None, None, None)
self.pluginLocation = ''
self.bulkSubmissionFlag = False
# for the threading
self.transQueue = Queue()
self.transInQueue = []
self.transInThread = {}
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
# # replica manager
self.replicaManager = ReplicaManager()
# # transformation client
self.transClient = TransformationClient()
# # wms client
self.wmsClient = WMSClient()
# # request client
self.requestClient = RequestClient()
# # file catalog clinet
self.metadataClient = FileCatalogClient()
# # placeholders for CS options
# # transformations types
self.transformationTypes = None
# # directory locations
self.directoryLocations = None
# # transformation metadata
self.transfidmeta = None
# # archive periof in days
self.archiveAfter = None
# # active SEs
self.activeStorages = None
# # transformation log SEs
self.logSE = None
# # enable/disable execution
self.enableFlag = None
def __init__(self, *args, **kwargs):
''' c'tor
'''
AgentModule.__init__(self, *args, **kwargs)
self.rsClient = None
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
# # replica manager
self.replicaManager = ReplicaManager()
# # transformation client
self.transClient = TransformationClient()
# # wms client
self.wmsClient = WMSClient()
# # request client
self.requestClient = RequestClient()
# # file catalog clinet
self.metadataClient = FileCatalogClient()
# # placeholders for CS options
# # transformations types
self.transformationTypes = None
# # directory locations
self.directoryLocations = None
# # transformation metadata
self.transfidmeta = None
# # archive periof in days
self.archiveAfter = None
# # active SEs
self.activeStorages = None
# # transformation log SEs
self.logSE = None
# # enable/disable execution
self.enableFlag = None
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.shifterProxy = None
# # transformation client
self.transClient = None
# # wms client
self.wmsClient = None
# # request client
self.reqClient = None
# # file catalog client
self.metadataClient = None
# # transformations types
self.transformationTypes = None
# # directory locations
self.directoryLocations = ['TransformationDB', 'MetadataCatalog']
# # transformation metadata
self.transfidmeta = 'TransformationID'
# # archive periof in days
self.archiveAfter = 7
# # transformation log SEs
self.logSE = 'LogSE'
# # enable/disable execution
self.enableFlag = 'True'
self.dataProcTTypes = ['MCSimulation', 'Merge']
self.dataManipTTypes = ['Replication', 'Removal']
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
self.rsClient = None
def __init__(self, *args, **kwargs):
"""Initialize the agent, clients, default values."""
AgentModule.__init__(self, *args, **kwargs)
self.name = 'MonitorAgents'
self.setup = "Production"
self.enabled = False
self.restartAgents = False
self.restartExecutors = False
self.restartServices = False
self.controlComponents = False
self.commitURLs = False
self.diracLocation = "/opt/dirac/pro"
self.sysAdminClient = SystemAdministratorClient(socket.gethostname())
self.jobMonClient = JobMonitoringClient()
self.nClient = NotificationClient()
self.csAPI = None
self.agents = dict()
self.executors = dict()
self.services = dict()
self.errors = list()
self.accounting = defaultdict(dict)
self.addressTo = ["*****@*****.**"]
self.addressFrom = "*****@*****.**"
self.emailSubject = "MonitorAgents on %s" % socket.gethostname()
def __init__(self, *args, **kwargs):
"""Initialize the agent, clients, default values."""
AgentModule.__init__(self, *args, **kwargs)
self.name = "ComponentSupervisionAgent"
self.setup = "DIRAC-Production"
self.enabled = False
self.restartAgents = False
self.restartExecutors = False
self.restartServices = False
self.controlComponents = False
self.commitURLs = False
self.doNotRestartInstancePattern = ["RequestExecutingAgent"]
self.diracLocation = rootPath
self.sysAdminClient = SystemAdministratorClient(socket.getfqdn())
self.jobMonClient = JobMonitoringClient()
self.nClient = NotificationClient()
self.csAPI = None
self.agents = dict()
self.executors = dict()
self.services = dict()
self._tornadoPort = "8443"
self.errors = list()
self.accounting = defaultdict(dict)
self.addressTo = []
self.addressFrom = ""
self.emailSubject = "ComponentSupervisionAgent on %s" % socket.getfqdn(
)
def __init__(self, *args, **kwargs):
""" c'tor
Always call this in the extension agent
"""
AgentModule.__init__(self, *args, **kwargs)
TransformationAgentsUtilities.__init__(self)
self.transClient = None
self.jobManagerClient = None
self.transType = []
self.tasksPerLoop = 50
self.maxParametricJobs = 20 # will be updated in execute()
# credentials
self.shifterProxy = None
self.credentials = None
self.credTuple = (None, None, None)
self.pluginLocation = ''
self.bulkSubmissionFlag = False
# for the threading
self.transQueue = Queue()
self.transInQueue = []
self.transInThread = {}
def __init__( self, *args, **kwargs ):
""" c'tor
"""
AgentModule.__init__( self, *args, **kwargs )
self.queueDict = {}
self.queueCECache = {}
self.queueSlots = {}
self.failedQueues = defaultdict( int )
self.firstPass = True
self.maxJobsInFillMode = MAX_JOBS_IN_FILLMODE
self.maxPilotsToSubmit = MAX_PILOTS_TO_SUBMIT
self.gridEnv = ''
self.vo = ''
self.group = ''
# self.voGroups contain all the eligible user groups for pilots submitted by this SiteDirector
self.voGroups = []
self.pilotDN = ''
self.pilotGroup = ''
self.platforms = []
self.sites = []
self.proxy = None
self.updateStatus = True
self.getOutput = False
self.sendAccounting = True
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.shifterProxy = None
# # transformation client
self.transClient = None
# # wms client
self.wmsClient = None
# # request client
self.reqClient = None
# # file catalog client
self.metadataClient = None
# # transformations types
self.transformationTypes = None
# # directory locations
self.directoryLocations = ['TransformationDB', 'MetadataCatalog']
# # transformation metadata
self.transfidmeta = 'TransformationID'
# # archive periof in days
self.archiveAfter = 7
# # transformation log SEs
self.logSE = 'LogSE'
# # enable/disable execution
self.enableFlag = 'True'
self.dataProcTTypes = ['MCSimulation', 'Merge']
self.dataManipTTypes = ['Replication', 'Removal']
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.bkClient = None
self.prodReq = None
def __init__( self, *args, **kwargs ):
""" c'tor
"""
AgentModule.__init__( self, *args, **kwargs )
# # data manager
self.dm = None
# # transformation client
self.transClient = None
# # wms client
self.wmsClient = None
# # request client
self.reqClient = None
# # file catalog client
self.metadataClient = None
# # transformations types
self.transformationTypes = None
# # directory locations
self.directoryLocations = None
# # transformation metadata
self.transfidmeta = None
# # archive periof in days
self.archiveAfter = None
# # active SEs
self.activeStorages = None
# # transformation log SEs
self.logSE = None
# # enable/disable execution
self.enableFlag = None
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
TransformationAgentsUtilities.__init__(self)
self.bkQueriesToBeChecked = Queue.Queue()
self.bkQueriesInCheck = []
self.fullUpdatePeriod = 86400
self.bkUpdateLatency = 7200
self.debug = False
self.transInThread = {}
self.pickleFile = 'BookkeepingWatchAgent.pkl'
self.chunkSize = 1000
self.pluginsWithNoRunInfo = ['LHCbStandard', 'ReplicateDataset', 'ArchiveDataset',
'LHCbMCDSTBroadcastRandom', 'ReplicateToLocalSE',
'RemoveReplicas', 'RemoveReplicasWhenProcessed',
'RemoveReplicasWithAncestors', 'ReplicateWithAncestors',
'ReduceReplicas', 'RemoveDatasetFromDisk',
'DestroyDataset', 'DestroyDatasetWhenProcessed',
'BySize', 'Standard']
self.timeLog = {}
self.fullTimeLog = {}
self.bkQueries = {}
self.transClient = None
self.bkClient = None
def __init__(self, *args, **kwargs):
"""c'tor"""
AgentModule.__init__(self, *args, **kwargs)
TransformationAgentsUtilities.__init__(self)
# few parameters
self.pluginLocation = ""
self.transformationStatus = []
self.maxFiles = 0
self.transformationTypes = []
# clients (out of the threads)
self.transfClient = None
# parameters for caching
self.workDirectory = ""
self.cacheFile = ""
self.controlDirectory = ""
self.lastFileOffset = {}
# Validity of the cache
self.replicaCache = None
self.replicaCacheValidity = None
self.writingCache = False
self.removedFromCache = 0
self.noUnusedDelay = 0
self.unusedFiles = {}
self.unusedTimeStamp = {}
self.debug = False
self.pluginTimeout = {}
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.vmTypeDict = {}
self.vmTypeCECache = {}
self.vmTypeSlots = {}
self.failedVMTypes = defaultdict(int)
self.firstPass = True
self.vo = ''
self.group = ''
# self.voGroups contain all the eligible user groups for clouds submitted by this SiteDirector
self.voGroups = []
self.cloudDN = ''
self.cloudGroup = ''
self.platforms = []
self.sites = []
self.wmsClient = WMSAdministratorClient()
self.proxy = None
self.updateStatus = True
self.getOutput = False
self.sendAccounting = True
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.jobDB = None
self.pilotDB = None
self.diracadmin = None
def __init__( self, agentName, baseAgentName, properties ): AgentModule.__init__( self, agentName, baseAgentName, properties ) self.addressTo = '' self.addressFrom = '' self.voName = '' self.csAPI = CSAPI() self.subject = "CE2CSAgent" self.alternativeBDIIs = []
def __init__( self, *args, **kwargs ):
""" Constructor.
"""
AgentModule.__init__( self, *args, **kwargs )
self.rsClient = None
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.rawIntegrityDB = None
self.fileCatalog = None
self.onlineRequestMgr = None
def __init__( self, agentName, loadName, baseAgentName=False, properties=dict() ):
""" c'tor
:param self: self reference
:param str agentName: name of agent
:param bool baseAgentName: whatever
:param dict properties: whatever else
"""
AgentModule.__init__( self, agentName, loadName, baseAgentName, properties )
def __init__(self, *args, **kwargs):
AgentModule.__init__(self, *args, **kwargs)
# ElementType, to be defined among Site, Resource or Node
self.sitesToBeChecked = None
self.threadPool = None
self.siteClient = None
self.clients = {}
def __init__(self, *args, **kwargs):
AgentModule.__init__(self, *args, **kwargs)
self.commands = {}
self.clients = {}
self.cCaller = None
self.rmClient = None
def __init__( self, *args, **kwargs ):
AgentModule.__init__( self, *args, **kwargs )
# ElementType, to be defined among Site, Resource or Node
self.sitesToBeChecked = None
self.threadPool = None
self.siteClient = None
self.clients = {}
def __init__(self, *args, **kwargs):
"""c'tor"""
AgentModule.__init__(self, *args, **kwargs)
# ElementType, to be defined among Resource or Node
self.elementType = "Resource"
self.rsClient = None
self.clients = {}
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
self.taskManager = None
self.shifterProxy = ''
self.transClient = TransformationClient()
self.transType = []
def __init__( self, *args, **kwargs ):
AgentModule.__init__( self, *args, **kwargs )
self.commands = {}
self.clients = {}
self.cCaller = None
self.rmClient = None
def __init__(self, agentName, baseAgentName=False, properties=dict()):
""" c'tor
:param self: self reference
:param str agentName: name of agent
:param bool baseAgentName: whatever
:param dict properties: whatever else
"""
AgentModule.__init__(self, agentName, baseAgentName, properties)
def __init__( self, *args, **kwargs ):
""" Constructor
"""
AgentModule.__init__( self, *args, **kwargs )
self.opHelper = None
self.stopAgentPath = ''
self.cfgToUpdate = ''
def __init__( self, *args, **kwargs ):
AgentModule.__init__( self, *args, **kwargs )
self.diracAdmin = None
self.default_value = None
if 'DIRAC' in os.environ:
self.cacheFile = os.path.join( os.getenv('DIRAC'), 'work/ResourceStatus/cache.db' )
else:
self.cacheFile = os.path.realpath('cache.db')
def __init__( self, *args ): """ Initialize the base class and define some extra data members """ AgentModule.__init__( self, *args ) self.requestDBClient = None self.replicaManager = None self.maxNumberOfThreads = 4 self.maxRequestsInQueue = 100 self.threadPool = None
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.transClient = None
self.bkClient = None
self.notifyClient = None
self.operations = None
self.failedTransIDs = []
def __init__( self, *args, **kwargs ):
AgentModule.__init__( self, *args, **kwargs )
self.diracAdmin = None
self.default_value = None
if 'DIRAC' in os.environ:
self.cacheFile = os.path.join( os.getenv('DIRAC'), 'work/ResourceStatus/cache.db' )
else:
self.cacheFile = os.path.realpath('cache.db')
def __init__(self, *args, **kwargs):
"""c'tor"""
AgentModule.__init__(self, *args, **kwargs)
# clients
self.jobDB = None
self.maxJobsAtOnce = 500
self.prodTypes = []
self.removeStatusDelay = {}
self.removeStatusDelayHB = {}
def __init__(self, *args, **kwargs):
"""c'tor"""
AgentModule.__init__(self, *args, **kwargs)
self.notifyHours = 12
self.adminMail = ""
self.rsClient = None
self.tokenDict = {}
self.diracAdmin = None
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.jobDB = None
self.logDB = None
self.startingMinorStatus = None
self.startingMajorStatus = JobStatus.CHECKING
self.failedStatus = None
self.requiredJobInfo = 'jdl'
self._initResult = None
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.jobDB = None
self.logDB = None
self.matchedTime = 7200
self.rescheduledTime = 600
self.completedTime = 86400
self.submittingTime = 300
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.jobDB = None self.logDB = None self.startingMinorStatus = None self.startingMajorStatus = "Checking" self.failedStatus = None self.requiredJobInfo = 'jdl' self._initResult = None
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.jobDB = None
self.logDB = None
self.matchedTime = 7200
self.rescheduledTime = 600
self.completedTime = 86400
self.submittingTime = 300
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.jobDB = None
self.logDB = None
self.matchedTime = 7200
self.rescheduledTime = 600
self.submittingTime = 300
self.stalledJobsTolerantSites = []
def __init__( self, *args, **kwargs ):
""" c'tor
"""
AgentModule.__init__( self, *args, **kwargs )
self.queueDict = {}
self.queueCECache = {}
self.queueSlots = {}
self.failedQueues = defaultdict( int )
self.firstPass = True
self.maxJobsInFillMode = MAX_JOBS_IN_FILLMODE
self.maxPilotsToSubmit = MAX_PILOTS_TO_SUBMIT
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
self.maxCacheLifetime = self.__maxCacheLifetime
self.maxHistoryLifetime = self.__maxHistoryLifetime
self.maxLogLifetime = self.__maxLogLifetime
self.rsClient = None
self.rmClient = None
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
self.notifyHours = 12
self.adminMail = ''
self.rsClient = None
self.tokenDict = {}
self.diracAdmin = None
def __init__( self, *args, **kwargs ):
""" c'tor
:param self: self reference
:param str agentName: name of agent
:param bool baseAgentName: whatever
:param dict properties: whatever else
"""
AgentModule.__init__( self, *args, **kwargs )
self.transClient = TransformationClient()
self.transfStatuses = self.am_getOption( 'TransformationStatuses', ['Active', 'Stopped'] )
def __init__( self, *args, **kwargs ):
""" c'tor
"""
AgentModule.__init__( self, *args, **kwargs )
# ElementType, to be defined among Site, Resource or Node
self.elementType = ''
self.elementsToBeChecked = None
self.threadPool = None
self.rsClient = None
self.clients = {}
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.notifyHours = self.__notifyHours
self.adminMail = self.__adminMail
self.rsClient = None
self.tokenDict = None
self.diracAdmin = None
def __init__(self, *args):
"""
Initialize the base class and define some extra data members
"""
AgentModule.__init__(self, *args)
self.requestDBClient = None
self.replicaManager = None
self.maxNumberOfThreads = 4
self.maxRequestsInQueue = 100
self.threadPool = None
self.timeOutCounter = 0
self.pendingRequests = True
def __init__( self, *args, **kwargs ): ''' Constructor ''' AgentModule.__init__( self, *args, **kwargs ) self.name = "TarTheProdLogsAgent" self.log = gLogger self.basepath = "" self.baselogpath = "" self.transclient = None self.ops = None self.storageElement = None self.baselfn = ""
def __init__( self, agentName, loadName, baseAgentName, properties = {} ):
''' c'tor
'''
AgentModule.__init__( self, agentName, loadName, baseAgentName, properties )
self.transClient = TransformationClient()
agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
if agentTSTypes:
self.transformationTypes = sortList( agentTSTypes )
else:
self.transformationTypes = sortList( Operations().getValue( 'Transformations/ExtendableTransfTypes',
['MCSimulation', 'Simulation'] ) )
self.maxIterationTasks = self.am_getOption( 'TasksPerIteration', 50 )
self.maxFailRate = self.am_getOption( 'MaxFailureRate', 30 )
self.maxWaitingJobs = self.am_getOption( 'MaxWaitingJobs', 1000 )
def __init__( self, *args, **kwargs ):
""" c'tor
:param self: self reference
:param str agentName: name of agent
:param str loadName: name of module
:param bool baseAgentName: whatever
:param dict properties: whatever else
"""
AgentModule.__init__( self, *args, **kwargs )
agentName = args[0]
## save config path
self.__configPath = PathFinder.getAgentSection( agentName )
self.log.info( "Will use %s config path" % self.__configPath )
## ProcessPool related stuff
self.__requestsPerCycle = self.am_getOption( "RequestsPerCycle", 10 )
self.log.info("requests/cycle = %d" % self.__requestsPerCycle )
self.__minProcess = self.am_getOption( "MinProcess", 1 )
self.log.info("ProcessPool min process = %d" % self.__minProcess )
self.__maxProcess = self.am_getOption( "MaxProcess", 4 )
self.log.info("ProcessPool max process = %d" % self.__maxProcess )
self.__queueSize = self.am_getOption( "ProcessPoolQueueSize", 10 )
self.log.info("ProcessPool queue size = %d" % self.__queueSize )
self.__poolTimeout = int( self.am_getOption( "ProcessPoolTimeout", 300 ) )
self.log.info("ProcessPool timeout = %d seconds" % self.__poolTimeout )
self.__taskTimeout = int( self.am_getOption( "ProcessTaskTimeout", 300 ) )
self.log.info("ProcessTask timeout = %d seconds" % self.__taskTimeout )
## request type
self.__requestType = self.am_getOption( "RequestType", self.__requestType )
self.log.info( "Will process '%s' request type." % str( self.__requestType ) )
## shifter proxy
self.am_setOption( "shifterProxy", "DataManager" )
self.log.info( "Will use DataManager proxy by default." )
## common monitor activity
self.monitor.registerActivity( "Iteration", "Agent Loops",
self.__class__.__name__, "Loops/min", gMonitor.OP_SUM )
self.monitor.registerActivity( "Execute", "Request Processed",
self.__class__.__name__, "Requests/min", gMonitor.OP_SUM )
self.monitor.registerActivity( "Done", "Request Completed",
self.__class__.__name__, "Requests/min", gMonitor.OP_SUM )
## create request dict
self.__requestHolder = dict()
def initialize( self ):
self.RequestDBClient = RequestClient()
backend = self.am_getOption( 'Backend', '' )
self.RequestDB = False
if backend == 'mysql':
from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL
requestDB = RequestDBMySQL()
if requestDB._connected:
self.RequestDB = requestDB
gMonitor.registerActivity( "Iteration", "Agent Loops", "DISETForwardingAgent", "Loops/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Attempted", "Request Processed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Successful", "Request Forward Successful", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Failed", "Request Forward Failed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM )
self.local = PathFinder.getServiceURL( "RequestManagement/localURL" )
if not self.local:
self.local = AgentModule.am_getOption( self, 'localURL', '' )
if not self.local:
errStr = 'The RequestManagement/localURL option must be defined.'
gLogger.fatal( errStr )
return S_ERROR( errStr )
return S_OK()
def __init__( self, *args, **kwargs ):
""" c'tor
"""
AgentModule.__init__( self, *args, **kwargs )
#clients
# FIXME: shouldn't we avoid using the DBs directly, and instead go through the service?
self.jobDB = None
self.taskQueueDB = None
self.jobLoggingDB = None
self.maxJobsAtOnce = 100
self.jobByJob = False
self.throttlingPeriod = 0.
self.removeStatusDelay = {}
