def __init__(self, **kwargs):
""" Constructor
"""
if 'hosts' in kwargs:
self.__hosts = kwargs['hosts']
del kwargs['hosts']
else:
result = Registry.getHosts()
if result['OK']:
self.__hosts = result['Value']
else:
self.__hosts = []
self.__kwargs = dict(kwargs)
self.__pool = ThreadPool(len(self.__hosts))
self.__resultDict = {}
def __updateServiceConfiguration(self, urlSet, fromMaster=False):
"""
Update configuration in a set of service in parallel
:param set urlSet: a set of service URLs
:param fromMaster: flag to force updating from the master CS
:return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs
"""
pool = ThreadPool(len(urlSet))
for url in urlSet:
pool.generateJobAndQueueIt(self.__forceServiceUpdate,
args=[url, fromMaster],
kwargs={},
oCallback=self.__processResults)
pool.processAllResults()
return S_OK(self.__updateResultDict)
def initialize(self):
self.RequestDBClient = RequestClient()
self.ReplicaManager = ReplicaManager()
self.DataLog = DataLoggingClient()
self.maxNumberOfThreads = self.am_getOption('NumberOfThreads', 1)
self.threadPoolDepth = self.am_getOption('ThreadPoolDepth', 1)
self.threadPool = ThreadPool(1, self.maxNumberOfThreads)
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def __addPool(self, poolName):
# create a new thread Pool, by default it has 2 executing threads and 40 requests
# in the Queue
if not poolName:
return None
if poolName in self.pools:
return None
pool = ThreadPool(self.am_getOption('minThreadsInPool'),
self.am_getOption('maxThreadsInPool'),
self.am_getOption('totalThreadsInPool'))
# Daemonize except "Default" pool
if poolName != 'Default':
pool.daemonize()
self.pools[poolName] = pool
return poolName
def initialize(self):
""" Standard constructor
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.ResourcesToBeChecked = Queue.Queue()
self.ResourceNamesInCheck = []
self.maxNumberOfThreads = self.am_getOption('maxThreadsInPool', 1)
self.threadPool = ThreadPool(self.maxNumberOfThreads,
self.maxNumberOfThreads)
if not self.threadPool:
self.log.error('Can not create Thread Pool')
return S_ERROR('Can not create Thread Pool')
self.setup = getSetup()['Value']
self.VOExtension = getExt()
configModule = __import__(
self.VOExtension +
"DIRAC.ResourceStatusSystem.Policy.Configurations", globals(),
locals(), ['*'])
self.Resources_check_freq = copy.deepcopy(
configModule.Resources_check_freq)
self.nc = NotificationClient()
self.diracAdmin = DiracAdmin()
self.csAPI = CSAPI()
for i in xrange(self.maxNumberOfThreads):
self.threadPool.generateJobAndQueueIt(self._executeCheck,
args=(None, ))
return S_OK()
except Exception:
errorStr = "RSInspectorAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def initialize(self):
requiredLifeTime = self.am_getOption("MinimumLifeTime", 3600)
renewedLifeTime = self.am_getOption("RenewedLifeTime", 54000)
myProxyServer = gConfig.getValue("/DIRAC/VOPolicy/MyProxyServer",
"myproxy.cern.ch")
self.proxyDB = ProxyDB(requireVoms=True, useMyProxy=True)
gLogger.info("Minimum Life time : %s" % requiredLifeTime)
gLogger.info("Life time on renew : %s" % renewedLifeTime)
gLogger.info("MyProxy server : %s" %
self.proxyDB.getMyProxyServer())
gLogger.info("MyProxy max proxy time : %s" %
self.proxyDB.getMyProxyMaxLifeTime())
self.__threadPool = ThreadPool(1, 10)
return S_OK()
def initialize( self ):
#Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR( "Could not build service URL for %s" % GatewayService.GATEWAY_NAME )
gLogger.verbose( "Service URL is %s" % self._url )
#Discover Handler
self._initMonitoring()
self._threadPool = ThreadPool( 1,
max( 0, self._cfg.getMaxThreads() ),
self._cfg.getMaxWaitingPetitions() )
self._threadPool.daemonize()
self._msgBroker = MessageBroker( "%sMSB" % GatewayService.GATEWAY_NAME, threadPool = self._threadPool )
self._msgBroker.useMessageObjects( False )
getGlobalMessageBroker().useMessageObjects( False )
self._msgForwarder = MessageForwarder( self._msgBroker )
return S_OK()
def initialize( self ):
""" Standard initialize.
"""
maxNumberOfThreads = self.am_getOption( 'maxNumberOfThreads', self.__maxNumberOfThreads )
self.threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads )
self.elementType = self.am_getOption( 'elementType', self.elementType )
self.rsClient = ResourceStatusClient()
self.clients[ 'ResourceStatusClient' ] = self.rsClient
self.clients[ 'ResourceManagementClient' ] = ResourceManagementClient()
if not self.elementType:
return S_ERROR( 'Missing elementType' )
return S_OK()
def initialize(self):
""" standard initialize
"""
# few parameters
self.pluginLocation = self.am_getOption('PluginLocation',
'DIRAC.TransformationSystem.Agent.TransformationPlugin')
self.transformationStatus = self.am_getOption('transformationStatus', ['Active', 'Completing', 'Flush'])
# Prepare to change the name of the CS option as MaxFiles is ambiguous
self.maxFiles = self.am_getOption('MaxFilesToProcess', self.am_getOption('MaxFiles', 5000))
agentTSTypes = self.am_getOption('TransformationTypes', [])
if agentTSTypes:
self.transformationTypes = sorted(agentTSTypes)
else:
dataProc = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge'])
dataManip = Operations().getValue('Transformations/DataManipulation', ['Replication', 'Removal'])
self.transformationTypes = sorted(dataProc + dataManip)
# clients
self.transfClient = TransformationClient()
# for caching using a pickle file
self.workDirectory = self.am_getWorkDirectory()
self.cacheFile = os.path.join(self.workDirectory, 'ReplicaCache.pkl')
self.controlDirectory = self.am_getControlDirectory()
# remember the offset if any in TS
self.lastFileOffset = {}
# Validity of the cache
self.replicaCache = {}
self.replicaCacheValidity = self.am_getOption('ReplicaCacheValidity', 2)
self.noUnusedDelay = self.am_getOption('NoUnusedDelay', 6)
# Get it threaded
maxNumberOfThreads = self.am_getOption('maxThreadsInPool', 1)
threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads)
self.log.info("Multithreaded with %d threads" % maxNumberOfThreads)
for i in range(maxNumberOfThreads):
threadPool.generateJobAndQueueIt(self._execute, [i])
self.log.info("Will treat the following transformation types: %s" % str(self.transformationTypes))
return S_OK()
def initialize(self):
"""Sets default parameters
"""
self.jobDB = JobDB()
self.logDB = JobLoggingDB()
self.am_setOption('PollingTime', 60 * 60)
if not self.am_getOption('Enable', True):
self.log.info('Stalled Job Agent running in disabled mode')
# setting up the threading
maxNumberOfThreads = self.am_getOption('MaxNumberOfThreads', 15)
threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads)
self.log.verbose("Multithreaded with %d threads" % maxNumberOfThreads)
for _ in range(maxNumberOfThreads):
threadPool.generateJobAndQueueIt(self._execute)
return S_OK()
def initialize(self):
""" Agent initialization.
The extensions MUST provide in the initialize method the following data members:
- TransformationClient objects (self.transClient),
- set the shifterProxy if different from the default one set here ('ProductionManager')
- list of transformation types to be looked (self.transType)
"""
gMonitor.registerActivity("SubmittedTasks",
"Automatically submitted tasks",
"Transformation Monitoring", "Tasks",
gMonitor.OP_ACUM)
self.pluginLocation = self.am_getOption(
'PluginLocation',
'DIRAC.TransformationSystem.Client.TaskManagerPlugin')
# Default clients
self.transClient = TransformationClient()
self.jobManagerClient = JobManagerClient()
# Bulk submission flag
self.bulkSubmissionFlag = self.am_getOption('BulkSubmission',
self.bulkSubmissionFlag)
# Shifter credentials to use, could replace the use of shifterProxy eventually
self.shifterProxy = self.am_getOption('shifterProxy',
self.shifterProxy)
self.credentials = self.am_getOption('ShifterCredentials',
self.credentials)
resCred = self.__getCredentials()
if not resCred['OK']:
return resCred
# setting up the threading
maxNumberOfThreads = self.am_getOption('maxNumberOfThreads', 15)
threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads)
self.log.verbose("Multithreaded with %d threads" % maxNumberOfThreads)
for i in range(maxNumberOfThreads):
threadPool.generateJobAndQueueIt(self._execute, [i])
return S_OK()
def initialize(self):
""" Make the necessary initializations.
The ThreadPool is created here, the _execute() method is what each thread will execute.
"""
self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod', self.fullUpdatePeriod)
self.bkUpdateLatency = self.am_getOption('BKUpdateLatency', self.bkUpdateLatency)
self.debug = self.am_getOption('verbose', self.debug)
self.pickleFile = os.path.join(self.am_getWorkDirectory(), self.pickleFile)
self.chunkSize = self.am_getOption('maxFilesPerChunk', self.chunkSize)
self.pluginsWithNoRunInfo = Operations().getValue('TransformationPlugins/PluginsWithNoRunInfo',
self.pluginsWithNoRunInfo)
self._logInfo('Full Update Period: %d seconds' % self.fullUpdatePeriod)
self._logInfo('BK update latency : %d seconds' % self.bkUpdateLatency)
self._logInfo('Plugins with no run info: %s' % ', '.join(self.pluginsWithNoRunInfo))
self.transClient = TransformationClient()
self.bkClient = BookkeepingClient()
try:
with open(self.pickleFile, 'r') as pf:
self.timeLog = pickle.load(pf)
self.fullTimeLog = pickle.load(pf)
self.bkQueries = pickle.load(pf)
self._logInfo("successfully loaded Log from", self.pickleFile, "initialize")
except (EOFError, IOError):
self._logInfo("failed loading Log from", self.pickleFile, "initialize")
self.timeLog = {}
self.fullTimeLog = {}
self.bkQueries = {}
maxNumberOfThreads = self.am_getOption('maxThreadsInPool', 1)
threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads)
for i in xrange(maxNumberOfThreads):
threadPool.generateJobAndQueueIt(self._execute, [i])
gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM)
return S_OK()
def initialize( self ):
""" standard initialize
"""
self.__readCache()
self.dateWriteCache = datetime.datetime.utcnow()
self.am_setOption( 'shifterProxy', 'ProductionManager' )
# Get it threaded
maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 )
threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads )
self.log.info( "Multithreaded with %d threads" % maxNumberOfThreads )
for i in xrange( maxNumberOfThreads ):
threadPool.generateJobAndQueueIt( self._execute, [i] )
self.log.info( "Will treat the following transformation types: %s" % str( self.transformationTypes ) )
return S_OK()
def initialize(self):
''' Standard initialize.
Uses the ProductionManager shifterProxy to modify the ResourceStatus DB
'''
self.maxNumberOfThreads = self.am_getOption('maxNumberOfThreads',
self.maxNumberOfThreads)
self.elementType = self.am_getOption('elementType', self.elementType)
self.checkingFreqs = self.am_getOption('checkingFreqs',
self.checkingFreqs)
self.limitQueueFeeder = self.am_getOption('limitQueueFeeder',
self.limitQueueFeeder)
self.elementsToBeChecked = Queue.Queue()
self.threadPool = ThreadPool(self.maxNumberOfThreads,
self.maxNumberOfThreads)
self.rsClient = ResourceStatusClient()
self.clients['ResourceStatusClient'] = self.rsClient
self.clients['ResourceManagementClient'] = ResourceManagementClient()
return S_OK()
def __init__(self, csPath=""):
self.log = gLogger.getSubLogger("OutputDataExecutor")
if not csPath:
vo = gConfig.getValue("/DIRAC/VirtualOrganization", "")
self.__transfersCSPath = '/Operations/%s/OutputData' % vo
else:
self.__transfersCSPath = csPath
self.log.verbose("Reading transfer paths from %s" %
self.__transfersCSPath)
self.__requiredCSOptions = [
'InputPath', 'InputFC', 'OutputPath', 'OutputFC', 'OutputSE'
]
self.__threadPool = ThreadPool(
gConfig.getValue("%s/MinTransfers" % self.__transfersCSPath, 1),
gConfig.getValue("%s/MaxTransfers" % self.__transfersCSPath, 4),
gConfig.getValue("%s/MaxQueuedTransfers" % self.__transfersCSPath,
100))
self.__threadPool.daemonize()
self.__processingFiles = set()
self.__okTransferredFiles = 0
self.__okTransferredBytes = 0
self.__failedFiles = {}
def initialize(self):
""" This replaces the standard initialize from Service
"""
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % GatewayService.GATEWAY_NAME)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result['OK']:
return result
self._handler = result['Value']
# Discover Handler
self._threadPool = ThreadPool(1,
max(0, self._cfg.getMaxThreads()),
self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % GatewayService.GATEWAY_NAME, threadPool=self._threadPool)
self._msgBroker.useMessageObjects(False)
getGlobalMessageBroker().useMessageObjects(False)
self._msgForwarder = MessageForwarder(self._msgBroker)
return S_OK()
def initialize(self):
""" Standard initialize.
"""
maxNumberOfThreads = self.am_getOption('maxNumberOfThreads',
self.__maxNumberOfThreads)
self.threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads)
self.elementType = self.am_getOption('elementType', self.elementType)
res = ObjectLoader().loadObject(
'DIRAC.ResourceStatusSystem.Client.ResourceStatusClient',
'ResourceStatusClient')
if not res['OK']:
self.log.error('Failed to load ResourceStatusClient class: %s' %
res['Message'])
return res
rsClass = res['Value']
res = ObjectLoader().loadObject(
'DIRAC.ResourceStatusSystem.Client.ResourceManagementClient',
'ResourceManagementClient')
if not res['OK']:
self.log.error(
'Failed to load ResourceManagementClient class: %s' %
res['Message'])
return res
rmClass = res['Value']
self.rsClient = rsClass()
self.clients['ResourceStatusClient'] = rsClass()
self.clients['ResourceManagementClient'] = rmClass()
if not self.elementType:
return S_ERROR('Missing elementType')
return S_OK()
def initialize(self):
""" Standard constructor
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.ServicesToBeChecked = Queue.Queue()
self.ServiceNamesInCheck = []
self.maxNumberOfThreads = self.am_getOption('maxThreadsInPool', 1)
self.threadPool = ThreadPool(self.maxNumberOfThreads,
self.maxNumberOfThreads)
if not self.threadPool:
self.log.error('Can not create Thread Pool')
return S_ERROR('Can not create Thread Pool')
self.setup = getSetup()['Value']
self.VOExtension = getExt()
self.ServicesFreqs = CheckingFreqs['ServicesFreqs']
self.nc = NotificationClient()
self.diracAdmin = DiracAdmin()
self.csAPI = CSAPI()
for _i in xrange(self.maxNumberOfThreads):
self.threadPool.generateJobAndQueueIt(self._executeCheck,
args=(None, ))
return S_OK()
except Exception:
errorStr = "SeSInspectorAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def initialize( self ):
""" Agent initialization.
The extensions MUST provide in the initialize method the following data members:
- TransformationClient objects (self.transClient),
- set the shifterProxy if different from the default one set here ('ProductionManager')
- list of transformation types to be looked (self.transType)
"""
gMonitor.registerActivity( "SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks",
gMonitor.OP_ACUM )
# Default clients
self.transClient = TransformationClient()
#setting up the threading
maxNumberOfThreads = self.am_getOption( 'maxNumberOfThreads', 15 )
threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads )
self.log.verbose( "Multithreaded with %d threads" % maxNumberOfThreads )
for i in xrange( maxNumberOfThreads ):
threadPool.generateJobAndQueueIt( self._execute, [i] )
return S_OK()
def initialize(self):
#Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
#Load handler
result = self._loadHandlerInit()
if not result['OK']:
return result
self._handler = result['Value']
#Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
self._initMonitoring()
self._threadPool = ThreadPool(1, max(0, self._cfg.getMaxThreads()),
self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % self._name,
threadPool=self._threadPool)
#Create static dict
self._serviceInfoDict = {
'serviceName':
self._name,
'serviceSectionPath':
PathFinder.getServiceSection(self._name),
'URL':
self._cfg.getURL(),
'messageSender':
MessageSender(self._name, self._msgBroker),
'validNames':
self._validNames,
'csPaths': [
PathFinder.getServiceSection(svcName)
for svcName in self._validNames
]
}
#Call static initialization function
try:
self._handler['class']._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager,
self._msgBroker, self._monitor)
if self._handler['init']:
for initFunc in self._handler['init']:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception, excp:
gLogger.exception(
"Exception while calling initialization function")
return S_ERROR(
"Exception while calling initialization function: %s"
% str(excp))
if not isReturnStructure(result):
return S_ERROR(
"Service initialization function %s must return S_OK/S_ERROR"
% initFunc)
if not result['OK']:
return S_ERROR("Error while initializing %s: %s" %
(self._name, result['Message']))
except Exception, e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(errMsg)
return S_ERROR(errMsg)
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result['OK']:
return result
self._handler = result['Value']
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
# TODO: remove ThreadPool
if useThreadPoolExecutor:
self._threadPool = ThreadPoolExecutor(
max(0, self._cfg.getMaxThreads()))
else:
self._threadPool = ThreadPool(max(1, self._cfg.getMinThreads()),
max(0, self._cfg.getMaxThreads()),
self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % self._name,
threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {
'serviceName':
self._name,
'serviceSectionPath':
PathFinder.getServiceSection(self._name),
'URL':
self._cfg.getURL(),
'messageSender':
MessageSender(self._name, self._msgBroker),
'validNames':
self._validNames,
'csPaths': [
PathFinder.getServiceSection(svcName)
for svcName in self._validNames
]
}
# Initialize Monitoring
# This is a flag used to check whether "EnableActivityMonitoring" is enabled or not from the config file.
self.activityMonitoring = (Operations().getValue(
"EnableActivityMonitoring", False) or getServiceOption(
self._serviceInfoDict, "EnableActivityMonitoring", False))
if self.activityMonitoring:
# The import needs to be here because of the CS must be initialized before importing
# this class (see https://github.com/DIRACGrid/DIRAC/issues/4793)
from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter
self.activityMonitoringReporter = MonitoringReporter(
monitoringType="ComponentMonitoring")
gThreadScheduler.addPeriodicTask(
100, self.__activityMonitoringReporting)
elif self._standalone:
self._monitor = gMonitor
else:
self._monitor = MonitoringClient()
self._initMonitoring()
# Call static initialization function
try:
if self.activityMonitoring:
self._handler['class']._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager,
self._msgBroker, self.activityMonitoringReporter)
else:
self._handler['class']._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager,
self._msgBroker, self._monitor)
if self._handler['init']:
for initFunc in self._handler['init']:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception as excp:
gLogger.exception(
"Exception while calling initialization function",
lException=excp)
return S_ERROR(
"Exception while calling initialization function: %s"
% str(excp))
if not isReturnStructure(result):
return S_ERROR(
"Service initialization function %s must return S_OK/S_ERROR"
% initFunc)
if not result['OK']:
return S_ERROR("Error while initializing %s: %s" %
(self._name, result['Message']))
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(e)
gLogger.exception(errMsg)
return S_ERROR(errMsg)
# Load actions after the handler has initialized itself
result = self._loadActions()
if not result['OK']:
return result
self._actions = result['Value']
if not self.activityMonitoring:
gThreadScheduler.addPeriodicTask(30,
self.__reportThreadPoolContents)
return S_OK()
def threadPool(self):
""" thread pool getter """
if not self.__threadPool:
self.__threadPool = ThreadPool(self.MIN_THREADS, self.MAX_THREADS)
self.__threadPool.daemonize()
return self.__threadPool
def initialize(self):
self.RequestDBClient = RequestClient()
self.ReplicaManager = ReplicaManager()
self.DataLog = DataLoggingClient()
gMonitor.registerActivity("Iteration", "Agent Loops", "TransferAgent",
"Loops/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Execute", "Request Processed",
"TransferAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Done", "Request Completed", "TransferAgent",
"Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Replicate and register",
"Replicate and register operations",
"TransferAgent", "Attempts/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Replicate", "Replicate operations",
"TransferAgent", "Attempts/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Put and register",
"Put and register operations",
"TransferAgent", "Attempts/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Put", "Put operations", "TransferAgent",
"Attempts/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Replication successful",
"Successful replications", "TransferAgent",
"Successful/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Put successful", "Successful puts",
"TransferAgent", "Successful/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Replication failed", "Failed replications",
"TransferAgent", "Failed/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Put failed", "Failed puts", "TransferAgent",
"Failed/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Replica registration successful",
"Successful replica registrations",
"TransferAgent", "Successful/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("File registration successful",
"Successful file registrations",
"TransferAgent", "Successful/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Replica registration failed",
"Failed replica registrations",
"TransferAgent", "Failed/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("File registration failed",
"Failed file registrations", "TransferAgent",
"Failed/min", gMonitor.OP_SUM)
self.maxNumberOfThreads = self.am_getOption('NumberOfThreads', 1)
self.threadPoolDepth = self.am_getOption('ThreadPoolDepth', 1)
self.threadPool = ThreadPool(1, self.maxNumberOfThreads)
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def initialize(self):
"""
Called by the framework upon startup, before any cycle (execute method bellow)
"""
self.requestDBClient = RequestClient()
# the RequestAgentMixIn needs the capitalized version, until is is fixed keep this.
self.RequestDBClient = self.requestDBClient
self.replicaManager = ReplicaManager()
gMonitor.registerActivity("Iteration", "Agent Loops", "RemovalAgent",
"Loops/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Execute", "Request Processed",
"RemovalAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Done", "Request Completed", "RemovalAgent",
"Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("PhysicalRemovalAtt",
"Physical removals attempted",
"RemovalAgent", "Removal/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("PhysicalRemovalDone",
"Successful physical removals",
"RemovalAgent", "Removal/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("PhysicalRemovalFail",
"Failed physical removals", "RemovalAgent",
"Removal/min", gMonitor.OP_SUM)
gMonitor.registerActivity("PhysicalRemovalSize",
"Physically removed size", "RemovalAgent",
"Bytes", gMonitor.OP_ACUM)
gMonitor.registerActivity("ReplicaRemovalAtt",
"Replica removal attempted", "RemovalAgent",
"Removal/min", gMonitor.OP_SUM)
gMonitor.registerActivity("ReplicaRemovalDone",
"Successful replica removals",
"RemovalAgent", "Removal/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("ReplicaRemovalFail",
"Failed replica removals", "RemovalAgent",
"Removal/min", gMonitor.OP_SUM)
gMonitor.registerActivity("RemoveFileAtt", "File removal attempted",
"RemovalAgent", "Removal/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RemoveFileDone", "File removal done",
"RemovalAgent", "Removal/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RemoveFileFail", "File removal failed",
"RemovalAgent", "Removal/min",
gMonitor.OP_SUM)
self.maxNumberOfThreads = self.am_getOption('NumberOfThreads',
self.maxNumberOfThreads)
self.maxRequestsInQueue = self.am_getOption('RequestsInQueue',
self.maxRequestsInQueue)
self.threadPool = ThreadPool(1, self.maxNumberOfThreads,
self.maxRequestsInQueue)
# Set the ThreadPool in daemon mode to process new ThreadedJobs as they are inserted
self.threadPool.daemonize()
self.maxRequests = self.am_getOption('MaxRequestsPerCycle', 1200.)
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
