def __init__(self):
"""c'tor
just setting validation order
"""
self.validator = (
self._hasRequestName,
self._hasOwner,
self._hasOperations,
self._hasType,
self._hasFiles,
self._hasRequiredAttrs,
self._hasChecksumAndChecksumType,
)
configPath = PathFinder.getAgentSection(
"RequestManagement/RequestExecutingAgent")
# # operation handlers over here
opHandlersPath = "%s/%s" % (configPath, "OperationHandlers")
opHandlers = gConfig.getSections(opHandlersPath)
if not opHandlers["OK"]:
gLogger.error(opHandlers["Message"])
else:
self.opHandlers = set(opHandlers["Value"])
def __init__(self, *args, **kwargs):
""" c'tor
:param self: self reference
:param str agentName: name of agent
:param str loadName: name of module
:param bool baseAgentName: whatever
:param dict properties: whatever else
"""
AgentModule.__init__(self, *args, **kwargs)
agentName = args[0]
## save config path
self.__configPath = PathFinder.getAgentSection(agentName)
self.log.info("Will use %s config path" % self.__configPath)
## ProcessPool related stuff
self.__requestsPerCycle = self.am_getOption("RequestsPerCycle", 10)
self.log.info("requests/cycle = %d" % self.__requestsPerCycle)
self.__minProcess = self.am_getOption("MinProcess", 1)
self.log.info("ProcessPool min process = %d" % self.__minProcess)
self.__maxProcess = self.am_getOption("MaxProcess", 4)
self.log.info("ProcessPool max process = %d" % self.__maxProcess)
self.__queueSize = self.am_getOption("ProcessPoolQueueSize", 10)
self.log.info("ProcessPool queue size = %d" % self.__queueSize)
self.__poolTimeout = int(self.am_getOption("ProcessPoolTimeout", 300))
self.log.info("ProcessPool timeout = %d seconds" % self.__poolTimeout)
self.__taskTimeout = int(self.am_getOption("ProcessTaskTimeout", 300))
self.log.info("ProcessTask timeout = %d seconds" % self.__taskTimeout)
## request type
self.__requestType = self.am_getOption("RequestType",
self.__requestType)
self.log.info("Will process '%s' request type." %
str(self.__requestType))
## shifter proxy
self.am_setOption("shifterProxy", "DataManager")
self.log.info("Will use DataManager proxy by default.")
## common monitor activity
self.monitor.registerActivity("Iteration", "Agent Loops",
self.__class__.__name__, "Loops/min",
gMonitor.OP_SUM)
self.monitor.registerActivity("Execute", "Request Processed",
self.__class__.__name__, "Requests/min",
gMonitor.OP_SUM)
self.monitor.registerActivity("Done", "Request Completed",
self.__class__.__name__, "Requests/min",
gMonitor.OP_SUM)
## create request dict
self.__requestHolder = dict()
def initialize(self):
self.section = PathFinder.getAgentSection(AGENT_NAME)
self.RequestDB = RequestDBMySQL()
self.TransferDB = TransferDB()
self.DataLog = DataLoggingClient()
self.factory = StorageFactory()
self.rm = ReplicaManager()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def initialize( self ):
self.section = PathFinder.getAgentSection( AGENT_NAME )
self.RequestDB = RequestDBMySQL()
self.TransferDB = TransferDB()
self.DataLog = DataLoggingClient()
self.factory = StorageFactory()
self.rm = ReplicaManager()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def __init__( self, *args, **kwargs ):
""" c'tor
:param self: self reference
:param str agentName: name of agent
:param str loadName: name of module
:param bool baseAgentName: whatever
:param dict properties: whatever else
"""
AgentModule.__init__( self, *args, **kwargs )
agentName = args[0]
## save config path
self.__configPath = PathFinder.getAgentSection( agentName )
self.log.info( "Will use %s config path" % self.__configPath )
## ProcessPool related stuff
self.__requestsPerCycle = self.am_getOption( "RequestsPerCycle", 10 )
self.log.info("requests/cycle = %d" % self.__requestsPerCycle )
self.__minProcess = self.am_getOption( "MinProcess", 1 )
self.log.info("ProcessPool min process = %d" % self.__minProcess )
self.__maxProcess = self.am_getOption( "MaxProcess", 4 )
self.log.info("ProcessPool max process = %d" % self.__maxProcess )
self.__queueSize = self.am_getOption( "ProcessPoolQueueSize", 10 )
self.log.info("ProcessPool queue size = %d" % self.__queueSize )
self.__poolTimeout = int( self.am_getOption( "ProcessPoolTimeout", 300 ) )
self.log.info("ProcessPool timeout = %d seconds" % self.__poolTimeout )
self.__taskTimeout = int( self.am_getOption( "ProcessTaskTimeout", 300 ) )
self.log.info("ProcessTask timeout = %d seconds" % self.__taskTimeout )
## request type
self.__requestType = self.am_getOption( "RequestType", self.__requestType )
self.log.info( "Will process '%s' request type." % str( self.__requestType ) )
## shifter proxy
self.am_setOption( "shifterProxy", "DataManager" )
self.log.info( "Will use DataManager proxy by default." )
## common monitor activity
self.monitor.registerActivity( "Iteration", "Agent Loops",
self.__class__.__name__, "Loops/min", gMonitor.OP_SUM )
self.monitor.registerActivity( "Execute", "Request Processed",
self.__class__.__name__, "Requests/min", gMonitor.OP_SUM )
self.monitor.registerActivity( "Done", "Request Completed",
self.__class__.__name__, "Requests/min", gMonitor.OP_SUM )
## create request dict
self.__requestHolder = dict()
def initialize( self ):
self.logger = gLogger.getSubLogger( "Monitoring" )
self.logger.debug( "Initializing Monitoring Client" )
self.sourceDict[ 'setup' ] = gConfig.getValue( "/DIRAC/Setup" )
self.sourceDict[ 'site' ] = DIRAC.siteName()
if self.sourceDict[ 'componentType' ] == self.COMPONENT_SERVICE:
self.cfgSection = PathFinder.getSystemSection( self.sourceDict[ 'componentName' ] )
elif self.sourceDict[ 'componentType' ] == self.COMPONENT_AGENT:
self.cfgSection = PathFinder.getAgentSection( self.sourceDict[ 'componentName' ] )
self.setComponentLocation( Network.getFQDN() )
elif self.sourceDict[ 'componentType' ] == self.COMPONENT_WEB:
self.cfgSection = "/Website"
self.setComponentLocation( 'http://%s' % Network.getFQDN() )
self.setComponentName( 'Web' )
elif self.sourceDict[ 'componentType' ] == self.COMPONENT_SCRIPT:
self.cfgSection = "/Script"
else:
raise Exception( "Component type has not been defined" )
gMonitoringFlusher.registerMonitoringClient( self )
#ExitCallback.registerExitCallback( self.forceFlush )
self.__initialized = True
def initialize( self ):
self.logger = gLogger.getSubLogger( "Monitoring" )
self.logger.debug( "Initializing Monitoring Client" )
self.sourceDict[ 'setup' ] = gConfig.getValue( "/DIRAC/Setup" )
self.sourceDict[ 'site' ] = DIRAC.siteName()
if self.sourceDict[ 'componentType' ] == self.COMPONENT_SERVICE:
self.cfgSection = PathFinder.getSystemSection( self.sourceDict[ 'componentName' ] )
elif self.sourceDict[ 'componentType' ] == self.COMPONENT_AGENT:
self.cfgSection = PathFinder.getAgentSection( self.sourceDict[ 'componentName' ] )
self.setComponentLocation( Network.getFQDN() )
elif self.sourceDict[ 'componentType' ] == self.COMPONENT_WEB:
self.cfgSection = "/WebApp"
self.setComponentLocation( 'http://%s' % Network.getFQDN() )
self.setComponentName( 'WebApp' )
elif self.sourceDict[ 'componentType' ] == self.COMPONENT_SCRIPT:
self.cfgSection = "/Script"
else:
raise Exception( "Component type has not been defined" )
gMonitoringFlusher.registerMonitoringClient( self )
# ExitCallback.registerExitCallback( self.forceFlush )
self.__initialized = True
def __init__(self):
""" c'tor
just setting validation order
"""
self.validator = (self._hasRequestName,
self._hasOwner,
self._hasOperations,
self._hasType,
self._hasFiles,
self._hasRequiredAttrs,
self._hasChecksumAndChecksumType)
configPath = PathFinder.getAgentSection("RequestManagement/RequestExecutingAgent")
# # operation handlers over here
opHandlersPath = "%s/%s" % (configPath, "OperationHandlers")
opHandlers = gConfig.getSections(opHandlersPath)
if not opHandlers["OK"]:
gLogger.error(opHandlers["Message"])
else:
self.opHandlers = set(opHandlers["Value"])
def initialize(self):
self.logger = gLogger.getSubLogger("Monitoring")
self.logger.debug("Initializing Monitoring Client")
self.sourceDict["setup"] = gConfig.getValue("/DIRAC/Setup")
self.sourceDict["site"] = DIRAC.siteName()
if self.sourceDict["componentType"] == self.COMPONENT_SERVICE:
self.cfgSection = PathFinder.getSystemSection(
self.sourceDict["componentName"])
elif self.sourceDict["componentType"] == self.COMPONENT_AGENT:
self.cfgSection = PathFinder.getAgentSection(
self.sourceDict["componentName"])
self.setComponentLocation(Network.getFQDN())
elif self.sourceDict["componentType"] == self.COMPONENT_WEB:
self.cfgSection = "/WebApp"
self.setComponentLocation("http://%s" % Network.getFQDN())
self.setComponentName("WebApp")
elif self.sourceDict["componentType"] == self.COMPONENT_SCRIPT:
self.cfgSection = "/Script"
elif self.sourceDict["componentType"] == self.COMPONENT_TORNADO:
self.cfgSection = "/Tornado"
else:
raise Exception("Component type has not been defined")
gMonitoringFlusher.registerMonitoringClient(self)
self.__initialized = True
def __init__( self, *args, **kwargs ):
""" c'tor """
# # call base class ctor
AgentModule.__init__( self, *args, **kwargs )
# # ProcessPool related stuff
self.__requestsPerCycle = self.am_getOption( "RequestsPerCycle", self.__requestsPerCycle )
self.log.info( "Requests/cycle = %d" % self.__requestsPerCycle )
self.__minProcess = self.am_getOption( "MinProcess", self.__minProcess )
self.log.info( "ProcessPool min process = %d" % self.__minProcess )
self.__maxProcess = self.am_getOption( "MaxProcess", 4 )
self.log.info( "ProcessPool max process = %d" % self.__maxProcess )
self.__queueSize = self.am_getOption( "ProcessPoolQueueSize", self.__queueSize )
self.log.info( "ProcessPool queue size = %d" % self.__queueSize )
self.__poolTimeout = int( self.am_getOption( "ProcessPoolTimeout", self.__poolTimeout ) )
self.log.info( "ProcessPool timeout = %d seconds" % self.__poolTimeout )
self.__poolSleep = int( self.am_getOption( "ProcessPoolSleep", self.__poolSleep ) )
self.log.info( "ProcessPool sleep time = %d seconds" % self.__poolSleep )
self.__taskTimeout = int( self.am_getOption( "ProcessTaskTimeout", self.__taskTimeout ) )
self.log.info( "ProcessTask timeout = %d seconds" % self.__taskTimeout )
self.__bulkRequest = self.am_getOption( "BulkRequest", 0 )
self.log.info( "Bulk request size = %d" % self.__bulkRequest )
# # keep config path and agent name
self.agentName = self.am_getModuleParam( "fullName" )
self.__configPath = PathFinder.getAgentSection( self.agentName )
# # operation handlers over here
opHandlersPath = "%s/%s" % ( self.__configPath, "OperationHandlers" )
opHandlers = gConfig.getSections( opHandlersPath )
if not opHandlers["OK"]:
self.log.error( opHandlers["Message" ] )
raise AgentConfigError( "OperationHandlers section not found in CS under %s" % self.__configPath )
opHandlers = opHandlers["Value"]
self.timeOuts = dict()
# # handlers dict
self.handlersDict = dict()
for opHandler in opHandlers:
opHandlerPath = "%s/%s/Location" % ( opHandlersPath, opHandler )
opLocation = gConfig.getValue( opHandlerPath, "" )
if not opLocation:
self.log.error( "%s not set for %s operation handler" % ( opHandlerPath, opHandler ) )
continue
self.timeOuts[opHandler] = { "PerFile": self.__fileTimeout, "PerOperation": self.__operationTimeout }
opTimeout = gConfig.getValue( "%s/%s/TimeOut" % ( opHandlersPath, opHandler ), 0 )
if opTimeout:
self.timeOuts[opHandler]["PerOperation"] = opTimeout
fileTimeout = gConfig.getValue( "%s/%s/TimeOutPerFile" % ( opHandlersPath, opHandler ), 0 )
if fileTimeout:
self.timeOuts[opHandler]["PerFile"] = fileTimeout
self.handlersDict[opHandler] = opLocation
self.log.info( "Operation handlers:" )
for item in enumerate ( self.handlersDict.items() ):
opHandler = item[1][0]
self.log.info( "[%s] %s: %s (timeout: %d s + %d s per file)" % ( item[0], item[1][0], item[1][1],
self.timeOuts[opHandler]['PerOperation'],
self.timeOuts[opHandler]['PerFile'] ) )
# # common monitor activity
gMonitor.registerActivity( "Iteration", "Agent Loops",
"RequestExecutingAgent", "Loops/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Processed", "Request Processed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Done", "Request Completed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
# # create request dict
self.__requestCache = dict()
self.FTSMode = self.am_getOption( "FTSMode", False )
def __init__( self, agentName, loadName, baseAgentName = False, properties = {} ):
"""
Common __init__ method for all Agents.
All Agent modules must define:
__doc__
__RCSID__
They are used to populate __codeProperties
The following Options are used from the Configuration:
- /LocalSite/InstancePath
- /DIRAC/Setup
- Status
- Enabled
- PollingTime default = 120
- MaxCycles default = 500
- ControlDirectory control/SystemName/AgentName
- WorkDirectory work/SystemName/AgentName
- shifterProxy ''
- shifterProxyLocation WorkDirectory/SystemName/AgentName/.shifterCred
It defines the following default Options that can be set via Configuration (above):
- MonitoringEnabled True
- Enabled True if Status == Active
- PollingTime 120
- MaxCycles 500
- ControlDirectory control/SystemName/AgentName
- WorkDirectory work/SystemName/AgentName
- shifterProxy False
- shifterProxyLocation work/SystemName/AgentName/.shifterCred
different defaults can be set in the initialize() method of the Agent using am_setOption()
In order to get a shifter proxy in the environment during the execute()
the configuration Option 'shifterProxy' must be set, a default may be given
in the initialize() method.
"""
if baseAgentName and agentName == baseAgentName:
self.log = gLogger
standaloneModule = True
else:
self.log = gLogger.getSubLogger( agentName, child = False )
standaloneModule = False
self.__basePath = gConfig.getValue( '/LocalSite/InstancePath', rootPath )
self.__agentModule = None
self.__codeProperties = {}
self.__getCodeInfo()
self.__moduleProperties = { 'fullName' : agentName,
'loadName' : loadName,
'section' : PathFinder.getAgentSection( agentName ),
'loadSection' : PathFinder.getAgentSection( loadName ),
'standalone' : standaloneModule,
'cyclesDone' : 0,
'totalElapsedTime' : 0,
'setup' : gConfig.getValue( "/DIRAC/Setup", "Unknown" ),
'alive' : True }
self.__moduleProperties[ 'system' ], self.__moduleProperties[ 'agentName' ] = agentName.split( "/" )
self.__configDefaults = {}
self.__configDefaults[ 'MonitoringEnabled'] = True
self.__configDefaults[ 'Enabled'] = self.am_getOption( "Status", "Active" ).lower() in ( 'active' )
self.__configDefaults[ 'PollingTime'] = self.am_getOption( "PollingTime", 120 )
self.__configDefaults[ 'MaxCycles'] = self.am_getOption( "MaxCycles", 500 )
self.__configDefaults[ 'ControlDirectory' ] = os.path.join( self.__basePath,
'control',
*agentName.split( "/" ) )
self.__configDefaults[ 'WorkDirectory' ] = os.path.join( self.__basePath,
'work',
*agentName.split( "/" ) )
self.__configDefaults[ 'shifterProxy' ] = ''
self.__configDefaults[ 'shifterProxyLocation' ] = os.path.join( self.__configDefaults[ 'WorkDirectory' ],
'.shifterCred' )
if isinstance( properties, dict):
for key in properties:
self.__moduleProperties[ key ] = properties[ key ]
self.__moduleProperties[ 'executors' ] = [ ( self.execute, () ) ]
self.__moduleProperties[ 'shifterProxy' ] = False
self.__monitorLastStatsUpdate = -1
self.monitor = None
self.__initializeMonitor()
self.__initialized = False
def setUp( self ):
""" test setup """
self.configPath = PathFinder.getAgentSection( "DataManagement/TransferAgent" )
self.channels = { 1L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'CERN', 'ChannelName': 'CERN-CERN', 'Size': 0},
2L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'CERN', 'ChannelName': 'CERN-CNAF', 'Size': 0},
3L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'CERN', 'ChannelName': 'CERN-GRIDKA', 'Size': 0},
4L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'CERN', 'ChannelName': 'CERN-IN2P3', 'Size': 0},
5L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'CERN', 'ChannelName': 'CERN-NIKHEF', 'Size': 0},
6L: {'Status': 'Active', 'Files': 2, 'Destination': 'PIC', 'Source': 'CERN', 'ChannelName': 'CERN-PIC', 'Size': 0},
7L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'CERN', 'ChannelName': 'CERN-RAL', 'Size': 0},
8L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'CNAF', 'ChannelName': 'CNAF-CERN', 'Size': 0},
9L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'CNAF', 'ChannelName': 'CNAF-CNAF', 'Size': 0},
10L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'CNAF', 'ChannelName': 'CNAF-GRIDKA', 'Size': 0},
11L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'CNAF', 'ChannelName': 'CNAF-IN2P3', 'Size': 0},
12L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'CNAF', 'ChannelName': 'CNAF-NIKHEF', 'Size': 0},
13L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'CNAF', 'ChannelName': 'CNAF-PIC', 'Size': 0},
14L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'CNAF', 'ChannelName': 'CNAF-RAL', 'Size': 0},
15L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-CERN', 'Size': 0},
16L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-CNAF', 'Size': 0},
17L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-GRIDKA', 'Size': 0},
18L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-IN2P3', 'Size': 0},
19L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-NIKHEF', 'Size': 0},
20L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-PIC', 'Size': 0},
21L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-RAL', 'Size': 0},
22L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-CERN', 'Size': 0},
23L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-CNAF', 'Size': 0},
24L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-GRIDKA', 'Size': 0},
25L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-IN2P3', 'Size': 0},
26L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-NIKHEF', 'Size': 0},
27L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-PIC', 'Size': 0},
28L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-RAL', 'Size': 0},
29L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-CERN', 'Size': 0},
30L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-CNAF', 'Size': 0},
31L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-GRIDKA', 'Size': 0},
32L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-IN2P3', 'Size': 0},
33L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-NIKHEF', 'Size': 0},
34L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-PIC', 'Size': 0},
35L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-RAL', 'Size': 0},
36L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'PIC', 'ChannelName': 'PIC-CERN', 'Size': 0},
37L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'PIC', 'ChannelName': 'PIC-CNAF', 'Size': 0},
38L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'PIC', 'ChannelName': 'PIC-GRIDKA', 'Size': 0},
39L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'PIC', 'ChannelName': 'PIC-IN2P3', 'Size': 0},
40L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'PIC', 'ChannelName': 'PIC-NIKHEF', 'Size': 0},
41L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'PIC', 'ChannelName': 'PIC-PIC', 'Size': 0},
42L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'PIC', 'ChannelName': 'PIC-RAL', 'Size': 0},
43L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'RAL', 'ChannelName': 'RAL-CERN', 'Size': 0},
44L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'RAL', 'ChannelName': 'RAL-CNAF', 'Size': 0},
45L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'RAL', 'ChannelName': 'RAL-GRIDKA', 'Size': 0},
46L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'RAL', 'ChannelName': 'RAL-IN2P3', 'Size': 0},
47L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'RAL', 'ChannelName': 'RAL-NIKHEF', 'Size': 0},
48L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'RAL', 'ChannelName': 'RAL-PIC', 'Size': 0},
49L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'RAL', 'ChannelName': 'RAL-RAL', 'Size': 0},
50L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'CERN', 'ChannelName': 'CERN-SARA', 'Size': 0},
51L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'SARA', 'ChannelName': 'SARA-CERN', 'Size': 0},
52L: {'Status': 'Active', 'Files': 0, 'Destination': 'CNAF', 'Source': 'SARA', 'ChannelName': 'SARA-CNAF', 'Size': 0},
53L: {'Status': 'Active', 'Files': 0, 'Destination': 'GRIDKA', 'Source': 'SARA', 'ChannelName': 'SARA-GRIDKA', 'Size': 0},
54L: {'Status': 'Active', 'Files': 0, 'Destination': 'IN2P3', 'Source': 'SARA', 'ChannelName': 'SARA-IN2P3', 'Size': 0},
55L: {'Status': 'Active', 'Files': 0, 'Destination': 'NIKHEF', 'Source': 'SARA', 'ChannelName': 'SARA-NIKHEF', 'Size': 0},
56L: {'Status': 'Active', 'Files': 0, 'Destination': 'PIC', 'Source': 'SARA', 'ChannelName': 'SARA-PIC', 'Size': 0},
57L: {'Status': 'Active', 'Files': 0, 'Destination': 'RAL', 'Source': 'SARA', 'ChannelName': 'SARA-RAL', 'Size': 0},
58L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'SARA', 'ChannelName': 'SARA-SARA', 'Size': 0},
59L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'CNAF', 'ChannelName': 'CNAF-SARA', 'Size': 0},
60L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'GRIDKA', 'ChannelName': 'GRIDKA-SARA', 'Size': 0},
61L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'IN2P3', 'ChannelName': 'IN2P3-SARA', 'Size': 0},
62L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'NIKHEF', 'ChannelName': 'NIKHEF-SARA', 'Size': 0},
63L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'PIC', 'ChannelName': 'PIC-SARA', 'Size': 0},
64L: {'Status': 'Active', 'Files': 0, 'Destination': 'SARA', 'Source': 'RAL', 'ChannelName': 'RAL-SARA', 'Size': 0},
65L: {'Status': 'Active', 'Files': 0, 'Destination': 'CERN', 'Source': 'RAL-HEP', 'ChannelName': 'CERN-RAL-HEP', 'Size':0 } }
self.bands = dict.fromkeys( self.channels.keys() )
for k in self.bands:
self.bands[k] = { 'SuccessfulFiles': 0, 'FailedFiles': 0, 'Throughput': 0, 'Fileput': 0 }
#self.bands[1]["Fileput"] = 10.0
self.bands[14]["Fileput"] = 10.0
self.failedFiles = dict.fromkeys( self.channels.keys(), 0 )
def __init__( self, *args, **kwargs ):
""" c'tor """
# # call base class ctor
AgentModule.__init__( self, *args, **kwargs )
# # ProcessPool related stuff
self.__requestsPerCycle = self.am_getOption( "RequestsPerCycle", self.__requestsPerCycle )
self.log.info( "Requests/cycle = %d" % self.__requestsPerCycle )
self.__minProcess = self.am_getOption( "MinProcess", self.__minProcess )
self.log.info( "ProcessPool min process = %d" % self.__minProcess )
self.__maxProcess = self.am_getOption( "MaxProcess", 4 )
self.log.info( "ProcessPool max process = %d" % self.__maxProcess )
self.__queueSize = self.am_getOption( "ProcessPoolQueueSize", self.__queueSize )
self.log.info( "ProcessPool queue size = %d" % self.__queueSize )
self.__poolTimeout = int( self.am_getOption( "ProcessPoolTimeout", self.__poolTimeout ) )
self.log.info( "ProcessPool timeout = %d seconds" % self.__poolTimeout )
self.__poolSleep = int( self.am_getOption( "ProcessPoolSleep", self.__poolSleep ) )
self.log.info( "ProcessPool sleep time = %d seconds" % self.__poolSleep )
self.__bulkRequest = self.am_getOption( "BulkRequest", 0 )
self.log.info( "Bulk request size = %d" % self.__bulkRequest )
# # keep config path and agent name
self.agentName = self.am_getModuleParam( "fullName" )
self.__configPath = PathFinder.getAgentSection( self.agentName )
# # operation handlers over here
opHandlersPath = "%s/%s" % ( self.__configPath, "OperationHandlers" )
opHandlers = gConfig.getSections( opHandlersPath )
if not opHandlers["OK"]:
self.log.error( opHandlers["Message" ] )
raise AgentConfigError( "OperationHandlers section not found in CS under %s" % self.__configPath )
opHandlers = opHandlers["Value"]
self.timeOuts = dict()
# # handlers dict
self.handlersDict = dict()
for opHandler in opHandlers:
opHandlerPath = "%s/%s/Location" % ( opHandlersPath, opHandler )
opLocation = gConfig.getValue( opHandlerPath, "" )
if not opLocation:
self.log.error( "%s not set for %s operation handler" % ( opHandlerPath, opHandler ) )
continue
self.timeOuts[opHandler] = { "PerFile": self.__fileTimeout, "PerOperation": self.__operationTimeout }
opTimeout = gConfig.getValue( "%s/%s/TimeOut" % ( opHandlersPath, opHandler ), 0 )
if opTimeout:
self.timeOuts[opHandler]["PerOperation"] = opTimeout
fileTimeout = gConfig.getValue( "%s/%s/TimeOutPerFile" % ( opHandlersPath, opHandler ), 0 )
if fileTimeout:
self.timeOuts[opHandler]["PerFile"] = fileTimeout
self.handlersDict[opHandler] = opLocation
self.log.info( "Operation handlers:" )
for item in enumerate ( self.handlersDict.items() ):
opHandler = item[1][0]
self.log.info("[%s] %s: %s (timeout: %d s + %d s per file)" % (item[0], item[1][0], item[1][1],
self.timeOuts[opHandler]['PerOperation'],
self.timeOuts[opHandler]['PerFile']))
# # common monitor activity
gMonitor.registerActivity( "Iteration", "Agent Loops",
"RequestExecutingAgent", "Loops/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Processed", "Request Processed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Done", "Request Completed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
# # create request dict
self.__requestCache = dict()
# ?? Probably should be removed
self.FTSMode = self.am_getOption( "FTSMode", False )
def __init__(self,
agentName,
loadName,
baseAgentName=False,
properties={}):
"""
Common __init__ method for all Agents.
All Agent modules must define:
__doc__
__RCSID__
They are used to populate __codeProperties
The following Options are used from the Configuration:
- /LocalSite/InstancePath
- /DIRAC/Setup
- Status
- Enabled
- PollingTime default = 120
- MaxCycles default = 500
- WatchdogTime default = 0 (disabled)
- ControlDirectory control/SystemName/AgentName
- WorkDirectory work/SystemName/AgentName
- shifterProxy ''
- shifterProxyLocation WorkDirectory/SystemName/AgentName/.shifterCred
It defines the following default Options that can be set via Configuration (above):
- MonitoringEnabled True
- Enabled True if Status == Active
- PollingTime 120
- MaxCycles 500
- ControlDirectory control/SystemName/AgentName
- WorkDirectory work/SystemName/AgentName
- shifterProxy False
- shifterProxyLocation work/SystemName/AgentName/.shifterCred
different defaults can be set in the initialize() method of the Agent using am_setOption()
In order to get a shifter proxy in the environment during the execute()
the configuration Option 'shifterProxy' must be set, a default may be given
in the initialize() method.
"""
if baseAgentName and agentName == baseAgentName:
self.log = gLogger
standaloneModule = True
else:
self.log = gLogger.getSubLogger(agentName, child=False)
standaloneModule = False
self.__basePath = gConfig.getValue("/LocalSite/InstancePath", rootPath)
self.__agentModule = None
self.__codeProperties = {}
self.__getCodeInfo()
self.__moduleProperties = {
"fullName": agentName,
"loadName": loadName,
"section": PathFinder.getAgentSection(agentName),
"loadSection": PathFinder.getAgentSection(loadName),
"standalone": standaloneModule,
"cyclesDone": 0,
"totalElapsedTime": 0,
"setup": gConfig.getValue("/DIRAC/Setup", "Unknown"),
"alive": True,
}
self.__moduleProperties["system"], self.__moduleProperties[
"agentName"] = agentName.split("/")
self.__configDefaults = {}
self.__configDefaults["MonitoringEnabled"] = True
self.__configDefaults["Enabled"] = self.am_getOption(
"Status", "Active").lower() in ("active")
self.__configDefaults["PollingTime"] = self.am_getOption(
"PollingTime", 120)
self.__configDefaults["MaxCycles"] = self.am_getOption(
"MaxCycles", 500)
self.__configDefaults["WatchdogTime"] = self.am_getOption(
"WatchdogTime", 0)
self.__configDefaults["ControlDirectory"] = os.path.join(
self.__basePath, "control", *agentName.split("/"))
self.__configDefaults["WorkDirectory"] = os.path.join(
self.__basePath, "work", *agentName.split("/"))
self.__configDefaults["shifterProxy"] = ""
self.__configDefaults["shifterProxyLocation"] = os.path.join(
self.__configDefaults["WorkDirectory"], ".shifterCred")
if isinstance(properties, dict):
for key in properties:
self.__moduleProperties[key] = properties[key]
self.__moduleProperties["executors"] = [(self.execute, ())]
self.__moduleProperties["shifterProxy"] = False
self.__monitorLastStatsUpdate = -1
self.monitor = None
self.__initializeMonitor()
self.__initialized = False
def showChannels():
""" print info about the last hour performance of FTS system """
global gProblematic
taSection = PathFinder.getAgentSection("DataManagement/TransferAgent")
timeScale = gConfig.getOption(taSection + "/ThroughputTimescale", 3600)
if not timeScale["OK"]:
gLogger.error(timeScale["Message"])
DIRAC.exit(1)
timeScale = int(timeScale["Value"])
accFailureRate = gConfig.getOption(
taSection + "/StrategyHandler/AcceptableFailureRate", 75)
if not accFailureRate["OK"]:
gLogger.error(accFailureRate["Message"])
accFailureRate = int(accFailureRate["Value"])
accFailedFiles = gConfig.getOption(
taSection + "/StrategyHandler/AcceptableFailedFiles", 5)
if not accFailedFiles["OK"]:
gLogger.error(accFailedFiles["Message"])
accFailedFiles = int(accFailedFiles["Value"])
scInfo = "timescale = %s s\nacc failure rate = %s %%\nacc distinct failed files = %s" % (
timeScale, accFailureRate, accFailedFiles)
## db monitor
transferDB = RPCClient("DataManagement/TransferDBMonitoring")
## get channels
channels = transferDB.getChannelQueues()
if not channels["OK"]:
gLogger.error(channels["Message"])
DIRAC.exit(1)
channels = channels["Value"]
## gend bandwidths
bands = transferDB.getChannelObservedThroughput(timeScale)
if not bands["OK"]:
gLogger.error(bands["Message"])
DIRAC.exit(1)
bands = bands["Value"]
## get failed files
badFiles = transferDB.getCountFileToFTS(timeScale, "Failed")
if not badFiles["OK"]:
gLogger.error(badFiles["Message"])
DIRAC.exit(1)
badFiles = badFiles["Value"] if badFiles["Value"] else {}
colorize = sys.stdout.isatty()
header = " %2s | %-15s | %8s | %8s | %8s | %8s | %8s | %12s | %8s | %8s" % (
"Id", "Name", "Status", "Waiting", "Success", "Failed", "FilePut",
"ThroughPut", "FailRate", "FailedFiles")
dashLine = "-" * len(header)
lineTemplate = " %2s | %-15s | %8s | %8d | %8d | %8d | %8.2f | %12.2f | %8.2f | %8d"
printOut = []
for chId, channel in channels.items():
name = channel["ChannelName"]
color = None
status = channel["Status"]
if status == "Active":
status = "OK"
waitingFiles = channel["Files"]
waitingSize = channel["Size"]
failedFiles = successFiles = filePut = througPut = fRate = 0
fFiles = 0
if chId in badFiles:
fFiles = int(badFiles[chId])
if chId in bands:
band = bands[chId]
failedFiles = int(band["FailedFiles"])
successFiles = int(band["SuccessfulFiles"])
filePut = band["Fileput"]
throughPut = band["Throughput"]
if failedFiles or successFiles:
fRate = 100.0 * float(failedFiles) / (float(failedFiles) +
float(successFiles))
if fRate > 0 and colorize:
color = "yellow"
status = "Poor"
if fRate > accFailureRate and fFiles > accFailedFiles:
status = "Closed"
if colorize:
color = "red"
if gProblematic and not fRate:
continue
if colorize and color:
line = colors[color] % lineTemplate
else:
line = lineTemplate
printOut.append(
line %
(chId, name, status, waitingFiles if waitingFiles else 0,
successFiles if successFiles else 0, failedFiles if failedFiles
else 0, filePut if filePut else 0, throughPut if througPut else 0,
fRate if fRate else 0, fFiles if fFiles else 0))
if printOut:
printOut = [scInfo, header, dashLine] + printOut
for line in printOut:
gLogger.always(line)
else:
gLogger.always("Noting to display...")
def showChannels():
""" print info about the last hour performance of FTS system """
global gProblematic
taSection = PathFinder.getAgentSection("DataManagement/TransferAgent")
timeScale = gConfig.getOption( taSection + "/ThroughputTimescale", 3600 )
if not timeScale["OK"]:
gLogger.error( timeScale["Message"] )
DIRAC.exit(1)
timeScale = int( timeScale["Value"] )
accFailureRate = gConfig.getOption( taSection + "/StrategyHandler/AcceptableFailureRate", 75 )
if not accFailureRate["OK"]:
gLogger.error( accFailureRate["Message"] )
accFailureRate = int( accFailureRate["Value"] )
accFailedFiles = gConfig.getOption( taSection + "/StrategyHandler/AcceptableFailedFiles", 5 )
if not accFailedFiles["OK"]:
gLogger.error( accFailedFiles["Message"] )
accFailedFiles = int( accFailedFiles["Value"] )
scInfo = "timescale = %s s\nacc failure rate = %s %%\nacc distinct failed files = %s" % ( timeScale,
accFailureRate,
accFailedFiles )
## db monitor
transferDB = RPCClient( "DataManagement/TransferDBMonitoring" )
## get channels
channels = transferDB.getChannelQueues()
if not channels["OK"]:
gLogger.error( channels["Message"] )
DIRAC.exit(1)
channels = channels["Value"]
## gend bandwidths
bands = transferDB.getChannelObservedThroughput( timeScale )
if not bands["OK"]:
gLogger.error( bands["Message"] )
DIRAC.exit(1)
bands = bands["Value"]
## get failed files
badFiles = transferDB.getCountFileToFTS( timeScale, "Failed" )
if not badFiles["OK"]:
gLogger.error( badFiles["Message"] )
DIRAC.exit(1)
badFiles = badFiles["Value"] if badFiles["Value"] else {}
colorize = sys.stdout.isatty()
header = " %2s | %-15s | %8s | %8s | %8s | %8s | %8s | %12s | %8s | %8s" % ( "Id", "Name", "Status",
"Waiting", "Success", "Failed",
"FilePut", "ThroughPut", "FailRate",
"FailedFiles" )
dashLine = "-"*len(header)
lineTemplate = " %2s | %-15s | %8s | %8d | %8d | %8d | %8.2f | %12.2f | %8.2f | %8d"
printOut = []
for chId, channel in channels.items():
name = channel["ChannelName"]
color = None
status = channel["Status"]
if status == "Active":
status = "OK"
waitingFiles = channel["Files"]
waitingSize = channel["Size"]
failedFiles = successFiles = filePut = througPut = fRate = 0
fFiles = 0
if chId in badFiles:
fFiles = int(badFiles[chId])
if chId in bands:
band = bands[chId]
failedFiles = int(band["FailedFiles"])
successFiles = int(band["SuccessfulFiles"])
filePut = band["Fileput"]
throughPut = band["Throughput"]
if failedFiles or successFiles:
fRate = 100.0 * float( failedFiles ) / ( float(failedFiles) + float( successFiles) )
if fRate > 0 and colorize:
color = "yellow"
status = "Poor"
if fRate > accFailureRate and fFiles > accFailedFiles:
status = "Closed"
if colorize:
color = "red"
if gProblematic and not fRate:
continue
if colorize and color:
line = colors[color] % lineTemplate
else:
line = lineTemplate
printOut.append( line % ( chId, name, status,
waitingFiles if waitingFiles else 0,
successFiles if successFiles else 0,
failedFiles if failedFiles else 0,
filePut if filePut else 0,
throughPut if througPut else 0,
fRate if fRate else 0,
fFiles if fFiles else 0 ) )
if printOut:
printOut = [ scInfo, header, dashLine ] + printOut
for line in printOut:
gLogger.always( line )
else:
gLogger.always("Noting to display...")
def __init__(self, agentName, baseAgentName=False, properties={}):
"""
Common __init__ method for all Agents.
All Agent modules must define:
__doc__
__RCSID__
They are used to populate __codeProperties
The following Options are used from the Configuration:
- /LocalSite/InstancePath
- /DIRAC/Setup
- Status
- Enabled
- PollingTime default = 120
- MaxCycles default = 500
- ControlDirectory control/SystemName/AgentName
- WorkDirectory work/SystemName/AgentName
- shifterProxy ''
- shifterProxyLocation WorkDirectory/SystemName/AgentName/.shifterCred
It defines the following default Options that can be set via Configuration (above):
- MonitoringEnabled True
- Enabled True if Status == Active
- PollingTime 120
- MaxCycles 500
- ControlDirectory control/SystemName/AgentName
- WorkDirectory work/SystemName/AgentName
- shifterProxy False
- shifterProxyLocation work/SystemName/AgentName/.shifterCred
different defaults can be set in the initialize() method of the Agent using am_setOption()
In order to get a shifter proxy in the environment during the execute()
the configuration Option 'shifterProxy' must be set, a default may be given
in the initialize() method.
"""
if baseAgentName and agentName == baseAgentName:
self.log = gLogger
standaloneModule = True
else:
self.log = gLogger.getSubLogger(agentName, child=False)
standaloneModule = False
self.__basePath = gConfig.getValue('/LocalSite/InstancePath', rootPath)
self.__agentModule = None
self.__codeProperties = {}
self.__getCodeInfo()
self.__moduleProperties = {
'fullName': agentName,
'section': PathFinder.getAgentSection(agentName),
'standalone': standaloneModule,
'cyclesDone': 0,
'totalElapsedTime': 0,
'setup': gConfig.getValue("/DIRAC/Setup", "Unknown")
}
self.__moduleProperties['system'], self.__moduleProperties[
'agentName'] = agentName.split("/")
self.__configDefaults = {}
self.__configDefaults['MonitoringEnabled'] = True
self.__configDefaults['Enabled'] = self.am_getOption(
"Status", "Active").lower() in ('active')
self.__configDefaults['PollingTime'] = self.am_getOption(
"PollingTime", 120)
self.__configDefaults['MaxCycles'] = self.am_getOption(
"MaxCycles", 500)
self.__configDefaults['ControlDirectory'] = os.path.join(
self.__basePath, 'control', *agentName.split("/"))
self.__configDefaults['WorkDirectory'] = os.path.join(
self.__basePath, 'work', *agentName.split("/"))
self.__configDefaults['shifterProxy'] = ''
self.__configDefaults['shifterProxyLocation'] = os.path.join(
self.__configDefaults['WorkDirectory'], '.shifterCred')
if type(properties) == types.DictType:
for key in properties:
self.__moduleProperties[key] = properties[key]
self.__moduleProperties['executors'] = [(self.execute, ())]
self.__moduleProperties['alive'] = True
self.__moduleProperties['shifterProxy'] = False
self.__monitorLastStatsUpdate = -1
self.monitor = None
self.__initializeMonitor()
self.__initialized = False
def initialize(self):
"""initialize agent"""
# # ProcessPool related stuff
self.__requestsPerCycle = self.am_getOption("RequestsPerCycle", self.__requestsPerCycle)
self.log.info("Requests/cycle = %d" % self.__requestsPerCycle)
self.__minProcess = self.am_getOption("MinProcess", self.__minProcess)
self.log.info("ProcessPool min process = %d" % self.__minProcess)
self.__maxProcess = self.am_getOption("MaxProcess", self.__maxProcess)
self.log.info("ProcessPool max process = %d" % self.__maxProcess)
self.__queueSize = self.am_getOption("ProcessPoolQueueSize", self.__queueSize)
self.log.info("ProcessPool queue size = %d" % self.__queueSize)
self.__poolTimeout = int(self.am_getOption("ProcessPoolTimeout", self.__poolTimeout))
self.log.info("ProcessPool timeout = %d seconds" % self.__poolTimeout)
self.__poolSleep = int(self.am_getOption("ProcessPoolSleep", self.__poolSleep))
self.log.info("ProcessPool sleep time = %d seconds" % self.__poolSleep)
self.__bulkRequest = self.am_getOption("BulkRequest", self.__bulkRequest)
self.log.info("Bulk request size = %d" % self.__bulkRequest)
# Check if monitoring is enabled
if "Monitoring" in Operations().getMonitoringBackends(monitoringType="RMSMonitoring"):
# Enable RMS monitoring
self.__rmsMonitoring = True
self.log.info("Enable ES RMS Monitoring = %s" % self.__rmsMonitoring)
# # keep config path and agent name
self.agentName = self.am_getModuleParam("fullName")
self.__configPath = PathFinder.getAgentSection(self.agentName)
# # operation handlers over here
opHandlersPath = "%s/%s" % (self.__configPath, "OperationHandlers")
opHandlers = gConfig.getSections(opHandlersPath)
if not opHandlers["OK"]:
self.log.error(opHandlers["Message"])
raise AgentConfigError("OperationHandlers section not found in CS under %s" % self.__configPath)
opHandlers = opHandlers["Value"]
self.timeOuts = dict()
# # handlers dict
self.handlersDict = dict()
for opHandler in opHandlers:
opHandlerPath = "%s/%s/Location" % (opHandlersPath, opHandler)
opLocation = gConfig.getValue(opHandlerPath, "")
if not opLocation:
self.log.error("%s not set for %s operation handler" % (opHandlerPath, opHandler))
continue
self.timeOuts[opHandler] = {"PerFile": self.__fileTimeout, "PerOperation": self.__operationTimeout}
opTimeout = gConfig.getValue("%s/%s/TimeOut" % (opHandlersPath, opHandler), 0)
if opTimeout:
self.timeOuts[opHandler]["PerOperation"] = opTimeout
fileTimeout = gConfig.getValue("%s/%s/TimeOutPerFile" % (opHandlersPath, opHandler), 0)
if fileTimeout:
self.timeOuts[opHandler]["PerFile"] = fileTimeout
self.handlersDict[opHandler] = opLocation
self.log.info("Operation handlers:")
for item in enumerate(self.handlersDict.items()):
opHandler = item[1][0]
self.log.info(
"[%s] %s: %s (timeout: %d s + %d s per file)"
% (
item[0],
item[1][0],
item[1][1],
self.timeOuts[opHandler]["PerOperation"],
self.timeOuts[opHandler]["PerFile"],
)
)
if self.__rmsMonitoring:
self.rmsMonitoringReporter = MonitoringReporter(monitoringType="RMSMonitoring")
gThreadScheduler.addPeriodicTask(100, self.__rmsMonitoringReporting)
# # create request dict
self.__requestCache = dict()
return S_OK()