def initializeReportGeneratorHandler( serviceInfo ):
global gAccountingDB
gAccountingDB = AccountingDB( readOnly = True )
#Get data location
reportSection = PathFinder.getServiceSection( "Accounting/ReportGenerator" )
dataPath = gConfig.getValue( "%s/DataLocation" % reportSection, "data/accountingGraphs" )
dataPath = dataPath.strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) )
gLogger.info( "Data will be written into %s" % dataPath )
try:
os.makedirs( dataPath )
except:
pass
try:
testFile = "%s/acc.jarl.test" % dataPath
fd = file( testFile, "w" )
fd.close()
os.unlink( testFile )
except IOError:
gLogger.fatal( "Can't write to %s" % dataPath )
return S_ERROR( "Data location is not writable" )
gDataCache.setGraphsLocation( dataPath )
gMonitor.registerActivity( "plotsDrawn", "Drawn plot images", "Accounting reports", "plots", gMonitor.OP_SUM )
gMonitor.registerActivity( "reportsRequested", "Generated reports", "Accounting reports", "reports", gMonitor.OP_SUM )
return S_OK()
def loadHandler( pluginPath ):
""" Create an instance of requested plugin class, loading and importing it when needed.
This function could raise ImportError when plugin cannot be find or TypeError when
loaded class object isn't inherited from BaseOperation class.
:param str pluginName: dotted path to plugin, specified as in import statement, i.e.
"DIRAC.CheesShopSystem.private.Cheddar" or alternatively in 'normal' path format
"DIRAC/CheesShopSystem/private/Cheddar"
:return: object instance
This function try to load and instantiate an object from given path. It is assumed that:
- :pluginPath: is pointing to module directory "importable" by python interpreter, i.e.: it's
package's top level directory is in $PYTHONPATH env variable,
- the module should consist a class definition following module name,
- the class itself is inherited from DIRAC.RequestManagementSystem.private.BaseOperation.BaseOperation
If above conditions aren't meet, function is throwing exceptions:
- ImportError when class cannot be imported
- TypeError when class isn't inherited from OperationHandlerBase
"""
if "/" in pluginPath:
pluginPath = ".".join( [ chunk for chunk in pluginPath.split( "/" ) if chunk ] )
pluginName = pluginPath.split( "." )[-1]
if pluginName not in globals():
mod = __import__( pluginPath, globals(), fromlist = [ pluginName ] )
pluginClassObj = getattr( mod, pluginName )
else:
pluginClassObj = globals()[pluginName]
if not issubclass( pluginClassObj, OperationHandlerBase ):
raise TypeError( "operation handler '%s' isn't inherited from OperationHandlerBase class" % pluginName )
for key, status in ( ( "Att", "Attempted" ), ( "OK", "Successful" ) , ( "Fail", "Failed" ) ):
gMonitor.registerActivity( "%s%s" % ( pluginName, key ), "%s operations %s" % ( pluginName, status ),
"RequestExecutingAgent", "Operations/min", gMonitor.OP_SUM )
# # return an instance
return pluginClassObj
def initialize(self):
self.pollingTime = self.am_getOption('PollingTime', 86400)
gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM)
#self.transClient = TransformationClient('TransformationDB')
self.transClient = TransformationClient()
self.fcc = FileCatalogClient()
return S_OK()
def initializePlottingHandler( serviceInfo ):
#Get data location
plottingSection = PathFinder.getServiceSection( "Framework/Plotting" )
dataPath = gConfig.getValue( "%s/DataLocation" % plottingSection, "data/graphs" )
dataPath = dataPath.strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) )
gLogger.info( "Data will be written into %s" % dataPath )
try:
os.makedirs( dataPath )
except:
pass
try:
testFile = "%s/plot__.test" % dataPath
fd = file( testFile, "w" )
fd.close()
os.unlink( testFile )
except IOError:
gLogger.fatal( "Can't write to %s" % dataPath )
return S_ERROR( "Data location is not writable" )
gPlotCache.setPlotsLocation( dataPath )
gMonitor.registerActivity( "plotsDrawn", "Drawn plot images", "Plotting requests", "plots", gMonitor.OP_SUM )
return S_OK()
def initialize( self ):
""" Agent initialization.
The extensions MUST provide in the initialize method the following data members:
- TransformationClient objects (self.transClient),
- set the shifterProxy if different from the default one set here ('ProductionManager')
- list of transformation types to be looked (self.transType)
"""
gMonitor.registerActivity( "SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks",
gMonitor.OP_ACUM )
self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Client.TaskManagerPlugin' )
# Default clients
self.transClient = TransformationClient()
# setting up the threading
maxNumberOfThreads = self.am_getOption( 'maxNumberOfThreads', 15 )
threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads )
self.log.verbose( "Multithreaded with %d threads" % maxNumberOfThreads )
for i in xrange( maxNumberOfThreads ):
threadPool.generateJobAndQueueIt( self._execute, [i] )
return S_OK()
def initialize(self):
self.pollingTime = self.am_getOption('PollingTime', 86400)
gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM)
self.ppc = ProcessProdClient()
self.dirac = DiracILC()
self.diracadmin = DiracAdmin()
self.am_setOption( 'shifterProxy', 'Admin' )
return S_OK()
def __init__( self, argumentsDict ): """ Standard constructor """ self.name = COMPONENT_NAME self.log = gLogger.getSubLogger( self.name ) self.argumentsDict = argumentsDict self.rm = False gMonitor.registerActivity( "Put and register", "Put and register operations", "PutAndRegister", "Attempts/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "Put failed", "Failed puts", "PutAndRegister", "Failed/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "Put successful", "Successful puts", "PutAndRegister", "Successful/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "File registration successful", "Successful file registrations", "PutAndRegister", "Successful/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "File registration failed", "Failed file registrations", "TransferAgent", "Failed/min", gMonitor.OP_SUM )
def initialize(self):
""" Make the necessary initializations """
self.fileLog = {}
self.timeLog = {}
self.fullTimeLog = {}
self.pollingTime = self.am_getOption('PollingTime',120)
self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod',86400)
gMonitor.registerActivity("Iteration","Agent Loops",AGENT_NAME,"Loops/min",gMonitor.OP_SUM)
self.transClient = TransformationClient('TransformationDB')
self.metadataClient = FileCatalogClient()
return S_OK()
def initializeMatcherHandler(serviceInfo):
""" Matcher Service initialization
"""
global jobDB
global jobLoggingDB
global taskQueueDB
jobDB = JobDB()
jobLoggingDB = JobLoggingDB()
taskQueueDB = TaskQueueDB()
gMonitor.registerActivity("matchTime", "Job matching time", "Matching", "secs", gMonitor.OP_MEAN, 300)
gMonitor.registerActivity(
"matchTaskQueues", "Task queues checked per job", "Matching", "task queues", gMonitor.OP_MEAN, 300
)
gMonitor.registerActivity("matchesDone", "Job Matches", "Matching", "matches", gMonitor.OP_MEAN, 300)
gMonitor.registerActivity("numTQs", "Number of Task Queues", "Matching", "tqsk queues", gMonitor.OP_MEAN, 300)
taskQueueDB.recalculateTQSharesForAll()
gThreadScheduler.addPeriodicTask(120, taskQueueDB.recalculateTQSharesForAll)
gThreadScheduler.addPeriodicTask(120, sendNumTaskQueues)
sendNumTaskQueues()
return S_OK()
def __init__(self, operation=None, csPath=None):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: cs config path
"""
DMSRequestOperationsBase.__init__(self, operation, csPath)
# # gMonitor stuff
gMonitor.registerActivity("PhysicalRemovalAtt",
"Physical file removals attempted",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("PhysicalRemovalOK",
"Successful file physical removals",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("PhysicalRemovalFail",
"Failed file physical removals",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("PhysicalRemovalSize",
"Physically removed size",
"RequestExecutingAgent", "Bytes",
gMonitor.OP_ACUM)
def initialize(self):
self.RequestDBClient = RequestClient()
backend = self.am_getOption('Backend', '')
self.RequestDB = False
if backend == 'mysql':
from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL
requestDB = RequestDBMySQL()
if requestDB._connected:
self.RequestDB = requestDB
gMonitor.registerActivity("Iteration", "Agent Loops",
"DISETForwardingAgent", "Loops/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Attempted", "Request Processed",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Successful", "Request Forward Successful",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Failed", "Request Forward Failed",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
#self.local = PathFinder.getServiceURL( "RequestManagement/localURL" )
#if not self.local:
# self.local = AgentModule.am_getOption( self, 'localURL', '' )
#if not self.local:
# errStr = 'The RequestManagement/localURL option must be defined.'
# gLogger.fatal( errStr )
# return S_ERROR( errStr )
return S_OK()
def initializeMatcherHandler( serviceInfo ):
""" Matcher Service initialization
"""
global gJobDB
global gJobLoggingDB
global gTaskQueueDB
global gPilotAgentsDB
gJobDB = JobDB()
gJobLoggingDB = JobLoggingDB()
gTaskQueueDB = TaskQueueDB()
gPilotAgentsDB = PilotAgentsDB()
gMonitor.registerActivity( 'matchTime', "Job matching time",
'Matching', "secs" , gMonitor.OP_MEAN, 300 )
gMonitor.registerActivity( 'matchesDone', "Job Match Request",
'Matching', "matches" , gMonitor.OP_RATE, 300 )
gMonitor.registerActivity( 'matchesOK', "Matched jobs",
'Matching', "matches" , gMonitor.OP_RATE, 300 )
gMonitor.registerActivity( 'numTQs', "Number of Task Queues",
'Matching', "tqsk queues" , gMonitor.OP_MEAN, 300 )
gTaskQueueDB.recalculateTQSharesForAll()
gThreadScheduler.addPeriodicTask( 120, gTaskQueueDB.recalculateTQSharesForAll )
gThreadScheduler.addPeriodicTask( 60, sendNumTaskQueues )
sendNumTaskQueues()
return S_OK()
def initialize(self):
self.RequestDBClient = RequestClient()
backend = self.am_getOption('Backend', '')
self.RequestDB = False
if backend == 'mysql':
from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL
requestDB = RequestDBMySQL()
if requestDB._connected:
self.RequestDB = requestDB
gMonitor.registerActivity("Iteration", "Agent Loops",
"DISETForwardingAgent", "Loops/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Attempted", "Request Processed",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Successful", "Request Forward Successful",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Failed", "Request Forward Failed",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
#self.local = PathFinder.getServiceURL( "RequestManagement/localURL" )
#if not self.local:
# self.local = AgentModule.am_getOption( self, 'localURL', '' )
#if not self.local:
# errStr = 'The RequestManagement/localURL option must be defined.'
# gLogger.fatal( errStr )
# return S_ERROR( errStr )
return S_OK()
def initializeMatcherHandler(serviceInfo):
""" Matcher Service initialization
"""
global gJobDB
global gJobLoggingDB
global gTaskQueueDB
global gPilotAgentsDB
gJobDB = JobDB()
gJobLoggingDB = JobLoggingDB()
gTaskQueueDB = TaskQueueDB()
gPilotAgentsDB = PilotAgentsDB()
gMonitor.registerActivity('matchTime', "Job matching time", 'Matching',
"secs", gMonitor.OP_MEAN, 300)
gMonitor.registerActivity('matchesDone', "Job Match Request", 'Matching',
"matches", gMonitor.OP_RATE, 300)
gMonitor.registerActivity('matchesOK', "Matched jobs", 'Matching',
"matches", gMonitor.OP_RATE, 300)
gMonitor.registerActivity('numTQs', "Number of Task Queues", 'Matching',
"tqsk queues", gMonitor.OP_MEAN, 300)
gTaskQueueDB.recalculateTQSharesForAll()
gThreadScheduler.addPeriodicTask(120,
gTaskQueueDB.recalculateTQSharesForAll)
gThreadScheduler.addPeriodicTask(60, sendNumTaskQueues)
sendNumTaskQueues()
return S_OK()
def initialize( self ):
self.jobDB = JobDB()
for status in MONITOR_STATUS:
for site in MONITOR_SITES:
gLogger.verbose( "Registering activity %s-%s" % ( status, site ) )
gLogger.verbose( "Jobs in %s state at %s" % ( status, site ) )
gMonitor.registerActivity( "%s-%s" % ( status, site ), "Jobs in %s state at %s" % ( status, site ),
"JobHistoryAgent", "Jobs/minute", gMonitor.OP_MEAN )
self.last_update = 0
self.resultDB = None
self.reportPeriod = 60
return S_OK()
def initialize(self):
self.jobDB = JobDB()
for status in MONITOR_STATUS:
for site in MONITOR_SITES:
gLogger.verbose("Registering activity %s-%s" % (status, site))
gLogger.verbose("Jobs in %s state at %s" % (status, site))
gMonitor.registerActivity(
"%s-%s" % (status, site),
"Jobs in %s state at %s" % (status, site),
"JobHistoryAgent", "Jobs/minute", gMonitor.OP_MEAN)
self.last_update = 0
self.resultDB = None
self.reportPeriod = 60
return S_OK()
def initialize( self ):
''' agent initialization
'''
if not self.taskManager:
return S_ERROR( 'No task manager provided!' )
if not self.shifterProxy:
return S_ERROR( 'No shifter proxy provided!' )
self.am_setOption( 'shifterProxy', self.shifterProxy )
if not self.transType:
return S_ERROR( 'No transformation types to look for!' )
gLogger.info( "Looking for %s" % self.transType )
gMonitor.registerActivity( "SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks",
gMonitor.OP_ACUM )
return S_OK()
def initialize( self ):
""" agent initialization """
self.DEL_GRACE_DAYS = self.am_getOption( "DeleteGraceDays", self.DEL_GRACE_DAYS )
self.log.info( "Delete grace period = %s days" % self.DEL_GRACE_DAYS )
self.DEL_LIMIT = self.am_getOption( "DeleteLimitPerCycle", self.DEL_LIMIT )
self.log.info( "Delete FTSJob limit = %s" % self.DEL_LIMIT )
self.KICK_ASSIGNED_HOURS = self.am_getOption( "KickAssignedHours", self.KICK_ASSIGNED_HOURS )
self.log.info( "Kick assigned period = %s hours" % self.KICK_ASSIGNED_HOURS )
self.KICK_LIMIT = self.am_getOption( "KickLimitPerCycle", self.KICK_LIMIT )
self.log.info( "Kick FTSJobs limit = %s" % self.KICK_LIMIT )
gMonitor.registerActivity( "KickedFTSJobs", "Assigned FTSJobs kicked",
"CleanFTSDBAgent", "FTSJobs/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "DeletedFTSJobs", "Deleted FTSJobs",
"CleanFTSDBAgent", "FTSJobs/min", gMonitor.OP_SUM )
return S_OK()
def initialize( self, tsClient = None, taskManager = None ):
self.section = self.am_getOption( "section" )
gMonitor.registerActivity( "SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks", gMonitor.OP_ACUM )
if not tsClient:
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
self.transClient = TransformationClient()
else:
self.transClient = tsClient
self.taskManager = taskManager
if not taskManager:
return S_ERROR('No task manager provided !')
self.transType = self.am_getOption( "TransType", ['MCSimulation'] )
gLogger.info( "Looking for %s" % self.transType )
return S_OK()
def initialize( self ):
''' Make the necessary initializations
'''
gMonitor.registerActivity( "Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM )
agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
if agentTSTypes:
self.transformationTypes = sorted( agentTSTypes )
else:
dataProc = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
dataManip = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
self.transformationTypes = sorted( dataProc + dataManip )
extendables = Operations().getValue( 'Transformations/ExtendableTransfTypes', [])
if extendables:
for extendable in extendables:
if extendable in self.transformationTypes:
self.transformationTypes.remove(extendable)
#This is because the Extendables do not use this Agent (have no Input data query)
return S_OK()
def initialize( self ):
""" initialization """
self.DEL_GRACE_DAYS = self.am_getOption( "DeleteGraceDays", self.DEL_GRACE_DAYS )
self.log.info( "Delete grace period = %s days" % self.DEL_GRACE_DAYS )
self.DEL_LIMIT = self.am_getOption( "DeleteLimit", self.DEL_LIMIT )
self.log.info( "Delete limit = %s request/cycle" % self.DEL_LIMIT )
self.DEL_FAILED = self.am_getOption( "DeleteFailed", self.DEL_FAILED )
self.log.info( "Delete failed requests: %s" % { True: "yes", False: "no"}[self.DEL_FAILED] )
self.KICK_GRACE_HOURS = self.am_getOption( "KickGraceHours", self.KICK_GRACE_HOURS )
self.log.info( "Kick assigned requests period = %s hours" % self.KICK_GRACE_HOURS )
self.KICK_LIMIT = self.am_getOption( "KickLimit", self.KICK_LIMIT )
self.log.info( "Kick limit = %s request/cycle" % self.KICK_LIMIT )
# # gMonitor stuff
gMonitor.registerActivity( "DeletedRequests", "Deleted finished requests",
"CleanReqDBAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "KickedRequests", "Assigned requests kicked",
"CleanReqDBAgent", "Requests/min", gMonitor.OP_SUM )
return S_OK()
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
# # base class ctor
OperationHandlerBase.__init__( self, operation, csPath )
# # gMonitor stuff
gMonitor.registerActivity( "PutAtt", "File put attempts",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "PutFail", "Failed file puts",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "PutOK", "Successful file puts",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterOK", "Successful file registrations",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterFail", "Failed file registrations",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
def loadHandler(pluginPath):
""" Create an instance of requested plugin class, loading and importing it when needed.
This function could raise ImportError when plugin cannot be find or TypeError when
loaded class object isn't inherited from BaseOperation class.
:param str pluginName: dotted path to plugin, specified as in import statement, i.e.
"DIRAC.CheesShopSystem.private.Cheddar" or alternatively in 'normal' path format
"DIRAC/CheesShopSystem/private/Cheddar"
:return: object instance
This function try to load and instantiate an object from given path. It is assumed that:
- :pluginPath: is pointing to module directory "importable" by python interpreter, i.e.: it's
package's top level directory is in $PYTHONPATH env variable,
- the module should consist a class definition following module name,
- the class itself is inherited from DIRAC.RequestManagementSystem.private.BaseOperation.BaseOperation
If above conditions aren't meet, function is throwing exceptions:
- ImportError when class cannot be imported
- TypeError when class isn't inherited from OperationHandlerBase
"""
if "/" in pluginPath:
pluginPath = ".".join(
[chunk for chunk in pluginPath.split("/") if chunk])
pluginName = pluginPath.split(".")[-1]
if pluginName not in globals():
mod = __import__(pluginPath, globals(), fromlist=[pluginName])
pluginClassObj = getattr(mod, pluginName)
else:
pluginClassObj = globals()[pluginName]
if not issubclass(pluginClassObj, OperationHandlerBase):
raise TypeError(
"operation handler '%s' isn't inherited from OperationHandlerBase class"
% pluginName)
for key, status in (("Att", "Attempted"), ("OK", "Successful"),
("Fail", "Failed")):
gMonitor.registerActivity(
"%s%s" % (pluginName, key),
"%s operations %s" % (pluginName, status),
"RequestExecutingAgent", "Operations/min", gMonitor.OP_SUM)
# # return an instance
return pluginClassObj
def __init__(self, argumentsDict):
""" Standard constructor """
self.name = COMPONENT_NAME
self.log = gLogger.getSubLogger(self.name)
self.argumentsDict = argumentsDict
self.rm = False
gMonitor.registerActivity("Put and register",
"Put and register operations",
"PutAndRegister", "Attempts/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Put failed", "Failed puts",
"PutAndRegister", "Failed/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Put successful", "Successful puts",
"PutAndRegister", "Successful/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("File registration successful",
"Successful file registrations",
"PutAndRegister", "Successful/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("File registration failed",
"Failed file registrations", "TransferAgent",
"Failed/min", gMonitor.OP_SUM)
def initialize( self ):
""" Agent initialization.
The extensions MUST provide in the initialize method the following data members:
- TransformationClient objects (self.transClient),
- set the shifterProxy if different from the default one set here ('ProductionManager')
- list of transformation types to be looked (self.transType)
"""
gMonitor.registerActivity( "SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks",
gMonitor.OP_ACUM )
# Default clients
self.transClient = TransformationClient()
#setting up the threading
maxNumberOfThreads = self.am_getOption( 'maxNumberOfThreads', 15 )
threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads )
self.log.verbose( "Multithreaded with %d threads" % maxNumberOfThreads )
for i in xrange( maxNumberOfThreads ):
threadPool.generateJobAndQueueIt( self._execute, [i] )
return S_OK()
def initialize(self):
""" agent initialization """
self.DEL_GRACE_DAYS = self.am_getOption("DeleteGraceDays",
self.DEL_GRACE_DAYS)
self.log.info("Delete grace period = %s days" % self.DEL_GRACE_DAYS)
self.DEL_LIMIT = self.am_getOption("DeleteLimitPerCycle",
self.DEL_LIMIT)
self.log.info("Delete FTSJob limit = %s" % self.DEL_LIMIT)
self.KICK_ASSIGNED_HOURS = self.am_getOption("KickAssignedHours",
self.KICK_ASSIGNED_HOURS)
self.log.info("Kick assigned period = %s hours" %
self.KICK_ASSIGNED_HOURS)
self.KICK_LIMIT = self.am_getOption("KickLimitPerCycle",
self.KICK_LIMIT)
self.log.info("Kick FTSJobs limit = %s" % self.KICK_LIMIT)
gMonitor.registerActivity("KickedFTSJobs", "Assigned FTSJobs kicked",
"CleanFTSDBAgent", "FTSJobs/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("DeletedFTSJobs", "Deleted FTSJobs",
"CleanFTSDBAgent", "FTSJobs/min",
gMonitor.OP_SUM)
return S_OK()
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: cs config path
"""
OperationHandlerBase.__init__( self, operation, csPath )
# # gMonitor stuff
gMonitor.registerActivity( "PhysicalRemovalAtt", "Physical file removals attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "PhysicalRemovalOK", "Successful file physical removals",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "PhysicalRemovalFail", "Failed file physical removals",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "PhysicalRemovalSize", "Physically removed size",
"RequestExecutingAgent", "Bytes", gMonitor.OP_ACUM )
def __init__(self,
requestJSON,
handlersDict,
csPath,
agentName,
standalone=False,
requestClient=None):
"""c'tor
:param self: self reference
:param str requestJSON: request serialized to JSON
:param dict opHandlers: operation handlers
"""
self.request = Request(requestJSON)
# # csPath
self.csPath = csPath
# # agent name
self.agentName = agentName
# # standalone flag
self.standalone = standalone
# # handlers dict
self.handlersDict = handlersDict
# # handlers class def
self.handlers = {}
# # own sublogger
self.log = gLogger.getSubLogger(
"pid_%s/%s" % (os.getpid(), self.request.RequestName))
# # get shifters info
self.__managersDict = {}
shifterProxies = self.__setupManagerProxies()
if not shifterProxies["OK"]:
self.log.error(shifterProxies["Message"])
# # initialize gMonitor
gMonitor.setComponentType(gMonitor.COMPONENT_AGENT)
gMonitor.setComponentName(self.agentName)
gMonitor.initialize()
# # own gMonitor activities
gMonitor.registerActivity("RequestAtt", "Requests processed",
"RequestExecutingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RequestFail", "Requests failed",
"RequestExecutingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RequestOK", "Requests done",
"RequestExecutingAgent", "Requests/min",
gMonitor.OP_SUM)
if requestClient is None:
self.requestClient = ReqClient()
else:
self.requestClient = requestClient
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
OperationHandlerBase.__init__( self, operation, csPath )
# # RegisterFile specific monitor info
gMonitor.registerActivity( "RegisterAtt", "Attempted file registrations",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterOK", "Successful file registrations",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterFail", "Failed file registrations",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param Operation operation: operation to execute
:param str csPath: CS path for this handler
"""
# # base class ctor
DMSRequestOperationsBase.__init__( self, operation, csPath )
# # gMonitor stuff
gMonitor.registerActivity( "FileReTransferAtt", "File retransfers attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FileReTransferOK", "File retransfers successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FileReTransferFail", "File retransfers failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param Operation operation: operation to execute
:param str csPath: CS path for this handler
"""
# # base class ctor
DMSRequestOperationsBase.__init__( self, operation, csPath )
# # gMonitor stuff
gMonitor.registerActivity( "RemoveReplicaAtt", "Replica removals attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveReplicaeOK", "Successful replica removals",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveReplicaFail", "Failed replica removals",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param Operation operation: operation to execute
:param str csPath: CS path for this handler
"""
# # base class ctor
OperationHandlerBase.__init__( self, operation, csPath )
# # gMonitor stuff
gMonitor.registerActivity( "RemoveReplicaAtt", "Replica removals attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveReplicaeOK", "Successful replica removals",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveReplicaFail", "Failed replica removals",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
def initialize(self):
self.RequestDBClient = RequestClient()
gMonitor.registerActivity("Iteration", "Agent Loops", "ZuziaAgent", "Loops/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Attempted", "Request Processed", "ZuziaRAgent", "Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Successful", "Request Forward Successful", "ZuziaAgent", "Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Failed", "Request Forward Failed", "ZuziaAgent", "Requests/min", gMonitor.OP_SUM)
self.local = PathFinder.getServiceURL("RequestManagement/localURL")
if not self.local:
self.local = AgentModule.am_getOption(self,'localURL','')
if not self.local:
errStr = 'The RequestManagement/localURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
self.central = PathFinder.getServiceURL("RequestManagement/centralURL")
if not self.central:
errStr = 'The RequestManagement/centralURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
return S_OK()
def initializeMatcherHandler( serviceInfo ): """ Matcher Service initialization """ global jobDB global jobLoggingDB global taskQueueDB jobDB = JobDB() jobLoggingDB = JobLoggingDB() taskQueueDB = TaskQueueDB() gMonitor.registerActivity( 'matchTime', "Job matching time", 'Matching', "secs" , gMonitor.OP_MEAN, 300 ) gMonitor.registerActivity( 'matchTaskQueues', "Task queues checked per job", 'Matching', "task queues" , gMonitor.OP_MEAN, 300 ) gMonitor.registerActivity( 'matchesDone', "Job Matches", 'Matching', "matches" , gMonitor.OP_MEAN, 300 ) gMonitor.registerActivity( 'numTQs', "Number of Task Queues", 'Matching', "tqsk queues" , gMonitor.OP_MEAN, 300 ) taskQueueDB.recalculateTQSharesForAll() gThreadScheduler.addPeriodicTask( 120, taskQueueDB.recalculateTQSharesForAll ) gThreadScheduler.addPeriodicTask( 120, sendNumTaskQueues ) sendNumTaskQueues() return S_OK()
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param string csPath: CS path for this handler
"""
# # base class ctor
OperationHandlerBase.__init__( self, operation, csPath )
# # gMonitor stuff
gMonitor.registerActivity( "LogUploadAtt", "Log upload attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "LogUploadOK", "Replications successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "LogUploadFail", "Replications failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
self.workDirectory = os.environ.get( 'LOGUPLOAD_CACHE', os.environ.get( 'AGENT_WORKDIRECTORY', '/tmp/LogUpload' ) )
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param Operation operation: Operation to execute
:param str csPath: CS path for this handler
"""
# # call base class ctor
OperationHandlerBase.__init__( self, operation, csPath )
# # gMOnitor stuff goes here
gMonitor.registerActivity( "RemoveFileAtt", "File removals attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveFileOK", "Successful file removals",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RemoveFileFail", "Failed file removals",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
# # re pattern for not existing files
self.reNotExisting = re.compile( "(no|not) such file or directory", re.IGNORECASE )
def __init__( self, requestJSON, handlersDict, csPath, agentName, standalone = False, requestClient = None ):
"""c'tor
:param self: self reference
:param str requestJSON: request serialized to JSON
:param dict opHandlers: operation handlers
"""
self.request = Request( requestJSON )
# # csPath
self.csPath = csPath
# # agent name
self.agentName = agentName
# # standalone flag
self.standalone = standalone
# # handlers dict
self.handlersDict = handlersDict
# # handlers class def
self.handlers = {}
# # own sublogger
self.log = gLogger.getSubLogger( "pid_%s/%s" % ( os.getpid(), self.request.RequestName ) )
# # get shifters info
self.__managersDict = {}
shifterProxies = self.__setupManagerProxies()
if not shifterProxies["OK"]:
self.log.error( shifterProxies["Message"] )
# # initialize gMonitor
gMonitor.setComponentType( gMonitor.COMPONENT_AGENT )
gMonitor.setComponentName( self.agentName )
gMonitor.initialize()
# # own gMonitor activities
gMonitor.registerActivity( "RequestAtt", "Requests processed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RequestFail", "Requests failed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RequestOK", "Requests done",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
if requestClient is None:
self.requestClient = ReqClient()
else:
self.requestClient = requestClient
def __init__(self, operation=None, csPath=None):
"""c'tor
:param self: self reference
:param Operation operation: operation to execute
:param str csPath: CS path for this handler
"""
# # base class ctor
OperationHandlerBase.__init__(self, operation, csPath)
# # gMonitor stuff
gMonitor.registerActivity("FileReTransferAtt",
"File retransfers attempted",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FileReTransferOK",
"File retransfers successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FileReTransferFail",
"File retransfers failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
def initialize(self):
self.section = self.am_getOption("section")
gMonitor.registerActivity("SubmittedTasks","Automatically submitted tasks","Transformation Monitoring","Tasks", gMonitor.OP_ACUM)
self.transClient = TransformationClient()
return S_OK()
def __init__( self, *args, **kwargs ):
""" c'tor """
# # call base class ctor
AgentModule.__init__( self, *args, **kwargs )
# # ProcessPool related stuff
self.__requestsPerCycle = self.am_getOption( "RequestsPerCycle", self.__requestsPerCycle )
self.log.info( "Requests/cycle = %d" % self.__requestsPerCycle )
self.__minProcess = self.am_getOption( "MinProcess", self.__minProcess )
self.log.info( "ProcessPool min process = %d" % self.__minProcess )
self.__maxProcess = self.am_getOption( "MaxProcess", 4 )
self.log.info( "ProcessPool max process = %d" % self.__maxProcess )
self.__queueSize = self.am_getOption( "ProcessPoolQueueSize", self.__queueSize )
self.log.info( "ProcessPool queue size = %d" % self.__queueSize )
self.__poolTimeout = int( self.am_getOption( "ProcessPoolTimeout", self.__poolTimeout ) )
self.log.info( "ProcessPool timeout = %d seconds" % self.__poolTimeout )
self.__poolSleep = int( self.am_getOption( "ProcessPoolSleep", self.__poolSleep ) )
self.log.info( "ProcessPool sleep time = %d seconds" % self.__poolSleep )
self.__taskTimeout = int( self.am_getOption( "ProcessTaskTimeout", self.__taskTimeout ) )
self.log.info( "ProcessTask timeout = %d seconds" % self.__taskTimeout )
self.__bulkRequest = self.am_getOption( "BulkRequest", 0 )
self.log.info( "Bulk request size = %d" % self.__bulkRequest )
# # keep config path and agent name
self.agentName = self.am_getModuleParam( "fullName" )
self.__configPath = PathFinder.getAgentSection( self.agentName )
# # operation handlers over here
opHandlersPath = "%s/%s" % ( self.__configPath, "OperationHandlers" )
opHandlers = gConfig.getSections( opHandlersPath )
if not opHandlers["OK"]:
self.log.error( opHandlers["Message" ] )
raise AgentConfigError( "OperationHandlers section not found in CS under %s" % self.__configPath )
opHandlers = opHandlers["Value"]
self.timeOuts = dict()
# # handlers dict
self.handlersDict = dict()
for opHandler in opHandlers:
opHandlerPath = "%s/%s/Location" % ( opHandlersPath, opHandler )
opLocation = gConfig.getValue( opHandlerPath, "" )
if not opLocation:
self.log.error( "%s not set for %s operation handler" % ( opHandlerPath, opHandler ) )
continue
self.timeOuts[opHandler] = { "PerFile": self.__fileTimeout, "PerOperation": self.__operationTimeout }
opTimeout = gConfig.getValue( "%s/%s/TimeOut" % ( opHandlersPath, opHandler ), 0 )
if opTimeout:
self.timeOuts[opHandler]["PerOperation"] = opTimeout
fileTimeout = gConfig.getValue( "%s/%s/TimeOutPerFile" % ( opHandlersPath, opHandler ), 0 )
if fileTimeout:
self.timeOuts[opHandler]["PerFile"] = fileTimeout
self.handlersDict[opHandler] = opLocation
self.log.info( "Operation handlers:" )
for item in enumerate ( self.handlersDict.items() ):
opHandler = item[1][0]
self.log.info( "[%s] %s: %s (timeout: %d s + %d s per file)" % ( item[0], item[1][0], item[1][1],
self.timeOuts[opHandler]['PerOperation'],
self.timeOuts[opHandler]['PerFile'] ) )
# # common monitor activity
gMonitor.registerActivity( "Iteration", "Agent Loops",
"RequestExecutingAgent", "Loops/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Processed", "Request Processed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Done", "Request Completed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
# # create request dict
self.__requestCache = dict()
self.FTSMode = self.am_getOption( "FTSMode", False )
def initialize(self):
self.ReplicaManager = ReplicaManager()
self.DataLog = DataLoggingClient()
self.DataIntegrityClient = DataIntegrityClient()
if self.am_getOption('DirectDB', False):
from DIRAC.StorageManagementSystem.DB.MigrationMonitoringDB import MigrationMonitoringDB
self.MigrationMonitoringDB = MigrationMonitoringDB()
else:
from DIRAC.StorageManagementSystem.Client.MigrationMonitoringClient import MigrationMonitoringClient
self.MigrationMonitoringDB = MigrationMonitoringClient()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
self.userName = '******'
self.storageElements = self.am_getOption('StorageElements',
['CERN-RAW'])
self.lastMonitors = {}
gMonitor.registerActivity("Iteration", "Agent Loops/min",
"MigrationMonitoringAgent", "Loops",
gMonitor.OP_SUM)
if self.storageElements:
gLogger.info(
"Agent will be initialised to monitor the following SEs:")
for se in self.storageElements:
gLogger.info(se)
self.lastMonitors[se] = datetime.datetime.utcfromtimestamp(0.0)
gMonitor.registerActivity("Iteration%s" % se,
"Agent Loops/min",
"MigrationMonitoringAgent", "Loops",
gMonitor.OP_SUM)
gMonitor.registerActivity("MigratingFiles%s" % se,
"Files waiting for migration",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_MEAN)
gMonitor.registerActivity("MigratedFiles%s" % se,
"Newly migrated files",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_SUM)
gMonitor.registerActivity("TotalMigratedFiles%s" % se,
"Total migrated files",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_ACUM)
gMonitor.registerActivity("TotalMigratedSize%s" % se,
"Total migrated file size",
"MigrationMonitoringAgent", "GB",
gMonitor.OP_ACUM)
gMonitor.registerActivity("ChecksumMatches%s" % se,
"Successfully migrated files",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_SUM)
gMonitor.registerActivity("TotalChecksumMatches%s" % se,
"Total successfully migrated files",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_ACUM)
gMonitor.registerActivity("ChecksumMismatches%s" % se,
"Erroneously migrated files",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_SUM)
gMonitor.registerActivity("TotalChecksumMismatches%s" % se,
"Total erroneously migrated files",
"MigrationMonitoringAgent", "Files",
gMonitor.OP_ACUM)
gMonitor.registerActivity("MigrationTime%s" % se,
"Average migration time",
"MigrationMonitoringAgent",
"Seconds", gMonitor.OP_MEAN)
return S_OK()
def __init__(self, operation=None, csPath=None):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
super(ReplicateAndRegister, self).__init__(operation, csPath)
# # own gMonitor stuff for files
gMonitor.registerActivity("ReplicateAndRegisterAtt",
"Replicate and register attempted",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("ReplicateOK", "Replications successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("ReplicateFail", "Replications failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterOK", "Registrations successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterFail", "Registrations failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
# # for FTS
gMonitor.registerActivity("FTSScheduleAtt", "Files schedule attempted",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FTSScheduleOK", "File schedule successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FTSScheduleFail", "File schedule failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
# # SE cache
self.seCache = {}
# Clients
self.fc = FileCatalog()
if hasattr(self, "FTSMode") and getattr(self, "FTSMode"):
self.ftsClient = FTSClient()
def initialize(self):
"""
Called by the framework upon startup, before any cycle (execute method bellow)
"""
self.requestDBClient = RequestClient()
# the RequestAgentMixIn needs the capitalized version, until is is fixed keep this.
self.RequestDBClient = self.requestDBClient
self.replicaManager = ReplicaManager()
gMonitor.registerActivity("Iteration", "Agent Loops", "RemovalAgent",
"Loops/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Execute", "Request Processed",
"RemovalAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Done", "Request Completed", "RemovalAgent",
"Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("PhysicalRemovalAtt",
"Physical removals attempted",
"RemovalAgent", "Removal/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("PhysicalRemovalDone",
"Successful physical removals",
"RemovalAgent", "Removal/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("PhysicalRemovalFail",
"Failed physical removals", "RemovalAgent",
"Removal/min", gMonitor.OP_SUM)
gMonitor.registerActivity("PhysicalRemovalSize",
"Physically removed size", "RemovalAgent",
"Bytes", gMonitor.OP_ACUM)
gMonitor.registerActivity("ReplicaRemovalAtt",
"Replica removal attempted", "RemovalAgent",
"Removal/min", gMonitor.OP_SUM)
gMonitor.registerActivity("ReplicaRemovalDone",
"Successful replica removals",
"RemovalAgent", "Removal/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("ReplicaRemovalFail",
"Failed replica removals", "RemovalAgent",
"Removal/min", gMonitor.OP_SUM)
gMonitor.registerActivity("RemoveFileAtt", "File removal attempted",
"RemovalAgent", "Removal/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RemoveFileDone", "File removal done",
"RemovalAgent", "Removal/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RemoveFileFail", "File removal failed",
"RemovalAgent", "Removal/min",
gMonitor.OP_SUM)
self.maxNumberOfThreads = self.am_getOption('NumberOfThreads',
self.maxNumberOfThreads)
self.maxRequestsInQueue = self.am_getOption('RequestsInQueue',
self.maxRequestsInQueue)
self.threadPool = ThreadPool(1, self.maxNumberOfThreads,
self.maxRequestsInQueue)
# Set the ThreadPool in daemon mode to process new ThreadedJobs as they are inserted
self.threadPool.daemonize()
self.maxRequests = self.am_getOption('MaxRequestsPerCycle', 1200.)
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def initializeFileCatalogHandler(serviceInfo):
""" handler initialisation """
global gFileCatalogDB
dbLocation = getServiceOption(serviceInfo, 'Database',
'DataManagement/FileCatalogDB')
gFileCatalogDB = FileCatalogDB(dbLocation)
databaseConfig = {}
# Obtain the plugins to be used for DB interaction
gLogger.info("Initializing with FileCatalog with following managers:")
defaultManagers = {
'UserGroupManager': 'UserAndGroupManagerDB',
'SEManager': 'SEManagerDB',
'SecurityManager': 'NoSecurityManager',
'DirectoryManager': 'DirectoryLevelTree',
'FileManager': 'FileManager',
'DirectoryMetadata': 'DirectoryMetadata',
'FileMetadata': 'FileMetadata',
'DatasetManager': 'DatasetManager'
}
for configKey in sortList(defaultManagers.keys()):
defaultValue = defaultManagers[configKey]
configValue = getServiceOption(serviceInfo, configKey, defaultValue)
gLogger.info("%-20s : %-20s" % (str(configKey), str(configValue)))
databaseConfig[configKey] = configValue
# Obtain some general configuration of the database
gLogger.info(
"Initializing the FileCatalog with the following configuration:")
defaultConfig = {
'UniqueGUID': False,
'GlobalReadAccess': True,
'LFNPFNConvention': 'Strong',
'ResolvePFN': True,
'DefaultUmask': 0775,
'ValidFileStatus': ['AprioriGood', 'Trash', 'Removing', 'Probing'],
'ValidReplicaStatus': ['AprioriGood', 'Trash', 'Removing', 'Probing'],
'VisibleFileStatus': ['AprioriGood'],
'VisibleReplicaStatus': ['AprioriGood']
}
for configKey in sortList(defaultConfig.keys()):
defaultValue = defaultConfig[configKey]
configValue = getServiceOption(serviceInfo, configKey, defaultValue)
gLogger.info("%-20s : %-20s" % (str(configKey), str(configValue)))
databaseConfig[configKey] = configValue
res = gFileCatalogDB.setConfig(databaseConfig)
gMonitor.registerActivity("AddFile", "Amount of addFile calls",
"FileCatalogHandler", "calls/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("AddFileSuccessful", "Files successfully added",
"FileCatalogHandler", "files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("AddFileFailed", "Files failed to add",
"FileCatalogHandler", "files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RemoveFile", "Amount of removeFile calls",
"FileCatalogHandler", "calls/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RemoveFileSuccessful",
"Files successfully removed",
"FileCatalogHandler", "files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RemoveFileFailed", "Files failed to remove",
"FileCatalogHandler", "files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("AddReplica", "Amount of addReplica calls",
"FileCatalogHandler", "calls/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("AddReplicaSuccessful",
"Replicas successfully added",
"FileCatalogHandler", "replicas/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("AddReplicaFailed", "Replicas failed to add",
"FileCatalogHandler", "replicas/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RemoveReplica", "Amount of removeReplica calls",
"FileCatalogHandler", "calls/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RemoveReplicaSuccessful",
"Replicas successfully removed",
"FileCatalogHandler", "replicas/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RemoveReplicaFailed",
"Replicas failed to remove",
"FileCatalogHandler", "replicas/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("ListDirectory", "Amount of listDirectory calls",
"FileCatalogHandler", "calls/min",
gMonitor.OP_SUM)
return res
def initialize(self):
""" agent's initialization """
log = self.log.getSubLogger("initialize")
self.FTSGRAPH_REFRESH = self.am_getOption("FTSGraphValidityPeriod",
self.FTSGRAPH_REFRESH)
log.info("FTSGraph validity period = %s s" %
self.FTSGRAPH_REFRESH)
self.RW_REFRESH = self.am_getOption("RWAccessValidityPeriod",
self.RW_REFRESH)
log.info("SEs R/W access validity period = %s s" % self.RW_REFRESH)
self.STAGE_FILES = self.am_getOption("StageFiles", self.STAGE_FILES)
log.info("Stage files before submission = %s" % {
True: "yes",
False: "no"
}[bool(self.STAGE_FILES)])
self.MAX_ACTIVE_JOBS = self.am_getOption("MaxActiveJobsPerRoute",
self.MAX_ACTIVE_JOBS)
log.info("Max active FTSJobs/route = %s" % self.MAX_ACTIVE_JOBS)
self.MAX_FILES_PER_JOB = self.am_getOption("MaxFilesPerJob",
self.MAX_FILES_PER_JOB)
log.info("Max FTSFiles/FTSJob = %d" %
self.MAX_FILES_PER_JOB)
self.MAX_ATTEMPT = self.am_getOption("MaxTransferAttempts",
self.MAX_ATTEMPT)
log.info("Max transfer attempts = %s" % self.MAX_ATTEMPT)
# # thread pool
self.MIN_THREADS = self.am_getOption("MinThreads", self.MIN_THREADS)
self.MAX_THREADS = self.am_getOption("MaxThreads", self.MAX_THREADS)
minmax = (abs(self.MIN_THREADS), abs(self.MAX_THREADS))
self.MIN_THREADS, self.MAX_THREADS = min(minmax), max(minmax)
log.info("ThreadPool min threads = %s" % self.MIN_THREADS)
log.info("ThreadPool max threads = %s" % self.MAX_THREADS)
log.info("initialize: creation of FTSGraph...")
createGraph = self.resetFTSGraph()
if not createGraph["OK"]:
log.error("initialize: %s" % createGraph["Message"])
return createGraph
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
log.info("will use DataManager proxy")
# # gMonitor stuff here
gMonitor.registerActivity("RequestsAtt",
"Attempted requests executions", "FTSAgent",
"Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("RequestsOK",
"Successful requests executions", "FTSAgent",
"Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("RequestsFail", "Failed requests executions",
"FTSAgent", "Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("FTSJobsSubAtt", "FTSJobs creation attempts",
"FTSAgent", "Created FTSJobs/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FTSJobsSubOK",
"FTSJobs submitted successfully", "FTSAgent",
"Successful FTSJobs submissions/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FTSJobsSubFail",
"FTSJobs submissions failed", "FTSAgent",
"Failed FTSJobs submissions/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FTSJobsMonAtt", "FTSJobs monitored",
"FTSAgent", "FTSJobs/min", gMonitor.OP_SUM)
gMonitor.registerActivity("FTSJobsMonOK",
"FTSJobs monitored successfully", "FTSAgent",
"FTSJobs/min", gMonitor.OP_SUM)
gMonitor.registerActivity("FTSJobsMonFail", "FTSJobs attempts failed",
"FTSAgent", "FTSJobs/min", gMonitor.OP_SUM)
gMonitor.registerActivity("FTSMonitorFail",
"Failed FTS monitor executions", "FTSAgent",
"Execution/mins", gMonitor.OP_SUM)
pollingTime = self.am_getOption("PollingTime", 60)
for status in list(FTSJob.INITSTATES + FTSJob.TRANSSTATES +
FTSJob.FAILEDSTATES + FTSJob.FINALSTATES):
gMonitor.registerActivity("FTSJobs%s" % status,
"FTSJobs %s" % status, "FTSAgent",
"FTSJobs/cycle", gMonitor.OP_ACUM,
pollingTime)
gMonitor.registerActivity("FtSJobsPerRequest",
"Average FTSJobs per request", "FTSAgent",
"FTSJobs/Request", gMonitor.OP_MEAN)
gMonitor.registerActivity("FTSFilesPerJob", "FTSFiles per FTSJob",
"FTSAgent", "Number of FTSFiles per FTSJob",
gMonitor.OP_MEAN)
gMonitor.registerActivity("FTSSizePerJob",
"Average FTSFiles size per FTSJob",
"FTSAgent",
"Average submitted size per FTSJob",
gMonitor.OP_MEAN)
return S_OK()
def initialize( self ):
""" agent's initialization """
# # data manager
self.dataManager = DataManager()
log = self.log.getSubLogger( "initialize" )
self.FTSPLACEMENT_REFRESH = self.am_getOption( "FTSPlacementValidityPeriod", self.FTSPLACEMENT_REFRESH )
log.info( "FTSPlacement validity period = %s s" % self.FTSPLACEMENT_REFRESH )
self.SUBMIT_COMMAND = self.am_getOption( "SubmitCommand", self.SUBMIT_COMMAND )
log.info( "FTS submit command = %s" % self.SUBMIT_COMMAND )
self.MONITOR_COMMAND = self.am_getOption( "MonitorCommand", self.MONITOR_COMMAND )
log.info( "FTS commands: submit = %s monitor %s" % ( self.SUBMIT_COMMAND, self.MONITOR_COMMAND ) )
self.PIN_TIME = self.am_getOption( "PinTime", self.PIN_TIME )
log.info( "Stage files before submission = ", {True: "yes", False: "no"}[bool( self.PIN_TIME )] )
self.MAX_ACTIVE_JOBS = self.am_getOption( "MaxActiveJobsPerRoute", self.MAX_ACTIVE_JOBS )
log.info( "Max active FTSJobs/route = ", str( self.MAX_ACTIVE_JOBS ) )
self.MAX_FILES_PER_JOB = self.am_getOption( "MaxFilesPerJob", self.MAX_FILES_PER_JOB )
log.info( "Max FTSFiles/FTSJob = ", str( self.MAX_FILES_PER_JOB ) )
self.MAX_ATTEMPT = self.am_getOption( "MaxTransferAttempts", self.MAX_ATTEMPT )
log.info( "Max transfer attempts = ", str( self.MAX_ATTEMPT ) )
# # thread pool
self.MIN_THREADS = self.am_getOption( "MinThreads", self.MIN_THREADS )
self.MAX_THREADS = self.am_getOption( "MaxThreads", self.MAX_THREADS )
minmax = ( abs( self.MIN_THREADS ), abs( self.MAX_THREADS ) )
self.MIN_THREADS, self.MAX_THREADS = min( minmax ), max( minmax )
log.info( "ThreadPool min threads = ", str( self.MIN_THREADS ) )
log.info( "ThreadPool max threads = ", str( self.MAX_THREADS ) )
self.MAX_REQUESTS = self.am_getOption( "MaxRequests", self.MAX_REQUESTS )
log.info( "Max Requests fetched = ", str( self.MAX_REQUESTS ) )
self.MONITORING_INTERVAL = self.am_getOption( "MonitoringInterval", self.MONITORING_INTERVAL )
log.info( "Minimum monitoring interval = ", str( self.MONITORING_INTERVAL ) )
self.__ftsVersion = Operations().getValue( 'DataManagement/FTSVersion', 'FTS2' )
log.info( "FTSVersion : %s" % self.__ftsVersion )
log.info( "initialize: creation of FTSPlacement..." )
createPlacement = self.resetFTSPlacement()
if not createPlacement["OK"]:
log.error( "initialize:", createPlacement["Message"] )
return createPlacement
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
log.info( "will use DataManager proxy" )
self.registrationProtocols = getRegistrationProtocols()
# # gMonitor stuff here
gMonitor.registerActivity( "RequestsAtt", "Attempted requests executions",
"FTSAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RequestsOK", "Successful requests executions",
"FTSAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RequestsFail", "Failed requests executions",
"FTSAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSJobsSubAtt", "FTSJobs creation attempts",
"FTSAgent", "Created FTSJobs/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSJobsSubOK", "FTSJobs submitted successfully",
"FTSAgent", "Successful FTSJobs submissions/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSJobsSubFail", "FTSJobs submissions failed",
"FTSAgent", "Failed FTSJobs submissions/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSJobsMonAtt", "FTSJobs monitored",
"FTSAgent", "FTSJobs/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSJobsMonOK", "FTSJobs monitored successfully",
"FTSAgent", "FTSJobs/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSJobsMonFail", "FTSJobs attempts failed",
"FTSAgent", "FTSJobs/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSMonitorFail", "Failed FTS monitor executions",
"FTSAgent", "Execution/mins", gMonitor.OP_SUM )
pollingTime = self.am_getOption( "PollingTime", 60 )
for status in list( FTSJob.INITSTATES + FTSJob.TRANSSTATES + FTSJob.FAILEDSTATES + FTSJob.FINALSTATES ):
gMonitor.registerActivity( "FTSJobs%s" % status, "FTSJobs %s" % status ,
"FTSAgent", "FTSJobs/cycle", gMonitor.OP_ACUM, pollingTime )
gMonitor.registerActivity( "FtSJobsPerRequest", "Average FTSJobs per request",
"FTSAgent", "FTSJobs/Request", gMonitor.OP_MEAN )
gMonitor.registerActivity( "FTSFilesPerJob", "FTSFiles per FTSJob",
"FTSAgent", "Number of FTSFiles per FTSJob", gMonitor.OP_MEAN )
gMonitor.registerActivity( "FTSSizePerJob", "Average FTSFiles size per FTSJob",
"FTSAgent", "Average submitted size per FTSJob", gMonitor.OP_MEAN )
return S_OK()
def initDIRAC( rootPath, enableDebug = False ):
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
configDict = { 'webConfig' : {} }
configDict[ 'webConfig' ]['dirac.webroot'] = rootPath
diracRootPath = os.path.realpath( os.path.dirname( os.path.dirname( rootPath ) ) )
configDict[ 'webConfig' ]['dirac.root'] = diracRootPath
if diracRootPath not in sys.path:
sys.path.append( diracRootPath )
from DIRAC.FrameworkSystem.Client.Logger import gLogger
gLogger.registerBackends( [ 'stderr' ] )
from DIRAC.Core.Base import Script
Script.registerSwitch( "r", "reload", "Reload for pylons" )
Script.localCfg.addDefaultEntry( "/DIRAC/Security/UseServerCertificate", "yes" )
Script.localCfg.addDefaultEntry( "LogColor", True )
Script.initialize( script = "Website", ignoreErrors = True, initializeMonitor = False )
gLogger._systemName = "Framework"
gLogger.initialize( "Web", "/Website" )
gLogger.setLevel( "VERBOSE" )
from DIRAC import gMonitor, gConfig, rootPath as droot
from DIRAC.Core.Utilities import CFG
from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions
gMonitor.setComponentType( gMonitor.COMPONENT_WEB )
gMonitor.initialize()
gMonitor.registerActivity( "pagesServed", "Pages served", "Framework", "pages", gMonitor.OP_SUM )
gLogger.info( "DIRAC Initialized" )
configDict['portalVersion'] = portalVersion( rootPath )
gLogger.info( "DIRAC portal version: %s" % configDict['portalVersion'] )
extModules = [ '%sDIRAC' % module for module in getCSExtensions() ]
#Load web.cfg of modules
cfgFilePaths = [ os.path.join( droot, "etc", "web.cfg" ) ]
for extModule in extModules:
gLogger.info( "Adding web.cfg for %s extension" % extModule )
extModulePath = os.path.join( diracRootPath, extModule )
webCFGPath = os.path.join( extModulePath, "Web", "web.cfg" )
cfgFilePaths.append( webCFGPath )
for systemDir in os.listdir( extModulePath ):
webCFGSystemPath = os.path.join( extModulePath, systemDir, "Web", "web.cfg" )
cfgFilePaths.append( webCFGSystemPath )
webCFG = CFG.CFG()
for webCFGPath in cfgFilePaths:
if not os.path.isfile( webCFGPath ):
gLogger.warn( "%s does not exist" % webCFGPath )
else:
gLogger.info( "Loading %s" % webCFGPath )
modCFG = CFG.CFG().loadFromFile( webCFGPath )
if modCFG.getOption( 'Website/AbsoluteDefinition', False ):
gLogger.info( "CFG %s is absolute" % webCFGPath )
webCFG = modCFG
else:
webCFG = webCFG.mergeWith( modCFG )
gConfig.loadCFG( webCFG )
gLogger.showHeaders( True )
gLogger._gLogger__initialized = False
gLogger.initialize( "Web", "/Website" )
#Define the controllers, templates and public directories
for type in ( 'controllers', 'templates', 'public' ):
configDict[ type ] = []
for extModule in extModules:
extModulePath = os.path.join( diracRootPath, extModule )
typePath = os.path.join( extModulePath, "Web", type )
if os.path.isdir( typePath ):
gLogger.info( "Adding %s path for module %s" % ( type, extModule ) )
configDict[ type ].append( typePath )
for systemDir in os.listdir( extModulePath ):
systemTypePath = os.path.join( extModulePath, systemDir, "Web", type )
if os.path.isdir( systemTypePath ):
gLogger.info( "Adding %s path for system %s in module %s" % ( type, systemDir, extModule ) )
configDict[ type ].append( systemTypePath )
#End of extensions
configDict[ type ].append( os.path.join( rootPath, type ) )
#Load debug.cfg?
if enableDebug:
debugCFGPath = os.path.join( rootPath, "debug.cfg" )
if os.path.isfile( debugCFGPath ):
gLogger.info( "Loading debug cfg file at %s" % debugCFGPath )
gConfig.loadFile( debugCFGPath )
gLogger.info( "Extension modules loaded" )
return configDict
def __init__( self, *args, **kwargs ):
""" c'tor """
# # call base class ctor
AgentModule.__init__( self, *args, **kwargs )
# # ProcessPool related stuff
self.__requestsPerCycle = self.am_getOption( "RequestsPerCycle", self.__requestsPerCycle )
self.log.info( "Requests/cycle = %d" % self.__requestsPerCycle )
self.__minProcess = self.am_getOption( "MinProcess", self.__minProcess )
self.log.info( "ProcessPool min process = %d" % self.__minProcess )
self.__maxProcess = self.am_getOption( "MaxProcess", 4 )
self.log.info( "ProcessPool max process = %d" % self.__maxProcess )
self.__queueSize = self.am_getOption( "ProcessPoolQueueSize", self.__queueSize )
self.log.info( "ProcessPool queue size = %d" % self.__queueSize )
self.__poolTimeout = int( self.am_getOption( "ProcessPoolTimeout", self.__poolTimeout ) )
self.log.info( "ProcessPool timeout = %d seconds" % self.__poolTimeout )
self.__poolSleep = int( self.am_getOption( "ProcessPoolSleep", self.__poolSleep ) )
self.log.info( "ProcessPool sleep time = %d seconds" % self.__poolSleep )
self.__taskTimeout = int( self.am_getOption( "ProcessTaskTimeout", self.__taskTimeout ) )
self.log.info( "ProcessTask timeout = %d seconds" % self.__taskTimeout )
self.__bulkRequest = self.am_getOption( "BulkRequest", 0 )
self.log.info( "Bulk request size = %d" % self.__bulkRequest )
# # keep config path and agent name
self.agentName = self.am_getModuleParam( "fullName" )
self.__configPath = PathFinder.getAgentSection( self.agentName )
# # operation handlers over here
opHandlersPath = "%s/%s" % ( self.__configPath, "OperationHandlers" )
opHandlers = gConfig.getSections( opHandlersPath )
if not opHandlers["OK"]:
self.log.error( opHandlers["Message" ] )
raise AgentConfigError( "OperationHandlers section not found in CS under %s" % self.__configPath )
opHandlers = opHandlers["Value"]
self.timeOuts = dict()
# # handlers dict
self.handlersDict = dict()
for opHandler in opHandlers:
opHandlerPath = "%s/%s/Location" % ( opHandlersPath, opHandler )
opLocation = gConfig.getValue( opHandlerPath, "" )
if not opLocation:
self.log.error( "%s not set for %s operation handler" % ( opHandlerPath, opHandler ) )
continue
self.timeOuts[opHandler] = { "PerFile": self.__fileTimeout, "PerOperation": self.__operationTimeout }
opTimeout = gConfig.getValue( "%s/%s/TimeOut" % ( opHandlersPath, opHandler ), 0 )
if opTimeout:
self.timeOuts[opHandler]["PerOperation"] = opTimeout
fileTimeout = gConfig.getValue( "%s/%s/TimeOutPerFile" % ( opHandlersPath, opHandler ), 0 )
if fileTimeout:
self.timeOuts[opHandler]["PerFile"] = fileTimeout
self.handlersDict[opHandler] = opLocation
self.log.info( "Operation handlers:" )
for item in enumerate ( self.handlersDict.items() ):
opHandler = item[1][0]
self.log.info( "[%s] %s: %s (timeout: %d s + %d s per file)" % ( item[0], item[1][0], item[1][1],
self.timeOuts[opHandler]['PerOperation'],
self.timeOuts[opHandler]['PerFile'] ) )
# # common monitor activity
gMonitor.registerActivity( "Iteration", "Agent Loops",
"RequestExecutingAgent", "Loops/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Processed", "Request Processed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Done", "Request Completed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM )
# # create request dict
self.__requestCache = dict()
self.FTSMode = self.am_getOption( "FTSMode", False )
def initialize( self ): ''' Make the necessary initializations ''' gMonitor.registerActivity( "Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM ) return S_OK()
def __init__(self, operation=None, csPath=None):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
OperationHandlerBase.__init__(self, operation, csPath)
# # own gMonitor stuff for files
gMonitor.registerActivity("ReplicateAndRegisterAtt",
"Replicate and register attempted",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("ReplicateOK", "Replications successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("ReplicateFail", "Replications failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterOK", "Registrations successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterFail", "Registrations failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
# # for FTS
gMonitor.registerActivity("FTSScheduleAtt", "Files schedule attempted",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FTSScheduleOK", "File schedule successful",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("FTSScheduleFail", "File schedule failed",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
# # SE cache
self.seCache = {}
def initDIRAC(rootPath, enableDebug=False):
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
configDict = {'webConfig': {}}
configDict['webConfig']['dirac.webroot'] = rootPath
diracRootPath = os.path.realpath(os.path.dirname(
os.path.dirname(rootPath)))
configDict['webConfig']['dirac.root'] = diracRootPath
if diracRootPath not in sys.path:
sys.path.append(diracRootPath)
from DIRAC.FrameworkSystem.Client.Logger import gLogger
gLogger.registerBackends(['stderr'])
from DIRAC.Core.Base import Script
Script.registerSwitch("r", "reload", "Reload for pylons")
Script.localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate",
"yes")
Script.localCfg.addDefaultEntry("LogColor", True)
Script.initialize(script="Website",
ignoreErrors=True,
initializeMonitor=False)
gLogger._systemName = "Framework"
gLogger.initialize("Web", "/Website")
gLogger.setLevel("VERBOSE")
from DIRAC import gMonitor, gConfig, rootPath as droot
from DIRAC.Core.Utilities import CFG
from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions
gMonitor.setComponentType(gMonitor.COMPONENT_WEB)
gMonitor.initialize()
gMonitor.registerActivity("pagesServed", "Pages served", "Framework",
"pages", gMonitor.OP_SUM)
gLogger.info("DIRAC Initialized")
configDict['portalVersion'] = portalVersion(rootPath)
gLogger.info("DIRAC portal version: %s" % configDict['portalVersion'])
extModules = ['%sDIRAC' % module for module in getCSExtensions()]
#Load web.cfg of modules
cfgFilePaths = [os.path.join(droot, "etc", "web.cfg")]
for extModule in extModules:
gLogger.info("Adding web.cfg for %s extension" % extModule)
extModulePath = os.path.join(diracRootPath, extModule)
webCFGPath = os.path.join(extModulePath, "Web", "web.cfg")
cfgFilePaths.append(webCFGPath)
for systemDir in os.listdir(extModulePath):
webCFGSystemPath = os.path.join(extModulePath, systemDir, "Web",
"web.cfg")
cfgFilePaths.append(webCFGSystemPath)
webCFG = CFG.CFG()
for webCFGPath in cfgFilePaths:
if not os.path.isfile(webCFGPath):
gLogger.warn("%s does not exist" % webCFGPath)
else:
gLogger.info("Loading %s" % webCFGPath)
modCFG = CFG.CFG().loadFromFile(webCFGPath)
if modCFG.getOption('Website/AbsoluteDefinition', False):
gLogger.info("CFG %s is absolute" % webCFGPath)
webCFG = modCFG
else:
webCFG = webCFG.mergeWith(modCFG)
gConfig.loadCFG(webCFG)
gLogger.showHeaders(True)
gLogger._gLogger__initialized = False
gLogger.initialize("Web", "/Website")
#Define the controllers, templates and public directories
for type in ('controllers', 'templates', 'public'):
configDict[type] = []
for extModule in extModules:
extModulePath = os.path.join(diracRootPath, extModule)
typePath = os.path.join(extModulePath, "Web", type)
if os.path.isdir(typePath):
gLogger.info("Adding %s path for module %s" %
(type, extModule))
configDict[type].append(typePath)
for systemDir in os.listdir(extModulePath):
systemTypePath = os.path.join(extModulePath, systemDir, "Web",
type)
if os.path.isdir(systemTypePath):
gLogger.info("Adding %s path for system %s in module %s" %
(type, systemDir, extModule))
configDict[type].append(systemTypePath)
#End of extensions
configDict[type].append(os.path.join(rootPath, type))
#Load debug.cfg?
if enableDebug:
debugCFGPath = os.path.join(rootPath, "debug.cfg")
if os.path.isfile(debugCFGPath):
gLogger.info("Loading debug cfg file at %s" % debugCFGPath)
gConfig.loadFile(debugCFGPath)
gLogger.info("Extension modules loaded")
return configDict
def initialize(self):
self.RequestDBClient = RequestClient()
self.ReplicaManager = ReplicaManager()
self.DataLog = DataLoggingClient()
gMonitor.registerActivity("Iteration", "Agent Loops", "TransferAgent",
"Loops/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Execute", "Request Processed",
"TransferAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Done", "Request Completed", "TransferAgent",
"Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Replicate and register",
"Replicate and register operations",
"TransferAgent", "Attempts/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Replicate", "Replicate operations",
"TransferAgent", "Attempts/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Put and register",
"Put and register operations",
"TransferAgent", "Attempts/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Put", "Put operations", "TransferAgent",
"Attempts/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Replication successful",
"Successful replications", "TransferAgent",
"Successful/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Put successful", "Successful puts",
"TransferAgent", "Successful/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Replication failed", "Failed replications",
"TransferAgent", "Failed/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Put failed", "Failed puts", "TransferAgent",
"Failed/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Replica registration successful",
"Successful replica registrations",
"TransferAgent", "Successful/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("File registration successful",
"Successful file registrations",
"TransferAgent", "Successful/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Replica registration failed",
"Failed replica registrations",
"TransferAgent", "Failed/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("File registration failed",
"Failed file registrations", "TransferAgent",
"Failed/min", gMonitor.OP_SUM)
self.maxNumberOfThreads = self.am_getOption('NumberOfThreads', 1)
self.threadPoolDepth = self.am_getOption('ThreadPoolDepth', 1)
self.threadPool = ThreadPool(1, self.maxNumberOfThreads)
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
