def _ex_initialize(cls, exeName, loadName):
cls.__properties = {
'fullName': exeName,
'loadName': loadName,
'section': PathFinder.getExecutorSection(exeName),
'loadSection': PathFinder.getExecutorSection(loadName),
'messagesProcessed': 0,
'reconnects': 0,
'setup': gConfig.getValue("/DIRAC/Setup", "Unknown")
}
cls.__basePath = gConfig.getValue('/LocalSite/InstancePath', rootPath)
cls.__defaults = {}
cls.__defaults['MonitoringEnabled'] = True
cls.__defaults['Enabled'] = True
cls.__defaults['ControlDirectory'] = os.path.join(
cls.__basePath, 'control', *exeName.split("/"))
cls.__defaults['WorkDirectory'] = os.path.join(cls.__basePath, 'work',
*exeName.split("/"))
cls.__defaults['ReconnectRetries'] = 10
cls.__defaults['ReconnectSleep'] = 5
cls.__properties['shifterProxy'] = ''
cls.__properties['shifterProxyLocation'] = os.path.join(
cls.__defaults['WorkDirectory'], '.shifterCred')
cls.__mindName = False
cls.__mindExtraArgs = False
cls.__freezeTime = 0
cls.__fastTrackEnabled = True
cls.log = gLogger.getSubLogger(exeName, child=False)
try:
result = cls.initialize()
except Exception, excp:
gLogger.exception("Exception while initializing %s" % loadName)
return S_ERROR("Exception while initializing: %s" % str(excp))
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result['OK']:
return result
self._handler = result['Value']
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
self._initMonitoring()
self._threadPool = ThreadPool(max(1, self._cfg.getMinThreads()),
max(0, self._cfg.getMaxThreads()),
self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {'serviceName': self._name,
'serviceSectionPath': PathFinder.getServiceSection(self._name),
'URL': self._cfg.getURL(),
'messageSender': MessageSender(self._name, self._msgBroker),
'validNames': self._validNames,
'csPaths': [PathFinder.getServiceSection(svcName) for svcName in self._validNames]
}
# Call static initialization function
try:
self._handler['class']._rh__initializeClass(dict(self._serviceInfoDict),
self._lockManager,
self._msgBroker,
self._monitor)
if self._handler['init']:
for initFunc in self._handler['init']:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception as excp:
gLogger.exception("Exception while calling initialization function", lException=excp)
return S_ERROR("Exception while calling initialization function: %s" % str(excp))
if not isReturnStructure(result):
return S_ERROR("Service initialization function %s must return S_OK/S_ERROR" % initFunc)
if not result['OK']:
return S_ERROR("Error while initializing %s: %s" % (self._name, result['Message']))
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(e)
gLogger.exception(errMsg)
return S_ERROR(errMsg)
# Load actions after the handler has initialized itself
result = self._loadActions()
if not result['OK']:
return result
self._actions = result['Value']
gThreadScheduler.addPeriodicTask(30, self.__reportThreadPoolContents)
return S_OK()
def __init__(self, useCertificates=False):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__(self)
## setup logger
self.log = gLogger.getSubLogger("RequestManagement/RequestClient")
## dict to store all RPC clients for easy reuse
self.__requestRPCClientsDict = {}
## local if any defined
local = PathFinder.getServiceURL("RequestManagement/localURL")
if local:
self.__requestRPCClientsDict.setdefault(
"local", [self.__requestRPCClient(local)])
## central if any defined
central = PathFinder.getServiceURL("RequestManagement/centralURL")
if central:
self.__requestRPCClientsDict.setdefault(
"central", [self.__requestRPCClient(central)])
## voboxes if any defined
voBoxUrls = fromChar(
PathFinder.getServiceURL("RequestManagement/voBoxURLs"))
if voBoxUrls:
self.__requestRPCClientsDict.setdefault("voboxes", [])
for voBoxURL in randomize(voBoxUrls):
self.__requestRPCClientsDict["voboxes"].append(
self.__requestRPCClient(voBoxURL))
self.setServer('RequestManagement/centralURL')
def _ex_initialize(cls, exeName, loadName):
cls.__properties = {
"fullName": exeName,
"loadName": loadName,
"section": PathFinder.getExecutorSection(exeName),
"loadSection": PathFinder.getExecutorSection(loadName),
"messagesProcessed": 0,
"reconnects": 0,
"setup": gConfig.getValue("/DIRAC/Setup", "Unknown"),
}
cls.__defaults = {}
cls.__defaults["MonitoringEnabled"] = True
cls.__defaults["Enabled"] = True
cls.__defaults["ControlDirectory"] = os.path.join(rootPath, "control", *exeName.split("/"))
cls.__defaults["WorkDirectory"] = os.path.join(rootPath, "work", *exeName.split("/"))
cls.__defaults["ReconnectRetries"] = 10
cls.__defaults["ReconnectSleep"] = 5
cls.__defaults["shifterProxy"] = ""
cls.__defaults["shifterProxyLocation"] = os.path.join(cls.__defaults["WorkDirectory"], ".shifterCred")
cls.__properties["shifterProxy"] = ""
cls.__properties["shifterProxyLocation"] = os.path.join(cls.__defaults["WorkDirectory"], ".shifterCred")
cls.__mindName = False
cls.__mindExtraArgs = False
cls.__freezeTime = 0
cls.__fastTrackEnabled = True
cls.log = gLogger.getSubLogger(exeName)
try:
result = cls.initialize() # pylint: disable=no-member
except Exception as excp:
gLogger.exception("Exception while initializing %s" % loadName, lException=excp)
return S_ERROR("Exception while initializing: %s" % str(excp))
if not isReturnStructure(result):
return S_ERROR("Executor %s does not return an S_OK/S_ERROR after initialization" % loadName)
return result
def __init__( self, useCertificates = False ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self )
## setup logger
self.log = gLogger.getSubLogger( "RequestManagement/RequestClient" )
## dict to store all RPC clients for easy reuse
self.__requestRPCClientsDict = {}
## local if any defined
local = PathFinder.getServiceURL( "RequestManagement/localURL" )
if local:
self.__requestRPCClientsDict.setdefault( "local" , [ self.__requestRPCClient( local ) ] )
## central if any defined
central = PathFinder.getServiceURL( "RequestManagement/centralURL" )
if central:
self.__requestRPCClientsDict.setdefault( "central", [ self.__requestRPCClient( central ) ] )
## voboxes if any defined
voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) )
if voBoxUrls:
self.__requestRPCClientsDict.setdefault( "voboxes", [] )
for voBoxURL in randomize( voBoxUrls ):
self.__requestRPCClientsDict["voboxes"].append( self.__requestRPCClient( voBoxURL ) )
self.setServer( 'RequestManagement/centralURL' )
def _ex_initialize( cls, exeName, loadName ):
cls.__properties = { 'fullName' : exeName,
'loadName' : loadName,
'section' : PathFinder.getExecutorSection( exeName ),
'loadSection' : PathFinder.getExecutorSection( loadName ),
'messagesProcessed' : 0,
'reconnects' : 0,
'setup' : gConfig.getValue( "/DIRAC/Setup", "Unknown" ) }
cls.__basePath = gConfig.getValue( '/LocalSite/InstancePath', rootPath )
cls.__defaults = {}
cls.__defaults[ 'MonitoringEnabled' ] = True
cls.__defaults[ 'Enabled' ] = True
cls.__defaults[ 'ControlDirectory' ] = os.path.join( cls.__basePath,
'control',
*exeName.split( "/" ) )
cls.__defaults[ 'WorkDirectory' ] = os.path.join( cls.__basePath,
'work',
*exeName.split( "/" ) )
cls.__defaults[ 'ReconnectRetries' ] = 10
cls.__defaults[ 'ReconnectSleep' ] = 5
cls.__properties[ 'shifterProxy' ] = ''
cls.__properties[ 'shifterProxyLocation' ] = os.path.join( cls.__defaults[ 'WorkDirectory' ],
'.shifterCred' )
cls.__mindName = False
cls.__mindExtraArgs = False
cls.__freezeTime = 0
cls.__fastTrackEnabled = True
cls.log = gLogger.getSubLogger( exeName, child = False )
try:
result = cls.initialize()
except Exception, excp:
gLogger.exception( "Exception while initializing %s" % loadName )
return S_ERROR( "Exception while initializing: %s" % str( excp ) )
def initialize(self):
self.RequestDBClient = RequestClient()
gMonitor.registerActivity("Iteration", "Agent Loops", "ZuziaAgent",
"Loops/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Attempted", "Request Processed",
"ZuziaRAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Successful", "Request Forward Successful",
"ZuziaAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Failed", "Request Forward Failed",
"ZuziaAgent", "Requests/min",
gMonitor.OP_SUM)
self.local = PathFinder.getServiceURL("RequestManagement/localURL")
if not self.local:
self.local = AgentModule.am_getOption(self, 'localURL', '')
if not self.local:
errStr = 'The RequestManagement/localURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
self.central = PathFinder.getServiceURL("RequestManagement/centralURL")
if not self.central:
errStr = 'The RequestManagement/centralURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
return S_OK()
def initialize( self ):
#Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR( "Could not build service URL for %s" % self._name )
gLogger.verbose( "Service URL is %s" % self._url )
#Load handler
result = self._loadHandlerInit()
if not result[ 'OK' ]:
return result
self._handler = result[ 'Value' ]
#Initialize lock manager
self._lockManager = LockManager( self._cfg.getMaxWaitingPetitions() )
self._initMonitoring()
self._threadPool = ThreadPool( max( 1, self._cfg.getMinThreads() ),
max( 0, self._cfg.getMaxThreads() ),
self._cfg.getMaxWaitingPetitions() )
self._threadPool.daemonize()
self._msgBroker = MessageBroker( "%sMSB" % self._name, threadPool = self._threadPool )
#Create static dict
self._serviceInfoDict = { 'serviceName' : self._name,
'serviceSectionPath' : PathFinder.getServiceSection( self._name ),
'URL' : self._cfg.getURL(),
'messageSender' : MessageSender( self._name, self._msgBroker ),
'validNames' : self._validNames,
'csPaths' : [ PathFinder.getServiceSection( svcName ) for svcName in self._validNames ]
}
#Call static initialization function
try:
self._handler[ 'class' ]._rh__initializeClass( dict( self._serviceInfoDict ),
self._lockManager,
self._msgBroker,
self._monitor )
if self._handler[ 'init' ]:
for initFunc in self._handler[ 'init' ]:
gLogger.verbose( "Executing initialization function" )
try:
result = initFunc( dict( self._serviceInfoDict ) )
except Exception as excp:
gLogger.exception( "Exception while calling initialization function", lException = excp )
return S_ERROR( "Exception while calling initialization function: %s" % str( excp ) )
if not isReturnStructure( result ):
return S_ERROR( "Service initialization function %s must return S_OK/S_ERROR" % initFunc )
if not result[ 'OK' ]:
return S_ERROR( "Error while initializing %s: %s" % ( self._name, result[ 'Message' ] ) )
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception( e )
gLogger.exception( errMsg )
return S_ERROR( errMsg )
#Load actions after the handler has initialized itself
result = self._loadActions()
if not result[ 'OK' ]:
return result
self._actions = result[ 'Value' ]
gThreadScheduler.addPeriodicTask( 30, self.__reportThreadPoolContents )
return S_OK()
def execute(self):
""" This agent is the smallest and (cutest) of all the DIRAC agents in existence.
"""
gMonitor.addMark("Iteration", 1)
central = PathFinder.getServiceURL("RequestManagement/centralURL")
if central:
self.central = central
local = PathFinder.getServiceURL("RequestManagement/localURL")
if local:
self.local = local
res = self.RequestDBClient.serveRequest(url=self.local)
if not res['OK']:
gLogger.error(
"ZuziaAgent.execute: Failed to get request from database.",
self.local)
return S_OK()
elif not res['Value']:
gLogger.info(
"ZuziaAgent.execute: No requests to be executed found.")
return S_OK()
gMonitor.addMark("Attempted", 1)
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
gLogger.info("ZuziaAgent.execute: Obtained request %s" % requestName)
gLogger.info("ZuziaAgent.execute: Attempting to set request to %s." %
self.central)
res = self.RequestDBClient.setRequest(requestName, requestString,
self.central)
if res['OK']:
gMonitor.addMark("Successful", 1)
gLogger.info("ZuziaAgent.execute: Successfully put request.")
else:
gMonitor.addMark("Failed", 1)
gLogger.error("ZuziaAgent.execute: Failed to set request to",
self.central)
gLogger.info(
"ZuziaAgent.execute: Attempting to set request to %s." %
self.local)
res = self.RequestDBClient.setRequest(requestName, requestString,
self.local)
if res['OK']:
gLogger.info("ZuziaAgent.execute: Successfully put request.")
else:
gLogger.error("ZuziaAgent.execute: Failed to set request to",
self.local)
while not res['OK']:
gLogger.info(
"ZuziaAgent.execute: Attempting to set request to anywhere."
)
res = self.RequestDBClient.setRequest(
requestName, requestString)
gLogger.info(
"ZuziaAgent.execute: Successfully put request to %s." %
res["Server"])
return S_OK()
def __initializeService(cls, relativeUrl, absoluteUrl):
"""
Initialize a service.
The work is only perform once at the first request.
:param relativeUrl: relative URL, e.g. ``/<System>/<Component>``
:param absoluteUrl: full URL e.g. ``https://<host>:<port>/<System>/<Component>``
:returns: S_OK
"""
# If the initialization was already done successfuly,
# we can just return
if cls.__init_done:
return S_OK()
# Otherwise, do the work but with a lock
with cls.__init_lock:
# Check again that the initialization was not done by another thread
# while we were waiting for the lock
if cls.__init_done:
return S_OK()
# Url starts with a "/", we just remove it
serviceName = relativeUrl[1:]
cls._startTime = datetime.utcnow()
sLog.info("First use, initializing service...", "%s" % relativeUrl)
cls._authManager = AuthManager(
"%s/Authorization" % PathFinder.getServiceSection(serviceName))
cls._initMonitoring(serviceName, absoluteUrl)
cls._serviceName = serviceName
cls._validNames = [serviceName]
serviceInfo = {
'serviceName': serviceName,
'serviceSectionPath':
PathFinder.getServiceSection(serviceName),
'csPaths': [PathFinder.getServiceSection(serviceName)],
'URL': absoluteUrl
}
cls._serviceInfoDict = serviceInfo
cls.__monitorLastStatsUpdate = time.time()
cls.initializeHandler(serviceInfo)
cls.__init_done = True
return S_OK()
def initializeReportGeneratorHandler( serviceInfo ):
global gAccountingDB
gAccountingDB = AccountingDB( readOnly = True )
#Get data location
reportSection = PathFinder.getServiceSection( "Accounting/ReportGenerator" )
dataPath = gConfig.getValue( "%s/DataLocation" % reportSection, "data/accountingGraphs" )
dataPath = dataPath.strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) )
gLogger.info( "Data will be written into %s" % dataPath )
try:
os.makedirs( dataPath )
except:
pass
try:
testFile = "%s/acc.jarl.test" % dataPath
fd = file( testFile, "w" )
fd.close()
os.unlink( testFile )
except IOError:
gLogger.fatal( "Can't write to %s" % dataPath )
return S_ERROR( "Data location is not writable" )
gDataCache.setGraphsLocation( dataPath )
gMonitor.registerActivity( "plotsDrawn", "Drawn plot images", "Accounting reports", "plots", gMonitor.OP_SUM )
gMonitor.registerActivity( "reportsRequested", "Generated reports", "Accounting reports", "reports", gMonitor.OP_SUM )
return S_OK()
def initialize(self):
self.RequestDBClient = RequestClient()
backend = self.am_getOption('Backend', '')
self.RequestDB = False
if backend == 'mysql':
from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL
requestDB = RequestDBMySQL()
if requestDB._connected:
self.RequestDB = requestDB
gMonitor.registerActivity("Iteration", "Agent Loops",
"DISETForwardingAgent", "Loops/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Attempted", "Request Processed",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Successful", "Request Forward Successful",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Failed", "Request Forward Failed",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
self.local = PathFinder.getServiceURL("RequestManagement/localURL")
if not self.local:
self.local = AgentModule.am_getOption(self, 'localURL', '')
if not self.local:
errStr = 'The RequestManagement/localURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
return S_OK()
def initializeDataLoggingHandler(serviceInfo):
global dataPath
global logDB
logDB = DataLoggingDB()
monitoringSection = PathFinder.getServiceSection(
"DataManagement/DataLogging")
#Get data location
retDict = gConfig.getOption("%s/DataLocation" % monitoringSection,
"dataLoggingPlots")
if not retDict['OK']:
return retDict
dataPath = retDict['Value'].strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath(
"%s/%s" %
(gConfig.getValue('/LocalSite/InstancePath', rootPath), dataPath))
gLogger.info("Data will be written into %s" % dataPath)
try:
os.makedirs(dataPath)
except:
pass
try:
testFile = "%s/mon.jarl.test" % dataPath
fd = file(testFile, "w")
fd.close()
os.unlink(testFile)
except IOError:
gLogger.fatal("Can't write to %s" % dataPath)
return S_ERROR("Data location is not writable")
return S_OK()
def initialize( self ):
self.RequestDBClient = RequestClient()
backend = self.am_getOption( 'Backend', '' )
self.RequestDB = False
if backend == 'mysql':
from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL
requestDB = RequestDBMySQL()
if requestDB._connected:
self.RequestDB = requestDB
gMonitor.registerActivity( "Iteration", "Agent Loops", "DISETForwardingAgent", "Loops/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Attempted", "Request Processed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Successful", "Request Forward Successful", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Failed", "Request Forward Failed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM )
self.local = PathFinder.getServiceURL( "RequestManagement/localURL" )
if not self.local:
self.local = AgentModule.am_getOption( self, 'localURL', '' )
if not self.local:
errStr = 'The RequestManagement/localURL option must be defined.'
gLogger.fatal( errStr )
return S_ERROR( errStr )
return S_OK()
def _checkServiceURL(self, serviceName, options):
"""Ensure service URL is properly configured in the CS."""
url = self._getURL(serviceName, options)
system = options["System"]
module = options["Module"]
self.log.info("Checking URLs for %s/%s" % (system, module))
urlsConfigPath = Path.cfgPath(
PathFinder.getSystemURLSection(system=system, setup=self.setup),
module)
urls = gConfig.getValue(urlsConfigPath, [])
self.log.debug("Found configured URLs for %s: %s" % (module, urls))
self.log.debug("This URL is %s" % url)
runitStatus = options["RunitStatus"]
wouldHave = "Would have " if not self.commitURLs else ""
if runitStatus == "Run" and url not in urls:
urls.append(url)
message = "%sAdded URL %s to URLs for %s/%s" % (wouldHave, url,
system, module)
self.log.info(message)
self.accounting[serviceName + "/URL"]["Treatment"] = message
self.csAPI.modifyValue(urlsConfigPath, ",".join(urls))
if runitStatus == "Down" and url in urls:
urls.remove(url)
message = "%sRemoved URL %s from URLs for %s/%s" % (wouldHave, url,
system, module)
self.log.info(message)
self.accounting[serviceName + "/URL"]["Treatment"] = message
self.csAPI.modifyValue(urlsConfigPath, ",".join(urls))
def loadModules(self, modulesList, hideExceptions=False):
"""
Load all modules required in moduleList
"""
for modName in modulesList:
gLogger.verbose("Checking %s" % modName)
# if it's a executor modName name just load it and be done with it
if "/" in modName:
gLogger.verbose(
"Module %s seems to be a valid name. Try to load it!" %
modName)
result = self.loadModule(modName,
hideExceptions=hideExceptions)
if not result["OK"]:
return result
continue
# Check if it's a system name
# Look in the CS
system = modName
# Can this be generated with sectionFinder?
csPath = "%s/Executors" % PathFinder.getSystemSection(system)
gLogger.verbose("Exploring %s to discover modules" % csPath)
result = gConfig.getSections(csPath)
if result["OK"]:
# Add all modules in the CS :P
for modName in result["Value"]:
result = self.loadModule("%s/%s" % (system, modName),
hideExceptions=hideExceptions)
if not result["OK"]:
return result
# Look what is installed
parentModule = None
for rootModule in extensionsByPriority():
if not system.endswith("System"):
system += "System"
parentImport = "%s.%s.%s" % (rootModule, system,
self.__csSuffix)
# HERE!
result = recurseImport(parentImport)
if not result["OK"]:
return result
parentModule = result["Value"]
if parentModule:
break
if not parentModule:
continue
parentPath = parentModule.__path__[0]
gLogger.notice("Found modules path at %s" % parentImport)
for entry in os.listdir(parentPath):
if entry == "__init__.py" or not entry.endswith(".py"):
continue
if not os.path.isfile(os.path.join(parentPath, entry)):
continue
modName = "%s/%s" % (system, entry[:-3])
gLogger.verbose("Trying to import %s" % modName)
result = self.loadModule(modName,
hideExceptions=hideExceptions,
parentModule=parentModule)
return S_OK()
def export_sendMail( self, address, subject, body, fromAddress, avoidSpam = False ):
""" Send an email with supplied body to the specified address using the Mail utility.
if avoidSpam is True, then emails are first added to a set so that duplicates are removed,
and sent every hour.
"""
gLogger.verbose( 'Received signal to send the following mail to %s:\nSubject = %s\n%s' % ( address, subject, body ) )
eMail = Mail()
notificationSection = PathFinder.getServiceSection( "Framework/Notification" )
csSection = notificationSection + '/SMTP'
eMail._smtpHost = gConfig.getValue( '%s/Host' % csSection )
eMail._smtpPort = gConfig.getValue( '%s/Port' % csSection )
eMail._smtpLogin = gConfig.getValue( '%s/Login' % csSection )
eMail._smtpPasswd = gConfig.getValue( '%s/Password' % csSection )
eMail._smtpPtcl = gConfig.getValue( '%s/Protocol' % csSection )
eMail._subject = subject
eMail._message = body
eMail._mailAddress = address
if not fromAddress == 'None':
eMail._fromAddress = fromAddress
if gConfig.getValue( '%s/FromAddress' % csSection ):
eMail._fromAddress = gConfig.getValue( '%s/FromAddress' % csSection )
if avoidSpam:
gMailSet.add(eMail)
return S_OK("Mail added to gMailSet")
else:
result = eMail._send()
if not result['OK']:
gLogger.warn( 'Could not send mail with the following message:\n%s' % result['Message'] )
else:
gLogger.info( 'Mail sent successfully to %s with subject %s' % ( address, subject ) )
gLogger.debug( result['Value'] )
return result
def export_sendMail( self, address, subject, body, fromAddress, avoidSpam = False ):
""" Send an email with supplied body to the specified address using the Mail utility.
if avoidSpam is True, then emails are first added to a set so that duplicates are removed,
and sent every hour.
"""
gLogger.verbose( 'Received signal to send the following mail to %s:\nSubject = %s\n%s' % ( address, subject, body ) )
eMail = Mail()
notificationSection = PathFinder.getServiceSection( "Framework/Notification" )
csSection = notificationSection + '/SMTPServer'
eMail._smtpHost = gConfig.getValue( '%s/Host' % csSection )
eMail._smtpPort = gConfig.getValue( '%s/Port' % csSection )
eMail._smtpLogin = gConfig.getValue( '%s/Login' % csSection )
eMail._smtpPasswd = gConfig.getValue( '%s/Password' % csSection )
eMail._smtpPtcl = gConfig.getValue( '%s/Protocol' % csSection )
eMail._subject = subject
eMail._message = body
eMail._mailAddress = address
if not fromAddress == 'None':
eMail._fromAddress = fromAddress
if gConfig.getValue( '%s/FromAddress' % csSection ):
eMail._fromAddress = gConfig.getValue( '%s/FromAddress' % csSection )
if avoidSpam:
gMailSet.add(eMail)
return S_OK("Mail added to gMailSet")
else:
result = eMail._send()
if not result['OK']:
gLogger.warn( 'Could not send mail with the following message:\n%s' % result['Message'] )
else:
gLogger.info( 'Mail sent successfully to %s with subject %s' % ( address, subject ) )
gLogger.debug( result['Value'] )
return result
def __init__(self, url=False, useCertificates=False):
""" Constructor of the LCGFileCatalogProxy client class
"""
self.name = 'LFCProxy'
self.valid = False
try:
if url:
self.url = url
else:
url = PathFinder.getServiceURL(
'DataManagement/LcgFileCatalogProxy')
if not url:
return
self.url = url
self.server = RPCClient(self.url,
timeout=120,
useCertificates=useCertificates)
if not self.server:
return
else:
self.valid = True
except Exception, x:
gLogger.exception(
'Exception while creating connection to LcgFileCatalog proxy server',
'', x)
return
def __init__(self, serviceData):
"""
Init the variables for the service
:param serviceData: dict with modName, standalone, loadName, moduleObj, classObj. e.g.:
{'modName': 'Framework/serviceName',
'standalone': True,
'loadName': 'Framework/serviceName',
'moduleObj': <module 'serviceNameHandler' from '/home/DIRAC/FrameworkSystem/Service/serviceNameHandler.pyo'>,
'classObj': <class 'serviceNameHandler.serviceHandler'>}
"""
self._svcData = serviceData
self._name = serviceData["modName"]
self._startTime = datetime.datetime.utcnow()
self._validNames = [serviceData["modName"]]
if serviceData["loadName"] not in self._validNames:
self._validNames.append(serviceData["loadName"])
self._cfg = ServiceConfiguration(list(self._validNames))
self._standalone = serviceData["standalone"]
self.__monitorLastStatsUpdate = time.time()
self._stats = {"queries": 0, "connections": 0}
self._authMgr = AuthManager(
"%s/Authorization" %
PathFinder.getServiceSection(serviceData["loadName"]))
self._transportPool = getGlobalTransportPool()
self.__cloneId = 0
self.__maxFD = 0
self.activityMonitoring = False
# Check if monitoring is enabled
if "Monitoring" in Operations().getMonitoringBackends(
monitoringType="ServiceMonitoring"):
self.activityMonitoring = True
def initialize(self):
''' NotifyAgent initialization
'''
try:
with sqlite3.connect(self.cacheFile) as conn:
conn.execute(
'''CREATE TABLE IF NOT EXISTS ProductionManagementCache(
reqId VARCHAR(64) NOT NULL DEFAULT "",
reqType VARCHAR(64) NOT NULL DEFAULT "",
reqWG VARCHAR(64) NOT NULL DEFAULT "",
reqName VARCHAR(64) NOT NULL DEFAULT "",
SimCondition VARCHAR(64) NOT NULL DEFAULT "",
ProPath VARCHAR(64) NOT NULL DEFAULT "",
thegroup VARCHAR(64) NOT NULL DEFAULT "",
reqInform VARCHAR(64) NOT NULL DEFAULT ""
);''')
except sqlite3.OperationalError:
self.log.error('Email cache database is locked')
self.diracAdmin = DiracAdmin()
self.csS = PathFinder.getServiceSection(
'ProductionManagement/ProductionRequest')
self.fromAddress = gConfig.getValue('%s/fromAddress' % self.csS, '')
if not self.fromAddress:
self.log.info(
'No fromAddress is defined, a default value will be used instead'
)
self.fromAddress = '*****@*****.**'
return S_OK()
def export_sendSMS( self, userName, body, fromAddress ):
""" Send an SMS with supplied body to the specified DIRAC user using the Mail utility via an SMS switch.
"""
gLogger.verbose( 'Received signal to send the following SMS to %s:\n%s' % ( userName, body ) )
mobile = gConfig.getValue( '/Registry/Users/%s/Mobile' % userName, '' )
if not mobile:
return S_ERROR( 'No registered mobile number for %s' % userName )
csSection = PathFinder.getServiceSection( 'Framework/Notification' )
smsSwitch = gConfig.getValue( '%s/SMSSwitch' % csSection, '' )
if not smsSwitch:
return S_ERROR( 'No SMS switch is defined in CS path %s/SMSSwitch' % csSection )
address = '%s@%s' % ( mobile, smsSwitch )
subject = 'DIRAC SMS'
m = Mail()
m._subject = subject
m._message = body
m._mailAddress = address
if not fromAddress == 'None':
m._fromAddress = fromAddress
result = m._send()
if not result['OK']:
gLogger.warn( 'Could not send SMS to %s with the following message:\n%s' % ( userName, result['Message'] ) )
else:
gLogger.info( 'SMS sent successfully to %s ' % ( userName ) )
gLogger.debug( result['Value'] )
return result
def __init__(self, serviceData):
"""
Init the variables for the service
:param serviceData: dict with modName, standalone, loadName, moduleObj, classObj. e.g.:
{'modName': 'Framework/serviceName',
'standalone': True,
'loadName': 'Framework/serviceName',
'moduleObj': <module 'serviceNameHandler' from '/home/DIRAC/FrameworkSystem/Service/serviceNameHandler.pyo'>,
'classObj': <class 'serviceNameHandler.serviceHandler'>}
Standalone is true if there is only one service started
If it's false, every service is linked to a different MonitoringClient
"""
self._svcData = serviceData
self._name = serviceData['modName']
self._startTime = Time.dateTime()
self._validNames = [serviceData['modName']]
if serviceData['loadName'] not in self._validNames:
self._validNames.append(serviceData['loadName'])
self._cfg = ServiceConfiguration(list(self._validNames))
if serviceData['standalone']:
self._monitor = gMonitor
else:
self._monitor = MonitoringClient()
self.__monitorLastStatsUpdate = time.time()
self._stats = {'queries': 0, 'connections': 0}
self._authMgr = AuthManager(
"%s/Authorization" %
PathFinder.getServiceSection(serviceData['loadName']))
self._transportPool = getGlobalTransportPool()
self.__cloneId = 0
self.__maxFD = 0
def __init__(self):
"""c'tor
just setting validation order
"""
self.validator = (
self._hasRequestName,
self._hasOwner,
self._hasOperations,
self._hasType,
self._hasFiles,
self._hasRequiredAttrs,
self._hasChecksumAndChecksumType,
)
configPath = PathFinder.getAgentSection(
"RequestManagement/RequestExecutingAgent")
# # operation handlers over here
opHandlersPath = "%s/%s" % (configPath, "OperationHandlers")
opHandlers = gConfig.getSections(opHandlersPath)
if not opHandlers["OK"]:
gLogger.error(opHandlers["Message"])
else:
self.opHandlers = set(opHandlers["Value"])
def __init__(self, nameList):
self.serviceName = nameList[0]
self.serviceURL = None
self.nameList = nameList
self.pathList = []
for svcName in nameList:
self.pathList.append(PathFinder.getServiceSection(svcName))
def initializeDataLoggingHandler( serviceInfo ):
global dataPath
global logDB
logDB = DataLoggingDB()
monitoringSection = PathFinder.getServiceSection( "DataManagement/DataLogging" )
#Get data location
retDict = gConfig.getOption( "%s/DataLocation" % monitoringSection, "dataLoggingPlots" )
if not retDict[ 'OK' ]:
return retDict
dataPath = retDict[ 'Value' ].strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) )
gLogger.info( "Data will be written into %s" % dataPath )
try:
os.makedirs( dataPath )
except:
pass
try:
testFile = "%s/mon.jarl.test" % dataPath
fd = file( testFile, "w" )
fd.close()
os.unlink( testFile )
except IOError:
gLogger.fatal( "Can't write to %s" % dataPath )
return S_ERROR( "Data location is not writable" )
return S_OK()
def initializeMonitoringHandler( serviceInfo ):
#Check that the path is writable
monitoringSection = PathFinder.getServiceSection( "Framework/Monitoring" )
#Get data location
dataPath = gConfig.getValue( "%s/DataLocation" % monitoringSection, "data/monitoring" )
dataPath = dataPath.strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) )
gLogger.info( "Data will be written into %s" % dataPath )
try:
os.makedirs( dataPath )
except:
pass
try:
testFile = "%s/mon.jarl.test" % dataPath
fd = file( testFile, "w" )
fd.close()
os.unlink( testFile )
except IOError:
gLogger.fatal( "Can't write to %s" % dataPath )
return S_ERROR( "Data location is not writable" )
#Define globals
gServiceInterface.initialize( dataPath )
if not gServiceInterface.initializeDB():
return S_ERROR( "Can't start db engine" )
gMonitor.registerActivity( "cachedplots", "Cached plot images", "Monitoring plots", "plots", gMonitor.OP_SUM )
gMonitor.registerActivity( "drawnplots", "Drawn plot images", "Monitoring plots", "plots", gMonitor.OP_SUM )
return S_OK()
def discoverHandlers(self):
"""
Force the discovery of URL, automatic call when we try to get handlers for the first time.
You can disable the automatic call with autoDiscovery=False at initialization
"""
gLogger.debug("Trying to auto-discover the handlers for Tornado")
# Look in config
diracSystems = gConfig.getSections("/Systems")
serviceList = []
if diracSystems["OK"]:
for system in diracSystems["Value"]:
try:
instance = PathFinder.getSystemInstance(system)
services = gConfig.getSections("/Systems/%s/%s/Services" % (system, instance))
if services["OK"]:
for service in services["Value"]:
newservice = "%s/%s" % (system, service)
# We search in the CS all handlers which used HTTPS as protocol
isHTTPS = gConfig.getValue(
"/Systems/%s/%s/Services/%s/Protocol" % (system, instance, service)
)
if isHTTPS and isHTTPS.lower() == "https":
serviceList.append(newservice)
# On systems sometime you have things not related to services...
except RuntimeError:
pass
return self.loadHandlersByServiceName(serviceList)
def export_sendSMS(self, userName, body, fromAddress):
""" Send an SMS with supplied body to the specified DIRAC user using the Mail utility via an SMS switch.
"""
gLogger.verbose(
'Received signal to send the following SMS to %s:\n%s' %
(userName, body))
mobile = gConfig.getValue('/Registry/Users/%s/Mobile' % userName, '')
if not mobile:
return S_ERROR('No registered mobile number for %s' % userName)
csSection = PathFinder.getServiceSection('Framework/Notification')
smsSwitch = gConfig.getValue('%s/SMSSwitch' % csSection, '')
if not smsSwitch:
return S_ERROR('No SMS switch is defined in CS path %s/SMSSwitch' %
csSection)
address = '%s@%s' % (mobile, smsSwitch)
subject = 'DIRAC SMS'
m = Mail()
m._subject = subject
m._message = body
m._mailAddress = address
if not fromAddress == 'None':
m._fromAddress = fromAddress
result = m._send()
if not result['OK']:
gLogger.warn(
'Could not send SMS to %s with the following message:\n%s' %
(userName, result['Message']))
else:
gLogger.info('SMS sent successfully to %s ' % (userName))
gLogger.debug(result['Value'])
return result
def initializePlottingHandler( serviceInfo ):
#Get data location
plottingSection = PathFinder.getServiceSection( "Framework/Plotting" )
dataPath = gConfig.getValue( "%s/DataLocation" % plottingSection, "data/graphs" )
dataPath = dataPath.strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath( "%s/%s" % ( gConfig.getValue( '/LocalSite/InstancePath', rootPath ), dataPath ) )
gLogger.info( "Data will be written into %s" % dataPath )
try:
os.makedirs( dataPath )
except:
pass
try:
testFile = "%s/plot__.test" % dataPath
fd = file( testFile, "w" )
fd.close()
os.unlink( testFile )
except IOError:
gLogger.fatal( "Can't write to %s" % dataPath )
return S_ERROR( "Data location is not writable" )
gPlotCache.setPlotsLocation( dataPath )
gMonitor.registerActivity( "plotsDrawn", "Drawn plot images", "Plotting requests", "plots", gMonitor.OP_SUM )
return S_OK()
def initializeHandler(cls, serviceInfo):
multiPath = PathFinder.getDatabaseSection("Accounting/MultiDB")
cls.__acDB = MultiAccountingDB(multiPath, readOnly=True)
# Get data location
reportSection = serviceInfo["serviceSectionPath"]
dataPath = gConfig.getValue("%s/DataLocation" % reportSection,
"data/accountingGraphs")
dataPath = dataPath.strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath("%s/%s" % (gConfig.getValue(
"/LocalSite/InstancePath", rootPath), dataPath))
gLogger.info("Data will be written into %s" % dataPath)
mkDir(dataPath)
try:
testFile = "%s/acc.jarl.test" % dataPath
with open(testFile, "w"):
pass
os.unlink(testFile)
except IOError:
gLogger.fatal("Can't write to %s" % dataPath)
return S_ERROR("Data location is not writable")
gDataCache.setGraphsLocation(dataPath)
gMonitor.registerActivity("plotsDrawn", "Drawn plot images",
"Accounting reports", "plots",
gMonitor.OP_SUM)
gMonitor.registerActivity("reportsRequested", "Generated reports",
"Accounting reports", "reports",
gMonitor.OP_SUM)
return S_OK()
def __init__( self, nameList ):
self.serviceName = nameList[0]
self.serviceURL = False
self.nameList = nameList
self.pathList = []
for svcName in nameList:
self.pathList.append( PathFinder.getServiceSection( svcName ) )
def initializePlottingHandler(serviceInfo):
#Get data location
plottingSection = PathFinder.getServiceSection("Framework/Plotting")
dataPath = gConfig.getValue("%s/DataLocation" % plottingSection,
"data/graphs")
dataPath = dataPath.strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath(
"%s/%s" %
(gConfig.getValue('/LocalSite/InstancePath', rootPath), dataPath))
gLogger.info("Data will be written into %s" % dataPath)
try:
os.makedirs(dataPath)
except:
pass
try:
testFile = "%s/plot__.test" % dataPath
fd = file(testFile, "w")
fd.close()
os.unlink(testFile)
except IOError:
gLogger.fatal("Can't write to %s" % dataPath)
return S_ERROR("Data location is not writable")
gPlotCache.setPlotsLocation(dataPath)
gMonitor.registerActivity("plotsDrawn", "Drawn plot images",
"Plotting requests", "plots", gMonitor.OP_SUM)
return S_OK()
def initializeReportGeneratorHandler(serviceInfo):
global gAccountingDB
gAccountingDB = AccountingDB(readOnly=True)
#Get data location
reportSection = PathFinder.getServiceSection("Accounting/ReportGenerator")
dataPath = gConfig.getValue("%s/DataLocation" % reportSection,
"data/accountingGraphs")
dataPath = dataPath.strip()
if "/" != dataPath[0]:
dataPath = os.path.realpath(
"%s/%s" %
(gConfig.getValue('/LocalSite/InstancePath', rootPath), dataPath))
gLogger.info("Data will be written into %s" % dataPath)
try:
os.makedirs(dataPath)
except:
pass
try:
testFile = "%s/acc.jarl.test" % dataPath
fd = file(testFile, "w")
fd.close()
os.unlink(testFile)
except IOError:
gLogger.fatal("Can't write to %s" % dataPath)
return S_ERROR("Data location is not writable")
gDataCache.setGraphsLocation(dataPath)
gMonitor.registerActivity("plotsDrawn", "Drawn plot images",
"Accounting reports", "plots", gMonitor.OP_SUM)
gMonitor.registerActivity("reportsRequested", "Generated reports",
"Accounting reports", "reports", gMonitor.OP_SUM)
return S_OK()
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result["OK"]:
return result
self._handler = result["Value"]
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
self._initMonitoring()
self._threadPool = ThreadPool(1, max(0, self._cfg.getMaxThreads()), self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {
"serviceName": self._name,
"serviceSectionPath": PathFinder.getServiceSection(self._name),
"URL": self._cfg.getURL(),
"messageSender": MessageSender(self._name, self._msgBroker),
"validNames": self._validNames,
"csPaths": [PathFinder.getServiceSection(svcName) for svcName in self._validNames],
}
# Call static initialization function
try:
self._handler["class"]._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager, self._msgBroker, self._monitor
)
if self._handler["init"]:
for initFunc in self._handler["init"]:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception, excp:
gLogger.exception("Exception while calling initialization function")
return S_ERROR("Exception while calling initialization function: %s" % str(excp))
if not isReturnStructure(result):
return S_ERROR("Service initialization function %s must return S_OK/S_ERROR" % initFunc)
if not result["OK"]:
return S_ERROR("Error while initializing %s: %s" % (self._name, result["Message"]))
except Exception, e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(errMsg)
return S_ERROR(errMsg)
def _getCSAuthorizarionSection(cls, apiName):
"""Search endpoint auth section.
:param str apiName: API name, see :py:meth:`_getFullComponentName`
:return: str
"""
return "%s/Authorization" % PathFinder.getAPISection(apiName)
def ftsManager( cls, timeout = 300 ):
""" get FTSManager instance """
if not cls.__ftsManager:
url = PathFinder.getServiceURL( "DataManagement/FTSManager" )
if not url:
raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" )
cls.__ftsManager = RPCClient( url, timeout = timeout )
return cls.__ftsManager
def _getCSAuthorizarionSection(cls, serviceName):
"""Search service auth section.
:param str serviceName: service name, see :py:meth:`_getFullComponentName`
:return: str
"""
return "%s/Authorization" % PathFinder.getServiceSection(serviceName)
def __init__( self):
Client.__init__( self )
self.setServer( "DataManagement/Test" )
url = PathFinder.getServiceURL( "DataManagement/Test" )
if not url:
raise RuntimeError( "CS option DataManagement/Test URL is not set!" )
self.testManager = RPCClient( url )
def requestManager( self, timeout = 120 ):
""" facade for RequestManager RPC client """
if not self.__requestManager:
url = PathFinder.getServiceURL( "RequestManagement/RequestManager" )
if not url:
raise RuntimeError( "CS option RequestManagement/RequestManager URL is not set!" )
self.__requestManager = RPCClient( url, timeout = timeout )
return self.__requestManager
def requestManager( self, timeout = 120 ):
""" facade for RequestManager RPC client """
if not self.__requestManager:
url = PathFinder.getServiceURL( "RequestManagement/ReqManager" )
if not url:
raise RuntimeError( "CS option RequestManagement/ReqManager URL is not set!" )
self.__requestManager = RPCClient( url, timeout = timeout )
return self.__requestManager
def execute(self):
""" This agent is the smallest and (cutest) of all the DIRAC agents in existence.
"""
gMonitor.addMark("Iteration",1)
central = PathFinder.getServiceURL("RequestManagement/centralURL")
if central:
self.central = central
local = PathFinder.getServiceURL("RequestManagement/localURL")
if local:
self.local = local
res = self.RequestDBClient.serveRequest(url=self.local)
if not res['OK']:
gLogger.error("ZuziaAgent.execute: Failed to get request from database.",self.local)
return S_OK()
elif not res['Value']:
gLogger.info("ZuziaAgent.execute: No requests to be executed found.")
return S_OK()
gMonitor.addMark("Attempted",1)
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
gLogger.info("ZuziaAgent.execute: Obtained request %s" % requestName)
gLogger.info("ZuziaAgent.execute: Attempting to set request to %s." % self.central)
res = self.RequestDBClient.setRequest(requestName,requestString,self.central)
if res['OK']:
gMonitor.addMark("Successful",1)
gLogger.info("ZuziaAgent.execute: Successfully put request.")
else:
gMonitor.addMark("Failed",1)
gLogger.error("ZuziaAgent.execute: Failed to set request to", self.central)
gLogger.info("ZuziaAgent.execute: Attempting to set request to %s." % self.local)
res = self.RequestDBClient.setRequest(requestName,requestString,self.local)
if res['OK']:
gLogger.info("ZuziaAgent.execute: Successfully put request.")
else:
gLogger.error("ZuziaAgent.execute: Failed to set request to", self.local)
while not res['OK']:
gLogger.info("ZuziaAgent.execute: Attempting to set request to anywhere.")
res = self.RequestDBClient.setRequest(requestName,requestString)
gLogger.info("ZuziaAgent.execute: Successfully put request to %s." % res["Server"])
return S_OK()
def __init__( self, storageName, parameters ):
StorageBase.__init__( self, storageName, parameters )
self.pluginName = 'Proxy'
self.isok = True
self.url = PathFinder.getServiceURL( "DataManagement/StorageElementProxy" )
if not self.url:
self.isok = False
def export_sendMail(self,
address,
subject,
body,
fromAddress,
avoidSpam=False):
""" Send an email with supplied body to the specified address using the Mail utility.
:param basestring address: recipient addresses
:param basestring subject: subject of letter
:param basestring body: body of letter
:param basestring fromAddress: sender address, if None, will be used default from CS
:param bool avoidSpam: if True, then emails are first added to a set so that duplicates are removed,
and sent every hour.
:return: S_OK(basestring)/S_ERROR() -- basestring is status message
"""
gLogger.verbose(
'Received signal to send the following mail to %s:\nSubject = %s\n%s'
% (address, subject, body))
if gMailCache.exists(hash(address + subject + body)) and not avoidSpam:
return S_OK(
'Email with the same content already sent today to current addresses, come back tomorrow'
)
eMail = Mail()
notificationSection = PathFinder.getServiceSection(
"Framework/Notification")
csSection = notificationSection + '/SMTP'
eMail._smtpHost = gConfig.getValue('%s/Host' % csSection)
eMail._smtpPort = gConfig.getValue('%s/Port' % csSection)
eMail._smtpLogin = gConfig.getValue('%s/Login' % csSection)
eMail._smtpPasswd = gConfig.getValue('%s/Password' % csSection)
eMail._smtpPtcl = gConfig.getValue('%s/Protocol' % csSection)
eMail._subject = subject
eMail._message = body
eMail._mailAddress = address
if not fromAddress == 'None':
eMail._fromAddress = fromAddress
eMail._fromAddress = gConfig.getValue(
'%s/FromAddress' % csSection) or eMail._fromAddress
if avoidSpam and eMail in gMailSet:
return S_OK("Mail already sent")
else:
result = eMail._send()
if not result['OK']:
gLogger.warn(
'Could not send mail with the following message:\n%s' %
result['Message'])
else:
gMailCache.add(hash(address + subject + body), 3600 * 24)
gLogger.info('Mail sent successfully to %s with subject %s' %
(address, subject))
gLogger.debug(result['Value'])
if avoidSpam:
gMailSet.add(eMail)
return result
def initializeHandler( cls, svcInfoDict ):
multiPath = PathFinder.getDatabaseSection( "Accounting/MultiDB" )
cls.__acDB = MultiAccountingDB( multiPath )
cls.__acDB.autoCompactDB()
result = cls.__acDB.markAllPendingRecordsAsNotTaken()
if not result[ 'OK' ]:
return result
gThreadScheduler.addPeriodicTask( 60, cls.__acDB.loadPendingRecords )
return S_OK()
def loadModules( self, modulesList, hideExceptions = False ):
"""
Load all modules required in moduleList
"""
for modName in modulesList:
gLogger.verbose( "Checking %s" % modName )
#if it's a executor modName name just load it and be done with it
if modName.find( "/" ) > -1:
gLogger.verbose( "Module %s seems to be a valid name. Try to load it!" % modName )
result = self.loadModule( modName, hideExceptions = hideExceptions )
if not result[ 'OK' ]:
return result
continue
#Check if it's a system name
#Look in the CS
system = modName
#Can this be generated with sectionFinder?
csPath = "%s/Executors" % PathFinder.getSystemSection ( system, ( system, ) )
gLogger.verbose( "Exploring %s to discover modules" % csPath )
result = gConfig.getSections( csPath )
if result[ 'OK' ]:
#Add all modules in the CS :P
for modName in result[ 'Value' ]:
result = self.loadModule( "%s/%s" % ( system, modName ), hideExceptions = hideExceptions )
if not result[ 'OK' ]:
return result
#Look what is installed
parentModule = None
for rootModule in getInstalledExtensions():
if system.find( "System" ) != len( system ) - 6:
parentImport = "%s.%sSystem.%s" % ( rootModule, system, self.__csSuffix )
else:
parentImport = "%s.%s.%s" % ( rootModule, system, self.__csSuffix )
#HERE!
result = self.__recurseImport( parentImport )
if not result[ 'OK' ]:
return result
parentModule = result[ 'Value' ]
if parentModule:
break
if not parentModule:
continue
parentPath = parentModule.__path__[0]
gLogger.notice( "Found modules path at %s" % parentImport )
for entry in os.listdir( parentPath ):
if entry[-3:] != ".py" or entry == "__init__.py":
continue
if not os.path.isfile( os.path.join( parentPath, entry ) ):
continue
modName = "%s/%s" % ( system, entry[:-3] )
gLogger.verbose( "Trying to import %s" % modName )
result = self.loadModule( modName,
hideExceptions = hideExceptions,
parentModule = parentModule )
return S_OK()
def __init__( self, useCertificates = False ):
""" Constructor of the RequestClient class
"""
voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) )
self.voBoxUrls = []
if voBoxUrls:
self.voBoxUrls = randomize( voBoxUrls )
local = PathFinder.getServiceURL( "RequestManagement/localURL" )
self.local = False
if local:
self.local = local
central = PathFinder.getServiceURL( "RequestManagement/centralURL" )
self.central = False
if central:
self.central = central
self.setServer( 'RequestManagement/centralURL' )
def requestProxies( self, timeout = 120 ):
""" get request proxies dict """
if not self.__requestProxiesDict:
self.__requestProxiesDict = {}
proxiesURLs = fromChar( PathFinder.getServiceURL( "RequestManagement/ReqProxyURLs" ) )
if not proxiesURLs:
self.log.warn( "CS option RequestManagement/ReqProxyURLs is not set!" )
for proxyURL in proxiesURLs:
self.log.debug( "creating RequestProxy for url = %s" % proxyURL )
self.__requestProxiesDict[proxyURL] = RPCClient( proxyURL, timeout = timeout )
return self.__requestProxiesDict
def __getRequestDBPath( self ):
""" Obtain the root of the requestDB from the configuration
"""
csSection = csSection = PathFinder.getServiceSection( "RequestManagement/RequestManager" )
root = gConfig.getValue( '%s/Path' % csSection )
if not root:
diracRoot = gConfig.getValue( '/LocalSite/InstancePath', rootPath )
root = diracRoot + '/requestdb'
if not os.path.exists( root ):
os.makedirs( root )
return root
def __init__(self,url=False,useCertificates=False):
""" Constructor of the DataLogging client
"""
try:
if not url:
self.url = PathFinder.getServiceURL('DataManagement/DataLogging')
else:
self.url = url
except Exception, x:
errStr = "DataLoggingClient.__init__: Exception while obtaining service URL."
gLogger.exception(errStr,lException=x)
def __init__( self, csSection = False ):
threading.Thread.__init__( self )
if csSection:
self.csSection = csSection
else:
self.csSection = PathFinder.getServiceSection( "Framework/SiteMap" )
self.refreshPeriod = self._getCSValue( "RefreshPeriod", 300 )
self.gridsToMap = []
self.history = []
self.sitesData = {}
self.refresh()
self.setDaemon( 1 )
def initialize( self ):
self.logger = gLogger.getSubLogger( "Monitoring" )
self.logger.debug( "Initializing Monitoring Client" )
self.sourceDict[ 'setup' ] = gConfig.getValue( "/DIRAC/Setup" )
self.sourceDict[ 'site' ] = DIRAC.siteName()
if self.sourceDict[ 'componentType' ] == self.COMPONENT_SERVICE:
self.cfgSection = PathFinder.getSystemSection( self.sourceDict[ 'componentName' ] )
elif self.sourceDict[ 'componentType' ] == self.COMPONENT_AGENT:
self.cfgSection = PathFinder.getAgentSection( self.sourceDict[ 'componentName' ] )
self.setComponentLocation( Network.getFQDN() )
elif self.sourceDict[ 'componentType' ] == self.COMPONENT_WEB:
self.cfgSection = "/WebApp"
self.setComponentLocation( 'http://%s' % Network.getFQDN() )
self.setComponentName( 'WebApp' )
elif self.sourceDict[ 'componentType' ] == self.COMPONENT_SCRIPT:
self.cfgSection = "/Script"
else:
raise Exception( "Component type has not been defined" )
gMonitoringFlusher.registerMonitoringClient( self )
# ExitCallback.registerExitCallback( self.forceFlush )
self.__initialized = True
def initializeHandler( cls, svcInfoDict ):
multiPath = PathFinder.getDatabaseSection( "Accounting/MultiDB" )
cls.__acDB = MultiAccountingDB( multiPath )
#we can run multiple services in read only mode. In that case we do not bucket
cls.runBucketing = getServiceOption( svcInfoDict, 'RunBucketing', True )
if cls.runBucketing:
cls.__acDB.autoCompactDB() #pylint: disable=no-member
result = cls.__acDB.markAllPendingRecordsAsNotTaken() #pylint: disable=no-member
if not result[ 'OK' ]:
return result
gThreadScheduler.addPeriodicTask( 60, cls.__acDB.loadPendingRecords ) #pylint: disable=no-member
return S_OK()
def __getRequestDBPath():
""" get the fs root path of the requestDB from the DIRAC configuration
:warn: if root path doesn't exist, it will be created
"""
csSection = csSection = PathFinder.getServiceSection("RequestManagement/RequestManager")
root = gConfig.getValue("%s/Path" % csSection, "requestdb")
diracRoot = gConfig.getValue("/LocalSite/InstancePath", rootPath)
# if the path return by the gConfig is absolute the following line does not change it,
# otherwise it makes it relative to diracRoot
root = os.path.join(diracRoot, root)
if not os.path.exists(root):
os.makedirs(root)
return root
def __init__( self, url = False ):
""" Constructor of the Bookkeeping catalogue client
"""
self.splitSize = 1000
self.name = 'BookkeepingDB'
self.valid = True
try:
if not url:
self.url = PathFinder.getServiceURL( 'Bookkeeping/NewBookkeepingManager' )
else:
self.url = url
except Exception, exceptionMessage:
gLogger.exception( 'BookkeepingDBClient.__init__: Exception while obtaining Bookkeeping service URL.', '', exceptionMessage )
self.valid = False
def initialize(self):
self.RequestDBClient = RequestClient()
gMonitor.registerActivity("Iteration", "Agent Loops", "ZuziaAgent", "Loops/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Attempted", "Request Processed", "ZuziaRAgent", "Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Successful", "Request Forward Successful", "ZuziaAgent", "Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Failed", "Request Forward Failed", "ZuziaAgent", "Requests/min", gMonitor.OP_SUM)
self.local = PathFinder.getServiceURL("RequestManagement/localURL")
if not self.local:
self.local = AgentModule.am_getOption(self,'localURL','')
if not self.local:
errStr = 'The RequestManagement/localURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
self.central = PathFinder.getServiceURL("RequestManagement/centralURL")
if not self.central:
errStr = 'The RequestManagement/centralURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
return S_OK()
def initialize( self ):
self.section = PathFinder.getAgentSection( AGENT_NAME )
self.RequestDB = RequestDBMySQL()
self.TransferDB = TransferDB()
self.DataLog = DataLoggingClient()
self.factory = StorageFactory()
self.rm = ReplicaManager()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()