def __init__(self, useCertificates=False):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__(self)
## setup logger
self.log = gLogger.getSubLogger("RequestManagement/RequestClient")
## dict to store all RPC clients for easy reuse
self.__requestRPCClientsDict = {}
## local if any defined
local = PathFinder.getServiceURL("RequestManagement/localURL")
if local:
self.__requestRPCClientsDict.setdefault(
"local", [self.__requestRPCClient(local)])
## central if any defined
central = PathFinder.getServiceURL("RequestManagement/centralURL")
if central:
self.__requestRPCClientsDict.setdefault(
"central", [self.__requestRPCClient(central)])
## voboxes if any defined
voBoxUrls = fromChar(
PathFinder.getServiceURL("RequestManagement/voBoxURLs"))
if voBoxUrls:
self.__requestRPCClientsDict.setdefault("voboxes", [])
for voBoxURL in randomize(voBoxUrls):
self.__requestRPCClientsDict["voboxes"].append(
self.__requestRPCClient(voBoxURL))
self.setServer('RequestManagement/centralURL')
def __init__( self, useCertificates = False ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self )
## setup logger
self.log = gLogger.getSubLogger( "RequestManagement/RequestClient" )
## dict to store all RPC clients for easy reuse
self.__requestRPCClientsDict = {}
## local if any defined
local = PathFinder.getServiceURL( "RequestManagement/localURL" )
if local:
self.__requestRPCClientsDict.setdefault( "local" , [ self.__requestRPCClient( local ) ] )
## central if any defined
central = PathFinder.getServiceURL( "RequestManagement/centralURL" )
if central:
self.__requestRPCClientsDict.setdefault( "central", [ self.__requestRPCClient( central ) ] )
## voboxes if any defined
voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) )
if voBoxUrls:
self.__requestRPCClientsDict.setdefault( "voboxes", [] )
for voBoxURL in randomize( voBoxUrls ):
self.__requestRPCClientsDict["voboxes"].append( self.__requestRPCClient( voBoxURL ) )
self.setServer( 'RequestManagement/centralURL' )
def initialize(self):
self.RequestDBClient = RequestClient()
gMonitor.registerActivity("Iteration", "Agent Loops", "ZuziaAgent",
"Loops/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Attempted", "Request Processed",
"ZuziaRAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Successful", "Request Forward Successful",
"ZuziaAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Failed", "Request Forward Failed",
"ZuziaAgent", "Requests/min",
gMonitor.OP_SUM)
self.local = PathFinder.getServiceURL("RequestManagement/localURL")
if not self.local:
self.local = AgentModule.am_getOption(self, 'localURL', '')
if not self.local:
errStr = 'The RequestManagement/localURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
self.central = PathFinder.getServiceURL("RequestManagement/centralURL")
if not self.central:
errStr = 'The RequestManagement/centralURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
return S_OK()
def execute(self):
""" This agent is the smallest and (cutest) of all the DIRAC agents in existence.
"""
gMonitor.addMark("Iteration", 1)
central = PathFinder.getServiceURL("RequestManagement/centralURL")
if central:
self.central = central
local = PathFinder.getServiceURL("RequestManagement/localURL")
if local:
self.local = local
res = self.RequestDBClient.serveRequest(url=self.local)
if not res['OK']:
gLogger.error(
"ZuziaAgent.execute: Failed to get request from database.",
self.local)
return S_OK()
elif not res['Value']:
gLogger.info(
"ZuziaAgent.execute: No requests to be executed found.")
return S_OK()
gMonitor.addMark("Attempted", 1)
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
gLogger.info("ZuziaAgent.execute: Obtained request %s" % requestName)
gLogger.info("ZuziaAgent.execute: Attempting to set request to %s." %
self.central)
res = self.RequestDBClient.setRequest(requestName, requestString,
self.central)
if res['OK']:
gMonitor.addMark("Successful", 1)
gLogger.info("ZuziaAgent.execute: Successfully put request.")
else:
gMonitor.addMark("Failed", 1)
gLogger.error("ZuziaAgent.execute: Failed to set request to",
self.central)
gLogger.info(
"ZuziaAgent.execute: Attempting to set request to %s." %
self.local)
res = self.RequestDBClient.setRequest(requestName, requestString,
self.local)
if res['OK']:
gLogger.info("ZuziaAgent.execute: Successfully put request.")
else:
gLogger.error("ZuziaAgent.execute: Failed to set request to",
self.local)
while not res['OK']:
gLogger.info(
"ZuziaAgent.execute: Attempting to set request to anywhere."
)
res = self.RequestDBClient.setRequest(
requestName, requestString)
gLogger.info(
"ZuziaAgent.execute: Successfully put request to %s." %
res["Server"])
return S_OK()
def initialize( self ):
self.RequestDBClient = RequestClient()
backend = self.am_getOption( 'Backend', '' )
self.RequestDB = False
if backend == 'mysql':
from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL
requestDB = RequestDBMySQL()
if requestDB._connected:
self.RequestDB = requestDB
gMonitor.registerActivity( "Iteration", "Agent Loops", "DISETForwardingAgent", "Loops/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Attempted", "Request Processed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Successful", "Request Forward Successful", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "Failed", "Request Forward Failed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM )
self.local = PathFinder.getServiceURL( "RequestManagement/localURL" )
if not self.local:
self.local = AgentModule.am_getOption( self, 'localURL', '' )
if not self.local:
errStr = 'The RequestManagement/localURL option must be defined.'
gLogger.fatal( errStr )
return S_ERROR( errStr )
return S_OK()
def __init__(self, url=False, useCertificates=False):
""" Constructor of the LCGFileCatalogProxy client class
"""
self.name = 'LFCProxy'
self.valid = False
try:
if url:
self.url = url
else:
url = PathFinder.getServiceURL(
'DataManagement/LcgFileCatalogProxy')
if not url:
return
self.url = url
self.server = RPCClient(self.url,
timeout=120,
useCertificates=useCertificates)
if not self.server:
return
else:
self.valid = True
except Exception, x:
gLogger.exception(
'Exception while creating connection to LcgFileCatalog proxy server',
'', x)
return
def initialize(self):
self.RequestDBClient = RequestClient()
backend = self.am_getOption('Backend', '')
self.RequestDB = False
if backend == 'mysql':
from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL
requestDB = RequestDBMySQL()
if requestDB._connected:
self.RequestDB = requestDB
gMonitor.registerActivity("Iteration", "Agent Loops",
"DISETForwardingAgent", "Loops/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Attempted", "Request Processed",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Successful", "Request Forward Successful",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Failed", "Request Forward Failed",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
self.local = PathFinder.getServiceURL("RequestManagement/localURL")
if not self.local:
self.local = AgentModule.am_getOption(self, 'localURL', '')
if not self.local:
errStr = 'The RequestManagement/localURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
return S_OK()
def requestManager( self, timeout = 120 ):
""" facade for RequestManager RPC client """
if not self.__requestManager:
url = PathFinder.getServiceURL( "RequestManagement/ReqManager" )
if not url:
raise RuntimeError( "CS option RequestManagement/ReqManager URL is not set!" )
self.__requestManager = RPCClient( url, timeout = timeout )
return self.__requestManager
def ftsManager( cls, timeout = 300 ):
""" get FTSManager instance """
if not cls.__ftsManager:
url = PathFinder.getServiceURL( "DataManagement/FTSManager" )
if not url:
raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" )
cls.__ftsManager = RPCClient( url, timeout = timeout )
return cls.__ftsManager
def requestManager( self, timeout = 120 ):
""" facade for RequestManager RPC client """
if not self.__requestManager:
url = PathFinder.getServiceURL( "RequestManagement/RequestManager" )
if not url:
raise RuntimeError( "CS option RequestManagement/RequestManager URL is not set!" )
self.__requestManager = RPCClient( url, timeout = timeout )
return self.__requestManager
def __init__( self):
Client.__init__( self )
self.setServer( "DataManagement/Test" )
url = PathFinder.getServiceURL( "DataManagement/Test" )
if not url:
raise RuntimeError( "CS option DataManagement/Test URL is not set!" )
self.testManager = RPCClient( url )
def execute(self):
""" This agent is the smallest and (cutest) of all the DIRAC agents in existence.
"""
gMonitor.addMark("Iteration",1)
central = PathFinder.getServiceURL("RequestManagement/centralURL")
if central:
self.central = central
local = PathFinder.getServiceURL("RequestManagement/localURL")
if local:
self.local = local
res = self.RequestDBClient.serveRequest(url=self.local)
if not res['OK']:
gLogger.error("ZuziaAgent.execute: Failed to get request from database.",self.local)
return S_OK()
elif not res['Value']:
gLogger.info("ZuziaAgent.execute: No requests to be executed found.")
return S_OK()
gMonitor.addMark("Attempted",1)
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
gLogger.info("ZuziaAgent.execute: Obtained request %s" % requestName)
gLogger.info("ZuziaAgent.execute: Attempting to set request to %s." % self.central)
res = self.RequestDBClient.setRequest(requestName,requestString,self.central)
if res['OK']:
gMonitor.addMark("Successful",1)
gLogger.info("ZuziaAgent.execute: Successfully put request.")
else:
gMonitor.addMark("Failed",1)
gLogger.error("ZuziaAgent.execute: Failed to set request to", self.central)
gLogger.info("ZuziaAgent.execute: Attempting to set request to %s." % self.local)
res = self.RequestDBClient.setRequest(requestName,requestString,self.local)
if res['OK']:
gLogger.info("ZuziaAgent.execute: Successfully put request.")
else:
gLogger.error("ZuziaAgent.execute: Failed to set request to", self.local)
while not res['OK']:
gLogger.info("ZuziaAgent.execute: Attempting to set request to anywhere.")
res = self.RequestDBClient.setRequest(requestName,requestString)
gLogger.info("ZuziaAgent.execute: Successfully put request to %s." % res["Server"])
return S_OK()
def __init__( self, storageName, parameters ):
StorageBase.__init__( self, storageName, parameters )
self.pluginName = 'Proxy'
self.isok = True
self.url = PathFinder.getServiceURL( "DataManagement/StorageElementProxy" )
if not self.url:
self.isok = False
def __init__( self, storageName, parameters ):
StorageBase.__init__( self, storageName, parameters )
self.pluginName = 'Proxy'
self.isok = True
self.url = PathFinder.getServiceURL( "DataManagement/StorageElementProxy" )
if not self.url:
self.isok = False
def __init__( self, useCertificates = False ):
""" Constructor of the RequestClient class
"""
voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) )
self.voBoxUrls = []
if voBoxUrls:
self.voBoxUrls = randomize( voBoxUrls )
local = PathFinder.getServiceURL( "RequestManagement/localURL" )
self.local = False
if local:
self.local = local
central = PathFinder.getServiceURL( "RequestManagement/centralURL" )
self.central = False
if central:
self.central = central
self.setServer( 'RequestManagement/centralURL' )
def requestProxies( self, timeout = 120 ):
""" get request proxies dict """
if not self.__requestProxiesDict:
self.__requestProxiesDict = {}
proxiesURLs = fromChar( PathFinder.getServiceURL( "RequestManagement/ReqProxyURLs" ) )
if not proxiesURLs:
self.log.warn( "CS option RequestManagement/ReqProxyURLs is not set!" )
for proxyURL in proxiesURLs:
self.log.debug( "creating RequestProxy for url = %s" % proxyURL )
self.__requestProxiesDict[proxyURL] = RPCClient( proxyURL, timeout = timeout )
return self.__requestProxiesDict
def __init__(self, useCertificates=False):
""" Constructor of the RequestClient class
"""
voBoxUrls = fromChar(
PathFinder.getServiceURL("RequestManagement/voBoxURLs"))
self.voBoxUrls = []
if voBoxUrls:
self.voBoxUrls = randomize(voBoxUrls)
local = PathFinder.getServiceURL("RequestManagement/localURL")
self.local = False
if local:
self.local = local
central = PathFinder.getServiceURL("RequestManagement/centralURL")
self.central = False
if central:
self.central = central
self.setServer('RequestManagement/centralURL')
def __init__(self,url=False,useCertificates=False):
""" Constructor of the DataLogging client
"""
try:
if not url:
self.url = PathFinder.getServiceURL('DataManagement/DataLogging')
else:
self.url = url
except Exception, x:
errStr = "DataLoggingClient.__init__: Exception while obtaining service URL."
gLogger.exception(errStr,lException=x)
def requestProxies(self, timeout=120):
""" get request proxies dict """
if not self.__requestProxiesDict:
self.__requestProxiesDict = {}
proxiesURLs = fromChar(PathFinder.getServiceURL("RequestManagement/ReqProxyURLs"))
if not proxiesURLs:
self.log.warn("CS option RequestManagement/ReqProxyURLs is not set!")
for proxyURL in proxiesURLs:
self.log.debug("creating RequestProxy for url = %s" % proxyURL)
self.__requestProxiesDict[proxyURL] = RPCClient(proxyURL, timeout=timeout)
return self.__requestProxiesDict
def __init__(self, url=False, useCertificates=False):
""" Constructor of the DataLogging client
"""
try:
if not url:
self.url = PathFinder.getServiceURL(
'DataManagement/DataLogging')
else:
self.url = url
except Exception, x:
errStr = "DataLoggingClient.__init__: Exception while obtaining service URL."
gLogger.exception(errStr, lException=x)
def initialize(self):
self.RequestDBClient = RequestClient()
gMonitor.registerActivity("Iteration", "Agent Loops", "ZuziaAgent", "Loops/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Attempted", "Request Processed", "ZuziaRAgent", "Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Successful", "Request Forward Successful", "ZuziaAgent", "Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("Failed", "Request Forward Failed", "ZuziaAgent", "Requests/min", gMonitor.OP_SUM)
self.local = PathFinder.getServiceURL("RequestManagement/localURL")
if not self.local:
self.local = AgentModule.am_getOption(self,'localURL','')
if not self.local:
errStr = 'The RequestManagement/localURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
self.central = PathFinder.getServiceURL("RequestManagement/centralURL")
if not self.central:
errStr = 'The RequestManagement/centralURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
return S_OK()
def __init__( self, url = False ):
""" Constructor of the Bookkeeping catalogue client
"""
self.splitSize = 1000
self.name = 'BookkeepingDB'
self.valid = True
try:
if not url:
self.url = PathFinder.getServiceURL( 'Bookkeeping/NewBookkeepingManager' )
else:
self.url = url
except Exception, exceptionMessage:
gLogger.exception( 'BookkeepingDBClient.__init__: Exception while obtaining Bookkeeping service URL.', '', exceptionMessage )
self.valid = False
def __init__( self, storageName, protocol, path, host, port, spaceToken, wspath ):
self.isok = True
self.protocolName = 'Proxy'
self.name = storageName
self.protocol = protocol
self.path = path
self.host = host
self.port = port
self.wspath = wspath
self.spaceToken = spaceToken
self.cwd = self.path
apply( StorageBase.__init__, ( self, self.name, self.path ) )
self.url = PathFinder.getServiceURL( "DataManagement/StorageElementProxy" )
if not self.url:
self.isok = False
def __removeRequests( self, requestIDs ):
""" This will remove requests from the (new) RMS system -
#FIXME: if the old system is still installed, it won't remove anything!!!
(we don't want to risk removing from the new RMS what is instead in the old)
"""
# FIXME: checking if the old system is still installed!
from DIRAC.ConfigurationSystem.Client import PathFinder
if PathFinder.getServiceURL( "RequestManagement/RequestManager" ):
self.log.warn( "NOT removing requests!!" )
return S_OK()
rIDs = [ int( long( j ) ) for j in requestIDs if long( j ) ]
for requestName in rIDs:
self.reqClient.deleteRequest( requestName )
return S_OK()
def __init__(self, url=False):
""" Constructor of the Bookkeeping catalogue client
"""
self.splitSize = 1000
self.name = 'BookkeepingDB'
self.valid = True
try:
if not url:
self.url = PathFinder.getServiceURL(
'Bookkeeping/NewBookkeepingManager')
else:
self.url = url
except Exception, exceptionMessage:
gLogger.exception(
'BookkeepingDBClient.__init__: Exception while obtaining Bookkeeping service URL.',
'', exceptionMessage)
self.valid = False
def __removeRequests( self, requestIDs ):
""" This will remove requests from the (new) RMS system -
#FIXME: if the old system is still installed, it won't remove anything!!!
(we don't want to risk removing from the new RMS what is instead in the old)
"""
# FIXME: checking if the old system is still installed!
from DIRAC.ConfigurationSystem.Client import PathFinder
if PathFinder.getServiceURL( "RequestManagement/RequestManager" ):
self.log.warn( "NOT removing requests!!" )
return S_OK()
rIDs = [ int( long( j ) ) for j in requestIDs if long( j ) ]
for requestName in rIDs:
self.reqClient.deleteRequest( requestName )
return S_OK()
def requestProxies(self, timeout=120):
"""get request proxies dict"""
# Forward all the connection options to the requestClient
# (e.g. the userDN to use)
kwargs = self.getClientKWArgs()
kwargs["timeout"] = timeout
if not self.__requestProxiesDict:
self.__requestProxiesDict = {}
proxiesURLs = fromChar(PathFinder.getServiceURL("RequestManagement/ReqProxyURLs"))
if not proxiesURLs:
self.log.warn("CS option RequestManagement/ReqProxyURLs is not set!")
for proxyURL in proxiesURLs:
self.log.debug("creating RequestProxy for url = %s" % proxyURL)
pc = Client(**kwargs)
pc.setServer(proxyURL)
self.__requestProxiesDict[proxyURL] = pc
return self.__requestProxiesDict
def __init__( self, useCertificates = False ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self )
self.log = gLogger.getSubLogger( "DataManagement/FTSClient" )
self.setServer( "DataManagement/FTSManager" )
# getting other clients
self.ftsValidator = FTSValidator()
self.dataManager = DataManager()
self.storageFactory = StorageFactory()
url = PathFinder.getServiceURL( "DataManagement/FTSManager" )
if not url:
raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" )
self.ftsManager = RPCClient( url )
def __init__( self, useCertificates = False ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self )
self.log = gLogger.getSubLogger( "DataManagement/FTSClient" )
self.setServer( "DataManagement/FTSManager" )
# getting other clients
self.ftsValidator = FTSValidator()
self.dataManager = DataManager()
self.storageFactory = StorageFactory()
url = PathFinder.getServiceURL( "DataManagement/FTSManager" )
if not url:
raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" )
self.ftsManager = RPCClient( url )
def __init__(self, url = False, useCertificates = False):
""" Constructor of the LCGFileCatalogProxy client class
"""
self.name = 'LFCProxy'
self.valid = False
try:
if url:
self.url = url
else:
url = PathFinder.getServiceURL('DataManagement/LcgFileCatalogProxy')
if not url:
return
self.url = url
self.server = RPCClient(self.url,timeout=120,useCertificates=useCertificates)
if not self.server:
return
else:
self.valid = True
except Exception,x:
gLogger.exception('Exception while creating connection to LcgFileCatalog proxy server','',x)
return
def export_getUsedPorts( self ):
"""
Retrieve the ports in use by services on this host
:return: Returns a dictionary containing, for each system, which port is being used by which service
"""
result = gComponentInstaller.getSetupComponents()
if not result[ 'OK' ]:
return result
services = result[ 'Value' ][ 'Services' ]
ports = {}
for system in services:
ports[ system ] = {}
for service in services[ system ]:
url = PathFinder.getServiceURL( '%s/%s' % ( system, service ) )
port = re.search( r':(\d{4,5})/', url )
if port:
ports[ system ][ service ] = port.group( 1 )
else:
ports[ system ][ service ] = 'None'
return S_OK( ports )
