def __init__(self, useCertificates=False): """c'tor :param self: self reference :param bool useCertificates: flag to enable/disable certificates """ Client.__init__(self) ## setup logger self.log = gLogger.getSubLogger("RequestManagement/RequestClient") ## dict to store all RPC clients for easy reuse self.__requestRPCClientsDict = {} ## local if any defined local = PathFinder.getServiceURL("RequestManagement/localURL") if local: self.__requestRPCClientsDict.setdefault( "local", [self.__requestRPCClient(local)]) ## central if any defined central = PathFinder.getServiceURL("RequestManagement/centralURL") if central: self.__requestRPCClientsDict.setdefault( "central", [self.__requestRPCClient(central)]) ## voboxes if any defined voBoxUrls = fromChar( PathFinder.getServiceURL("RequestManagement/voBoxURLs")) if voBoxUrls: self.__requestRPCClientsDict.setdefault("voboxes", []) for voBoxURL in randomize(voBoxUrls): self.__requestRPCClientsDict["voboxes"].append( self.__requestRPCClient(voBoxURL)) self.setServer('RequestManagement/centralURL')
def __init__( self, useCertificates = False ): """c'tor :param self: self reference :param bool useCertificates: flag to enable/disable certificates """ Client.__init__( self ) ## setup logger self.log = gLogger.getSubLogger( "RequestManagement/RequestClient" ) ## dict to store all RPC clients for easy reuse self.__requestRPCClientsDict = {} ## local if any defined local = PathFinder.getServiceURL( "RequestManagement/localURL" ) if local: self.__requestRPCClientsDict.setdefault( "local" , [ self.__requestRPCClient( local ) ] ) ## central if any defined central = PathFinder.getServiceURL( "RequestManagement/centralURL" ) if central: self.__requestRPCClientsDict.setdefault( "central", [ self.__requestRPCClient( central ) ] ) ## voboxes if any defined voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) ) if voBoxUrls: self.__requestRPCClientsDict.setdefault( "voboxes", [] ) for voBoxURL in randomize( voBoxUrls ): self.__requestRPCClientsDict["voboxes"].append( self.__requestRPCClient( voBoxURL ) ) self.setServer( 'RequestManagement/centralURL' )
def initialize(self): self.RequestDBClient = RequestClient() gMonitor.registerActivity("Iteration", "Agent Loops", "ZuziaAgent", "Loops/min", gMonitor.OP_SUM) gMonitor.registerActivity("Attempted", "Request Processed", "ZuziaRAgent", "Requests/min", gMonitor.OP_SUM) gMonitor.registerActivity("Successful", "Request Forward Successful", "ZuziaAgent", "Requests/min", gMonitor.OP_SUM) gMonitor.registerActivity("Failed", "Request Forward Failed", "ZuziaAgent", "Requests/min", gMonitor.OP_SUM) self.local = PathFinder.getServiceURL("RequestManagement/localURL") if not self.local: self.local = AgentModule.am_getOption(self, 'localURL', '') if not self.local: errStr = 'The RequestManagement/localURL option must be defined.' gLogger.fatal(errStr) return S_ERROR(errStr) self.central = PathFinder.getServiceURL("RequestManagement/centralURL") if not self.central: errStr = 'The RequestManagement/centralURL option must be defined.' gLogger.fatal(errStr) return S_ERROR(errStr) return S_OK()
def execute(self): """ This agent is the smallest and (cutest) of all the DIRAC agents in existence. """ gMonitor.addMark("Iteration", 1) central = PathFinder.getServiceURL("RequestManagement/centralURL") if central: self.central = central local = PathFinder.getServiceURL("RequestManagement/localURL") if local: self.local = local res = self.RequestDBClient.serveRequest(url=self.local) if not res['OK']: gLogger.error( "ZuziaAgent.execute: Failed to get request from database.", self.local) return S_OK() elif not res['Value']: gLogger.info( "ZuziaAgent.execute: No requests to be executed found.") return S_OK() gMonitor.addMark("Attempted", 1) requestString = res['Value']['RequestString'] requestName = res['Value']['RequestName'] gLogger.info("ZuziaAgent.execute: Obtained request %s" % requestName) gLogger.info("ZuziaAgent.execute: Attempting to set request to %s." % self.central) res = self.RequestDBClient.setRequest(requestName, requestString, self.central) if res['OK']: gMonitor.addMark("Successful", 1) gLogger.info("ZuziaAgent.execute: Successfully put request.") else: gMonitor.addMark("Failed", 1) gLogger.error("ZuziaAgent.execute: Failed to set request to", self.central) gLogger.info( "ZuziaAgent.execute: Attempting to set request to %s." % self.local) res = self.RequestDBClient.setRequest(requestName, requestString, self.local) if res['OK']: gLogger.info("ZuziaAgent.execute: Successfully put request.") else: gLogger.error("ZuziaAgent.execute: Failed to set request to", self.local) while not res['OK']: gLogger.info( "ZuziaAgent.execute: Attempting to set request to anywhere." ) res = self.RequestDBClient.setRequest( requestName, requestString) gLogger.info( "ZuziaAgent.execute: Successfully put request to %s." % res["Server"]) return S_OK()
def initialize( self ): self.RequestDBClient = RequestClient() backend = self.am_getOption( 'Backend', '' ) self.RequestDB = False if backend == 'mysql': from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL requestDB = RequestDBMySQL() if requestDB._connected: self.RequestDB = requestDB gMonitor.registerActivity( "Iteration", "Agent Loops", "DISETForwardingAgent", "Loops/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "Attempted", "Request Processed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "Successful", "Request Forward Successful", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "Failed", "Request Forward Failed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM ) self.local = PathFinder.getServiceURL( "RequestManagement/localURL" ) if not self.local: self.local = AgentModule.am_getOption( self, 'localURL', '' ) if not self.local: errStr = 'The RequestManagement/localURL option must be defined.' gLogger.fatal( errStr ) return S_ERROR( errStr ) return S_OK()
def __init__(self, url=False, useCertificates=False): """ Constructor of the LCGFileCatalogProxy client class """ self.name = 'LFCProxy' self.valid = False try: if url: self.url = url else: url = PathFinder.getServiceURL( 'DataManagement/LcgFileCatalogProxy') if not url: return self.url = url self.server = RPCClient(self.url, timeout=120, useCertificates=useCertificates) if not self.server: return else: self.valid = True except Exception, x: gLogger.exception( 'Exception while creating connection to LcgFileCatalog proxy server', '', x) return
def initialize(self): self.RequestDBClient = RequestClient() backend = self.am_getOption('Backend', '') self.RequestDB = False if backend == 'mysql': from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL requestDB = RequestDBMySQL() if requestDB._connected: self.RequestDB = requestDB gMonitor.registerActivity("Iteration", "Agent Loops", "DISETForwardingAgent", "Loops/min", gMonitor.OP_SUM) gMonitor.registerActivity("Attempted", "Request Processed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM) gMonitor.registerActivity("Successful", "Request Forward Successful", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM) gMonitor.registerActivity("Failed", "Request Forward Failed", "DISETForwardingAgent", "Requests/min", gMonitor.OP_SUM) self.local = PathFinder.getServiceURL("RequestManagement/localURL") if not self.local: self.local = AgentModule.am_getOption(self, 'localURL', '') if not self.local: errStr = 'The RequestManagement/localURL option must be defined.' gLogger.fatal(errStr) return S_ERROR(errStr) return S_OK()
def requestManager( self, timeout = 120 ): """ facade for RequestManager RPC client """ if not self.__requestManager: url = PathFinder.getServiceURL( "RequestManagement/ReqManager" ) if not url: raise RuntimeError( "CS option RequestManagement/ReqManager URL is not set!" ) self.__requestManager = RPCClient( url, timeout = timeout ) return self.__requestManager
def ftsManager( cls, timeout = 300 ): """ get FTSManager instance """ if not cls.__ftsManager: url = PathFinder.getServiceURL( "DataManagement/FTSManager" ) if not url: raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" ) cls.__ftsManager = RPCClient( url, timeout = timeout ) return cls.__ftsManager
def requestManager( self, timeout = 120 ): """ facade for RequestManager RPC client """ if not self.__requestManager: url = PathFinder.getServiceURL( "RequestManagement/RequestManager" ) if not url: raise RuntimeError( "CS option RequestManagement/RequestManager URL is not set!" ) self.__requestManager = RPCClient( url, timeout = timeout ) return self.__requestManager
def __init__( self): Client.__init__( self ) self.setServer( "DataManagement/Test" ) url = PathFinder.getServiceURL( "DataManagement/Test" ) if not url: raise RuntimeError( "CS option DataManagement/Test URL is not set!" ) self.testManager = RPCClient( url )
def execute(self): """ This agent is the smallest and (cutest) of all the DIRAC agents in existence. """ gMonitor.addMark("Iteration",1) central = PathFinder.getServiceURL("RequestManagement/centralURL") if central: self.central = central local = PathFinder.getServiceURL("RequestManagement/localURL") if local: self.local = local res = self.RequestDBClient.serveRequest(url=self.local) if not res['OK']: gLogger.error("ZuziaAgent.execute: Failed to get request from database.",self.local) return S_OK() elif not res['Value']: gLogger.info("ZuziaAgent.execute: No requests to be executed found.") return S_OK() gMonitor.addMark("Attempted",1) requestString = res['Value']['RequestString'] requestName = res['Value']['RequestName'] gLogger.info("ZuziaAgent.execute: Obtained request %s" % requestName) gLogger.info("ZuziaAgent.execute: Attempting to set request to %s." % self.central) res = self.RequestDBClient.setRequest(requestName,requestString,self.central) if res['OK']: gMonitor.addMark("Successful",1) gLogger.info("ZuziaAgent.execute: Successfully put request.") else: gMonitor.addMark("Failed",1) gLogger.error("ZuziaAgent.execute: Failed to set request to", self.central) gLogger.info("ZuziaAgent.execute: Attempting to set request to %s." % self.local) res = self.RequestDBClient.setRequest(requestName,requestString,self.local) if res['OK']: gLogger.info("ZuziaAgent.execute: Successfully put request.") else: gLogger.error("ZuziaAgent.execute: Failed to set request to", self.local) while not res['OK']: gLogger.info("ZuziaAgent.execute: Attempting to set request to anywhere.") res = self.RequestDBClient.setRequest(requestName,requestString) gLogger.info("ZuziaAgent.execute: Successfully put request to %s." % res["Server"]) return S_OK()
def __init__( self, storageName, parameters ): StorageBase.__init__( self, storageName, parameters ) self.pluginName = 'Proxy' self.isok = True self.url = PathFinder.getServiceURL( "DataManagement/StorageElementProxy" ) if not self.url: self.isok = False
def __init__( self, storageName, parameters ): StorageBase.__init__( self, storageName, parameters ) self.pluginName = 'Proxy' self.isok = True self.url = PathFinder.getServiceURL( "DataManagement/StorageElementProxy" ) if not self.url: self.isok = False
def __init__( self, useCertificates = False ): """ Constructor of the RequestClient class """ voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) ) self.voBoxUrls = [] if voBoxUrls: self.voBoxUrls = randomize( voBoxUrls ) local = PathFinder.getServiceURL( "RequestManagement/localURL" ) self.local = False if local: self.local = local central = PathFinder.getServiceURL( "RequestManagement/centralURL" ) self.central = False if central: self.central = central self.setServer( 'RequestManagement/centralURL' )
def requestProxies( self, timeout = 120 ): """ get request proxies dict """ if not self.__requestProxiesDict: self.__requestProxiesDict = {} proxiesURLs = fromChar( PathFinder.getServiceURL( "RequestManagement/ReqProxyURLs" ) ) if not proxiesURLs: self.log.warn( "CS option RequestManagement/ReqProxyURLs is not set!" ) for proxyURL in proxiesURLs: self.log.debug( "creating RequestProxy for url = %s" % proxyURL ) self.__requestProxiesDict[proxyURL] = RPCClient( proxyURL, timeout = timeout ) return self.__requestProxiesDict
def __init__(self, useCertificates=False): """ Constructor of the RequestClient class """ voBoxUrls = fromChar( PathFinder.getServiceURL("RequestManagement/voBoxURLs")) self.voBoxUrls = [] if voBoxUrls: self.voBoxUrls = randomize(voBoxUrls) local = PathFinder.getServiceURL("RequestManagement/localURL") self.local = False if local: self.local = local central = PathFinder.getServiceURL("RequestManagement/centralURL") self.central = False if central: self.central = central self.setServer('RequestManagement/centralURL')
def __init__(self,url=False,useCertificates=False): """ Constructor of the DataLogging client """ try: if not url: self.url = PathFinder.getServiceURL('DataManagement/DataLogging') else: self.url = url except Exception, x: errStr = "DataLoggingClient.__init__: Exception while obtaining service URL." gLogger.exception(errStr,lException=x)
def requestProxies(self, timeout=120): """ get request proxies dict """ if not self.__requestProxiesDict: self.__requestProxiesDict = {} proxiesURLs = fromChar(PathFinder.getServiceURL("RequestManagement/ReqProxyURLs")) if not proxiesURLs: self.log.warn("CS option RequestManagement/ReqProxyURLs is not set!") for proxyURL in proxiesURLs: self.log.debug("creating RequestProxy for url = %s" % proxyURL) self.__requestProxiesDict[proxyURL] = RPCClient(proxyURL, timeout=timeout) return self.__requestProxiesDict
def __init__(self, url=False, useCertificates=False): """ Constructor of the DataLogging client """ try: if not url: self.url = PathFinder.getServiceURL( 'DataManagement/DataLogging') else: self.url = url except Exception, x: errStr = "DataLoggingClient.__init__: Exception while obtaining service URL." gLogger.exception(errStr, lException=x)
def initialize(self): self.RequestDBClient = RequestClient() gMonitor.registerActivity("Iteration", "Agent Loops", "ZuziaAgent", "Loops/min", gMonitor.OP_SUM) gMonitor.registerActivity("Attempted", "Request Processed", "ZuziaRAgent", "Requests/min", gMonitor.OP_SUM) gMonitor.registerActivity("Successful", "Request Forward Successful", "ZuziaAgent", "Requests/min", gMonitor.OP_SUM) gMonitor.registerActivity("Failed", "Request Forward Failed", "ZuziaAgent", "Requests/min", gMonitor.OP_SUM) self.local = PathFinder.getServiceURL("RequestManagement/localURL") if not self.local: self.local = AgentModule.am_getOption(self,'localURL','') if not self.local: errStr = 'The RequestManagement/localURL option must be defined.' gLogger.fatal(errStr) return S_ERROR(errStr) self.central = PathFinder.getServiceURL("RequestManagement/centralURL") if not self.central: errStr = 'The RequestManagement/centralURL option must be defined.' gLogger.fatal(errStr) return S_ERROR(errStr) return S_OK()
def __init__( self, url = False ): """ Constructor of the Bookkeeping catalogue client """ self.splitSize = 1000 self.name = 'BookkeepingDB' self.valid = True try: if not url: self.url = PathFinder.getServiceURL( 'Bookkeeping/NewBookkeepingManager' ) else: self.url = url except Exception, exceptionMessage: gLogger.exception( 'BookkeepingDBClient.__init__: Exception while obtaining Bookkeeping service URL.', '', exceptionMessage ) self.valid = False
def __init__( self, storageName, protocol, path, host, port, spaceToken, wspath ): self.isok = True self.protocolName = 'Proxy' self.name = storageName self.protocol = protocol self.path = path self.host = host self.port = port self.wspath = wspath self.spaceToken = spaceToken self.cwd = self.path apply( StorageBase.__init__, ( self, self.name, self.path ) ) self.url = PathFinder.getServiceURL( "DataManagement/StorageElementProxy" ) if not self.url: self.isok = False
def __removeRequests( self, requestIDs ): """ This will remove requests from the (new) RMS system - #FIXME: if the old system is still installed, it won't remove anything!!! (we don't want to risk removing from the new RMS what is instead in the old) """ # FIXME: checking if the old system is still installed! from DIRAC.ConfigurationSystem.Client import PathFinder if PathFinder.getServiceURL( "RequestManagement/RequestManager" ): self.log.warn( "NOT removing requests!!" ) return S_OK() rIDs = [ int( long( j ) ) for j in requestIDs if long( j ) ] for requestName in rIDs: self.reqClient.deleteRequest( requestName ) return S_OK()
def __init__(self, url=False): """ Constructor of the Bookkeeping catalogue client """ self.splitSize = 1000 self.name = 'BookkeepingDB' self.valid = True try: if not url: self.url = PathFinder.getServiceURL( 'Bookkeeping/NewBookkeepingManager') else: self.url = url except Exception, exceptionMessage: gLogger.exception( 'BookkeepingDBClient.__init__: Exception while obtaining Bookkeeping service URL.', '', exceptionMessage) self.valid = False
def __removeRequests( self, requestIDs ): """ This will remove requests from the (new) RMS system - #FIXME: if the old system is still installed, it won't remove anything!!! (we don't want to risk removing from the new RMS what is instead in the old) """ # FIXME: checking if the old system is still installed! from DIRAC.ConfigurationSystem.Client import PathFinder if PathFinder.getServiceURL( "RequestManagement/RequestManager" ): self.log.warn( "NOT removing requests!!" ) return S_OK() rIDs = [ int( long( j ) ) for j in requestIDs if long( j ) ] for requestName in rIDs: self.reqClient.deleteRequest( requestName ) return S_OK()
def requestProxies(self, timeout=120): """get request proxies dict""" # Forward all the connection options to the requestClient # (e.g. the userDN to use) kwargs = self.getClientKWArgs() kwargs["timeout"] = timeout if not self.__requestProxiesDict: self.__requestProxiesDict = {} proxiesURLs = fromChar(PathFinder.getServiceURL("RequestManagement/ReqProxyURLs")) if not proxiesURLs: self.log.warn("CS option RequestManagement/ReqProxyURLs is not set!") for proxyURL in proxiesURLs: self.log.debug("creating RequestProxy for url = %s" % proxyURL) pc = Client(**kwargs) pc.setServer(proxyURL) self.__requestProxiesDict[proxyURL] = pc return self.__requestProxiesDict
def __init__( self, useCertificates = False ): """c'tor :param self: self reference :param bool useCertificates: flag to enable/disable certificates """ Client.__init__( self ) self.log = gLogger.getSubLogger( "DataManagement/FTSClient" ) self.setServer( "DataManagement/FTSManager" ) # getting other clients self.ftsValidator = FTSValidator() self.dataManager = DataManager() self.storageFactory = StorageFactory() url = PathFinder.getServiceURL( "DataManagement/FTSManager" ) if not url: raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" ) self.ftsManager = RPCClient( url )
def __init__( self, useCertificates = False ): """c'tor :param self: self reference :param bool useCertificates: flag to enable/disable certificates """ Client.__init__( self ) self.log = gLogger.getSubLogger( "DataManagement/FTSClient" ) self.setServer( "DataManagement/FTSManager" ) # getting other clients self.ftsValidator = FTSValidator() self.dataManager = DataManager() self.storageFactory = StorageFactory() url = PathFinder.getServiceURL( "DataManagement/FTSManager" ) if not url: raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" ) self.ftsManager = RPCClient( url )
def __init__(self, url = False, useCertificates = False): """ Constructor of the LCGFileCatalogProxy client class """ self.name = 'LFCProxy' self.valid = False try: if url: self.url = url else: url = PathFinder.getServiceURL('DataManagement/LcgFileCatalogProxy') if not url: return self.url = url self.server = RPCClient(self.url,timeout=120,useCertificates=useCertificates) if not self.server: return else: self.valid = True except Exception,x: gLogger.exception('Exception while creating connection to LcgFileCatalog proxy server','',x) return
def export_getUsedPorts( self ): """ Retrieve the ports in use by services on this host :return: Returns a dictionary containing, for each system, which port is being used by which service """ result = gComponentInstaller.getSetupComponents() if not result[ 'OK' ]: return result services = result[ 'Value' ][ 'Services' ] ports = {} for system in services: ports[ system ] = {} for service in services[ system ]: url = PathFinder.getServiceURL( '%s/%s' % ( system, service ) ) port = re.search( r':(\d{4,5})/', url ) if port: ports[ system ][ service ] = port.group( 1 ) else: ports[ system ][ service ] = 'None' return S_OK( ports )