def __init__(self, serviceData): """ Init the variables for the service :param serviceData: dict with modName, standalone, loadName, moduleObj, classObj. e.g.: {'modName': 'Framework/serviceName', 'standalone': True, 'loadName': 'Framework/serviceName', 'moduleObj': <module 'serviceNameHandler' from '/home/DIRAC/FrameworkSystem/Service/serviceNameHandler.pyo'>, 'classObj': <class 'serviceNameHandler.serviceHandler'>} """ self._svcData = serviceData self._name = serviceData["modName"] self._startTime = datetime.datetime.utcnow() self._validNames = [serviceData["modName"]] if serviceData["loadName"] not in self._validNames: self._validNames.append(serviceData["loadName"]) self._cfg = ServiceConfiguration(list(self._validNames)) self._standalone = serviceData["standalone"] self.__monitorLastStatsUpdate = time.time() self._stats = {"queries": 0, "connections": 0} self._authMgr = AuthManager( "%s/Authorization" % PathFinder.getServiceSection(serviceData["loadName"])) self._transportPool = getGlobalTransportPool() self.__cloneId = 0 self.__maxFD = 0 self.activityMonitoring = False # Check if monitoring is enabled if "Monitoring" in Operations().getMonitoringBackends( monitoringType="ServiceMonitoring"): self.activityMonitoring = True
def __init__(self, serviceData): """ Init the variables for the service :param serviceData: dict with modName, standalone, loadName, moduleObj, classObj. e.g.: {'modName': 'Framework/serviceName', 'standalone': True, 'loadName': 'Framework/serviceName', 'moduleObj': <module 'serviceNameHandler' from '/home/DIRAC/FrameworkSystem/Service/serviceNameHandler.pyo'>, 'classObj': <class 'serviceNameHandler.serviceHandler'>} Standalone is true if there is only one service started If it's false, every service is linked to a different MonitoringClient """ self._svcData = serviceData self._name = serviceData['modName'] self._startTime = Time.dateTime() self._validNames = [serviceData['modName']] if serviceData['loadName'] not in self._validNames: self._validNames.append(serviceData['loadName']) self._cfg = ServiceConfiguration(list(self._validNames)) if serviceData['standalone']: self._monitor = gMonitor else: self._monitor = MonitoringClient() self.__monitorLastStatsUpdate = time.time() self._stats = {'queries': 0, 'connections': 0} self._authMgr = AuthManager( "%s/Authorization" % PathFinder.getServiceSection(serviceData['loadName'])) self._transportPool = getGlobalTransportPool() self.__cloneId = 0 self.__maxFD = 0
def getGlobalMessageBroker(): global gMessageBroker from DIRAC.Core.DISET.private.TransportPool import getGlobalTransportPool if not gMessageBroker: gMessageBroker = MessageBroker("GlobalMessageBroker", transportPool=getGlobalTransportPool()) return gMessageBroker
def getGlobalMessageBroker(): global gMessageBroker from DIRAC.Core.DISET.private.TransportPool import getGlobalTransportPool if not gMessageBroker: gMessageBroker = MessageBroker('GlobalMessageBroker', transportPool=getGlobalTransportPool()) return gMessageBroker
def __init__( self, serviceName ): self._name = serviceName self._startTime = Time.dateTime() self._cfg = ServiceConfiguration( serviceName ) self._validNames = [ self._name ] self._monitor = MonitoringClient() self.__monitorLastStatsUpdate = time.time() self._stats = { 'queries' : 0, 'connections' : 0 } self._authMgr = AuthManager( "%s/Authorization" % self._cfg.getServicePath() ) self._transportPool = getGlobalTransportPool() self.__cloneId = 0
def _connect(self): self.__discoverExtraCredentials() if not self.__initStatus['OK']: return self.__initStatus if self.__enableThreadCheck: self.__checkThreadID() gLogger.debug("Connecting to: %s" % self.serviceURL) try: transport = gProtocolDict[self.__URLTuple[0]]['transport']( self.__URLTuple[1:3], **self.kwargs) #the socket timeout is the default value which is 1. #later we increase to 5 retVal = transport.initAsClient() if not retVal['OK']: if self.__retry < self.__nbOfRetry * self.__nbOfUrls - 1: url = "%s://%s:%d/%s" % ( self.__URLTuple[0], self.__URLTuple[1], int(self.__URLTuple[2]), self.__URLTuple[3]) if url not in self.__bannedUrls: self.__bannedUrls += [url] if len(self.__bannedUrls) < self.__nbOfUrls: gLogger.notice( "Non-responding URL temporarily banned", "%s" % url) self.__retry += 1 if self.__retryCounter == self.__nbOfRetry - 1: transport.setSocketTimeout( 5 ) # we increase the socket timeout in case the network is not good gLogger.info("Retry connection: ", "%d" % self.__retry) if len(self.__bannedUrls) == self.__nbOfUrls: self.__retryCounter += 1 self.__retryDelay = 3. / self.__nbOfUrls if self.__nbOfUrls > 1 else 2 # we run only one service! In that case we increase the retry delay. gLogger.info( "Waiting %f second before retry all service(s)" % self.__retryDelay) time.sleep(self.__retryDelay) self.__discoverURL() return self._connect() else: return retVal except Exception as e: return S_ERROR("Can't connect to %s: %s" % (self.serviceURL, repr(e))) trid = getGlobalTransportPool().add(transport) return S_OK((trid, transport))
def __init__( self, serviceData ): self._svcData = serviceData self._name = serviceData[ 'loadName' ] self._startTime = Time.dateTime() self._validNames = [ serviceData[ 'modName' ] ] if serviceData[ 'loadName' ] not in self._validNames: self._validNames.append( serviceData[ 'loadName' ] ) self._cfg = ServiceConfiguration( list( self._validNames ) ) if serviceData[ 'standalone' ]: self._monitor = gMonitor else: self._monitor = MonitoringClient() self.__monitorLastStatsUpdate = time.time() self._stats = { 'queries' : 0, 'connections' : 0 } self._authMgr = AuthManager( "%s/Authorization" % PathFinder.getServiceSection( serviceData[ 'loadName' ] ) ) self._transportPool = getGlobalTransportPool() self.__cloneId = 0 self.__maxFD = 0
def __init__( self, serviceData ): self._svcData = serviceData self._name = serviceData[ 'modName' ] self._startTime = Time.dateTime() self._validNames = [ serviceData[ 'modName' ] ] if serviceData[ 'loadName' ] not in self._validNames: self._validNames.append( serviceData[ 'loadName' ] ) self._cfg = ServiceConfiguration( list( self._validNames ) ) if serviceData[ 'standalone' ]: self._monitor = gMonitor else: self._monitor = MonitoringClient() self.__monitorLastStatsUpdate = time.time() self._stats = { 'queries' : 0, 'connections' : 0 } self._authMgr = AuthManager( "%s/Authorization" % PathFinder.getServiceSection( serviceData[ 'loadName' ] ) ) self._transportPool = getGlobalTransportPool() self.__cloneId = 0 self.__maxFD = 0
def __init__(self, name, transportPool=None, threadPool=None): self.__name = name self.__messageTransports = {} self.__msgCounter = 0 self.__msgCounterLock = threading.Lock() self.__responseCallbacks = {} self.__msgInTransport = {} self.__listenPersistConn = False self.__useMessageObjects = True self.__callbacksLock = threading.Condition() self.__trInOutLock = threading.Lock() self.__msgFactory = MessageFactory() self.__log = gLogger.getSubLogger("MSGBRK") if not transportPool: transportPool = getGlobalTransportPool() self.__trPool = transportPool if not threadPool: threadPool = getGlobalThreadPool() self.__threadPool = threadPool self.__listeningForMessages = False
def __init__(self, name, transportPool=None, threadPool=None): self.__name = name self.__messageTransports = {} self.__msgCounter = 0 self.__msgCounterLock = threading.Lock() self.__responseCallbacks = {} self.__msgInTransport = {} self.__listenPersistConn = False self.__useMessageObjects = True self.__callbacksLock = threading.Condition() self.__trInOutLock = threading.Lock() self.__msgFactory = MessageFactory() self.__log = gLogger.getSubLogger("MSGBRK") if not transportPool: transportPool = getGlobalTransportPool() self.__trPool = transportPool if not threadPool: threadPool = getGlobalThreadPool() self.__threadPool = threadPool self.__listeningForMessages = False
def _connect( self ): self.__discoverExtraCredentials() if not self.__initStatus[ 'OK' ]: return self.__initStatus if self.__enableThreadCheck: self.__checkThreadID() gLogger.debug( "Connecting to: %s" % self.serviceURL ) try: transport = gProtocolDict[ self.__URLTuple[0] ][ 'transport' ]( self.__URLTuple[1:3], **self.kwargs ) #the socket timeout is the default value which is 1. #later we increase to 5 retVal = transport.initAsClient() if not retVal[ 'OK' ]: if self.__retry < self.__nbOfRetry * self.__nbOfUrls - 1: url = "%s://%s:%d/%s" % ( self.__URLTuple[0], self.__URLTuple[1], int( self.__URLTuple[2] ), self.__URLTuple[3] ) if url not in self.__bannedUrls: self.__bannedUrls += [url] if len( self.__bannedUrls ) < self.__nbOfUrls: gLogger.notice( "Non-responding URL temporarily banned", "%s" % url ) self.__retry += 1 if self.__retryCounter == self.__nbOfRetry - 1: transport.setSocketTimeout( 5 ) # we increase the socket timeout in case the network is not good gLogger.info( "Retry connection: ", "%d" % self.__retry ) if len(self.__bannedUrls) == self.__nbOfUrls: self.__retryCounter += 1 self.__retryDelay = 3. / self.__nbOfUrls if self.__nbOfUrls > 1 else 2 # we run only one service! In that case we increase the retry delay. gLogger.info( "Waiting %f second before retry all service(s)" % self.__retryDelay ) time.sleep( self.__retryDelay ) self.__discoverURL() return self._connect() else: return retVal except Exception as e: return S_ERROR( "Can't connect to %s: %s" % ( self.serviceURL, repr( e ) ) ) trid = getGlobalTransportPool().add( transport ) return S_OK( ( trid, transport ) )
def getGlobalMessageBroker(): global gMessageBroker if not gMessageBroker: gMessageBroker = MessageBroker("GlobalMessageBroker", transportPool=getGlobalTransportPool()) return gMessageBroker
def _connect(self): """ Establish the connection. It uses the URL discovered in __discoverURL. In case the connection cannot be established, __discoverURL is called again, and _connect calls itself. We stop after trying self.__nbOfRetry * self.__nbOfUrls :return: S_OK()/S_ERROR() """ # Check if the useServerCertificate configuration changed # Note: I am not really sure that all this block makes # any sense at all since all these variables are # evaluated in __discoverCredentialsToUse if gConfig.useServerCertificate() != self.__useCertificates: if self.__forceUseCertificates is None: self.__useCertificates = gConfig.useServerCertificate() self.kwargs[self.KW_USE_CERTIFICATES] = self.__useCertificates # The server certificate use context changed, rechecking the transport sanity result = self.__checkTransportSanity() if not result['OK']: return result # Take all the extra credentials self.__discoverExtraCredentials() if not self.__initStatus['OK']: return self.__initStatus if self.__enableThreadCheck: self.__checkThreadID() gLogger.debug("Trying to connect to: %s" % self.serviceURL) try: # Calls the transport method of the apropriate protocol. # self.__URLTuple[1:3] = [server name, port, System/Component] transport = gProtocolDict[self.__URLTuple[0]]['transport'](self.__URLTuple[1:3], **self.kwargs) # the socket timeout is the default value which is 1. # later we increase to 5 retVal = transport.initAsClient() # We try at most __nbOfRetry each URLs if not retVal['OK']: gLogger.warn("Issue getting socket:", "%s : %s : %s" % (transport, self.__URLTuple, retVal['Message'])) # We try at most __nbOfRetry each URLs if self.__retry < self.__nbOfRetry * self.__nbOfUrls - 1: # Recompose the URL (why not using self.serviceURL ? ) url = "%s://%s:%d/%s" % (self.__URLTuple[0], self.__URLTuple[1], int(self.__URLTuple[2]), self.__URLTuple[3]) # Add the url to the list of banned URLs if it is not already there. (Can it happen ? I don't think so) if url not in self.__bannedUrls: gLogger.warn("Non-responding URL temporarily banned", "%s" % url) self.__bannedUrls += [url] # Increment the retry counter self.__retry += 1 # 16.07.20 CHRIS: I guess this setSocketTimeout does not behave as expected. # If the initasClient did not work, we anyway re-enter the whole method, # so a new transport object is created. # However, it migh be that this timeout value was propagated down to the # SocketInfoFactory singleton, and thus used, but that means that the timeout # specified in parameter was then void. # If it is our last attempt for each URL, we increase the timeout if self.__retryCounter == self.__nbOfRetry - 1: transport.setSocketTimeout(5) # we increase the socket timeout in case the network is not good gLogger.info("Retry connection", ": %d to %s" % (self.__retry, self.serviceURL)) # If we tried all the URL, we increase the global counter (__retryCounter), and sleep if len(self.__bannedUrls) == self.__nbOfUrls: self.__retryCounter += 1 # we run only one service! In that case we increase the retry delay. self.__retryDelay = 3. / self.__nbOfUrls if self.__nbOfUrls > 1 else 2 gLogger.info("Waiting %f seconds before retry all service(s)" % self.__retryDelay) time.sleep(self.__retryDelay) # rediscover the URL self.__discoverURL() # try to reconnect return self._connect() else: return retVal except Exception as e: gLogger.exception(lException=True, lExcInfo=True) return S_ERROR("Can't connect to %s: %s" % (self.serviceURL, repr(e))) # We add the connection to the transport pool gLogger.debug("Connected to: %s" % self.serviceURL) trid = getGlobalTransportPool().add(transport) return S_OK((trid, transport))
def _disconnect(self, trid): """ Disconnect the connection. :param str trid: Transport ID in the transportPool """ getGlobalTransportPool().close(trid)
def _connect(self): """ Establish the connection. It uses the URL discovered in __discoverURL. In case the connection cannot be established, __discoverURL is called again, and _connect calls itself. We stop after trying self.__nbOfRetry * self.__nbOfUrls """ # Check if the useServerCertificate configuration changed # Note: I am not really sure that all this block makes # any sense at all since all these variables are # evaluated in __discoverCredentialsToUse if gConfig.useServerCertificate() != self.__useCertificates: if self.__forceUseCertificates is None: self.__useCertificates = gConfig.useServerCertificate() self.kwargs[self.KW_USE_CERTIFICATES] = self.__useCertificates # The server certificate use context changed, rechecking the transport sanity result = self.__checkTransportSanity() if not result['OK']: return result # Take all the extra credentials self.__discoverExtraCredentials() if not self.__initStatus['OK']: return self.__initStatus if self.__enableThreadCheck: self.__checkThreadID() gLogger.debug("Connecting to: %s" % self.serviceURL) try: # Calls the transport method of the apropriate protocol. # self.__URLTuple[1:3] = [server name, port, System/Component] transport = gProtocolDict[self.__URLTuple[0]]['transport'](self.__URLTuple[1:3], **self.kwargs) # the socket timeout is the default value which is 1. # later we increase to 5 retVal = transport.initAsClient() # If we have an issue connecting if not retVal['OK']: # We try at most __nbOfRetry each URLs if self.__retry < self.__nbOfRetry * self.__nbOfUrls - 1: # Recompose the URL (why not using self.serviceURL ? ) url = "%s://%s:%d/%s" % (self.__URLTuple[0], self.__URLTuple[1], int(self.__URLTuple[2]), self.__URLTuple[3]) # Add the url to the list of banned URLs if it is not already there. (Can it happen ? I don't think so) if url not in self.__bannedUrls: self.__bannedUrls += [url] # Why only printing in this case ? if len(self.__bannedUrls) < self.__nbOfUrls: gLogger.notice("Non-responding URL temporarily banned", "%s" % url) # Increment the retry couunter self.__retry += 1 # If it is our last attempt for each URL, we increase the timeout if self.__retryCounter == self.__nbOfRetry - 1: transport.setSocketTimeout(5) # we increase the socket timeout in case the network is not good gLogger.info("Retry connection", ": %d to %s" % (self.__retry, self.serviceURL)) # If we tried all the URL, we increase the global counter (__retryCounter), and sleep if len(self.__bannedUrls) == self.__nbOfUrls: self.__retryCounter += 1 # we run only one service! In that case we increase the retry delay. self.__retryDelay = 3. / self.__nbOfUrls if self.__nbOfUrls > 1 else 2 gLogger.info("Waiting %f seconds before retry all service(s)" % self.__retryDelay) time.sleep(self.__retryDelay) # rediscover the URL self.__discoverURL() # try to reconnect return self._connect() else: return retVal except Exception as e: gLogger.exception(lException=True, lExcInfo=True) return S_ERROR("Can't connect to %s: %s" % (self.serviceURL, repr(e))) # We add the connection to the transport pool trid = getGlobalTransportPool().add(transport) return S_OK((trid, transport))
def _disconnect(self, trid): """ Disconnect the connection. :param trid: Transport ID in the transportPool """ getGlobalTransportPool().close(trid)
def _disconnect( self ): if self.__trid: getGlobalTransportPool().close( self.__trid )
def _disconnect(self): if self.__trid: getGlobalTransportPool().close(self.__trid)
if url not in self.__bannedUrls: gLogger.notice( "URL banned", "%s" % url ) self.__bannedUrls += [url] self.__retry += 1 gLogger.info( "Retry connection: ", "%d" % self.__retry ) if (len(self.__bannedUrls) == self.__nbOfUrls): self.__retryDelay = 3. / self.__nbOfUrls if self.__nbOfUrls > 1 else 5 # we run only one service! In that case we increase the retry delay. gLogger.info( "Waiting %f second before retry all service(s)" % self.__retryDelay ) time.sleep( self.__retryDelay ) self.__discoverURL() return self._connect() else: return S_ERROR( "Can't connect to %s: %s" % ( self.serviceURL, retVal ) ) except Exception, e: return S_ERROR( "Can't connect to %s: %s" % ( self.serviceURL, e ) ) trid = getGlobalTransportPool().add( transport ) return S_OK( ( trid, transport ) ) def _disconnect( self, trid ): getGlobalTransportPool().close( trid ) def _proposeAction( self, transport, action ): if not self.__initStatus[ 'OK' ]: return self.__initStatus stConnectionInfo = ( ( self.__URLTuple[3], self.setup, self.vo ), action, self.__extraCredentials ) retVal = transport.sendData( S_OK( stConnectionInfo ) ) if not retVal[ 'OK' ]: return retVal serverReturn = transport.receiveData()
def _disconnect( self, trid ): getGlobalTransportPool().close( trid )
def getGlobalMessageBroker(): global gMessageBroker if not gMessageBroker: gMessageBroker = MessageBroker( 'GlobalMessageBroker', transportPool = getGlobalTransportPool() ) return gMessageBroker