def __discoverURL(self): """ Calculate the final URL. It is called at initialization and in connect in case of issue It sets: * self.serviceURL: the url (dips) selected as target using __findServiceURL * self.__URLTuple: a split of serviceURL obtained by Network.splitURL * self._serviceName: the last part of URLTuple (typically System/Component) """ # Calculate final URL try: result = self.__findServiceURL() except Exception as e: return S_ERROR(repr(e)) if not result['OK']: return result self.serviceURL = result['Value'] retVal = Network.splitURL(self.serviceURL) if not retVal['OK']: return retVal self.__URLTuple = retVal['Value'] self._serviceName = self.__URLTuple[-1] res = gConfig.getOptionsDict("/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3]) if res['OK']: opts = res['Value'] for k in opts: if k not in self.kwargs: self.kwargs[k] = opts[k] return S_OK()
def __discoverURL(self): """Calculate the final URL. It is called at initialization and in connect in case of issue It sets: * self.serviceURL: the url (dips) selected as target using __findServiceURL * self.__URLTuple: a split of serviceURL obtained by Network.splitURL * self._serviceName: the last part of URLTuple (typically System/Component) WARNING: COPY PASTE FROM BaseClient """ # Calculate final URL try: result = self.__findServiceURL() except Exception as e: return S_ERROR(repr(e)) if not result["OK"]: return result self.serviceURL = result["Value"] retVal = Network.splitURL(self.serviceURL) if not retVal["OK"]: return retVal self.__URLTuple = retVal["Value"] self._serviceName = self.__URLTuple[-1] res = gConfig.getOptionsDict("/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3]) if res["OK"]: opts = res["Value"] for k in opts: if k not in self.kwargs: self.kwargs[k] = opts[k] return S_OK()
def __selectUrl(self, notselect, urls): """In case when multiple services are running in the same host, a new url has to be in a different host Note: If we do not have different host we will use the selected url... """ url = None for i in urls: retVal = Network.splitURL(i) if retVal['OK']: if retVal['Value'][1] != notselect[1]: # the hots are different url = i break else: gLogger.error(retVal['Message']) return url
def __selectUrl( self, notselect, urls ): """In case when multiple services are running in the same host, a new url has to be in a different host Note: If we do not have different host we will use the selected url... """ url = None for i in urls: retVal = Network.splitURL( i ) if retVal['OK']: if retVal['Value'][1] != notselect[1]: # the hots are different url = i break else: gLogger.error( retVal['Message'] ) return url
def __selectUrl(self, notselect, urls): """In case when multiple services are running in the same host, a new url has to be in a different host Note: If we do not have different host we will use the selected url... :param notselect: URL that should NOT be selected :param list urls: list of potential URLs :return: str -- selected URL """ url = None for i in urls: retVal = Network.splitURL(i) if retVal["OK"]: if retVal["Value"][1] != notselect[ 1]: # the hosts are different url = i break else: gLogger.error(retVal["Message"]) return url
def __discoverURL( self ): #Calculate final URL try: result = self.__findServiceURL() except Exception as e: return S_ERROR( repr( e ) ) if not result[ 'OK' ]: return result self.serviceURL = result[ 'Value' ] retVal = Network.splitURL( self.serviceURL ) if not retVal[ 'OK' ]: return retVal self.__URLTuple = retVal[ 'Value' ] self._serviceName = self.__URLTuple[-1] res = gConfig.getOptionsDict( "/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3] ) if res[ 'OK' ]: opts = res[ 'Value' ] for k in opts: if k not in self.kwargs: self.kwargs[k] = opts[k] return S_OK()
def __selectUrl(self, notselect, urls): """In case when multiple services are running in the same host, a new url has to be in a different host Note: If we do not have different host we will use the selected url... :param notselect: URL that should NOT be selected :param urls: list of potential URLs :return: selected URL WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient """ url = None for i in urls: retVal = Network.splitURL(i) if retVal['OK']: if retVal['Value'][1] != notselect[1]: # the hots are different url = i break else: gLogger.error(retVal['Message']) return url
def __discoverURL(self): #Calculate final URL try: result = self.__findServiceURL() except Exception as e: return S_ERROR(repr(e)) if not result['OK']: return result self.serviceURL = result['Value'] retVal = Network.splitURL(self.serviceURL) if not retVal['OK']: return retVal self.__URLTuple = retVal['Value'] self._serviceName = self.__URLTuple[-1] res = gConfig.getOptionsDict("/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3]) if res['OK']: opts = res['Value'] for k in opts: if k not in self.kwargs: self.kwargs[k] = opts[k] return S_OK()
class BaseClient: VAL_EXTRA_CREDENTIALS_HOST = "hosts" KW_USE_CERTIFICATES = "useCertificates" KW_EXTRA_CREDENTIALS = "extraCredentials" KW_TIMEOUT = "timeout" KW_SETUP = "setup" KW_VO = "VO" KW_DELEGATED_DN = "delegatedDN" KW_DELEGATED_GROUP = "delegatedGroup" KW_IGNORE_GATEWAYS = "ignoreGateways" KW_PROXY_LOCATION = "proxyLocation" KW_PROXY_STRING = "proxyString" KW_PROXY_CHAIN = "proxyChain" KW_SKIP_CA_CHECK = "skipCACheck" KW_KEEP_ALIVE_LAPSE = "keepAliveLapse" __threadConfig = ThreadConfig() def __init__( self, serviceName, **kwargs ): if type( serviceName ) not in types.StringTypes: raise TypeError( "Service name expected to be a string. Received %s type %s" % ( str( serviceName ), type( serviceName ) ) ) self._destinationSrv = serviceName self._serviceName = serviceName self.kwargs = kwargs self.__initStatus = S_OK() self.__idDict = {} self.__extraCredentials = "" self.__enableThreadCheck = False self.__retry = 0 self.__retryDelay = 0 self.__nbOfUrls = 1 #by default we always have 1 url for example: RPCClient('dips://volhcb38.cern.ch:9162/Framework/SystemAdministrator') self.__nbOfRetry = 3 # by default we try try times self.__bannedUrls = [] for initFunc in ( self.__discoverSetup, self.__discoverVO, self.__discoverTimeout, self.__discoverURL, self.__discoverCredentialsToUse, self.__checkTransportSanity, self.__setKeepAliveLapse ): result = initFunc() if not result[ 'OK' ] and self.__initStatus[ 'OK' ]: self.__initStatus = result self._initialize() #HACK for thread-safety: self.__allowedThreadID = False def _initialize( self ): pass def getDestinationService( self ): return self._destinationSrv def getServiceName( self ): return self._serviceName def __discoverSetup( self ): #Which setup to use? if self.KW_SETUP in self.kwargs and self.kwargs[ self.KW_SETUP ]: self.setup = str( self.kwargs[ self.KW_SETUP ] ) else: self.setup = self.__threadConfig.getSetup() if not self.setup: self.setup = gConfig.getValue( "/DIRAC/Setup", "Test" ) return S_OK() def __discoverVO( self ): #Which setup to use? if self.KW_VO in self.kwargs and self.kwargs[ self.KW_VO ]: self.vo = str( self.kwargs[ self.KW_VO ] ) else: self.vo = gConfig.getValue( "/DIRAC/VirtualOrganization", "unknown" ) return S_OK() def __discoverURL( self ): #Calculate final URL try: result = self.__findServiceURL() except Exception, e: return S_ERROR( str( e ) ) if not result[ 'OK' ]: return result self.serviceURL = result[ 'Value' ] retVal = Network.splitURL( self.serviceURL ) if not retVal[ 'OK' ]: return S_ERROR( "URL is malformed: %s" % retVal[ 'Message' ] ) self.__URLTuple = retVal[ 'Value' ] self._serviceName = self.__URLTuple[-1] res = gConfig.getOptionsDict( "/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3] ) if res[ 'OK' ]: opts = res[ 'Value' ] for k in opts: if k not in self.kwargs: self.kwargs[k] = opts[k] return S_OK()
def __findServiceURL(self): """ Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs If the site on which we run is configured to use gateways (/DIRAC/Gateways/<siteName>), these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS to False in kwargs. If self._destinationSrv (given as constructor attribute) is a properly formed URL, we just return this one. If we have to use a gateway, we just replace the server name in the url. The list of URLs defined in the CS (<System>/URLs/<Component>) is randomized This method also sets some attributes: * self.__nbOfUrls = number of URLs * self.__nbOfRetry removed in HTTPS (Managed by requests) * self.__bannedUrls is reinitialized if all the URLs are banned :return: the selected URL WARNING (Mostly) COPY PASTE FROM BaseClient (protocols list is changed to https) """ if not self.__initStatus["OK"]: return self.__initStatus # Load the Gateways URLs for the current site Name gatewayURL = False if not self.kwargs.get(self.KW_IGNORE_GATEWAYS): gatewayURLs = getGatewayURLs() if gatewayURLs: gatewayURL = "/".join(gatewayURLs[0].split("/")[:3]) # If what was given as constructor attribute is a properly formed URL, # we just return this one. # If we have to use a gateway, we just replace the server name in it if self._destinationSrv.startswith("https://"): gLogger.debug("Already given a valid url", self._destinationSrv) if not gatewayURL: return S_OK(self._destinationSrv) gLogger.debug("Reconstructing given URL to pass through gateway") path = "/".join(self._destinationSrv.split("/")[3:]) finalURL = "%s/%s" % (gatewayURL, path) gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL)) return S_OK(finalURL) if gatewayURL: gLogger.debug("Using gateway", gatewayURL) return S_OK("%s/%s" % (gatewayURL, self._destinationSrv)) # If nor url is given as constructor, we extract the list of URLs from the CS (System/URLs/Component) try: # We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component) urlsList = getServiceURLs(self._destinationSrv, setup=self.setup, failover=True) except Exception as e: return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e))) if not urlsList: return S_ERROR("URL for service %s not found" % self._destinationSrv) self.__nbOfUrls = len(urlsList) # __nbOfRetry removed in HTTPS (managed by requests) if self.__nbOfUrls == len(self.__bannedUrls): self.__bannedUrls = [] # retry all urls gLogger.debug("Retrying again all URLs") if self.__bannedUrls and len(urlsList) > 1: # we have host which is not accessible. We remove that host from the list. # We only remove if we have more than one instance for i in self.__bannedUrls: gLogger.debug("Removing banned URL", "%s" % i) urlsList.remove(i) sURL = urlsList[0] # If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL # is not on a host which is banned. If it is, we take the next one in the list using __selectUrl if self.__bannedUrls and self.__nbOfUrls > 2: # when we have multiple services then we can # have a situation when two services are running on the same machine with different ports... retVal = Network.splitURL(sURL) nexturl = None if retVal["OK"]: nexturl = retVal["Value"] found = False for i in self.__bannedUrls: retVal = Network.splitURL(i) if retVal["OK"]: bannedurl = retVal["Value"] else: break # We found a banned URL on the same host as the one we are running on if nexturl[1] == bannedurl[1]: found = True break if found: nexturl = self.__selectUrl(nexturl, urlsList[1:]) if nexturl: # an url found which is in different host sURL = nexturl gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL)) return S_OK(sURL)
def __findServiceURL(self): """ Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs If the site on which we run is configured to use gateways (/DIRAC/Gateways/<siteName>), these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS to False in kwargs. If self._destinationSrv (given as constructor attribute) is a properly formed URL, we just return this one. If we have to use a gateway, we just replace the server name in the url. The list of URLs defined in the CS (<System>/URLs/<Component>) is randomized This method also sets some attributes: * self.__nbOfUrls = number of URLs * self.__nbOfRetry = 2 if we have more than 2 urls, otherwise 3 * self.__bannedUrls is reinitialized if all the URLs are banned :return: S_OK(str)/S_ERROR() -- the selected URL """ if not self.__initStatus['OK']: return self.__initStatus # Load the Gateways URLs for the current site Name gatewayURL = False if not self.kwargs.get(self.KW_IGNORE_GATEWAYS): dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" % DIRAC.siteName()) if dRetVal['OK']: rawGatewayURL = List.randomize(List.fromChar(dRetVal['Value'], ","))[0] gatewayURL = "/".join(rawGatewayURL.split("/")[:3]) # If what was given as constructor attribute is a properly formed URL, # we just return this one. # If we have to use a gateway, we just replace the server name in it for protocol in gProtocolDict: if self._destinationSrv.find("%s://" % protocol) == 0: gLogger.debug("Already given a valid url", self._destinationSrv) if not gatewayURL: return S_OK(self._destinationSrv) gLogger.debug("Reconstructing given URL to pass through gateway") path = "/".join(self._destinationSrv.split("/")[3:]) finalURL = "%s/%s" % (gatewayURL, path) gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL)) return S_OK(finalURL) if gatewayURL: gLogger.debug("Using gateway", gatewayURL) return S_OK("%s/%s" % (gatewayURL, self._destinationSrv)) # We extract the list of URLs from the CS (System/URLs/Component) try: urls = getServiceURL(self._destinationSrv, setup=self.setup) except Exception as e: return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e))) if not urls: return S_ERROR("URL for service %s not found" % self._destinationSrv) failoverUrls = [] # Try if there are some failover URLs to use as last resort try: failoverUrlsStr = getServiceFailoverURL(self._destinationSrv, setup=self.setup) if failoverUrlsStr: failoverUrls = failoverUrlsStr.split(',') except Exception as e: pass # We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component) urlsList = List.randomize(List.fromChar(urls, ",")) + failoverUrls self.__nbOfUrls = len(urlsList) self.__nbOfRetry = 2 if self.__nbOfUrls > 2 else 3 # we retry 2 times all services, if we run more than 2 services if self.__nbOfUrls == len(self.__bannedUrls): self.__bannedUrls = [] # retry all urls gLogger.debug("Retrying again all URLs") if len(self.__bannedUrls) > 0 and len(urlsList) > 1: # we have host which is not accessible. We remove that host from the list. # We only remove if we have more than one instance for i in self.__bannedUrls: gLogger.debug("Removing banned URL", "%s" % i) urlsList.remove(i) # Take the first URL from the list # randUrls = List.randomize( urlsList ) + failoverUrls sURL = urlsList[0] # If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL # is not on a host which is banned. If it is, we take the next one in the list using __selectUrl # If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL # is not on a host which is banned. If it is, we take the next one in the list using __selectUrl if len(self.__bannedUrls) > 0 and self.__nbOfUrls > 2: # when we have multiple services then we can # have a situation when two services are running on the same machine with different ports... retVal = Network.splitURL(sURL) nexturl = None if retVal['OK']: nexturl = retVal['Value'] found = False for i in self.__bannedUrls: retVal = Network.splitURL(i) if retVal['OK']: bannedurl = retVal['Value'] else: break # We found a banned URL on the same host as the one we are running on if nexturl[1] == bannedurl[1]: found = True break if found: nexturl = self.__selectUrl(nexturl, urlsList[1:]) if nexturl: # an url found which is in different host sURL = nexturl gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL)) return S_OK(sURL)
def __findServiceURL(self): if not self.__initStatus['OK']: return self.__initStatus gatewayURL = False if self.KW_IGNORE_GATEWAYS not in self.kwargs or not self.kwargs[ self.KW_IGNORE_GATEWAYS]: dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" % DIRAC.siteName()) if dRetVal['OK']: rawGatewayURL = List.randomize( List.fromChar(dRetVal['Value'], ","))[0] gatewayURL = "/".join(rawGatewayURL.split("/")[:3]) for protocol in gProtocolDict.keys(): if self._destinationSrv.find("%s://" % protocol) == 0: gLogger.debug("Already given a valid url", self._destinationSrv) if not gatewayURL: return S_OK(self._destinationSrv) gLogger.debug( "Reconstructing given URL to pass through gateway") path = "/".join(self._destinationSrv.split("/")[3:]) finalURL = "%s/%s" % (gatewayURL, path) gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL)) return S_OK(finalURL) if gatewayURL: gLogger.debug("Using gateway", gatewayURL) return S_OK("%s/%s" % (gatewayURL, self._destinationSrv)) try: urls = getServiceURL(self._destinationSrv, setup=self.setup) except Exception as e: return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e))) if not urls: return S_ERROR("URL for service %s not found" % self._destinationSrv) urlsList = List.fromChar(urls, ",") self.__nbOfUrls = len(urlsList) self.__nbOfRetry = 2 if self.__nbOfUrls > 2 else 3 # we retry 2 times all services, if we run more than 2 services if len(urlsList) == len(self.__bannedUrls): self.__bannedUrls = [] # retry all urls gLogger.debug("Retrying again all URLs") if len(self.__bannedUrls) > 0 and len(urlsList) > 1: # we have host which is not accessible. We remove that host from the list. # We only remove if we have more than one instance for i in self.__bannedUrls: gLogger.debug("Removing banned URL", "%s" % i) urlsList.remove(i) randUrls = List.randomize(urlsList) sURL = randUrls[0] if len( self.__bannedUrls ) > 0 and self.__nbOfUrls > 2: # when we have multiple services then we can have a situation # when two service are running on the same machine with different port... retVal = Network.splitURL(sURL) nexturl = None if retVal['OK']: nexturl = retVal['Value'] found = False for i in self.__bannedUrls: retVal = Network.splitURL(i) if retVal['OK']: bannedurl = retVal['Value'] else: break if nexturl[1] == bannedurl[1]: found = True break if found: nexturl = self.__selectUrl(nexturl, randUrls[1:]) if nexturl: # an url found which is in different host sURL = nexturl gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL)) return S_OK(sURL)
class BaseClient: VAL_EXTRA_CREDENTIALS_HOST = "hosts" KW_USE_CERTIFICATES = "useCertificates" KW_EXTRA_CREDENTIALS = "extraCredentials" KW_TIMEOUT = "timeout" KW_SETUP = "setup" KW_VO = "VO" KW_DELEGATED_DN = "delegatedDN" KW_DELEGATED_GROUP = "delegatedGroup" KW_IGNORE_GATEWAYS = "ignoreGateways" KW_PROXY_LOCATION = "proxyLocation" KW_PROXY_STRING = "proxyString" KW_PROXY_CHAIN = "proxyChain" KW_SKIP_CA_CHECK = "skipCACheck" KW_KEEP_ALIVE_LAPSE = "keepAliveLapse" def __init__(self, serviceName, **kwargs): if type(serviceName) != types.StringType: raise TypeError( "Service name expected to be a string. Received %s type %s" % (str(serviceName), type(serviceName))) self._destinationSrv = serviceName self.kwargs = kwargs self.__initStatus = S_OK() self.__idDict = {} self.__enableThreadCheck = False for initFunc in (self.__discoverSetup, self.__discoverVO, self.__discoverTimeout, self.__discoverURL, self.__discoverCredentialsToUse, self.__discoverExtraCredentials, self.__checkTransportSanity, self.__setKeepAliveLapse): result = initFunc() if not result['OK'] and self.__initStatus['OK']: self.__initStatus = result self._initialize() #HACK for thread-safety: self.__allowedThreadID = False def _initialize(self): pass def getDestinationService(self): return self._destinationSrv def __discoverSetup(self): #Which setup to use? if self.KW_SETUP in self.kwargs and self.kwargs[self.KW_SETUP]: self.setup = str(self.kwargs[self.KW_SETUP]) else: self.setup = gConfig.getValue("/DIRAC/Setup", "Test") return S_OK() def __discoverVO(self): #Which setup to use? if self.KW_VO in self.kwargs and self.kwargs[self.KW_VO]: self.vo = str(self.kwargs[self.KW_VO]) else: self.vo = gConfig.getValue("/DIRAC/VirtualOrganization", "unknown") return S_OK() def __discoverURL(self): #Calculate final URL try: result = self.__findServiceURL() except Exception, e: return S_ERROR(str(e)) if not result['OK']: return result self.serviceURL = result['Value'] retVal = Network.splitURL(self.serviceURL) if not retVal['OK']: return S_ERROR("URL is malformed: %s" % retVal['Message']) self.__URLTuple = retVal['Value'] self._serviceName = self.__URLTuple[-1] return S_OK()
def __findServiceURL(self): """ Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs If the site on which we run is configured to use gateways (/DIRAC/Gateways/<siteName>), these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS to False in kwargs. If self._destinationSrv (given as constructor attribute) is a properly formed URL, we just return this one. If we have to use a gateway, we just replace the server name in the url. The list of URLs defined in the CS (<System>/URLs/<Component>) is randomized This method also sets some attributes: * self.__nbOfUrls = number of URLs * self.__nbOfRetry = 2 if we have more than 2 urls, otherwise 3 * self.__bannedUrls is reinitialized if all the URLs are banned :return: the selected URL """ if not self.__initStatus['OK']: return self.__initStatus # Load the Gateways URLs for the current site Name gatewayURL = False if self.KW_IGNORE_GATEWAYS not in self.kwargs or not self.kwargs[self.KW_IGNORE_GATEWAYS]: dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" % DIRAC.siteName()) if dRetVal['OK']: rawGatewayURL = List.randomize(List.fromChar(dRetVal['Value'], ","))[0] gatewayURL = "/".join(rawGatewayURL.split("/")[:3]) # If what was given as constructor attribute is a properly formed URL, # we just return this one. # If we have to use a gateway, we just replace the server name in it for protocol in gProtocolDict: if self._destinationSrv.find("%s://" % protocol) == 0: gLogger.debug("Already given a valid url", self._destinationSrv) if not gatewayURL: return S_OK(self._destinationSrv) gLogger.debug("Reconstructing given URL to pass through gateway") path = "/".join(self._destinationSrv.split("/")[3:]) finalURL = "%s/%s" % (gatewayURL, path) gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL)) return S_OK(finalURL) if gatewayURL: gLogger.debug("Using gateway", gatewayURL) return S_OK("%s/%s" % (gatewayURL, self._destinationSrv)) # We extract the list of URLs from the CS (System/URLs/Component) try: urls = getServiceURL(self._destinationSrv, setup=self.setup) except Exception as e: return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e))) if not urls: return S_ERROR("URL for service %s not found" % self._destinationSrv) failoverUrls = [] # Try if there are some failover URLs to use as last resort try: failoverUrlsStr = getServiceFailoverURL(self._destinationSrv, setup=self.setup) if failoverUrlsStr: failoverUrls = failoverUrlsStr.split(',') except Exception as e: pass # We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component) urlsList = List.randomize(List.fromChar(urls, ",")) + failoverUrls self.__nbOfUrls = len(urlsList) self.__nbOfRetry = 2 if self.__nbOfUrls > 2 else 3 # we retry 2 times all services, if we run more than 2 services if self.__nbOfUrls == len(self.__bannedUrls): self.__bannedUrls = [] # retry all urls gLogger.debug("Retrying again all URLs") if len(self.__bannedUrls) > 0 and len(urlsList) > 1: # we have host which is not accessible. We remove that host from the list. # We only remove if we have more than one instance for i in self.__bannedUrls: gLogger.debug("Removing banned URL", "%s" % i) urlsList.remove(i) # Take the first URL from the list #randUrls = List.randomize( urlsList ) + failoverUrls sURL = urlsList[0] # If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL # is not on a host which is banned. If it is, we take the next one in the list using __selectUrl # If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL # is not on a host which is banned. If it is, we take the next one in the list using __selectUrl if len(self.__bannedUrls) > 0 and self.__nbOfUrls > 2: # when we have multiple services then we can # have a situation when two services are running on the same machine with different ports... retVal = Network.splitURL(sURL) nexturl = None if retVal['OK']: nexturl = retVal['Value'] found = False for i in self.__bannedUrls: retVal = Network.splitURL(i) if retVal['OK']: bannedurl = retVal['Value'] else: break # We found a banned URL on the same host as the one we are running on if nexturl[1] == bannedurl[1]: found = True break if found: nexturl = self.__selectUrl(nexturl, urlsList[1:]) if nexturl: # an url found which is in different host sURL = nexturl gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL)) return S_OK(sURL)
def __findServiceURL( self ): if not self.__initStatus[ 'OK' ]: return self.__initStatus gatewayURL = False if self.KW_IGNORE_GATEWAYS not in self.kwargs or not self.kwargs[ self.KW_IGNORE_GATEWAYS ]: dRetVal = gConfig.getOption( "/DIRAC/Gateways/%s" % DIRAC.siteName() ) if dRetVal[ 'OK' ]: rawGatewayURL = List.randomize( List.fromChar( dRetVal[ 'Value'], "," ) )[0] gatewayURL = "/".join( rawGatewayURL.split( "/" )[:3] ) for protocol in gProtocolDict.keys(): if self._destinationSrv.find( "%s://" % protocol ) == 0: gLogger.debug( "Already given a valid url", self._destinationSrv ) if not gatewayURL: return S_OK( self._destinationSrv ) gLogger.debug( "Reconstructing given URL to pass through gateway" ) path = "/".join( self._destinationSrv.split( "/" )[3:] ) finalURL = "%s/%s" % ( gatewayURL, path ) gLogger.debug( "Gateway URL conversion:\n %s -> %s" % ( self._destinationSrv, finalURL ) ) return S_OK( finalURL ) if gatewayURL: gLogger.debug( "Using gateway", gatewayURL ) return S_OK( "%s/%s" % ( gatewayURL, self._destinationSrv ) ) try: urls = getServiceURL( self._destinationSrv, setup = self.setup ) except Exception as e: return S_ERROR( "Cannot get URL for %s in setup %s: %s" % ( self._destinationSrv, self.setup, repr( e ) ) ) if not urls: return S_ERROR( "URL for service %s not found" % self._destinationSrv ) urlsList = List.fromChar( urls, "," ) self.__nbOfUrls = len( urlsList ) self.__nbOfRetry = 2 if self.__nbOfUrls > 2 else 3 # we retry 2 times all services, if we run more than 2 services if len( urlsList ) == len( self.__bannedUrls ): self.__bannedUrls = [] # retry all urls gLogger.debug( "Retrying again all URLs" ) if len( self.__bannedUrls ) > 0 and len( urlsList ) > 1 : # we have host which is not accessible. We remove that host from the list. # We only remove if we have more than one instance for i in self.__bannedUrls: gLogger.debug( "Removing banned URL", "%s" % i ) urlsList.remove( i ) randUrls = List.randomize( urlsList ) sURL = randUrls[0] if len( self.__bannedUrls ) > 0 and self.__nbOfUrls > 2: # when we have multiple services then we can have a situation # when two service are running on the same machine with different port... retVal = Network.splitURL( sURL ) nexturl = None if retVal['OK']: nexturl = retVal['Value'] found = False for i in self.__bannedUrls: retVal = Network.splitURL( i ) if retVal['OK']: bannedurl = retVal['Value'] else: break if nexturl[1] == bannedurl[1]: found = True break if found: nexturl = self.__selectUrl( nexturl, randUrls[1:] ) if nexturl: # an url found which is in different host sURL = nexturl gLogger.debug( "Discovering URL for service", "%s -> %s" % ( self._destinationSrv, sURL ) ) return S_OK( sURL )