def __discoverURL(self):
""" Calculate the final URL. It is called at initialization and in connect in case of issue
It sets:
* self.serviceURL: the url (dips) selected as target using __findServiceURL
* self.__URLTuple: a split of serviceURL obtained by Network.splitURL
* self._serviceName: the last part of URLTuple (typically System/Component)
"""
# Calculate final URL
try:
result = self.__findServiceURL()
except Exception as e:
return S_ERROR(repr(e))
if not result['OK']:
return result
self.serviceURL = result['Value']
retVal = Network.splitURL(self.serviceURL)
if not retVal['OK']:
return retVal
self.__URLTuple = retVal['Value']
self._serviceName = self.__URLTuple[-1]
res = gConfig.getOptionsDict("/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3])
if res['OK']:
opts = res['Value']
for k in opts:
if k not in self.kwargs:
self.kwargs[k] = opts[k]
return S_OK()
def __discoverURL(self):
"""Calculate the final URL. It is called at initialization and in connect in case of issue
It sets:
* self.serviceURL: the url (dips) selected as target using __findServiceURL
* self.__URLTuple: a split of serviceURL obtained by Network.splitURL
* self._serviceName: the last part of URLTuple (typically System/Component)
WARNING: COPY PASTE FROM BaseClient
"""
# Calculate final URL
try:
result = self.__findServiceURL()
except Exception as e:
return S_ERROR(repr(e))
if not result["OK"]:
return result
self.serviceURL = result["Value"]
retVal = Network.splitURL(self.serviceURL)
if not retVal["OK"]:
return retVal
self.__URLTuple = retVal["Value"]
self._serviceName = self.__URLTuple[-1]
res = gConfig.getOptionsDict("/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3])
if res["OK"]:
opts = res["Value"]
for k in opts:
if k not in self.kwargs:
self.kwargs[k] = opts[k]
return S_OK()
def __selectUrl(self, notselect, urls):
"""In case when multiple services are running in the same host, a new url has to be in a different host
Note: If we do not have different host we will use the selected url...
"""
url = None
for i in urls:
retVal = Network.splitURL(i)
if retVal['OK']:
if retVal['Value'][1] != notselect[1]: # the hots are different
url = i
break
else:
gLogger.error(retVal['Message'])
return url
def __selectUrl( self, notselect, urls ):
"""In case when multiple services are running in the same host, a new url has to be in a different host
Note: If we do not have different host we will use the selected url...
"""
url = None
for i in urls:
retVal = Network.splitURL( i )
if retVal['OK']:
if retVal['Value'][1] != notselect[1]: # the hots are different
url = i
break
else:
gLogger.error( retVal['Message'] )
return url
def __selectUrl(self, notselect, urls):
"""In case when multiple services are running in the same host, a new url has to be in a different host
Note: If we do not have different host we will use the selected url...
:param notselect: URL that should NOT be selected
:param list urls: list of potential URLs
:return: str -- selected URL
"""
url = None
for i in urls:
retVal = Network.splitURL(i)
if retVal["OK"]:
if retVal["Value"][1] != notselect[
1]: # the hosts are different
url = i
break
else:
gLogger.error(retVal["Message"])
return url
def __discoverURL( self ):
#Calculate final URL
try:
result = self.__findServiceURL()
except Exception as e:
return S_ERROR( repr( e ) )
if not result[ 'OK' ]:
return result
self.serviceURL = result[ 'Value' ]
retVal = Network.splitURL( self.serviceURL )
if not retVal[ 'OK' ]:
return retVal
self.__URLTuple = retVal[ 'Value' ]
self._serviceName = self.__URLTuple[-1]
res = gConfig.getOptionsDict( "/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3] )
if res[ 'OK' ]:
opts = res[ 'Value' ]
for k in opts:
if k not in self.kwargs:
self.kwargs[k] = opts[k]
return S_OK()
def __selectUrl(self, notselect, urls):
"""In case when multiple services are running in the same host, a new url has to be in a different host
Note: If we do not have different host we will use the selected url...
:param notselect: URL that should NOT be selected
:param urls: list of potential URLs
:return: selected URL
WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient
"""
url = None
for i in urls:
retVal = Network.splitURL(i)
if retVal['OK']:
if retVal['Value'][1] != notselect[1]: # the hots are different
url = i
break
else:
gLogger.error(retVal['Message'])
return url
def __discoverURL(self):
#Calculate final URL
try:
result = self.__findServiceURL()
except Exception as e:
return S_ERROR(repr(e))
if not result['OK']:
return result
self.serviceURL = result['Value']
retVal = Network.splitURL(self.serviceURL)
if not retVal['OK']:
return retVal
self.__URLTuple = retVal['Value']
self._serviceName = self.__URLTuple[-1]
res = gConfig.getOptionsDict("/DIRAC/ConnConf/%s:%s" %
self.__URLTuple[1:3])
if res['OK']:
opts = res['Value']
for k in opts:
if k not in self.kwargs:
self.kwargs[k] = opts[k]
return S_OK()
class BaseClient:
VAL_EXTRA_CREDENTIALS_HOST = "hosts"
KW_USE_CERTIFICATES = "useCertificates"
KW_EXTRA_CREDENTIALS = "extraCredentials"
KW_TIMEOUT = "timeout"
KW_SETUP = "setup"
KW_VO = "VO"
KW_DELEGATED_DN = "delegatedDN"
KW_DELEGATED_GROUP = "delegatedGroup"
KW_IGNORE_GATEWAYS = "ignoreGateways"
KW_PROXY_LOCATION = "proxyLocation"
KW_PROXY_STRING = "proxyString"
KW_PROXY_CHAIN = "proxyChain"
KW_SKIP_CA_CHECK = "skipCACheck"
KW_KEEP_ALIVE_LAPSE = "keepAliveLapse"
__threadConfig = ThreadConfig()
def __init__( self, serviceName, **kwargs ):
if type( serviceName ) not in types.StringTypes:
raise TypeError( "Service name expected to be a string. Received %s type %s" %
( str( serviceName ), type( serviceName ) ) )
self._destinationSrv = serviceName
self._serviceName = serviceName
self.kwargs = kwargs
self.__initStatus = S_OK()
self.__idDict = {}
self.__extraCredentials = ""
self.__enableThreadCheck = False
self.__retry = 0
self.__retryDelay = 0
self.__nbOfUrls = 1 #by default we always have 1 url for example: RPCClient('dips://volhcb38.cern.ch:9162/Framework/SystemAdministrator')
self.__nbOfRetry = 3 # by default we try try times
self.__bannedUrls = []
for initFunc in ( self.__discoverSetup, self.__discoverVO, self.__discoverTimeout,
self.__discoverURL, self.__discoverCredentialsToUse,
self.__checkTransportSanity,
self.__setKeepAliveLapse ):
result = initFunc()
if not result[ 'OK' ] and self.__initStatus[ 'OK' ]:
self.__initStatus = result
self._initialize()
#HACK for thread-safety:
self.__allowedThreadID = False
def _initialize( self ):
pass
def getDestinationService( self ):
return self._destinationSrv
def getServiceName( self ):
return self._serviceName
def __discoverSetup( self ):
#Which setup to use?
if self.KW_SETUP in self.kwargs and self.kwargs[ self.KW_SETUP ]:
self.setup = str( self.kwargs[ self.KW_SETUP ] )
else:
self.setup = self.__threadConfig.getSetup()
if not self.setup:
self.setup = gConfig.getValue( "/DIRAC/Setup", "Test" )
return S_OK()
def __discoverVO( self ):
#Which setup to use?
if self.KW_VO in self.kwargs and self.kwargs[ self.KW_VO ]:
self.vo = str( self.kwargs[ self.KW_VO ] )
else:
self.vo = gConfig.getValue( "/DIRAC/VirtualOrganization", "unknown" )
return S_OK()
def __discoverURL( self ):
#Calculate final URL
try:
result = self.__findServiceURL()
except Exception, e:
return S_ERROR( str( e ) )
if not result[ 'OK' ]:
return result
self.serviceURL = result[ 'Value' ]
retVal = Network.splitURL( self.serviceURL )
if not retVal[ 'OK' ]:
return S_ERROR( "URL is malformed: %s" % retVal[ 'Message' ] )
self.__URLTuple = retVal[ 'Value' ]
self._serviceName = self.__URLTuple[-1]
res = gConfig.getOptionsDict( "/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3] )
if res[ 'OK' ]:
opts = res[ 'Value' ]
for k in opts:
if k not in self.kwargs:
self.kwargs[k] = opts[k]
return S_OK()
def __findServiceURL(self):
"""
Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs
If the site on which we run is configured to use gateways (/DIRAC/Gateways/<siteName>),
these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS
to False in kwargs.
If self._destinationSrv (given as constructor attribute) is a properly formed URL,
we just return this one. If we have to use a gateway, we just replace the server name in the url.
The list of URLs defined in the CS (<System>/URLs/<Component>) is randomized
This method also sets some attributes:
* self.__nbOfUrls = number of URLs
* self.__nbOfRetry removed in HTTPS (Managed by requests)
* self.__bannedUrls is reinitialized if all the URLs are banned
:return: the selected URL
WARNING (Mostly) COPY PASTE FROM BaseClient (protocols list is changed to https)
"""
if not self.__initStatus["OK"]:
return self.__initStatus
# Load the Gateways URLs for the current site Name
gatewayURL = False
if not self.kwargs.get(self.KW_IGNORE_GATEWAYS):
gatewayURLs = getGatewayURLs()
if gatewayURLs:
gatewayURL = "/".join(gatewayURLs[0].split("/")[:3])
# If what was given as constructor attribute is a properly formed URL,
# we just return this one.
# If we have to use a gateway, we just replace the server name in it
if self._destinationSrv.startswith("https://"):
gLogger.debug("Already given a valid url", self._destinationSrv)
if not gatewayURL:
return S_OK(self._destinationSrv)
gLogger.debug("Reconstructing given URL to pass through gateway")
path = "/".join(self._destinationSrv.split("/")[3:])
finalURL = "%s/%s" % (gatewayURL, path)
gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL))
return S_OK(finalURL)
if gatewayURL:
gLogger.debug("Using gateway", gatewayURL)
return S_OK("%s/%s" % (gatewayURL, self._destinationSrv))
# If nor url is given as constructor, we extract the list of URLs from the CS (System/URLs/Component)
try:
# We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component)
urlsList = getServiceURLs(self._destinationSrv, setup=self.setup, failover=True)
except Exception as e:
return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e)))
if not urlsList:
return S_ERROR("URL for service %s not found" % self._destinationSrv)
self.__nbOfUrls = len(urlsList)
# __nbOfRetry removed in HTTPS (managed by requests)
if self.__nbOfUrls == len(self.__bannedUrls):
self.__bannedUrls = [] # retry all urls
gLogger.debug("Retrying again all URLs")
if self.__bannedUrls and len(urlsList) > 1:
# we have host which is not accessible. We remove that host from the list.
# We only remove if we have more than one instance
for i in self.__bannedUrls:
gLogger.debug("Removing banned URL", "%s" % i)
urlsList.remove(i)
sURL = urlsList[0]
# If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL
# is not on a host which is banned. If it is, we take the next one in the list using __selectUrl
if self.__bannedUrls and self.__nbOfUrls > 2: # when we have multiple services then we can
# have a situation when two services are running on the same machine with different ports...
retVal = Network.splitURL(sURL)
nexturl = None
if retVal["OK"]:
nexturl = retVal["Value"]
found = False
for i in self.__bannedUrls:
retVal = Network.splitURL(i)
if retVal["OK"]:
bannedurl = retVal["Value"]
else:
break
# We found a banned URL on the same host as the one we are running on
if nexturl[1] == bannedurl[1]:
found = True
break
if found:
nexturl = self.__selectUrl(nexturl, urlsList[1:])
if nexturl: # an url found which is in different host
sURL = nexturl
gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL))
return S_OK(sURL)
def __findServiceURL(self):
""" Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs
If the site on which we run is configured to use gateways (/DIRAC/Gateways/<siteName>),
these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS
to False in kwargs.
If self._destinationSrv (given as constructor attribute) is a properly formed URL,
we just return this one. If we have to use a gateway, we just replace the server name in the url.
The list of URLs defined in the CS (<System>/URLs/<Component>) is randomized
This method also sets some attributes:
* self.__nbOfUrls = number of URLs
* self.__nbOfRetry = 2 if we have more than 2 urls, otherwise 3
* self.__bannedUrls is reinitialized if all the URLs are banned
:return: S_OK(str)/S_ERROR() -- the selected URL
"""
if not self.__initStatus['OK']:
return self.__initStatus
# Load the Gateways URLs for the current site Name
gatewayURL = False
if not self.kwargs.get(self.KW_IGNORE_GATEWAYS):
dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" % DIRAC.siteName())
if dRetVal['OK']:
rawGatewayURL = List.randomize(List.fromChar(dRetVal['Value'], ","))[0]
gatewayURL = "/".join(rawGatewayURL.split("/")[:3])
# If what was given as constructor attribute is a properly formed URL,
# we just return this one.
# If we have to use a gateway, we just replace the server name in it
for protocol in gProtocolDict:
if self._destinationSrv.find("%s://" % protocol) == 0:
gLogger.debug("Already given a valid url", self._destinationSrv)
if not gatewayURL:
return S_OK(self._destinationSrv)
gLogger.debug("Reconstructing given URL to pass through gateway")
path = "/".join(self._destinationSrv.split("/")[3:])
finalURL = "%s/%s" % (gatewayURL, path)
gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL))
return S_OK(finalURL)
if gatewayURL:
gLogger.debug("Using gateway", gatewayURL)
return S_OK("%s/%s" % (gatewayURL, self._destinationSrv))
# We extract the list of URLs from the CS (System/URLs/Component)
try:
urls = getServiceURL(self._destinationSrv, setup=self.setup)
except Exception as e:
return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e)))
if not urls:
return S_ERROR("URL for service %s not found" % self._destinationSrv)
failoverUrls = []
# Try if there are some failover URLs to use as last resort
try:
failoverUrlsStr = getServiceFailoverURL(self._destinationSrv, setup=self.setup)
if failoverUrlsStr:
failoverUrls = failoverUrlsStr.split(',')
except Exception as e:
pass
# We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component)
urlsList = List.randomize(List.fromChar(urls, ",")) + failoverUrls
self.__nbOfUrls = len(urlsList)
self.__nbOfRetry = 2 if self.__nbOfUrls > 2 else 3 # we retry 2 times all services, if we run more than 2 services
if self.__nbOfUrls == len(self.__bannedUrls):
self.__bannedUrls = [] # retry all urls
gLogger.debug("Retrying again all URLs")
if len(self.__bannedUrls) > 0 and len(urlsList) > 1:
# we have host which is not accessible. We remove that host from the list.
# We only remove if we have more than one instance
for i in self.__bannedUrls:
gLogger.debug("Removing banned URL", "%s" % i)
urlsList.remove(i)
# Take the first URL from the list
# randUrls = List.randomize( urlsList ) + failoverUrls
sURL = urlsList[0]
# If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL
# is not on a host which is banned. If it is, we take the next one in the list using __selectUrl
# If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL
# is not on a host which is banned. If it is, we take the next one in the list using __selectUrl
if len(self.__bannedUrls) > 0 and self.__nbOfUrls > 2: # when we have multiple services then we can
# have a situation when two services are running on the same machine with different ports...
retVal = Network.splitURL(sURL)
nexturl = None
if retVal['OK']:
nexturl = retVal['Value']
found = False
for i in self.__bannedUrls:
retVal = Network.splitURL(i)
if retVal['OK']:
bannedurl = retVal['Value']
else:
break
# We found a banned URL on the same host as the one we are running on
if nexturl[1] == bannedurl[1]:
found = True
break
if found:
nexturl = self.__selectUrl(nexturl, urlsList[1:])
if nexturl: # an url found which is in different host
sURL = nexturl
gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL))
return S_OK(sURL)
def __findServiceURL(self):
if not self.__initStatus['OK']:
return self.__initStatus
gatewayURL = False
if self.KW_IGNORE_GATEWAYS not in self.kwargs or not self.kwargs[
self.KW_IGNORE_GATEWAYS]:
dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" %
DIRAC.siteName())
if dRetVal['OK']:
rawGatewayURL = List.randomize(
List.fromChar(dRetVal['Value'], ","))[0]
gatewayURL = "/".join(rawGatewayURL.split("/")[:3])
for protocol in gProtocolDict.keys():
if self._destinationSrv.find("%s://" % protocol) == 0:
gLogger.debug("Already given a valid url",
self._destinationSrv)
if not gatewayURL:
return S_OK(self._destinationSrv)
gLogger.debug(
"Reconstructing given URL to pass through gateway")
path = "/".join(self._destinationSrv.split("/")[3:])
finalURL = "%s/%s" % (gatewayURL, path)
gLogger.debug("Gateway URL conversion:\n %s -> %s" %
(self._destinationSrv, finalURL))
return S_OK(finalURL)
if gatewayURL:
gLogger.debug("Using gateway", gatewayURL)
return S_OK("%s/%s" % (gatewayURL, self._destinationSrv))
try:
urls = getServiceURL(self._destinationSrv, setup=self.setup)
except Exception as e:
return S_ERROR("Cannot get URL for %s in setup %s: %s" %
(self._destinationSrv, self.setup, repr(e)))
if not urls:
return S_ERROR("URL for service %s not found" %
self._destinationSrv)
urlsList = List.fromChar(urls, ",")
self.__nbOfUrls = len(urlsList)
self.__nbOfRetry = 2 if self.__nbOfUrls > 2 else 3 # we retry 2 times all services, if we run more than 2 services
if len(urlsList) == len(self.__bannedUrls):
self.__bannedUrls = [] # retry all urls
gLogger.debug("Retrying again all URLs")
if len(self.__bannedUrls) > 0 and len(urlsList) > 1:
# we have host which is not accessible. We remove that host from the list.
# We only remove if we have more than one instance
for i in self.__bannedUrls:
gLogger.debug("Removing banned URL", "%s" % i)
urlsList.remove(i)
randUrls = List.randomize(urlsList)
sURL = randUrls[0]
if len(
self.__bannedUrls
) > 0 and self.__nbOfUrls > 2: # when we have multiple services then we can have a situation
# when two service are running on the same machine with different port...
retVal = Network.splitURL(sURL)
nexturl = None
if retVal['OK']:
nexturl = retVal['Value']
found = False
for i in self.__bannedUrls:
retVal = Network.splitURL(i)
if retVal['OK']:
bannedurl = retVal['Value']
else:
break
if nexturl[1] == bannedurl[1]:
found = True
break
if found:
nexturl = self.__selectUrl(nexturl, randUrls[1:])
if nexturl: # an url found which is in different host
sURL = nexturl
gLogger.debug("Discovering URL for service",
"%s -> %s" % (self._destinationSrv, sURL))
return S_OK(sURL)
class BaseClient:
VAL_EXTRA_CREDENTIALS_HOST = "hosts"
KW_USE_CERTIFICATES = "useCertificates"
KW_EXTRA_CREDENTIALS = "extraCredentials"
KW_TIMEOUT = "timeout"
KW_SETUP = "setup"
KW_VO = "VO"
KW_DELEGATED_DN = "delegatedDN"
KW_DELEGATED_GROUP = "delegatedGroup"
KW_IGNORE_GATEWAYS = "ignoreGateways"
KW_PROXY_LOCATION = "proxyLocation"
KW_PROXY_STRING = "proxyString"
KW_PROXY_CHAIN = "proxyChain"
KW_SKIP_CA_CHECK = "skipCACheck"
KW_KEEP_ALIVE_LAPSE = "keepAliveLapse"
def __init__(self, serviceName, **kwargs):
if type(serviceName) != types.StringType:
raise TypeError(
"Service name expected to be a string. Received %s type %s" %
(str(serviceName), type(serviceName)))
self._destinationSrv = serviceName
self.kwargs = kwargs
self.__initStatus = S_OK()
self.__idDict = {}
self.__enableThreadCheck = False
for initFunc in (self.__discoverSetup, self.__discoverVO,
self.__discoverTimeout, self.__discoverURL,
self.__discoverCredentialsToUse,
self.__discoverExtraCredentials,
self.__checkTransportSanity,
self.__setKeepAliveLapse):
result = initFunc()
if not result['OK'] and self.__initStatus['OK']:
self.__initStatus = result
self._initialize()
#HACK for thread-safety:
self.__allowedThreadID = False
def _initialize(self):
pass
def getDestinationService(self):
return self._destinationSrv
def __discoverSetup(self):
#Which setup to use?
if self.KW_SETUP in self.kwargs and self.kwargs[self.KW_SETUP]:
self.setup = str(self.kwargs[self.KW_SETUP])
else:
self.setup = gConfig.getValue("/DIRAC/Setup", "Test")
return S_OK()
def __discoverVO(self):
#Which setup to use?
if self.KW_VO in self.kwargs and self.kwargs[self.KW_VO]:
self.vo = str(self.kwargs[self.KW_VO])
else:
self.vo = gConfig.getValue("/DIRAC/VirtualOrganization", "unknown")
return S_OK()
def __discoverURL(self):
#Calculate final URL
try:
result = self.__findServiceURL()
except Exception, e:
return S_ERROR(str(e))
if not result['OK']:
return result
self.serviceURL = result['Value']
retVal = Network.splitURL(self.serviceURL)
if not retVal['OK']:
return S_ERROR("URL is malformed: %s" % retVal['Message'])
self.__URLTuple = retVal['Value']
self._serviceName = self.__URLTuple[-1]
return S_OK()
def __findServiceURL(self):
"""
Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs
If the site on which we run is configured to use gateways (/DIRAC/Gateways/<siteName>),
these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS
to False in kwargs.
If self._destinationSrv (given as constructor attribute) is a properly formed URL,
we just return this one. If we have to use a gateway, we just replace the server name in the url.
The list of URLs defined in the CS (<System>/URLs/<Component>) is randomized
This method also sets some attributes:
* self.__nbOfUrls = number of URLs
* self.__nbOfRetry = 2 if we have more than 2 urls, otherwise 3
* self.__bannedUrls is reinitialized if all the URLs are banned
:return: the selected URL
"""
if not self.__initStatus['OK']:
return self.__initStatus
# Load the Gateways URLs for the current site Name
gatewayURL = False
if self.KW_IGNORE_GATEWAYS not in self.kwargs or not self.kwargs[self.KW_IGNORE_GATEWAYS]:
dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" % DIRAC.siteName())
if dRetVal['OK']:
rawGatewayURL = List.randomize(List.fromChar(dRetVal['Value'], ","))[0]
gatewayURL = "/".join(rawGatewayURL.split("/")[:3])
# If what was given as constructor attribute is a properly formed URL,
# we just return this one.
# If we have to use a gateway, we just replace the server name in it
for protocol in gProtocolDict:
if self._destinationSrv.find("%s://" % protocol) == 0:
gLogger.debug("Already given a valid url", self._destinationSrv)
if not gatewayURL:
return S_OK(self._destinationSrv)
gLogger.debug("Reconstructing given URL to pass through gateway")
path = "/".join(self._destinationSrv.split("/")[3:])
finalURL = "%s/%s" % (gatewayURL, path)
gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL))
return S_OK(finalURL)
if gatewayURL:
gLogger.debug("Using gateway", gatewayURL)
return S_OK("%s/%s" % (gatewayURL, self._destinationSrv))
# We extract the list of URLs from the CS (System/URLs/Component)
try:
urls = getServiceURL(self._destinationSrv, setup=self.setup)
except Exception as e:
return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e)))
if not urls:
return S_ERROR("URL for service %s not found" % self._destinationSrv)
failoverUrls = []
# Try if there are some failover URLs to use as last resort
try:
failoverUrlsStr = getServiceFailoverURL(self._destinationSrv, setup=self.setup)
if failoverUrlsStr:
failoverUrls = failoverUrlsStr.split(',')
except Exception as e:
pass
# We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component)
urlsList = List.randomize(List.fromChar(urls, ",")) + failoverUrls
self.__nbOfUrls = len(urlsList)
self.__nbOfRetry = 2 if self.__nbOfUrls > 2 else 3 # we retry 2 times all services, if we run more than 2 services
if self.__nbOfUrls == len(self.__bannedUrls):
self.__bannedUrls = [] # retry all urls
gLogger.debug("Retrying again all URLs")
if len(self.__bannedUrls) > 0 and len(urlsList) > 1:
# we have host which is not accessible. We remove that host from the list.
# We only remove if we have more than one instance
for i in self.__bannedUrls:
gLogger.debug("Removing banned URL", "%s" % i)
urlsList.remove(i)
# Take the first URL from the list
#randUrls = List.randomize( urlsList ) + failoverUrls
sURL = urlsList[0]
# If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL
# is not on a host which is banned. If it is, we take the next one in the list using __selectUrl
# If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL
# is not on a host which is banned. If it is, we take the next one in the list using __selectUrl
if len(self.__bannedUrls) > 0 and self.__nbOfUrls > 2: # when we have multiple services then we can
# have a situation when two services are running on the same machine with different ports...
retVal = Network.splitURL(sURL)
nexturl = None
if retVal['OK']:
nexturl = retVal['Value']
found = False
for i in self.__bannedUrls:
retVal = Network.splitURL(i)
if retVal['OK']:
bannedurl = retVal['Value']
else:
break
# We found a banned URL on the same host as the one we are running on
if nexturl[1] == bannedurl[1]:
found = True
break
if found:
nexturl = self.__selectUrl(nexturl, urlsList[1:])
if nexturl: # an url found which is in different host
sURL = nexturl
gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL))
return S_OK(sURL)
def __findServiceURL( self ):
if not self.__initStatus[ 'OK' ]:
return self.__initStatus
gatewayURL = False
if self.KW_IGNORE_GATEWAYS not in self.kwargs or not self.kwargs[ self.KW_IGNORE_GATEWAYS ]:
dRetVal = gConfig.getOption( "/DIRAC/Gateways/%s" % DIRAC.siteName() )
if dRetVal[ 'OK' ]:
rawGatewayURL = List.randomize( List.fromChar( dRetVal[ 'Value'], "," ) )[0]
gatewayURL = "/".join( rawGatewayURL.split( "/" )[:3] )
for protocol in gProtocolDict.keys():
if self._destinationSrv.find( "%s://" % protocol ) == 0:
gLogger.debug( "Already given a valid url", self._destinationSrv )
if not gatewayURL:
return S_OK( self._destinationSrv )
gLogger.debug( "Reconstructing given URL to pass through gateway" )
path = "/".join( self._destinationSrv.split( "/" )[3:] )
finalURL = "%s/%s" % ( gatewayURL, path )
gLogger.debug( "Gateway URL conversion:\n %s -> %s" % ( self._destinationSrv, finalURL ) )
return S_OK( finalURL )
if gatewayURL:
gLogger.debug( "Using gateway", gatewayURL )
return S_OK( "%s/%s" % ( gatewayURL, self._destinationSrv ) )
try:
urls = getServiceURL( self._destinationSrv, setup = self.setup )
except Exception as e:
return S_ERROR( "Cannot get URL for %s in setup %s: %s" % ( self._destinationSrv, self.setup, repr( e ) ) )
if not urls:
return S_ERROR( "URL for service %s not found" % self._destinationSrv )
urlsList = List.fromChar( urls, "," )
self.__nbOfUrls = len( urlsList )
self.__nbOfRetry = 2 if self.__nbOfUrls > 2 else 3 # we retry 2 times all services, if we run more than 2 services
if len( urlsList ) == len( self.__bannedUrls ):
self.__bannedUrls = [] # retry all urls
gLogger.debug( "Retrying again all URLs" )
if len( self.__bannedUrls ) > 0 and len( urlsList ) > 1 :
# we have host which is not accessible. We remove that host from the list.
# We only remove if we have more than one instance
for i in self.__bannedUrls:
gLogger.debug( "Removing banned URL", "%s" % i )
urlsList.remove( i )
randUrls = List.randomize( urlsList )
sURL = randUrls[0]
if len( self.__bannedUrls ) > 0 and self.__nbOfUrls > 2: # when we have multiple services then we can have a situation
# when two service are running on the same machine with different port...
retVal = Network.splitURL( sURL )
nexturl = None
if retVal['OK']:
nexturl = retVal['Value']
found = False
for i in self.__bannedUrls:
retVal = Network.splitURL( i )
if retVal['OK']:
bannedurl = retVal['Value']
else:
break
if nexturl[1] == bannedurl[1]:
found = True
break
if found:
nexturl = self.__selectUrl( nexturl, randUrls[1:] )
if nexturl: # an url found which is in different host
sURL = nexturl
gLogger.debug( "Discovering URL for service", "%s -> %s" % ( self._destinationSrv, sURL ) )
return S_OK( sURL )
