def __refresh( self ):
self.__lastUpdateTime = time.time()
gLogger.debug( "Refreshing configuration..." )
gatewayList = getGatewayURLs( "Configuration/Server" )
updatingErrorsList = []
if gatewayList:
lInitialListOfServers = gatewayList
gLogger.debug( "Using configuration gateway", str( lInitialListOfServers[0] ) )
else:
lInitialListOfServers = gConfigurationData.getServers()
gLogger.debug( "Refreshing from list %s" % str( lInitialListOfServers ) )
lRandomListOfServers = List.randomize( lInitialListOfServers )
gLogger.debug( "Randomized server list is %s" % ", ".join( lRandomListOfServers ) )
for sServer in lRandomListOfServers:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient( sServer,
useCertificates = gConfigurationData.useServerCertificate(),
skipCACheck = gConfigurationData.skipCACheck() )
dRetVal = _updateFromRemoteLocation( oClient )
if dRetVal[ 'OK' ]:
return dRetVal
else:
updatingErrorsList.append( dRetVal[ 'Message' ] )
gLogger.warn( "Can't update from server", "Error while updating from %s: %s" % ( sServer, dRetVal[ 'Message' ] ) )
if dRetVal[ 'Message' ].find( "Insane environment" ) > -1:
break
return S_ERROR( "Reason(s):\n\t%s" % "\n\t".join( List.uniqueElements( updatingErrorsList ) ) )
def __refresh(self):
self.__lastUpdateTime = time.time()
gLogger.debug("Refreshing configuration...")
gatewayList = getGatewayURLs("Configuration/Server")
updatingErrorsList = []
if gatewayList:
lInitialListOfServers = gatewayList
gLogger.debug("Using configuration gateway",
str(lInitialListOfServers[0]))
else:
lInitialListOfServers = gConfigurationData.getServers()
gLogger.debug("Refreshing from list %s" %
str(lInitialListOfServers))
lRandomListOfServers = List.randomize(lInitialListOfServers)
gLogger.debug("Randomized server list is %s" %
", ".join(lRandomListOfServers))
for sServer in lRandomListOfServers:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient(
sServer,
useCertificates=gConfigurationData.useServerCertificate(),
skipCACheck=gConfigurationData.skipCACheck())
dRetVal = _updateFromRemoteLocation(oClient)
if dRetVal['OK']:
return dRetVal
else:
updatingErrorsList.append(dRetVal['Message'])
gLogger.warn(
"Can't update from server",
"Error while updating from %s: %s" %
(sServer, dRetVal['Message']))
return S_ERROR("Reason(s):\n\t%s" %
"\n\t".join(List.uniqueElements(updatingErrorsList)))
def _refresh(self, fromMaster=False):
"""
Refresh configuration
"""
self._lastUpdateTime = time.time()
gLogger.debug("Refreshing configuration...")
gatewayList = getGatewayURLs("Configuration/Server")
updatingErrorsList = []
if gatewayList:
initialServerList = gatewayList
gLogger.debug("Using configuration gateway",
str(initialServerList[0]))
elif fromMaster:
masterServer = gConfigurationData.getMasterServer()
initialServerList = [masterServer]
gLogger.debug("Refreshing from master %s" % masterServer)
else:
initialServerList = gConfigurationData.getServers()
gLogger.debug("Refreshing from list %s" % str(initialServerList))
# If no servers in the initial list, we are supposed to use the local configuration only
if not initialServerList:
return S_OK()
randomServerList = List.randomize(initialServerList)
gLogger.debug("Randomized server list is %s" %
", ".join(randomServerList))
for sServer in randomServerList:
from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient
oClient = ConfigurationClient(
url=sServer,
useCertificates=gConfigurationData.useServerCertificate(),
skipCACheck=gConfigurationData.skipCACheck(),
)
dRetVal = _updateFromRemoteLocation(oClient)
if dRetVal["OK"]:
self._refreshTime = gConfigurationData.getRefreshTime()
return dRetVal
else:
updatingErrorsList.append(dRetVal["Message"])
gLogger.warn(
"Can't update from server",
"Error while updating from %s: %s" %
(sServer, dRetVal["Message"]))
if dRetVal["Message"].find("Insane environment") > -1:
break
return S_ERROR("Reason(s):\n\t%s" %
"\n\t".join(List.uniqueElements(updatingErrorsList)))
def __refresh(self):
self.__lastUpdateTime = time.time()
gLogger.debug("Refreshing configuration...")
gatewayList = getGatewayURLs("Configuration/Server")
updatingErrorsList = []
if gatewayList:
initialServerList = gatewayList
gLogger.debug("Using configuration gateway",
str(initialServerList[0]))
else:
initialServerList = gConfigurationData.getServers()
gLogger.debug("Refreshing from list %s" % str(initialServerList))
# If no servers in the initial list, we are supposed to use the local configuration only
if not initialServerList:
return S_OK()
randomServerList = List.randomize(initialServerList)
gLogger.debug("Randomized server list is %s" %
", ".join(randomServerList))
for sServer in randomServerList:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient(
sServer,
useCertificates=gConfigurationData.useServerCertificate(),
skipCACheck=gConfigurationData.skipCACheck())
dRetVal = _updateFromRemoteLocation(oClient)
if dRetVal['OK']:
return dRetVal
else:
updatingErrorsList.append(dRetVal['Message'])
gLogger.warn(
"Can't update from server",
"Error while updating from %s: %s" %
(sServer, dRetVal['Message']))
if dRetVal['Message'].find("Insane environment") > -1:
break
return S_ERROR("Reason(s):\n\t%s" %
"\n\t".join(List.uniqueElements(updatingErrorsList)))
def __refresh(self):
self.__lastUpdateTime = time.time()
gLogger.debug("Refreshing configuration...")
gatewayList = getGatewayURLs("Configuration/Server")
updatingErrorsList = []
if gatewayList:
initialServerList = gatewayList
gLogger.debug("Using configuration gateway", str(initialServerList[0]))
else:
initialServerList = gConfigurationData.getServers()
gLogger.debug("Refreshing from list %s" % str(initialServerList))
# If no servers in the initial list, we are supposed to use the local configuration only
if not initialServerList:
return S_OK()
randomServerList = List.randomize(initialServerList)
gLogger.debug("Randomized server list is %s" % ", ".join(randomServerList))
for sServer in randomServerList:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient(
sServer,
useCertificates=gConfigurationData.useServerCertificate(),
skipCACheck=gConfigurationData.skipCACheck(),
)
dRetVal = _updateFromRemoteLocation(oClient)
if dRetVal["OK"]:
return dRetVal
else:
updatingErrorsList.append(dRetVal["Message"])
gLogger.warn(
"Can't update from server", "Error while updating from %s: %s" % (sServer, dRetVal["Message"])
)
if dRetVal["Message"].find("Insane environment") > -1:
break
return S_ERROR("Reason(s):\n\t%s" % "\n\t".join(List.uniqueElements(updatingErrorsList)))
def __findServiceURL(self):
"""
Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs
If the site on which we run is configured to use gateways (/DIRAC/Gateways/<siteName>),
these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS
to False in kwargs.
If self._destinationSrv (given as constructor attribute) is a properly formed URL,
we just return this one. If we have to use a gateway, we just replace the server name in the url.
The list of URLs defined in the CS (<System>/URLs/<Component>) is randomized
This method also sets some attributes:
* self.__nbOfUrls = number of URLs
* self.__nbOfRetry removed in HTTPS (Managed by requests)
* self.__bannedUrls is reinitialized if all the URLs are banned
:return: the selected URL
WARNING (Mostly) COPY PASTE FROM BaseClient (protocols list is changed to https)
"""
if not self.__initStatus["OK"]:
return self.__initStatus
# Load the Gateways URLs for the current site Name
gatewayURL = False
if not self.kwargs.get(self.KW_IGNORE_GATEWAYS):
gatewayURLs = getGatewayURLs()
if gatewayURLs:
gatewayURL = "/".join(gatewayURLs[0].split("/")[:3])
# If what was given as constructor attribute is a properly formed URL,
# we just return this one.
# If we have to use a gateway, we just replace the server name in it
if self._destinationSrv.startswith("https://"):
gLogger.debug("Already given a valid url", self._destinationSrv)
if not gatewayURL:
return S_OK(self._destinationSrv)
gLogger.debug("Reconstructing given URL to pass through gateway")
path = "/".join(self._destinationSrv.split("/")[3:])
finalURL = "%s/%s" % (gatewayURL, path)
gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL))
return S_OK(finalURL)
if gatewayURL:
gLogger.debug("Using gateway", gatewayURL)
return S_OK("%s/%s" % (gatewayURL, self._destinationSrv))
# If nor url is given as constructor, we extract the list of URLs from the CS (System/URLs/Component)
try:
# We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component)
urlsList = getServiceURLs(self._destinationSrv, setup=self.setup, failover=True)
except Exception as e:
return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e)))
if not urlsList:
return S_ERROR("URL for service %s not found" % self._destinationSrv)
self.__nbOfUrls = len(urlsList)
# __nbOfRetry removed in HTTPS (managed by requests)
if self.__nbOfUrls == len(self.__bannedUrls):
self.__bannedUrls = [] # retry all urls
gLogger.debug("Retrying again all URLs")
if self.__bannedUrls and len(urlsList) > 1:
# we have host which is not accessible. We remove that host from the list.
# We only remove if we have more than one instance
for i in self.__bannedUrls:
gLogger.debug("Removing banned URL", "%s" % i)
urlsList.remove(i)
sURL = urlsList[0]
# If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL
# is not on a host which is banned. If it is, we take the next one in the list using __selectUrl
if self.__bannedUrls and self.__nbOfUrls > 2: # when we have multiple services then we can
# have a situation when two services are running on the same machine with different ports...
retVal = Network.splitURL(sURL)
nexturl = None
if retVal["OK"]:
nexturl = retVal["Value"]
found = False
for i in self.__bannedUrls:
retVal = Network.splitURL(i)
if retVal["OK"]:
bannedurl = retVal["Value"]
else:
break
# We found a banned URL on the same host as the one we are running on
if nexturl[1] == bannedurl[1]:
found = True
break
if found:
nexturl = self.__selectUrl(nexturl, urlsList[1:])
if nexturl: # an url found which is in different host
sURL = nexturl
gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL))
return S_OK(sURL)
