def __refresh( self ):
self.__lastUpdateTime = time.time()
gLogger.debug( "Refreshing configuration..." )
gatewayList = getGatewayURLs( "Configuration/Server" )
updatingErrorsList = []
if gatewayList:
lInitialListOfServers = gatewayList
gLogger.debug( "Using configuration gateway", str( lInitialListOfServers[0] ) )
else:
lInitialListOfServers = gConfigurationData.getServers()
gLogger.debug( "Refreshing from list %s" % str( lInitialListOfServers ) )
lRandomListOfServers = List.randomize( lInitialListOfServers )
gLogger.debug( "Randomized server list is %s" % ", ".join( lRandomListOfServers ) )
for sServer in lRandomListOfServers:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient( sServer,
useCertificates = gConfigurationData.useServerCertificate(),
skipCACheck = gConfigurationData.skipCACheck() )
dRetVal = _updateFromRemoteLocation( oClient )
if dRetVal[ 'OK' ]:
return dRetVal
else:
updatingErrorsList.append( dRetVal[ 'Message' ] )
gLogger.warn( "Can't update from server", "Error while updating from %s: %s" % ( sServer, dRetVal[ 'Message' ] ) )
if dRetVal[ 'Message' ].find( "Insane environment" ) > -1:
break
return S_ERROR( "Reason(s):\n\t%s" % "\n\t".join( List.uniqueElements( updatingErrorsList ) ) )
def __refresh(self):
self.__lastUpdateTime = time.time()
gLogger.debug("Refreshing configuration...")
gatewayList = getGatewayURLs("Configuration/Server")
updatingErrorsList = []
if gatewayList:
lInitialListOfServers = gatewayList
gLogger.debug("Using configuration gateway",
str(lInitialListOfServers[0]))
else:
lInitialListOfServers = gConfigurationData.getServers()
gLogger.debug("Refreshing from list %s" %
str(lInitialListOfServers))
lRandomListOfServers = List.randomize(lInitialListOfServers)
gLogger.debug("Randomized server list is %s" %
", ".join(lRandomListOfServers))
for sServer in lRandomListOfServers:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient(
sServer,
useCertificates=gConfigurationData.useServerCertificate(),
skipCACheck=gConfigurationData.skipCACheck())
dRetVal = _updateFromRemoteLocation(oClient)
if dRetVal['OK']:
return dRetVal
else:
updatingErrorsList.append(dRetVal['Message'])
gLogger.warn(
"Can't update from server",
"Error while updating from %s: %s" %
(sServer, dRetVal['Message']))
return S_ERROR("Reason(s):\n\t%s" %
"\n\t".join(List.uniqueElements(updatingErrorsList)))
def __addUserDataToConfiguration(self):
if not self.isParsed:
self.__parseCommandLine()
errorsList = self.__loadCFGFiles()
if gConfigurationData.getServers():
retVal = self.syncRemoteConfiguration()
if not retVal['OK']:
return retVal
else:
gLogger.warn("Running without remote configuration")
try:
if self.componentType == "service":
self.__setDefaultSection(getServiceSection(self.componentName))
elif self.componentType == "agent":
self.__setDefaultSection(getAgentSection(self.componentName))
elif self.componentType == "executor":
self.__setDefaultSection(getExecutorSection(
self.componentName))
elif self.componentType == "web":
self.__setDefaultSection("/%s" % self.componentName)
elif self.componentType == "script":
if self.componentName and self.componentName[0] == "/":
self.__setDefaultSection(self.componentName)
self.componentName = self.componentName[1:]
else:
self.__setDefaultSection("/Scripts/%s" %
self.componentName)
else:
self.__setDefaultSection("/")
except Exception, e:
errorsList.append(str(e))
def __loadConfigurationData( self ):
try:
os.makedirs( os.path.join( DIRAC.rootPath, "etc", "csbackup" ) )
except:
pass
gConfigurationData.loadConfigurationData()
if gConfigurationData.isMaster():
bBuiltNewConfiguration = False
if not gConfigurationData.getName():
DIRAC.abort( 10, "Missing name for the configuration to be exported!" )
gConfigurationData.exportName()
sVersion = gConfigurationData.getVersion()
if sVersion == "0":
gLogger.info( "There's no version. Generating a new one" )
gConfigurationData.generateNewVersion()
bBuiltNewConfiguration = True
if self.sURL not in gConfigurationData.getServers():
gConfigurationData.setServers( self.sURL )
bBuiltNewConfiguration = True
gConfigurationData.setMasterServer( self.sURL )
if bBuiltNewConfiguration:
gConfigurationData.writeRemoteConfigurationToDisk()
def __loadConfigurationData(self):
try:
os.makedirs(os.path.join(DIRAC.rootPath, "etc", "csbackup"))
except:
pass
gConfigurationData.loadConfigurationData()
if gConfigurationData.isMaster():
bBuiltNewConfiguration = False
if not gConfigurationData.getName():
DIRAC.abort(
10, "Missing name for the configuration to be exported!")
gConfigurationData.exportName()
sVersion = gConfigurationData.getVersion()
if sVersion == "0":
gLogger.info("There's no version. Generating a new one")
gConfigurationData.generateNewVersion()
bBuiltNewConfiguration = True
if self.sURL not in gConfigurationData.getServers():
gConfigurationData.setServers(self.sURL)
bBuiltNewConfiguration = True
gConfigurationData.setMasterServer(self.sURL)
if bBuiltNewConfiguration:
gConfigurationData.writeRemoteConfigurationToDisk()
def __addUserDataToConfiguration( self ):
if not self.isParsed:
self.__parseCommandLine()
errorsList = self.__loadCFGFiles()
if gConfigurationData.getServers():
retVal = self.syncRemoteConfiguration()
if not retVal[ 'OK' ]:
return retVal
else:
gLogger.warn( "Running without remote configuration" )
try:
if self.componentType == "service":
self.__setDefaultSection( getServiceSection( self.componentName ) )
elif self.componentType == "agent":
self.__setDefaultSection( getAgentSection( self.componentName ) )
elif self.componentType == "executor":
self.__setDefaultSection( getExecutorSection( self.componentName ) )
elif self.componentType == "web":
self.__setDefaultSection( "/%s" % self.componentName )
elif self.componentType == "script":
if self.componentName and self.componentName[0] == "/":
self.__setDefaultSection( self.componentName )
self.componentName = self.componentName[1:]
else:
self.__setDefaultSection( "/Scripts/%s" % self.componentName )
else:
self.__setDefaultSection( "/" )
except Exception, e:
errorsList.append( str( e ) )
def __addUserDataToConfiguration(self):
if not self.isParsed:
self.__parseCommandLine()
errorsList = self.__loadCFGFiles()
if gConfigurationData.getServers():
retVal = self.syncRemoteConfiguration()
if not retVal['OK']:
return retVal
else:
gLogger.warn("Running without remote configuration")
try:
if self.componentType == "service":
self.__setDefaultSection(getServiceSection(self.componentName))
elif self.componentType == "agent":
self.__setDefaultSection(getAgentSection(self.componentName))
elif self.componentType == "executor":
self.__setDefaultSection(getExecutorSection(
self.componentName))
elif self.componentType == "web":
self.__setDefaultSection("/%s" % self.componentName)
elif self.componentType == "script":
if self.componentName and self.componentName[0] == "/":
self.__setDefaultSection(self.componentName)
self.componentName = self.componentName[1:]
else:
self.__setDefaultSection("/Scripts/%s" %
self.componentName)
else:
self.__setDefaultSection("/")
except Exception as e:
errorsList.append(str(e))
self.unprocessedSwitches = []
for optionName, optionValue in self.parsedOptionList:
optionName = optionName.lstrip("-")
for definedOptionTuple in self.commandOptionList:
if optionName == definedOptionTuple[0].replace(":", "") or \
optionName == definedOptionTuple[1].replace("=", ""):
if definedOptionTuple[3]:
retVal = definedOptionTuple[3](optionValue)
if not isinstance(retVal, dict):
errorsList.append(
"Callback for switch '%s' does not return S_OK or S_ERROR"
% optionName)
elif not retVal['OK']:
errorsList.append(retVal['Message'])
else:
self.unprocessedSwitches.append(
(optionName, optionValue))
if len(errorsList) > 0:
return S_ERROR("\n%s" % "\n".join(errorsList))
return S_OK()
def __addUserDataToConfiguration( self ):
if not self.isParsed:
self.__parseCommandLine()
errorsList = self.__loadCFGFiles()
if gConfigurationData.getServers():
retVal = self.syncRemoteConfiguration()
if not retVal[ 'OK' ]:
return retVal
else:
gLogger.warn( "Running without remote configuration" )
try:
if self.componentType == "service":
self.__setDefaultSection( getServiceSection( self.componentName ) )
elif self.componentType == "agent":
self.__setDefaultSection( getAgentSection( self.componentName ) )
elif self.componentType == "executor":
self.__setDefaultSection( getExecutorSection( self.componentName ) )
elif self.componentType == "web":
self.__setDefaultSection( "/%s" % self.componentName )
elif self.componentType == "script":
if self.componentName and self.componentName[0] == "/":
self.__setDefaultSection( self.componentName )
self.componentName = self.componentName[1:]
else:
self.__setDefaultSection( "/Scripts/%s" % self.componentName )
else:
self.__setDefaultSection( "/" )
except Exception as e:
errorsList.append( str( e ) )
self.unprocessedSwitches = []
for optionName, optionValue in self.parsedOptionList:
optionName = optionName.lstrip( "-" )
for definedOptionTuple in self.commandOptionList:
if optionName == definedOptionTuple[0].replace( ":", "" ) or \
optionName == definedOptionTuple[1].replace( "=", "" ):
if definedOptionTuple[3]:
retVal = definedOptionTuple[3]( optionValue )
if type( retVal ) != types.DictType:
errorsList.append( "Callback for switch '%s' does not return S_OK or S_ERROR" % optionName )
elif not retVal[ 'OK' ]:
errorsList.append( retVal[ 'Message' ] )
else:
self.unprocessedSwitches.append( ( optionName, optionValue ) )
if len( errorsList ) > 0:
return S_ERROR( "\n%s" % "\n".join( errorsList ) )
return S_OK()
def refreshConfigurationIfNeeded( self ):
if not self.__refreshEnabled or self.__automaticUpdate or not gConfigurationData.getServers():
return
self.__triggeredRefreshLock.acquire()
try:
if not self.__lastRefreshExpired():
return
self.__lastUpdateTime = time.time()
finally:
self.__triggeredRefreshLock.release()
#Launch the refresh
thd = threading.Thread( target = self.__refreshInThread )
thd.setDaemon( 1 )
thd.start()
def refreshConfigurationIfNeeded(self):
"""
Trigger an automatic refresh, most of the time nothing happens because automaticUpdate is enabled.
This function is called by gConfig.getValue most of the time.
We disable pylint error because this class must be instanciated by a mixin to define the missing methods
"""
if not self._refreshEnabled or self._automaticUpdate: # pylint: disable=no-member
return
if not gConfigurationData.getServers() or not self._lastRefreshExpired(
): # pylint: disable=no-member
return
self._lastUpdateTime = time.time()
_IOLoop.current().run_in_executor(None, self._refresh) # pylint: disable=no-member
def _refresh(self, fromMaster=False):
"""
Refresh configuration
"""
self._lastUpdateTime = time.time()
gLogger.debug("Refreshing configuration...")
gatewayList = getGatewayURLs("Configuration/Server")
updatingErrorsList = []
if gatewayList:
initialServerList = gatewayList
gLogger.debug("Using configuration gateway",
str(initialServerList[0]))
elif fromMaster:
masterServer = gConfigurationData.getMasterServer()
initialServerList = [masterServer]
gLogger.debug("Refreshing from master %s" % masterServer)
else:
initialServerList = gConfigurationData.getServers()
gLogger.debug("Refreshing from list %s" % str(initialServerList))
# If no servers in the initial list, we are supposed to use the local configuration only
if not initialServerList:
return S_OK()
randomServerList = List.randomize(initialServerList)
gLogger.debug("Randomized server list is %s" %
", ".join(randomServerList))
for sServer in randomServerList:
from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient
oClient = ConfigurationClient(
url=sServer,
useCertificates=gConfigurationData.useServerCertificate(),
skipCACheck=gConfigurationData.skipCACheck(),
)
dRetVal = _updateFromRemoteLocation(oClient)
if dRetVal["OK"]:
self._refreshTime = gConfigurationData.getRefreshTime()
return dRetVal
else:
updatingErrorsList.append(dRetVal["Message"])
gLogger.warn(
"Can't update from server",
"Error while updating from %s: %s" %
(sServer, dRetVal["Message"]))
if dRetVal["Message"].find("Insane environment") > -1:
break
return S_ERROR("Reason(s):\n\t%s" %
"\n\t".join(List.uniqueElements(updatingErrorsList)))
def refreshConfigurationIfNeeded(self):
if not self.__refreshEnabled or self.__automaticUpdate or not gConfigurationData.getServers(
):
return
self.__triggeredRefreshLock.acquire()
try:
if not self.__lastRefreshExpired():
return
self.__lastUpdateTime = time.time()
finally:
self.__triggeredRefreshLock.release()
#Launch the refresh
thd = threading.Thread(target=self.__refreshInThread)
thd.setDaemon(1)
thd.start()
def refreshConfigurationIfNeeded(self):
"""
Refresh the configuration if automatic updates are disabled, refresher is enabled and servers are defined
"""
if not self._refreshEnabled or self._automaticUpdate or not gConfigurationData.getServers():
return
self._triggeredRefreshLock.acquire()
try:
if not self._lastRefreshExpired():
return
self._lastUpdateTime = time.time()
finally:
try:
self._triggeredRefreshLock.release()
except thread.error:
pass
# Launch the refreshf
thd = threading.Thread(target=self._refreshInThread)
thd.setDaemon(1)
thd.start()
def __refresh(self):
self.__lastUpdateTime = time.time()
gLogger.debug("Refreshing configuration...")
gatewayList = getGatewayURLs("Configuration/Server")
updatingErrorsList = []
if gatewayList:
initialServerList = gatewayList
gLogger.debug("Using configuration gateway",
str(initialServerList[0]))
else:
initialServerList = gConfigurationData.getServers()
gLogger.debug("Refreshing from list %s" % str(initialServerList))
# If no servers in the initial list, we are supposed to use the local configuration only
if not initialServerList:
return S_OK()
randomServerList = List.randomize(initialServerList)
gLogger.debug("Randomized server list is %s" %
", ".join(randomServerList))
for sServer in randomServerList:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient(
sServer,
useCertificates=gConfigurationData.useServerCertificate(),
skipCACheck=gConfigurationData.skipCACheck())
dRetVal = _updateFromRemoteLocation(oClient)
if dRetVal['OK']:
return dRetVal
else:
updatingErrorsList.append(dRetVal['Message'])
gLogger.warn(
"Can't update from server",
"Error while updating from %s: %s" %
(sServer, dRetVal['Message']))
if dRetVal['Message'].find("Insane environment") > -1:
break
return S_ERROR("Reason(s):\n\t%s" %
"\n\t".join(List.uniqueElements(updatingErrorsList)))
def __refresh(self):
self.__lastUpdateTime = time.time()
gLogger.debug("Refreshing configuration...")
gatewayList = getGatewayURLs("Configuration/Server")
updatingErrorsList = []
if gatewayList:
initialServerList = gatewayList
gLogger.debug("Using configuration gateway", str(initialServerList[0]))
else:
initialServerList = gConfigurationData.getServers()
gLogger.debug("Refreshing from list %s" % str(initialServerList))
# If no servers in the initial list, we are supposed to use the local configuration only
if not initialServerList:
return S_OK()
randomServerList = List.randomize(initialServerList)
gLogger.debug("Randomized server list is %s" % ", ".join(randomServerList))
for sServer in randomServerList:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient(
sServer,
useCertificates=gConfigurationData.useServerCertificate(),
skipCACheck=gConfigurationData.skipCACheck(),
)
dRetVal = _updateFromRemoteLocation(oClient)
if dRetVal["OK"]:
return dRetVal
else:
updatingErrorsList.append(dRetVal["Message"])
gLogger.warn(
"Can't update from server", "Error while updating from %s: %s" % (sServer, dRetVal["Message"])
)
if dRetVal["Message"].find("Insane environment") > -1:
break
return S_ERROR("Reason(s):\n\t%s" % "\n\t".join(List.uniqueElements(updatingErrorsList)))
def getServersList(self):
return gConfigurationData.getServers()
def getServersList(self):
""" Get list of servers
:return: list
"""
return gConfigurationData.getServers()
def getServersList( self ): return gConfigurationData.getServers()