def __loadConfigurationData( self ):
try:
os.makedirs( os.path.join( DIRAC.rootPath, "etc", "csbackup" ) )
except:
pass
gConfigurationData.loadConfigurationData()
if gConfigurationData.isMaster():
bBuiltNewConfiguration = False
if not gConfigurationData.getName():
DIRAC.abort( 10, "Missing name for the configuration to be exported!" )
gConfigurationData.exportName()
sVersion = gConfigurationData.getVersion()
if sVersion == "0":
gLogger.info( "There's no version. Generating a new one" )
gConfigurationData.generateNewVersion()
bBuiltNewConfiguration = True
if self.sURL not in gConfigurationData.getServers():
gConfigurationData.setServers( self.sURL )
bBuiltNewConfiguration = True
gConfigurationData.setMasterServer( self.sURL )
if bBuiltNewConfiguration:
gConfigurationData.writeRemoteConfigurationToDisk()
def __loadConfigurationData(self):
try:
os.makedirs(os.path.join(DIRAC.rootPath, "etc", "csbackup"))
except:
pass
gConfigurationData.loadConfigurationData()
if gConfigurationData.isMaster():
bBuiltNewConfiguration = False
if not gConfigurationData.getName():
DIRAC.abort(
10, "Missing name for the configuration to be exported!")
gConfigurationData.exportName()
sVersion = gConfigurationData.getVersion()
if sVersion == "0":
gLogger.info("There's no version. Generating a new one")
gConfigurationData.generateNewVersion()
bBuiltNewConfiguration = True
if self.sURL not in gConfigurationData.getServers():
gConfigurationData.setServers(self.sURL)
bBuiltNewConfiguration = True
gConfigurationData.setMasterServer(self.sURL)
if bBuiltNewConfiguration:
gConfigurationData.writeRemoteConfigurationToDisk()
def do_connect(self, line):
"""connect
Connect to the CS
Usage: connect <URL> (Connect to the CS at the specified URL)
connect (Connect to the default CS URL of your config)
"""
if line == "":
self.serverURL = gConfigurationData.getMasterServer()
self.serverName = gConfigurationData.getName()
else:
self.serverURL = self.serverName = line
if self.serverURL == None:
print "Unable to connect to the default server. Maybe you don't have a proxy ?"
return self.do_disconnect("")
print "Trying to connect to " + self.serverURL + "...",
self.modificator = Modificator(RPCClient(self.serverURL))
rv = self.modificator.loadFromRemote()
rv2 = self.modificator.loadCredentials()
if rv['OK'] == False or rv2['OK'] == False:
print "failed: ",
if rv['OK'] == False: print rv['Message']
else: print rv2['Message']
self.connected = False
self.update_prompt()
def do_connect(self, line):
"""connect
Connect to the CS
Usage: connect <URL> (Connect to the CS at the specified URL)
connect (Connect to the default CS URL of your config)
"""
if line == "":
self.serverURL = gConfigurationData.getMasterServer()
self.serverName = gConfigurationData.getName()
else:
self.serverURL = self.serverName = line
if self.serverURL == None:
print "Unable to connect to the default server. Maybe you don't have a proxy ?"
return self.do_disconnect("")
print "Trying to connect to " + self.serverURL + "...",
self.modificator = Modificator(RPCClient(self.serverURL))
rv = self.modificator.loadFromRemote()
rv2 = self.modificator.loadCredentials()
if rv['OK'] == False or rv2['OK'] == False:
print "failed: ",
if rv['OK'] == False: print rv['Message']
else: print rv2['Message']
self.connected = False
self.update_prompt()
def do_connect(self, line):
"""connect
Connect to the CS
Usage: connect <URL> (Connect to the CS at the specified URL)
connect (Connect to the default CS URL of your config)
"""
if line == "":
self.serverURL = gConfigurationData.getMasterServer()
self.serverName = gConfigurationData.getName()
else:
self.serverURL = self.serverName = line
if self.serverURL is None:
print("Unable to connect to the default server. Maybe you don't have a proxy ?")
return self.do_disconnect("")
print("Trying to connect to " + self.serverURL + "...", end=" ")
self.modificator = Modificator(ConfigurationClient(url=self.serverURL))
rv = self.modificator.loadFromRemote()
rv2 = self.modificator.loadCredentials()
if rv["OK"] == False or rv2["OK"] == False:
print("failed: ", end=" ")
if rv["OK"] is False:
print(rv["Message"])
else:
print(rv2["Message"])
self.connected = False
self.update_prompt()
else:
self.connected = True
self.update_prompt()
print("done.")
def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False):
"""
Update the master configuration with the newly received changes
:param str sBuffer: newly received configuration data
:param str committer: the user name of the committer
:param bool updateVersionOption: flag to update the current configuration version
:return: S_OK/S_ERROR of the write-to-disk of the new configuration
"""
if not gConfigurationData.isMaster():
return S_ERROR("Configuration modification is not allowed in this server")
# Load the data in a ConfigurationData object
oRemoteConfData = ConfigurationData(False)
oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer)
if updateVersionOption:
oRemoteConfData.setVersion(gConfigurationData.getVersion())
# Test that remote and new versions are the same
sRemoteVersion = oRemoteConfData.getVersion()
sLocalVersion = gConfigurationData.getVersion()
gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion))
if sRemoteVersion != sLocalVersion:
if not gConfigurationData.mergingEnabled():
return S_ERROR("Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion))
else:
gLogger.info("AutoMerging new data!")
if updateVersionOption:
return S_ERROR("Cannot AutoMerge! version was overwritten")
result = self.__mergeIndependentUpdates(oRemoteConfData)
if not result['OK']:
gLogger.warn("Could not AutoMerge!", result['Message'])
return S_ERROR("AutoMerge failed: %s" % result['Message'])
requestedRemoteCFG = result['Value']
gLogger.info("AutoMerge successful!")
oRemoteConfData.setRemoteCFG(requestedRemoteCFG)
# Test that configuration names are the same
sRemoteName = oRemoteConfData.getName()
sLocalName = gConfigurationData.getName()
if sRemoteName != sLocalName:
return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName))
# Update and generate a new version
gLogger.info("Committing new data...")
gConfigurationData.lock()
gLogger.info("Setting the new CFG")
gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG())
gConfigurationData.unlock()
gLogger.info("Generating new version")
gConfigurationData.generateNewVersion()
# self.__checkSlavesStatus( forceWriteConfiguration = True )
gLogger.info("Writing new version to disk")
retVal = gConfigurationData.writeRemoteConfigurationToDisk("%s@%s" % (committer, gConfigurationData.getVersion()))
gLogger.info("New version", gConfigurationData.getVersion())
# Attempt to update the configuration on currently registered slave services
if gConfigurationData.getAutoSlaveSync():
result = self.forceSlavesUpdate()
if not result['OK']:
gLogger.warn('Failed to update slave servers')
return retVal
def autoRefreshAndPublish( self, sURL ):
gLogger.debug( "Setting configuration refresh as automatic" )
if not gConfigurationData.getAutoPublish():
gLogger.debug( "Slave server won't auto publish itself" )
if not gConfigurationData.getName():
import DIRAC
DIRAC.abort( 10, "Missing configuration name!" )
self.__url = sURL
self.__automaticUpdate = True
self.setDaemon( 1 )
self.start()
def autoRefreshAndPublish(self, sURL):
gLogger.debug("Setting configuration refresh as automatic")
if not gConfigurationData.getAutoPublish():
gLogger.debug("Slave server won't auto publish itself")
if not gConfigurationData.getName():
import DIRAC
DIRAC.abort(10, "Missing configuration name!")
self.__url = sURL
self.__automaticUpdate = True
self.setDaemon(1)
self.start()
def updateConfiguration(self,
sBuffer,
commiterDN="",
updateVersionOption=False):
if not gConfigurationData.isMaster():
return S_ERROR(
"Configuration modification is not allowed in this server")
#Load the data in a ConfigurationData object
oRemoteConfData = ConfigurationData(False)
oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer)
if updateVersionOption:
oRemoteConfData.setVersion(gConfigurationData.getVersion())
#Test that remote and new versions are the same
sRemoteVersion = oRemoteConfData.getVersion()
sLocalVersion = gConfigurationData.getVersion()
gLogger.info("Checking versions\nremote: %s\nlocal: %s" %
(sRemoteVersion, sLocalVersion))
if sRemoteVersion != sLocalVersion:
if not gConfigurationData.mergingEnabled():
return S_ERROR(
"Local and remote versions differ (%s vs %s). Cannot commit."
% (sLocalVersion, sRemoteVersion))
else:
gLogger.info("AutoMerging new data!")
if updateVersionOption:
return S_ERROR("Cannot AutoMerge! version was overwritten")
result = self.__mergeIndependentUpdates(oRemoteConfData)
if not result['OK']:
gLogger.warn("Could not AutoMerge!", result['Message'])
return S_ERROR("AutoMerge failed: %s" % result['Message'])
requestedRemoteCFG = result['Value']
gLogger.info("AutoMerge successful!")
oRemoteConfData.setRemoteCFG(requestedRemoteCFG)
#Test that configuration names are the same
sRemoteName = oRemoteConfData.getName()
sLocalName = gConfigurationData.getName()
if sRemoteName != sLocalName:
return S_ERROR("Names differ: Server is %s and remote is %s" %
(sLocalName, sRemoteName))
#Update and generate a new version
gLogger.info("Committing new data...")
gConfigurationData.lock()
gLogger.info("Setting the new CFG")
gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG())
gConfigurationData.unlock()
gLogger.info("Generating new version")
gConfigurationData.generateNewVersion()
#self.__checkSlavesStatus( forceWriteConfiguration = True )
gLogger.info("Writing new version to disk!")
retVal = gConfigurationData.writeRemoteConfigurationToDisk(
"%s@%s" % (commiterDN, gConfigurationData.getVersion()))
gLogger.info("New version it is!")
return retVal
def __getCfgBackups( self, basePath, date = "", subPath = "" ):
rs = re.compile( "^%s\..*%s.*\.zip$" % ( gConfigurationData.getName(), date ) )
fsEntries = os.listdir( "%s/%s" % ( basePath, subPath ) )
fsEntries.sort( reverse = True )
backupsList = []
for entry in fsEntries:
entryPath = "%s/%s/%s" % ( basePath, subPath, entry )
if os.path.isdir( entryPath ):
backupsList.extend( self.__getCfgBackups( basePath, date, "%s/%s" % ( subPath, entry ) ) )
elif os.path.isfile( entryPath ):
if rs.search( entry ):
backupsList.append( "%s/%s" % ( subPath, entry ) )
return backupsList
def __getCfgBackups(self, basePath, date="", subPath=""):
rs = re.compile(r"^%s\..*%s.*\.zip$" % (gConfigurationData.getName(), date))
fsEntries = os.listdir("%s/%s" % (basePath, subPath))
fsEntries.sort(reverse=True)
backupsList = []
for entry in fsEntries:
entryPath = "%s/%s/%s" % (basePath, subPath, entry)
if os.path.isdir(entryPath):
backupsList.extend(self.__getCfgBackups(basePath, date, "%s/%s" % (subPath, entry)))
elif os.path.isfile(entryPath):
if rs.search(entry):
backupsList.append("%s/%s" % (subPath, entry))
return backupsList
def updateConfiguration(self, sBuffer, commiter="", updateVersionOption=False):
if not gConfigurationData.isMaster():
return S_ERROR("Configuration modification is not allowed in this server")
# Load the data in a ConfigurationData object
oRemoteConfData = ConfigurationData(False)
oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer)
if updateVersionOption:
oRemoteConfData.setVersion(gConfigurationData.getVersion())
# Test that remote and new versions are the same
sRemoteVersion = oRemoteConfData.getVersion()
sLocalVersion = gConfigurationData.getVersion()
gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion))
if sRemoteVersion != sLocalVersion:
if not gConfigurationData.mergingEnabled():
return S_ERROR(
"Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion)
)
else:
gLogger.info("AutoMerging new data!")
if updateVersionOption:
return S_ERROR("Cannot AutoMerge! version was overwritten")
result = self.__mergeIndependentUpdates(oRemoteConfData)
if not result["OK"]:
gLogger.warn("Could not AutoMerge!", result["Message"])
return S_ERROR("AutoMerge failed: %s" % result["Message"])
requestedRemoteCFG = result["Value"]
gLogger.info("AutoMerge successful!")
oRemoteConfData.setRemoteCFG(requestedRemoteCFG)
# Test that configuration names are the same
sRemoteName = oRemoteConfData.getName()
sLocalName = gConfigurationData.getName()
if sRemoteName != sLocalName:
return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName))
# Update and generate a new version
gLogger.info("Committing new data...")
gConfigurationData.lock()
gLogger.info("Setting the new CFG")
gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG())
gConfigurationData.unlock()
gLogger.info("Generating new version")
gConfigurationData.generateNewVersion()
# self.__checkSlavesStatus( forceWriteConfiguration = True )
gLogger.info("Writing new version to disk!")
retVal = gConfigurationData.writeRemoteConfigurationToDisk(
"%s@%s" % (commiter, gConfigurationData.getVersion())
)
gLogger.info("New version it is!")
return retVal
def autoRefreshAndPublish(self, sURL):
"""
Start the autorefresh background task
:param str sURL: URL of the configuration server
"""
gLogger.debug("Setting configuration refresh as automatic")
if not gConfigurationData.getAutoPublish():
gLogger.debug("Slave server won't auto publish itself")
if not gConfigurationData.getName():
import DIRAC
DIRAC.abort(10, "Missing configuration name!")
self._url = sURL
self._automaticUpdate = True
self.setDaemon(1)
self.start()
def autoRefreshAndPublish(self, sURL):
"""
Start the autorefresh background task, called by ServiceInterface
(the class behind the Configuration/Server handler)
:param str sURL: URL of the configuration server
"""
gLogger.debug("Setting configuration refresh as automatic")
if not gConfigurationData.getAutoPublish():
gLogger.debug("Slave server won't auto publish itself")
if not gConfigurationData.getName():
import DIRAC
DIRAC.abort(10, "Missing configuration name!")
self._url = sURL
self._automaticUpdate = True
# Tornado replacement solution to the classic thread
# It start the method self.__refreshLoop on the next IOLoop iteration
_IOLoop.current().spawn_callback(self.__refreshLoop)
