def updateFlag():
# Here now setting the flag as the following inside /Operations/Defaults:
# in Operations/Defaults/Services/JobMonitoring/useESForJobParametersFlag
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
csAPI = CSAPI()
res = csAPI.createSection('Operations/Defaults/Services/')
if not res['OK']:
print(res['Message'])
exit(1)
res = csAPI.createSection('Operations/Defaults/Services/JobMonitoring/')
if not res['OK']:
print(res['Message'])
exit(1)
csAPI.setOption(
'Operations/Defaults/Services/JobMonitoring/useESForJobParametersFlag',
True)
csAPI.commit()
# Now we need to restart the services for the new configuration to be picked up
time.sleep(2)
os.system("dirac-restart-component WorkloadManagement JobMonitoring")
os.system("dirac-restart-component WorkloadManagement JobStateUpdate")
time.sleep(5)
# ProductionSandboxSE
# {
# BackendType = DISET
# AccessProtocol.1
# {
# Host = localhost
# Port = 9196
# ProtocolName = DIP
# Protocol = dips
# Path = /scratch/workspace/%s/sandboxes % setupName
# Access = remote
# SpaceToken =
# WSUrl =
# }
# }
res = csAPI.createSection( 'Resources/StorageElements/' )
if not res['OK']:
print res['Message']
exit( 1 )
res = csAPI.createSection( 'Resources/StorageElements/ProductionSandboxSE' )
if not res['OK']:
print res['Message']
exit( 1 )
csAPI.setOption( 'Resources/StorageElements/ProductionSandboxSE/BackendType', 'DISET' )
res = csAPI.createSection( 'Resources/StorageElements/ProductionSandboxSE/AccessProtocol.1' )
if not res['OK']:
print res['Message']
exit( 1 )
csAPI.setOption( 'Resources/StorageElements/ProductionSandboxSE/AccessProtocol.1/Host', 'localhost' )
# Databases
# {
# FileCatalogDB
# {
# DBName = FileCatalogDB
# }
# }
# }
# }
for sct in [
'Systems/DataManagement', 'Systems/DataManagement/Production',
'Systems/DataManagement/Production/Databases',
'Systems/DataManagement/Production/Databases/FileCatalogDB'
]:
res = csAPI.createSection(sct)
if not res['OK']:
print(res['Message'])
exit(1)
dbHost = os.environ['DB_HOST']
dbPort = os.environ['DB_PORT']
csAPI.setOption(
'Systems/DataManagement/Production/Databases/FileCatalogDB/DBName',
'FileCatalogDB')
csAPI.setOption(
'Systems/DataManagement/Production/Databases/FileCatalogDB/Host', dbHost)
csAPI.setOption(
'Systems/DataManagement/Production/Databases/FileCatalogDB/Port', dbPort)
# Now setting a SandboxSE as the following:
# ProductionSandboxSE
# {
# BackendType = DISET
# AccessProtocol = dips
# DIP
# {
# Host = localhost
# Port = 9196
# ProtocolName = DIP
# Protocol = dips
# Path = /scratch/workspace/%s/sandboxes % setupName
# Access = remote
# }
# }
res = csAPI.createSection('Resources/StorageElements/')
if not res['OK']:
print res['Message']
exit(1)
res = csAPI.createSection('Resources/StorageElements/ProductionSandboxSE')
if not res['OK']:
print res['Message']
exit(1)
csAPI.setOption('Resources/StorageElements/ProductionSandboxSE/BackendType',
'DISET')
csAPI.setOption('Resources/StorageElements/ProductionSandboxSE/AccessProtocol',
'dips')
res = csAPI.createSection('Resources/StorageElements/ProductionSandboxSE/DIP')
if not res['OK']:
class DDSimTarMaker( object ):
""" create a tarball of the DDSim release """
def __init__( self ):
self.detmodels = {}
self.lcgeo_env="lcgeo_DIR"
self.ddhep_env="DD4HEP"
self.softSec = "/Operations/Defaults/AvailableTarBalls"
self.version = ''
self.platform = 'x86_64-slc5-gcc43-opt'
self.comment = ""
self.name = "ddsim"
self.csapi = None
self.tarBallName = None
self.md5sum = None
def copyDetectorModels( self, basePath, folder, targetFolder ):
"""copy the compact folders to the targetFolder """
for root,dirs,_files in os.walk( os.path.join(basePath, folder) ):
for direct in dirs:
if root.endswith("compact"):
## the main xml file must have the same name as the folder, ILD and CLIC follow this convention already
xmlPath = os.path.join( root, direct, direct+".xml")
if os.path.exists( xmlPath ):
self.detmodels[direct] = "detectors/"+direct+"/"+direct+".xml"
copyFolder( os.path.join(root, direct), targetFolder )
def createTarBall( self, folder ):
"""create a tar ball from the folder
tar zcf $TARBALLNAME $LIBFOLDER/*
"""
##Create the Tarball
if os.path.exists(self.tarBallName):
os.remove(self.tarBallName)
gLogger.notice("Creating Tarball...")
myappTar = tarfile.open(self.tarBallName, "w:gz")
myappTar.add(folder, self.tarBallName[:-4])
myappTar.close()
self.md5sum = md5.md5(open( self.tarBallName, 'r' ).read()).hexdigest()
gLogger.notice("...Done")
return S_OK( "Created Tarball")
def parseArgs( self ):
""" parse the command line arguments"""
if len(sys.argv) != 3:
raise RuntimeError( "Wrong number of arguments in call: '%s'" % " ".join(sys.argv) )
self.name = sys.argv[1]
self.version = sys.argv[2]
self.tarBallName = "%s%s.tgz" % (self.name, self.version)
def checkEnvironment( self ):
""" check if dd4hep and lcgeo are in the environment """
for var in [ self.ddhep_env, self.lcgeo_env , 'ROOTSYS' ]:
if var not in os.environ:
raise RuntimeError( "%s is not set" % var )
return os.environ[self.ddhep_env], os.environ[self.lcgeo_env], os.environ['ROOTSYS']
def createCSEntry( self ):
"""add the entries for this version into the Configuration System
<version>
{
TarBall = ddsim<version>.tgz
AdditionalEnvVar
{
ROOTSYS = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/root/5.34.30
G4INSTALL = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/geant4/10.01
G4DATA = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/geant4/10.01/share/Geant4-10.1.0/data
}
Overwrite = True
}
Operations/DDSimDetectorModels/<Version>
{
CLIC_o2_v03 = detectors/CLIC_o2_v03/CLIC_o2_v03.xml
...
}
"""
#FIXME: Get root and geant4 location from environment, make sure it is cvmfs
csParameter = { "TarBall": self.tarBallName,
"AdditionalEnvVar": {
"ROOTSYS" : os.environ.get("ROOTSYS"),
"G4INSTALL" : os.environ.get("G4INSTALL"),
"G4DATA" : os.environ.get("G4DATA"),
},
"Md5Sum": self.md5sum,
}
pars = dict( platform=self.platform,
name="ddsim",
version=self.version
)
csPath = os.path.join( self.softSec , "%(platform)s/%(name)s/%(version)s/" % pars )
pprint(csParameter)
result = self.insertCSSection( csPath, csParameter )
csPathModels = "Operations/Defaults/DDSimDetectorModels"
csModels = { self.version : self.detmodels }
pprint(csModels)
result = self.insertCSSection( csPathModels, csModels )
if self.csapi is not None:
resProxy = checkOrGetGroupProxy( "diracAdmin" )
if not resProxy['OK']:
gLogger.error( "Failed to get AdminProxy", resProxy['Message'] )
raise RuntimeError( "Failed to get diracAdminProxy" )
self.csapi.commit()
if not result['OK']:
gLogger.error( "Failed to create CS Section", result['Message'] )
raise RuntimeError( "Failed to create CS Section" )
def insertCSSection( self, path, pardict ):
""" insert a section and values (or subsections) into the CS
:param str path: full path of the new section
:param str pardict: dictionary of key values in the new section, values can also be dictionaries
:return: S_OK(), S_ERROR()
"""
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
if self.csapi is None:
self.csapi = CSAPI()
for key, value in pardict.iteritems():
newSectionPath = os.path.join(path,key)
gLogger.debug( "Adding to cs %s : %s " % ( newSectionPath, value ) )
self.csapi.createSection( path )
if isinstance( value, dict ):
res = self.insertCSSection( newSectionPath, value )
else:
res = self.csapi.setOption( newSectionPath, value )
if not res['OK']:
return res
else:
gLogger.notice( "Added to CS: %s " % res['Value'] )
return S_OK("Added all things to cs")
def createDDSimTarBall( self ):
""" do everything to create the DDSim tarball"""
self.parseArgs()
ddBase, lcgeoBase, _rootsys = self.checkEnvironment()
realTargetFolder = os.path.join( os.getcwd(), self.name+self.version )
targetFolder = os.path.join( os.getcwd(), "temp", self.name+self.version )
for folder in (targetFolder, targetFolder+"/lib"):
try:
os.makedirs( folder )
except OSError:
pass
libraries = set()
rootmaps = set()
dd4hepLibPath = getLibraryPath( ddBase )
lcgeoPath = getLibraryPath( lcgeoBase )
self.copyDetectorModels( lcgeoBase, "CLIC" , targetFolder+"/detectors" )
self.copyDetectorModels( lcgeoBase, "ILD" , targetFolder+"/detectors" )
copyFolder( ddBase+"/DDDetectors/compact", realTargetFolder.rstrip("/")+"/DDDetectors")
libraries.update( getFiles( dd4hepLibPath, ".so") )
libraries.update( getFiles( lcgeoPath, ".so" ) )
rootmaps.update( getFiles( dd4hepLibPath, ".rootmap") )
rootmaps.update( getFiles( lcgeoPath, ".rootmap" ) )
pprint( libraries )
pprint( rootmaps )
allLibs = set()
for lib in libraries:
allLibs.update( getDependentLibraries(lib) )
### remote root and geant4 libraries, we pick them up from
allLibs = set( [ lib for lib in allLibs if not ( "/geant4/" in lib.lower() or "/root/" in lib.lower()) ] )
print allLibs
copyLibraries( libraries, targetFolder+"/lib" )
copyLibraries( allLibs, targetFolder+"/lib" )
copyLibraries( rootmaps, targetFolder+"/lib" )
getPythonStuff( ddBase+"/python" , targetFolder+"/lib/")
getPythonStuff( lcgeoBase+"/lib/python", targetFolder+"/lib/" )
getPythonStuff( lcgeoBase+"/bin/ddsim", targetFolder+"/bin/" )
##Should get this from CVMFS
#getRootStuff( rootsys, targetFolder+"/ROOT" )
copyFolder( targetFolder+"/", realTargetFolder.rstrip("/") )
killRPath( realTargetFolder )
resolveLinks( realTargetFolder+"/lib" )
removeSystemLibraries( realTargetFolder+"/lib" )
#removeSystemLibraries( realTargetFolder+"/ROOT/lib" )
print self.detmodels
self.createTarBall( realTargetFolder )
self.createCSEntry()
# Now setting a SandboxSE as the following:
# ProductionSandboxSE
# {
# BackendType = DISET
# AccessProtocol = dips
# DIP
# {
# Host = localhost
# Port = 9196
# ProtocolName = DIP
# Protocol = dips
# Path = /scratch/workspace/%s/sandboxes % setupName
# Access = remote
# }
# }
res = csAPI.createSection("Resources/StorageElements/")
if not res["OK"]:
print(res["Message"])
exit(1)
res = csAPI.createSection("Resources/StorageElements/ProductionSandboxSE")
if not res["OK"]:
print(res["Message"])
exit(1)
csAPI.setOption("Resources/StorageElements/ProductionSandboxSE/BackendType",
"DISET")
csAPI.setOption("Resources/StorageElements/ProductionSandboxSE/AccessProtocol",
"dips")
res = csAPI.createSection("Resources/StorageElements/ProductionSandboxSE/DIP")
if not res["OK"]:
class GOCDB2CSAgent(AgentModule):
""" Class to retrieve information about service endpoints
from GOCDB and update configuration stored by CS
"""
def __init__(self, *args, **kwargs):
""" c'tor
"""
super(GOCDB2CSAgent, self).__init__(*args, **kwargs)
self.GOCDBClient = None
self.csAPI = None
self.dryRun = False
def initialize(self):
""" Run at the agent initialization (normally every 500 cycles)
"""
# client to connect to GOCDB
self.GOCDBClient = GOCDBClient()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
# API needed to update configuration stored by CS
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
"""
Execute GOCDB queries according to the function map
and user request (options in configuration).
"""
# __functionMap is at the end of the class definition
for option, functionCall in GOCDB2CSAgent.__functionMap.iteritems():
optionValue = self.am_getOption(option, True)
if optionValue:
result = functionCall(self)
if not result['OK']:
self.log.error("%s() failed with message: %s" %
(functionCall.__name__, result['Message']))
else:
self.log.info("Successfully executed %s" %
functionCall.__name__)
return S_OK()
def updatePerfSONARConfiguration(self):
"""
Get current status of perfSONAR endpoints from GOCDB
and update CS configuration accordingly.
"""
log = self.log.getSubLogger('updatePerfSONAREndpoints')
log.debug('Begin function ...')
# get endpoints
result = self.__getPerfSONAREndpoints()
if not result['OK']:
log.error("__getPerfSONAREndpoints() failed with message: %s" %
result['Message'])
return S_ERROR('Unable to fetch perfSONAR endpoints from GOCDB.')
endpointList = result['Value']
# add DIRAC site name
result = self.__addDIRACSiteName(endpointList)
if not result['OK']:
log.error("__addDIRACSiteName() failed with message: %s" %
result['Message'])
return S_ERROR('Unable to extend the list with DIRAC site names.')
extendedEndpointList = result['Value']
# prepare dictionary with new configuration
result = self.__preparePerfSONARConfiguration(extendedEndpointList)
if not result['OK']:
log.error(
"__preparePerfSONARConfiguration() failed with message: %s" %
result['Message'])
return S_ERROR('Unable to prepare a new perfSONAR configuration.')
finalConfiguration = result['Value']
# update configuration according to the final status of endpoints
self.__updateConfiguration(finalConfiguration)
log.debug("Configuration updated succesfully")
log.debug('End function.')
return S_OK()
def __getPerfSONAREndpoints(self):
"""
Retrieve perfSONAR endpoint information directly from GOCDB.
:return: List of perfSONAR endpoints (dictionaries) as stored by GOCDB.
"""
log = self.log.getSubLogger('__getPerfSONAREndpoints')
log.debug('Begin function ...')
# get perfSONAR endpoints (latency and bandwidth) form GOCDB
endpointList = []
for endpointType in ['Latency', 'Bandwidth']:
result = self.GOCDBClient.getServiceEndpointInfo(
'service_type', 'net.perfSONAR.%s' % endpointType)
if not result['OK']:
log.error("getServiceEndpointInfo() failed with message: %s" %
result['Message'])
return S_ERROR('Could not fetch %s endpoints from GOCDB' %
endpointType.lower())
log.debug('Number of %s endpoints: %s' %
(endpointType.lower(), len(result['Value'])))
endpointList.extend(result['Value'])
log.debug('Number of perfSONAR endpoints: %s' % len(endpointList))
log.debug('End function.')
return S_OK(endpointList)
def __preparePerfSONARConfiguration(self, endpointList):
"""
Prepare a dictionary with a new CS configuration of perfSONAR endpoints.
:return: Dictionary where keys are configuration paths (options and sections)
and values are values of corresponding options
or None in case of a path pointing to a section.
"""
log = self.log.getSubLogger('__preparePerfSONARConfiguration')
log.debug('Begin function ...')
# static elements of a path
rootPath = '/Resources/Sites'
extPath = 'Network'
baseOptionName = 'Enabled'
options = {baseOptionName: 'True', 'ServiceType': 'perfSONAR'}
# enable GOCDB endpoints in configuration
newConfiguration = {}
for endpoint in endpointList:
if endpoint['DIRACSITENAME'] is None:
continue
split = endpoint['DIRACSITENAME'].split('.')
path = cfgPath(rootPath, split[0], endpoint['DIRACSITENAME'],
extPath, endpoint['HOSTNAME'])
for name, defaultValue in options.iteritems():
newConfiguration[cfgPath(path, name)] = defaultValue
# get current configuration
currentConfiguration = {}
for option in options.iterkeys():
result = gConfig.getConfigurationTree(rootPath, extPath + '/',
'/' + option)
if not result['OK']:
log.error("getConfigurationTree() failed with message: %s" %
result['Message'])
return S_ERROR('Unable to fetch perfSONAR endpoints from CS.')
currentConfiguration.update(result['Value'])
# disable endpoints that disappeared in GOCDB
removedElements = set(currentConfiguration) - set(newConfiguration)
newElements = set(newConfiguration) - set(currentConfiguration)
addedEndpoints = len(newElements) / len(options)
disabledEndpoints = 0
for path in removedElements:
if baseOptionName in path:
newConfiguration[path] = 'False'
if currentConfiguration[path] != 'False':
disabledEndpoints = disabledEndpoints + 1
# inform what will be changed
if addedEndpoints > 0:
self.log.info(
"%s new perfSONAR endpoints will be added to the configuration"
% addedEndpoints)
if disabledEndpoints > 0:
self.log.info(
"%s old perfSONAR endpoints will be disable in the configuration"
% disabledEndpoints)
if addedEndpoints == 0 and disabledEndpoints == 0:
self.log.info("perfSONAR configuration is up-to-date")
log.debug('End function.')
return S_OK(newConfiguration)
def __addDIRACSiteName(self, inputList):
"""
Extend given list of GOCDB endpoints with DIRAC site name, i.e.
add an entry "DIRACSITENAME" in dictionaries that describe endpoints.
If given site name could not be found "DIRACSITENAME" is set to 'None'.
:return: List of perfSONAR endpoints (dictionaries).
"""
log = self.log.getSubLogger('__addDIRACSiteName')
log.debug('Begin function ...')
# get site name dictionary
result = getDIRACGOCDictionary()
if not result['OK']:
log.error("getDIRACGOCDictionary() failed with message: %s" %
result['Message'])
return S_ERROR('Could not get site name dictionary')
# reverse the dictionary (assume 1 to 1 relation)
DIRACGOCDict = result['Value']
GOCDIRACDict = dict(zip(DIRACGOCDict.values(), DIRACGOCDict.keys()))
# add DIRAC site names
outputList = []
for entry in inputList:
try:
entry['DIRACSITENAME'] = GOCDIRACDict[entry['SITENAME']]
except KeyError:
self.log.warn("No dictionary entry for %s. " %
entry['SITENAME'])
entry['DIRACSITENAME'] = None
outputList.append(entry)
log.debug('End function.')
return S_OK(outputList)
def __updateConfiguration(self, setElements=None, delElements=None):
"""
Update configuration stored by CS.
"""
if setElements is None:
setElements = {}
if delElements is None:
delElements = []
log = self.log.getSubLogger('__updateConfiguration')
log.debug('Begin function ...')
# assure existence and proper value of a section or an option
for path, value in setElements.iteritems():
if value is None:
section = path
else:
split = path.rsplit('/', 1)
section = split[0]
try:
result = self.csAPI.createSection(section)
if not result['OK']:
log.error("createSection() failed with message: %s" %
result['Message'])
except Exception as e:
log.error("Exception in createSection(): %s" %
repr(e).replace(',)', ')'))
if value is not None:
try:
result = self.csAPI.setOption(path, value)
if not result['OK']:
log.error("setOption() failed with message: %s" %
result['Message'])
except Exception as e:
log.error("Exception in setOption(): %s" %
repr(e).replace(',)', ')'))
# delete elements in the configuration
for path in delElements:
result = self.csAPI.delOption(path)
if not result['OK']:
log.warn("delOption() failed with message: %s" %
result['Message'])
result = self.csAPI.delSection(path)
if not result['OK']:
log.warn("delSection() failed with message: %s" %
result['Message'])
if self.dryRun:
log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
# update configuration stored by CS
result = self.csAPI.commit()
if not result['OK']:
log.error("commit() failed with message: %s" %
result['Message'])
return S_ERROR("Could not commit changes to CS.")
else:
log.info("Committed changes to CS")
log.debug('End function.')
return S_OK()
# define mapping between an agent option in the configuration and a function call
__functionMap = {
'UpdatePerfSONARS': updatePerfSONARConfiguration,
}
# ProductionSandboxSE
# {
# BackendType = DISET
# AccessProtocol.1
# {
# Host = localhost
# Port = 9196
# ProtocolName = DIP
# Protocol = dips
# Path = /scratch/workspace/%s/sandboxes % setupName
# Access = remote
# SpaceToken =
# WSUrl =
# }
# }
res = csAPI.createSection( 'Resources/StorageElements/' )
if not res['OK']:
print res['Message']
exit( 1 )
res = csAPI.createSection( 'Resources/StorageElements/ProductionSandboxSE' )
if not res['OK']:
print res['Message']
exit( 1 )
csAPI.setOption( 'Resources/StorageElements/ProductionSandboxSE/BackendType', 'DISET' )
res = csAPI.createSection( 'Resources/StorageElements/ProductionSandboxSE/AccessProtocol.1' )
if not res['OK']:
print res['Message']
exit( 1 )
csAPI.setOption( 'Resources/StorageElements/ProductionSandboxSE/AccessProtocol.1/Host', 'localhost' )
# FileManager = FileManagerPS
# SecurityManager = FullSecurityManager
# }
# }
# Databases
# {
# FileCatalogDB
# {
# DBName = FileCatalogDB
# }
# }
# }
# }
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
csAPI = CSAPI()
for sct in ['Systems/DataManagement/Production/Services',
'Systems/DataManagement/Production/Services/FileCatalog' ]:
res = csAPI.createSection( sct )
if not res['OK']:
print res['Message']
exit( 1 )
csAPI.setOption( 'Systems/DataManagement/Production/Services/FileCatalog/DirectoryManager', 'DirectoryClosure' )
csAPI.setOption( 'Systems/DataManagement/Production/Services/FileCatalog/FileManager', 'FileManagerPs' )
csAPI.setOption( 'Systems/DataManagement/Production/Services/FileCatalog/SecurityManager', 'DirectorySecurityManagerWithDelete' )
csAPI.setOption( 'Systems/DataManagement/Production/Services/FileCatalog/UniqueGUID', True )
csAPI.commit()
# ProductionSandboxSE
# {
# BackendType = DISET
# AccessProtocol.1
# {
# Host = localhost
# Port = 9196
# ProtocolName = DIP
# Protocol = dips
# Path = /scratch/workspace/%s/sandboxes % setupName
# Access = remote
# SpaceToken =
# WSUrl =
# }
# }
res = csAPI.createSection('Resources/StorageElements/')
if not res['OK']:
print res['Message']
exit(1)
res = csAPI.createSection('Resources/StorageElements/ProductionSandboxSE')
if not res['OK']:
print res['Message']
exit(1)
csAPI.setOption('Resources/StorageElements/ProductionSandboxSE/BackendType',
'DISET')
res = csAPI.createSection(
'Resources/StorageElements/ProductionSandboxSE/AccessProtocol.1')
if not res['OK']:
print res['Message']
# Now setting a SandboxSE as the following:
# ProductionSandboxSE
# {
# BackendType = DISET
# AccessProtocol = dips
# DIP
# {
# Host = localhost
# Port = 9196
# ProtocolName = DIP
# Protocol = dips
# Path = /scratch/workspace/%s/sandboxes % setupName
# Access = remote
# }
# }
res = csAPI.createSection('Resources/StorageElements/')
if not res['OK']:
print res['Message']
exit(1)
res = csAPI.createSection('Resources/StorageElements/ProductionSandboxSE')
if not res['OK']:
print res['Message']
exit(1)
csAPI.setOption('Resources/StorageElements/ProductionSandboxSE/BackendType', 'DISET')
csAPI.setOption('Resources/StorageElements/ProductionSandboxSE/AccessProtocol', 'dips')
res = csAPI.createSection('Resources/StorageElements/ProductionSandboxSE/DIP')
if not res['OK']:
print res['Message']
exit(1)
class GOCDB2CSAgent (AgentModule):
""" Class to retrieve information about service endpoints
from GOCDB and update configuration stored by CS
"""
def __init__(self, *args, **kwargs):
""" c'tor
"""
super(GOCDB2CSAgent, self).__init__(*args, **kwargs)
self.GOCDBClient = None
self.csAPI = None
self.dryRun = False
def initialize(self):
""" Run at the agent initialization (normally every 500 cycles)
"""
# client to connect to GOCDB
self.GOCDBClient = GOCDBClient()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
# API needed to update configuration stored by CS
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute(self):
"""
Execute GOCDB queries according to the function map
and user request (options in configuration).
"""
# __functionMap is at the end of the class definition
for option, functionCall in GOCDB2CSAgent.__functionMap.iteritems():
optionValue = self.am_getOption(option, True)
if optionValue:
result = functionCall(self)
if not result['OK']:
self.log.error("%s() failed with message: %s" % (functionCall.__name__, result['Message']))
else:
self.log.info("Successfully executed %s" % functionCall.__name__)
return S_OK()
def updatePerfSONARConfiguration(self):
"""
Get current status of perfSONAR endpoints from GOCDB
and update CS configuration accordingly.
"""
log = self.log.getSubLogger('updatePerfSONAREndpoints')
log.debug('Begin function ...')
# get endpoints
result = self.__getPerfSONAREndpoints()
if not result['OK']:
log.error("__getPerfSONAREndpoints() failed with message: %s" % result['Message'])
return S_ERROR('Unable to fetch perfSONAR endpoints from GOCDB.')
endpointList = result['Value']
# add DIRAC site name
result = self.__addDIRACSiteName(endpointList)
if not result['OK']:
log.error("__addDIRACSiteName() failed with message: %s" % result['Message'])
return S_ERROR('Unable to extend the list with DIRAC site names.')
extendedEndpointList = result['Value']
# prepare dictionary with new configuration
result = self.__preparePerfSONARConfiguration(extendedEndpointList)
if not result['OK']:
log.error("__preparePerfSONARConfiguration() failed with message: %s" % result['Message'])
return S_ERROR('Unable to prepare a new perfSONAR configuration.')
finalConfiguration = result['Value']
# update configuration according to the final status of endpoints
self.__updateConfiguration(finalConfiguration)
log.debug("Configuration updated succesfully")
log.debug('End function.')
return S_OK()
def __getPerfSONAREndpoints(self):
"""
Retrieve perfSONAR endpoint information directly from GOCDB.
:return: List of perfSONAR endpoints (dictionaries) as stored by GOCDB.
"""
log = self.log.getSubLogger('__getPerfSONAREndpoints')
log.debug('Begin function ...')
# get perfSONAR endpoints (latency and bandwidth) form GOCDB
endpointList = []
for endpointType in ['Latency', 'Bandwidth']:
result = self.GOCDBClient.getServiceEndpointInfo('service_type', 'net.perfSONAR.%s' % endpointType)
if not result['OK']:
log.error("getServiceEndpointInfo() failed with message: %s" % result['Message'])
return S_ERROR('Could not fetch %s endpoints from GOCDB' % endpointType.lower())
log.debug('Number of %s endpoints: %s' % (endpointType.lower(), len(result['Value'])))
endpointList.extend(result['Value'])
log.debug('Number of perfSONAR endpoints: %s' % len(endpointList))
log.debug('End function.')
return S_OK(endpointList)
def __preparePerfSONARConfiguration(self, endpointList):
"""
Prepare a dictionary with a new CS configuration of perfSONAR endpoints.
:return: Dictionary where keys are configuration paths (options and sections)
and values are values of corresponding options
or None in case of a path pointing to a section.
"""
log = self.log.getSubLogger('__preparePerfSONARConfiguration')
log.debug('Begin function ...')
# static elements of a path
rootPath = '/Resources/Sites'
extPath = 'Network'
baseOptionName = 'Enabled'
options = {baseOptionName: 'True', 'ServiceType': 'perfSONAR'}
# enable GOCDB endpoints in configuration
newConfiguration = {}
for endpoint in endpointList:
if endpoint['DIRACSITENAME'] is None:
continue
split = endpoint['DIRACSITENAME'].split('.')
path = cfgPath(rootPath, split[0], endpoint['DIRACSITENAME'], extPath, endpoint['HOSTNAME'])
for name, defaultValue in options.iteritems():
newConfiguration[cfgPath(path, name)] = defaultValue
# get current configuration
currentConfiguration = {}
for option in options.iterkeys():
result = gConfig.getConfigurationTree(rootPath, extPath + '/', '/' + option)
if not result['OK']:
log.error("getConfigurationTree() failed with message: %s" % result['Message'])
return S_ERROR('Unable to fetch perfSONAR endpoints from CS.')
currentConfiguration.update(result['Value'])
# disable endpoints that disappeared in GOCDB
removedElements = set(currentConfiguration) - set(newConfiguration)
newElements = set(newConfiguration) - set(currentConfiguration)
addedEndpoints = len(newElements) / len(options)
disabledEndpoints = 0
for path in removedElements:
if baseOptionName in path:
newConfiguration[path] = 'False'
if currentConfiguration[path] != 'False':
disabledEndpoints = disabledEndpoints + 1
# inform what will be changed
if addedEndpoints > 0:
self.log.info("%s new perfSONAR endpoints will be added to the configuration" % addedEndpoints)
if disabledEndpoints > 0:
self.log.info("%s old perfSONAR endpoints will be disable in the configuration" % disabledEndpoints)
if addedEndpoints == 0 and disabledEndpoints == 0:
self.log.info("perfSONAR configuration is up-to-date")
log.debug('End function.')
return S_OK(newConfiguration)
def __addDIRACSiteName(self, inputList):
"""
Extend given list of GOCDB endpoints with DIRAC site name, i.e.
add an entry "DIRACSITENAME" in dictionaries that describe endpoints.
If given site name could not be found "DIRACSITENAME" is set to 'None'.
:return: List of perfSONAR endpoints (dictionaries).
"""
log = self.log.getSubLogger('__addDIRACSiteName')
log.debug('Begin function ...')
# get site name dictionary
result = getDIRACGOCDictionary()
if not result['OK']:
log.error("getDIRACGOCDictionary() failed with message: %s" % result['Message'])
return S_ERROR('Could not get site name dictionary')
# reverse the dictionary (assume 1 to 1 relation)
DIRACGOCDict = result['Value']
GOCDIRACDict = dict(zip(DIRACGOCDict.values(), DIRACGOCDict.keys()))
# add DIRAC site names
outputList = []
for entry in inputList:
try:
entry['DIRACSITENAME'] = GOCDIRACDict[entry['SITENAME']]
except KeyError:
self.log.warn("No dictionary entry for %s. " % entry['SITENAME'])
entry['DIRACSITENAME'] = None
outputList.append(entry)
log.debug('End function.')
return S_OK(outputList)
def __updateConfiguration(self, setElements=None, delElements=None):
"""
Update configuration stored by CS.
"""
if setElements is None:
setElements = {}
if delElements is None:
delElements = []
log = self.log.getSubLogger('__updateConfiguration')
log.debug('Begin function ...')
# assure existence and proper value of a section or an option
for path, value in setElements.iteritems():
if value is None:
section = path
else:
split = path.rsplit('/', 1)
section = split[0]
try:
result = self.csAPI.createSection(section)
if not result['OK']:
log.error("createSection() failed with message: %s" % result['Message'])
except Exception as e:
log.error("Exception in createSection(): %s" % repr(e).replace(',)', ')'))
if value is not None:
try:
result = self.csAPI.setOption(path, value)
if not result['OK']:
log.error("setOption() failed with message: %s" % result['Message'])
except Exception as e:
log.error("Exception in setOption(): %s" % repr(e).replace(',)', ')'))
# delete elements in the configuration
for path in delElements:
result = self.csAPI.delOption(path)
if not result['OK']:
log.warn("delOption() failed with message: %s" % result['Message'])
result = self.csAPI.delSection(path)
if not result['OK']:
log.warn("delSection() failed with message: %s" % result['Message'])
if self.dryRun:
log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
# update configuration stored by CS
result = self.csAPI.commit()
if not result['OK']:
log.error("commit() failed with message: %s" % result['Message'])
return S_ERROR("Could not commit changes to CS.")
else:
log.info("Committed changes to CS")
log.debug('End function.')
return S_OK()
# define mapping between an agent option in the configuration and a function call
__functionMap = {'UpdatePerfSONARS': updatePerfSONARConfiguration,
}