def _getCurrentConfig(self):
"""Return the current system configuration."""
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
gConfig.forceRefresh()
fullCfg = CFG()
setup = gConfig.getValue('/DIRAC/Setup', '')
setupList = gConfig.getSections('/DIRAC/Setups', [])
if not setupList['OK']:
return S_ERROR('Could not get /DIRAC/Setups sections')
setupList = setupList['Value']
if setup not in setupList:
return S_ERROR('Setup %s is not in allowed list: %s' %
(setup, ', '.join(setupList)))
serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
if not serviceSetups['OK']:
return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
serviceSetups = serviceSetups['Value'] # dict
for system, setup in serviceSetups.items():
if self.systems and system not in self.systems:
continue
systemCfg = gConfigurationData.remoteCFG.getAsCFG(
"/Systems/%s/%s" % (system, setup))
for section in systemCfg.listSections():
if section not in ('Agents', 'Services', 'Executors'):
systemCfg.deleteKey(section)
fullCfg.createNewSection("/%s" % system, contents=systemCfg)
return S_OK(fullCfg)
def export_setupComponent( self, componentType, system, component, componentModule='' ):
""" Setup the specified component for running with the runsvdir daemon
It implies installComponent
"""
result = InstallTools.setupComponent( componentType, system, component, getCSExtensions(), componentModule )
gConfig.forceRefresh()
return result
def export_setupComponent( self, componentType, system, component, componentModule = '' ):
""" Setup the specified component for running with the runsvdir daemon
It implies installComponent
"""
result = gComponentInstaller.setupComponent( componentType, system, component, getCSExtensions(), componentModule )
gConfig.forceRefresh()
return result
def _getCurrentConfig(self):
"""Return the current system configuration."""
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
gConfig.forceRefresh()
fullCfg = CFG()
setup = gConfig.getValue('/DIRAC/Setup', '')
setupList = gConfig.getSections('/DIRAC/Setups', [])
if not setupList['OK']:
return S_ERROR('Could not get /DIRAC/Setups sections')
setupList = setupList['Value']
if setup not in setupList:
return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList)))
serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
if not serviceSetups['OK']:
return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
serviceSetups = serviceSetups['Value'] # dict
for system, setup in serviceSetups.items():
if self.systems and system not in self.systems:
continue
systemCfg = gConfigurationData.remoteCFG.getAsCFG("/Systems/%s/%s" % (system, setup))
for section in systemCfg.listSections():
if section not in ('Agents', 'Services', 'Executors'):
systemCfg.deleteKey(section)
fullCfg.createNewSection("/%s" % system, contents=systemCfg)
return S_OK(fullCfg)
def execute(self):
""" General agent execution method
"""
self.voBdiiCEDict = {}
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn("Could not download a fresh copy of the CS data", result['Message'])
# Refresh the configuration from the master server
gConfig.forceRefresh(fromMaster=True)
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
return S_OK()
def execute( self ):
""" General agent execution method
"""
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn( "Could not download a fresh copy of the CS data", result[ 'Message' ] )
# Refresh the configuration from the master server
gConfig.forceRefresh( fromMaster = True )
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
if self.processSEs:
self.__lookForNewSEs()
self.__updateSEs()
return S_OK()
def checkURLs(self):
"""Ensure that the running services have their URL in the Config."""
self.log.info("Checking URLs")
# get services again, in case they were started/stop in controlComponents
gConfig.forceRefresh(fromMaster=True)
# get port used for https based services
try:
tornadoSystemInstance = PathFinder.getSystemInstance(
system="Tornado",
setup=self.setup,
)
self._tornadoPort = gConfig.getValue(
Path.cfgPath("/System/Tornado/", tornadoSystemInstance,
"Port"),
self._tornadoPort,
)
except RuntimeError:
pass
self.log.debug("Using Tornado Port:", self._tornadoPort)
res = self.getRunningInstances(instanceType="Services",
runitStatus="All")
if not res["OK"]:
return S_ERROR("Failure to get running services")
self.services = res["Value"]
for service, options in sorted(self.services.items()):
self.log.debug("Checking URL for %s with options %s" %
(service, options))
# ignore SystemAdministrator, does not have URLs
if "SystemAdministrator" in service:
continue
self._checkServiceURL(service, options)
if self.csAPI.csModified and self.commitURLs:
self.log.info("Commiting changes to the CS")
result = self.csAPI.commit()
if not result["OK"]:
self.logError("Commit to CS failed", result["Message"])
return S_ERROR("Failed to commit to CS")
return S_OK()
def getVOUserData(self, vo, refreshFlag=False):
""" Get a report for users of a given VO
:param str vo: VO name
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO(vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % vo)
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers(diracUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
def doOAuthMagic(self):
""" Magic method
:return: S_OK()/S_ERROR()
"""
nAuth = notebookAuth(self.__piParams.diracGroup, voms=self.__piParams.addVOMSExt, proxyPath=self.__piParams.proxyLoc)
result = nAuth.getToken()
if not result['OK']:
return result
aToken = result['Value'].get('access_token')
if not aToken:
return S_ERROR('Access token is absent in resporse.')
result = nAuth.getProxyWithToken(aToken)
if not result['OK']:
return result
result = Script.enableCS()
if not result['OK']:
return S_ERROR("Cannot contact CS to get user list")
threading.Thread(target=self.checkCAs).start()
gConfig.forceRefresh(fromMaster=True)
return S_OK(self.__piParams.proxyLoc)
def getVOUserData( self, vo, refreshFlag = False ):
""" Get a report for users of a given VO
:param str vo: VO name
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO( vo )
if not diracUsers:
return S_ERROR( "No VO users found for %s" % vo )
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers( diracUsers )
if not result['OK']:
self.log.error( 'Could not retrieve CS User description' )
return result
def getVOUserData(self, refreshFlag=False):
"""Get a report for users of a given VO
:param bool refreshFlag: flag to indicate that the configuration must be refreshed
before looking up user data
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO(self.vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % self.vo)
if refreshFlag:
result = self.csapi.downloadCSData()
if not result["OK"]:
return result
result = self.csapi.describeUsers(diracUsers)
if not result["OK"]:
self.log.error("Could not retrieve CS User description")
return result
def getVOUserData(self, refreshFlag=False):
""" Get a report for users of a given VO
:param bool refreshFlag: flag to indicate that the configuration must be refreshed
before looking up user data
:return: S_OK/S_ERROR, Value = user description dictionary
"""
if refreshFlag:
gConfig.forceRefresh()
# Get DIRAC users
diracUsers = getUsersInVO(self.vo)
if not diracUsers:
return S_ERROR("No VO users found for %s" % self.vo)
if refreshFlag:
result = self.csapi.downloadCSData()
if not result['OK']:
return result
result = self.csapi.describeUsers(diracUsers)
if not result['OK']:
self.log.error('Could not retrieve CS User description')
return result
def checkURLs(self):
"""Ensure that the running services have their URL in the Config."""
self.log.info("Checking URLs")
# get services again, in case they were started/stop in controlComponents
gConfig.forceRefresh(fromMaster=True)
res = self.getRunningInstances(instanceType='Services', runitStatus='All')
if not res["OK"]:
return S_ERROR("Failure to get running services")
self.services = res["Value"]
for service, options in self.services.iteritems():
self.log.debug("Checking URL for %s with options %s" % (service, options))
# ignore SystemAdministrator, does not have URLs
if 'SystemAdministrator' in service:
continue
self._checkServiceURL(service, options)
if self.csAPI.csModified and self.commitURLs:
self.log.info("Commiting changes to the CS")
result = self.csAPI.commit()
if not result['OK']:
self.logError('Commit to CS failed', result['Message'])
return S_ERROR("Failed to commit to CS")
return S_OK()
def __enableCS(self):
if not (result := Script.enableCS())["OK"] or not (result := gConfig.forceRefresh())["OK"]:
return S_ERROR(f"Cannot contact CS: {result['Message']}")
def getSiteUpdates( vo, bdiiInfo = None, log = None ):
""" Get all the necessary updates for the already defined sites and CEs
"""
def addToChangeSet( entry, changeSet ):
_section, _option, value, new_value = entry
if new_value and new_value != value:
changeSet.add( entry )
if log is None:
log = gLogger
ceBdiiDict = bdiiInfo
if bdiiInfo is None:
result = getBdiiCEInfo( vo )
if not result['OK']:
return result
ceBdiiDict = result['Value']
changeSet = set()
gConfig.forceRefresh()
for site in ceBdiiDict:
result = getDIRACSiteName( site )
if not result['OK']:
continue
siteNames = result['Value']
for siteName in siteNames:
siteSection = cfgPath( '/Resources', 'Sites', siteName.split('.')[0], siteName )
result = gConfig.getOptionsDict( siteSection )
if not result['OK']:
continue
siteDict = result['Value']
# Current CS values
coor = siteDict.get( 'Coordinates', 'Unknown' )
mail = siteDict.get( 'Mail', 'Unknown' ).replace( ' ','' )
description = siteDict.get( 'Description', 'Unknown' )
longitude = ceBdiiDict[site].get( 'GlueSiteLongitude', '' ).strip()
latitude = ceBdiiDict[site].get( 'GlueSiteLatitude', '' ).strip()
# Current BDII value
newcoor = ''
if longitude and latitude:
newcoor = "%s:%s" % ( longitude, latitude )
newmail = ceBdiiDict[site].get( 'GlueSiteSysAdminContact', '' ).replace( 'mailto:', '' ).strip()
newdescription = ceBdiiDict[site].get( 'GlueSiteDescription', '' ).strip()
# Adding site data to the changes list
addToChangeSet( ( siteSection, 'Coordinates', coor, newcoor ), changeSet )
addToChangeSet( ( siteSection, 'Mail', mail, newmail ), changeSet )
addToChangeSet( ( siteSection, 'Description', description, newdescription ), changeSet )
ces = gConfig.getValue( cfgPath( siteSection, 'CE' ), [] )
for ce in ces:
ceSection = cfgPath( siteSection, 'CEs', ce )
ceDict = {}
result = gConfig.getOptionsDict( ceSection )
if result['OK']:
ceDict = result['Value']
else:
if ceBdiiDict[site]['CEs'].get( ce, None ):
log.notice( "Adding new CE %s to site %s/%s" % (ce, siteName, site) )
ceInfo = ceBdiiDict[site]['CEs'].get( ce, None )
if ceInfo is None:
ceType = ceDict.get( 'CEType', '')
continue
# Current CS CE info
arch = ceDict.get( 'architecture', 'Unknown' )
OS = ceDict.get( 'OS', 'Unknown' )
si00 = ceDict.get( 'SI00', 'Unknown' )
ceType = ceDict.get( 'CEType', 'Unknown' )
ram = ceDict.get( 'HostRAM', 'Unknown' )
submissionMode = ceDict.get( 'SubmissionMode', 'Unknown' )
# Current BDII CE info
newarch = ceBdiiDict[site]['CEs'][ce].get( 'GlueHostArchitecturePlatformType', '' ).strip()
systemName = ceInfo.get( 'GlueHostOperatingSystemName', '' ).strip()
systemVersion = ceInfo.get( 'GlueHostOperatingSystemVersion', '' ).strip()
systemRelease = ceInfo.get( 'GlueHostOperatingSystemRelease', '' ).strip()
newOS = ''
if systemName and systemVersion and systemRelease:
newOS = '_'.join( ( systemName, systemVersion, systemRelease ) )
newsi00 = ceInfo.get( 'GlueHostBenchmarkSI00', '' ).strip()
newCEType = 'Unknown'
for queue in ceInfo['Queues']:
queueDict = ceInfo['Queues'][queue]
newCEType = queueDict.get( 'GlueCEImplementationName', '' ).strip()
if newCEType:
break
if newCEType=='ARC-CE':
newCEType = 'ARC'
newSubmissionMode = None
if newCEType in ['ARC','CREAM']:
newSubmissionMode = "Direct"
newRAM = ceInfo.get( 'GlueHostMainMemoryRAMSize', '' ).strip()
# Adding CE data to the change list
addToChangeSet( ( ceSection, 'architecture', arch, newarch ), changeSet )
addToChangeSet( ( ceSection, 'OS', OS, newOS ), changeSet )
addToChangeSet( ( ceSection, 'SI00', si00, newsi00 ), changeSet )
addToChangeSet( ( ceSection, 'CEType', ceType, newCEType ), changeSet )
addToChangeSet( ( ceSection, 'HostRAM', ram, newRAM ), changeSet )
if submissionMode == "Unknown" and newSubmissionMode:
addToChangeSet( ( ceSection, 'SubmissionMode', submissionMode, newSubmissionMode ), changeSet )
queues = ceInfo['Queues'].keys()
for queue in queues:
queueSection = cfgPath( ceSection, 'Queues', queue )
queueDict = {}
result = gConfig.getOptionsDict( queueSection )
if result['OK']:
queueDict = result['Value']
else:
log.notice( "Adding new queue %s to CE %s" % (queue, ce) )
queueInfo = ceInfo['Queues'][queue]
queueStatus = queueInfo['GlueCEStateStatus']
if queueStatus.lower() != "production":
continue
# Current CS queue info
maxCPUTime = queueDict.get( 'maxCPUTime', 'Unknown' )
si00 = queueDict.get( 'SI00', 'Unknown' )
maxTotalJobs = queueDict.get( 'MaxTotalJobs', 'Unknown' )
# Current BDII queue info
newMaxCPUTime = queueInfo.get( 'GlueCEPolicyMaxCPUTime', '' )
newSI00 = ''
caps = queueInfo['GlueCECapability']
if type( caps ) == type( '' ):
caps = [caps]
for cap in caps:
if 'CPUScalingReferenceSI00' in cap:
newSI00 = cap.split( '=' )[-1]
# Adding queue info to the CS
addToChangeSet( ( queueSection, 'maxCPUTime', maxCPUTime, newMaxCPUTime ), changeSet )
addToChangeSet( ( queueSection, 'SI00', si00, newSI00 ), changeSet )
if maxTotalJobs == "Unknown":
newTotalJobs = min( 1000, int( int( queueInfo.get( 'GlueCEInfoTotalCPUs', 0 ) )/2 ) )
newWaitingJobs = max( 2, int( newTotalJobs * 0.1 ) )
newTotalJobs = str( newTotalJobs )
newWaitingJobs = str( newWaitingJobs )
addToChangeSet( ( queueSection, 'MaxTotalJobs', '', newTotalJobs ), changeSet )
addToChangeSet( ( queueSection, 'MaxWaitingJobs', '', newWaitingJobs ), changeSet )
# Updating eligible VO list
VOs = set()
if queueDict.get( 'VO', '' ):
VOs = set( [ q.strip() for q in queueDict.get( 'VO', '' ).split( ',' ) if q ] )
if not vo in VOs:
VOs.add( vo )
VOs = list( VOs )
newVOs = ','.join( VOs )
addToChangeSet( ( queueSection, 'VO', '', newVOs ), changeSet )
return S_OK( changeSet )
def getSiteUpdates( vo, bdiiInfo = None, log = None ):
""" Get all the necessary updates for the already defined sites and CEs
"""
def addToChangeSet( entry, changeSet ):
_section, _option, value, new_value = entry
if new_value and new_value != value:
changeSet.add( entry )
if log is None:
log = gLogger
ceBdiiDict = bdiiInfo
if bdiiInfo is None:
result = getBdiiCEInfo( vo )
if not result['OK']:
return result
ceBdiiDict = result['Value']
changeSet = set()
gConfig.forceRefresh()
for site in ceBdiiDict:
result = getDIRACSiteName( site )
if not result['OK']:
continue
siteNames = result['Value']
for siteName in siteNames:
siteSection = cfgPath( '/Resources', 'Sites', siteName.split('.')[0], siteName )
result = gConfig.getOptionsDict( siteSection )
if not result['OK']:
continue
siteDict = result['Value']
# Current CS values
coor = siteDict.get( 'Coordinates', 'Unknown' )
mail = siteDict.get( 'Mail', 'Unknown' ).replace( ' ','' )
description = siteDict.get( 'Description', 'Unknown' )
longitude = ceBdiiDict[site].get( 'GlueSiteLongitude', '' ).strip()
latitude = ceBdiiDict[site].get( 'GlueSiteLatitude', '' ).strip()
# Current BDII value
newcoor = ''
if longitude and latitude:
newcoor = "%s:%s" % ( longitude, latitude )
newmail = ceBdiiDict[site].get( 'GlueSiteSysAdminContact', '' ).replace( 'mailto:', '' ).strip()
newdescription = ceBdiiDict[site].get( 'GlueSiteDescription', '' ).strip()
# Adding site data to the changes list
addToChangeSet( ( siteSection, 'Coordinates', coor, newcoor ), changeSet )
addToChangeSet( ( siteSection, 'Mail', mail, newmail ), changeSet )
addToChangeSet( ( siteSection, 'Description', description, newdescription ), changeSet )
ces = gConfig.getValue( cfgPath( siteSection, 'CE' ), [] )
for ce in ces:
ceSection = cfgPath( siteSection, 'CEs', ce )
ceDict = {}
result = gConfig.getOptionsDict( ceSection )
if result['OK']:
ceDict = result['Value']
else:
if ceBdiiDict[site]['CEs'].get( ce, None ):
log.notice( "Adding new CE %s to site %s/%s" % (ce, siteName, site) )
ceInfo = ceBdiiDict[site]['CEs'].get( ce, None )
if ceInfo is None:
ceType = ceDict.get( 'CEType', '')
continue
# Current CS CE info
arch = ceDict.get( 'architecture', 'Unknown' )
OS = ceDict.get( 'OS', 'Unknown' )
si00 = ceDict.get( 'SI00', 'Unknown' )
ceType = ceDict.get( 'CEType', 'Unknown' )
ram = ceDict.get( 'HostRAM', 'Unknown' )
submissionMode = ceDict.get( 'SubmissionMode', 'Unknown' )
# Current BDII CE info
newarch = ceBdiiDict[site]['CEs'][ce].get( 'GlueHostArchitecturePlatformType', '' ).strip()
systemName = ceInfo.get( 'GlueHostOperatingSystemName', '' ).strip()
systemVersion = ceInfo.get( 'GlueHostOperatingSystemVersion', '' ).strip()
systemRelease = ceInfo.get( 'GlueHostOperatingSystemRelease', '' ).strip()
newOS = ''
if systemName and systemVersion and systemRelease:
newOS = '_'.join( ( systemName, systemVersion, systemRelease ) )
newsi00 = ceInfo.get( 'GlueHostBenchmarkSI00', '' ).strip()
newCEType = 'Unknown'
for queue in ceInfo['Queues']:
queueDict = ceInfo['Queues'][queue]
newCEType = queueDict.get( 'GlueCEImplementationName', '' ).strip()
if newCEType:
break
if newCEType=='ARC-CE':
newCEType = 'ARC'
if newCEType in ['ARC','CREAM']:
newSubmissionMode = "Direct"
newRAM = ceInfo.get( 'GlueHostMainMemoryRAMSize', '' ).strip()
# Adding CE data to the change list
addToChangeSet( ( ceSection, 'architecture', arch, newarch ), changeSet )
addToChangeSet( ( ceSection, 'OS', OS, newOS ), changeSet )
addToChangeSet( ( ceSection, 'SI00', si00, newsi00 ), changeSet )
addToChangeSet( ( ceSection, 'CEType', ceType, newCEType ), changeSet )
addToChangeSet( ( ceSection, 'HostRAM', ram, newRAM ), changeSet )
if submissionMode == "Unknown":
addToChangeSet( ( ceSection, 'SubmissionMode', submissionMode, newSubmissionMode ), changeSet )
queues = ceInfo['Queues'].keys()
for queue in queues:
queueSection = cfgPath( ceSection, 'Queues', queue )
queueDict = {}
result = gConfig.getOptionsDict( queueSection )
if result['OK']:
queueDict = result['Value']
else:
log.notice( "Adding new queue %s to CE %s" % (queue, ce) )
queueInfo = ceInfo['Queues'][queue]
queueStatus = queueInfo['GlueCEStateStatus']
if queueStatus.lower() != "production":
continue
# Current CS queue info
maxCPUTime = queueDict.get( 'maxCPUTime', 'Unknown' )
si00 = queueDict.get( 'SI00', 'Unknown' )
maxTotalJobs = queueDict.get( 'MaxTotalJobs', 'Unknown' )
# Current BDII queue info
newMaxCPUTime = queueInfo.get( 'GlueCEPolicyMaxCPUTime', '' )
newSI00 = ''
caps = queueInfo['GlueCECapability']
if type( caps ) == type( '' ):
caps = [caps]
for cap in caps:
if 'CPUScalingReferenceSI00' in cap:
newSI00 = cap.split( '=' )[-1]
# Adding queue info to the CS
addToChangeSet( ( queueSection, 'maxCPUTime', maxCPUTime, newMaxCPUTime ), changeSet )
addToChangeSet( ( queueSection, 'SI00', si00, newSI00 ), changeSet )
if maxTotalJobs == "Unknown":
newTotalJobs = min( 1000, int( int( queueInfo.get( 'GlueCEInfoTotalCPUs', 0 ) )/2 ) )
newWaitingJobs = max( 2, int( newTotalJobs * 0.1 ) )
newTotalJobs = str( newTotalJobs )
newWaitingJobs = str( newWaitingJobs )
addToChangeSet( ( queueSection, 'MaxTotalJobs', '', newTotalJobs ), changeSet )
addToChangeSet( ( queueSection, 'MaxWaitingJobs', '', newWaitingJobs ), changeSet )
# Updating eligible VO list
VOs = set()
if queueDict.get( 'VO', '' ):
VOs = set( [ q.strip() for q in queueDict.get( 'VO', '' ).split( ',' ) if q ] )
if not vo in VOs:
VOs.add( vo )
VOs = list( VOs )
newVOs = ','.join( VOs )
addToChangeSet( ( queueSection, 'VO', '', newVOs ), changeSet )
return S_OK( changeSet )
