def __getBdiiCEInfo(self, vo):
if vo in self.voBdiiCEDict:
return S_OK(self.voBdiiCEDict[vo])
self.log.info("Check for available CEs for VO", vo)
totalResult = S_OK({})
message = ''
mainResult = getBdiiCEInfo(vo)
if not mainResult['OK']:
self.log.error("Failed getting information from default bdii",
mainResult['Message'])
message = mainResult['Message']
for bdii in reversed(self.alternativeBDIIs):
resultAlt = getBdiiCEInfo(vo, host=bdii)
if resultAlt['OK']:
totalResult['Value'].update(resultAlt['Value'])
else:
self.log.error("Failed getting information from %s " % bdii,
resultAlt['Message'])
message = (message + "\n" + resultAlt['Message']).strip()
if mainResult['OK']:
totalResult['Value'].update(mainResult['Value'])
if not totalResult[
'Value'] and message: ## Dict is empty and we have an error message
self.log.error("Error during BDII request", message)
totalResult = S_ERROR(message)
else:
self.voBdiiCEDict[vo] = totalResult['Value']
return totalResult
def __getBdiiCEInfo( self, vo ):
if vo in self.voBdiiCEDict:
return S_OK( self.voBdiiCEDict[vo] )
self.log.info( "Check for available CEs for VO", vo )
totalResult = S_OK( {} )
message = ''
mainResult = getBdiiCEInfo( vo )
if not mainResult['OK']:
self.log.error( "Failed getting information from default bdii", mainResult['Message'] )
message = mainResult['Message']
for bdii in reversed( self.alternativeBDIIs ):
resultAlt = getBdiiCEInfo( vo, host = bdii )
if resultAlt['OK']:
totalResult['Value'].update( resultAlt['Value'] )
else:
self.log.error( "Failed getting information from %s " % bdii, resultAlt['Message'] )
message = ( message + "\n" + resultAlt['Message'] ).strip()
if mainResult['OK']:
totalResult['Value'].update( mainResult['Value'] )
if not totalResult['Value'] and message: ## Dict is empty and we have an error message
self.log.error( "Error during BDII request", message )
totalResult = S_ERROR( message )
else:
self.voBdiiCEDict[vo] = totalResult['Value']
return totalResult
def __getBdiiCEInfo(self, vo):
if vo in self.voBdiiCEDict:
return S_OK(self.voBdiiCEDict[vo])
self.log.info("Check for available CEs for VO", vo)
totalResult = S_OK({})
message = ''
mainResult = getBdiiCEInfo(vo, host=self.host, glue2=self.glue2Only)
if not mainResult['OK']:
self.log.error("Failed getting information from default bdii",
mainResult['Message'])
message = mainResult['Message']
for bdii in reversed(self.alternativeBDIIs):
resultAlt = getBdiiCEInfo(vo, host=bdii, glue2=self.glue2Only)
if resultAlt['OK']:
totalResult['Value'].update(resultAlt['Value'])
else:
self.log.error("Failed getting information from %s " % bdii,
resultAlt['Message'])
message = (message + "\n" + resultAlt['Message']).strip()
for glue2URL in self.glue2URLs:
if self.glue2Only:
break
resultGlue2 = getBdiiCEInfo(vo, host=glue2URL, glue2=True)
if resultGlue2['OK']:
totalResult['Value'].update(resultGlue2['Value'])
else:
self.log.error(
"Failed getting GLUE2 information for",
"%s, %s: %s" % (glue2URL, vo, resultGlue2['Message']))
message = (message + "\n" + resultGlue2['Message']).strip()
if mainResult['OK']:
totalResult['Value'].update(mainResult['Value'])
if not totalResult[
'Value'] and message: # Dict is empty and we have an error message
self.log.error("Error during BDII request", message)
totalResult = S_ERROR(message)
else:
self.voBdiiCEDict[vo] = totalResult['Value']
self.__purgeSites(totalResult['Value'])
return totalResult
def getGridCEs(vo, bdiiInfo=None, ceBlackList=None, hostURL=None, glue2=False):
""" Get all the CEs available for a given VO and having queues in Production state
:param str vo: VO name
:param dict bddiInfo: information from BDII
:param list ceBlackList: CEs from black list
:param str hostURL: host URL
:param bool glue2: use glue2
:return: S_OK(set)/S_ERROR()
"""
knownCEs = set()
if ceBlackList is not None:
knownCEs = knownCEs.union(set(ceBlackList))
ceBdiiDict = bdiiInfo
if bdiiInfo is None:
result = getBdiiCEInfo(vo, host=hostURL, glue2=glue2)
if not result['OK']:
return result
ceBdiiDict = result['Value']
siteDict = {}
for site in ceBdiiDict:
siteCEs = set(ceBdiiDict[site]['CEs'].keys())
newCEs = siteCEs - knownCEs
if not newCEs:
continue
ceFullDict = {}
for ce in newCEs:
ceDict = {}
ceInfo = ceBdiiDict[site]['CEs'][ce]
ceType = 'Unknown'
ceDict['Queues'] = []
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get('GlueCEStateStatus', 'UnknownStatus')
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get('GlueCEImplementationName', '')
ceDict['Queues'].append(queue)
if not ceDict['Queues']:
continue
ceDict['CEType'] = ceType
ceDict['GOCSite'] = site
ceDict['CEID'] = ce
systemName = ceInfo.get('GlueHostOperatingSystemName', 'Unknown')
systemVersion = ceInfo.get('GlueHostOperatingSystemVersion', 'Unknown')
systemRelease = ceInfo.get('GlueHostOperatingSystemRelease', 'Unknown')
ceDict['System'] = (systemName, systemVersion, systemRelease)
ceFullDict[ce] = ceDict
siteDict[site] = ceFullDict
result = S_OK(siteDict)
result['BdiiInfo'] = ceBdiiDict
return result
def __getBdiiCEInfo( self, vo ):
if vo in self.voBdiiCEDict:
return S_OK( self.voBdiiCEDict[vo] )
self.log.info( "Check for available CEs for VO", vo )
result = getBdiiCEInfo( vo )
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs :
result = getBdiiCEInfo( vo, host = bdii )
if result['OK']:
break
if not result['OK']:
if message:
self.log.error( "Error during BDII request", message )
else:
self.log.error( "Error during BDII request", result['Message'] )
else:
self.voBdiiCEDict[vo] = result['Value']
return result
def getGridCEs(vo, bdiiInfo=None, ceBlackList=None, hostURL=None, glue2=False):
""" Get all the CEs available for a given VO and having queues in Production state
"""
knownCEs = set()
if ceBlackList is not None:
knownCEs = knownCEs.union(set(ceBlackList))
ceBdiiDict = bdiiInfo
if bdiiInfo is None:
result = getBdiiCEInfo(vo, host=hostURL, glue2=glue2)
if not result['OK']:
return result
ceBdiiDict = result['Value']
siteDict = {}
for site in ceBdiiDict:
siteCEs = set(ceBdiiDict[site]['CEs'].keys())
newCEs = siteCEs - knownCEs
if not newCEs:
continue
ceFullDict = {}
for ce in newCEs:
ceDict = {}
ceInfo = ceBdiiDict[site]['CEs'][ce]
ceType = 'Unknown'
ceDict['Queues'] = []
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get('GlueCEStateStatus', 'UnknownStatus')
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get('GlueCEImplementationName', '')
ceDict['Queues'].append(queue)
if not ceDict['Queues']:
continue
ceDict['CEType'] = ceType
ceDict['GOCSite'] = site
ceDict['CEID'] = ce
systemName = ceInfo.get('GlueHostOperatingSystemName', 'Unknown')
systemVersion = ceInfo.get('GlueHostOperatingSystemVersion', 'Unknown')
systemRelease = ceInfo.get('GlueHostOperatingSystemRelease', 'Unknown')
ceDict['System'] = (systemName, systemVersion, systemRelease)
ceFullDict[ce] = ceDict
siteDict[site] = ceFullDict
result = S_OK(siteDict)
result['BdiiInfo'] = ceBdiiDict
return result
def getSiteUpdates(vo, bdiiInfo=None, log=None):
""" Get all the necessary updates for the already defined sites and CEs
"""
def addToChangeSet(entry, changeSet):
""" Inner function to update changeSet with entry (a tuple)
:param tuple entry: entry to add to changeSet
:param set changeSet: set collecting stuff to change
"""
_section, _option, value, new_value = entry
if new_value and new_value != value:
changeSet.add(entry)
if log is None:
log = gLogger
ceBdiiDict = bdiiInfo
if bdiiInfo is None:
result = getBdiiCEInfo(vo)
if not result['OK']:
return result
ceBdiiDict = result['Value']
changeSet = set()
for site in ceBdiiDict:
result = getDIRACSiteName(site)
if not result['OK']:
continue
siteNames = result['Value']
for siteName in siteNames:
siteSection = cfgPath('/Resources', 'Sites',
siteName.split('.')[0], siteName)
result = gConfig.getOptionsDict(siteSection)
if not result['OK']:
continue
siteDict = result['Value']
# Current CS values
coor = siteDict.get('Coordinates', 'Unknown')
mail = siteDict.get('Mail', 'Unknown').replace(' ', '')
description = siteDict.get('Description', 'Unknown')
description = description.replace(' ,', ',')
longitude = ceBdiiDict[site].get('GlueSiteLongitude', '').strip()
latitude = ceBdiiDict[site].get('GlueSiteLatitude', '').strip()
# Current BDII value
newcoor = ''
if longitude and latitude:
newcoor = "%s:%s" % (longitude, latitude)
newmail = ceBdiiDict[site].get('GlueSiteSysAdminContact',
'').replace('mailto:', '').strip()
newdescription = ceBdiiDict[site].get('GlueSiteDescription',
'').strip()
newdescription = ", ".join(
[line.strip() for line in newdescription.split(",")])
# Adding site data to the changes list
addToChangeSet((siteSection, 'Coordinates', coor, newcoor),
changeSet)
addToChangeSet((siteSection, 'Mail', mail, newmail), changeSet)
addToChangeSet(
(siteSection, 'Description', description, newdescription),
changeSet)
ces = gConfig.getValue(cfgPath(siteSection, 'CE'), [])
for ce in ces:
ceSection = cfgPath(siteSection, 'CEs', ce)
ceDict = {}
result = gConfig.getOptionsDict(ceSection)
if result['OK']:
ceDict = result['Value']
else:
if ceBdiiDict[site]['CEs'].get(ce, None):
log.notice("Adding new CE",
"%s to site %s/%s" % (ce, siteName, site))
ceInfo = ceBdiiDict[site]['CEs'].get(ce, None)
if ceInfo is None:
ceType = ceDict.get('CEType', '')
continue
# Current CS CE info
arch = ceDict.get('architecture', 'Unknown')
OS = ceDict.get('OS', 'Unknown')
si00 = ceDict.get('SI00', 'Unknown')
ceType = ceDict.get('CEType', 'Unknown')
ram = ceDict.get('MaxRAM', 'Unknown')
submissionMode = ceDict.get('SubmissionMode', 'Unknown')
# Current BDII CE info
newarch = ceBdiiDict[site]['CEs'][ce].get(
'GlueHostArchitecturePlatformType', '').strip()
systemName = ceInfo.get('GlueHostOperatingSystemName',
'').strip()
systemVersion = ceInfo.get('GlueHostOperatingSystemVersion',
'').strip()
systemRelease = ceInfo.get('GlueHostOperatingSystemRelease',
'').strip()
newOS = ''
if systemName and systemVersion and systemRelease:
newOS = '_'.join(
(systemName, systemVersion, systemRelease))
newsi00 = ceInfo.get('GlueHostBenchmarkSI00', '').strip()
newCEType = 'Unknown'
for queue in ceInfo['Queues']:
queueDict = ceInfo['Queues'][queue]
newCEType = queueDict.get('GlueCEImplementationName',
'').strip()
if newCEType:
break
if newCEType == 'ARC-CE':
newCEType = 'ARC'
newSubmissionMode = None
if newCEType in ['ARC', 'CREAM']:
newSubmissionMode = "Direct"
newRAM = ceInfo.get('GlueHostMainMemoryRAMSize', '').strip()
# Protect from unreasonable values
if newRAM and int(newRAM) > 150000:
newRAM = ''
# Adding CE data to the change list
addToChangeSet((ceSection, 'architecture', arch, newarch),
changeSet)
addToChangeSet((ceSection, 'OS', OS, newOS), changeSet)
addToChangeSet((ceSection, 'SI00', si00, newsi00), changeSet)
addToChangeSet((ceSection, 'CEType', ceType, newCEType),
changeSet)
addToChangeSet((ceSection, 'MaxRAM', ram, newRAM), changeSet)
if submissionMode == "Unknown" and newSubmissionMode:
addToChangeSet((ceSection, 'SubmissionMode',
submissionMode, newSubmissionMode),
changeSet)
queues = ceInfo['Queues'].keys()
for queue in queues:
queueInfo = ceInfo['Queues'][queue]
queueStatus = queueInfo['GlueCEStateStatus']
queueSection = cfgPath(ceSection, 'Queues', queue)
queueDict = {}
result = gConfig.getOptionsDict(queueSection)
if result['OK']:
queueDict = result['Value']
else:
if queueStatus.lower() == "production":
log.notice("Adding new queue",
"%s to CE %s" % (queue, ce))
else:
continue
# Current CS queue info
maxCPUTime = queueDict.get('maxCPUTime', 'Unknown')
si00 = queueDict.get('SI00', 'Unknown')
maxTotalJobs = queueDict.get('MaxTotalJobs', 'Unknown')
# Current BDII queue info
newMaxCPUTime = queueInfo.get('GlueCEPolicyMaxCPUTime', '')
if newMaxCPUTime == "4" * len(
newMaxCPUTime) or newMaxCPUTime == "9" * len(
newMaxCPUTime):
newMaxCPUTime = ''
wallTime = queueInfo.get('GlueCEPolicyMaxWallClockTime',
'')
if wallTime == "4" * len(
wallTime) or wallTime == "9" * len(wallTime):
wallTime = ''
if wallTime and int(wallTime) > 0:
if not newMaxCPUTime:
newMaxCPUTime = str(int(0.8 * int(wallTime)))
else:
if int(wallTime) <= int(newMaxCPUTime):
newMaxCPUTime = str(int(0.8 * int(wallTime)))
newSI00 = ''
caps = queueInfo.get('GlueCECapability', [])
if isinstance(caps, basestring):
caps = [caps]
for cap in caps:
if 'CPUScalingReferenceSI00' in cap:
newSI00 = cap.split('=')[-1]
# Adding queue info to the CS
addToChangeSet((queueSection, 'maxCPUTime', maxCPUTime,
newMaxCPUTime), changeSet)
addToChangeSet((queueSection, 'SI00', si00, newSI00),
changeSet)
if maxTotalJobs == "Unknown":
newTotalJobs = min(
1000,
int(
int(queueInfo.get('GlueCEInfoTotalCPUs', 0)) /
2))
newWaitingJobs = max(2, int(newTotalJobs * 0.1))
newTotalJobs = str(newTotalJobs)
newWaitingJobs = str(newWaitingJobs)
addToChangeSet(
(queueSection, 'MaxTotalJobs', '', newTotalJobs),
changeSet)
addToChangeSet((queueSection, 'MaxWaitingJobs', '',
newWaitingJobs), changeSet)
# Updating eligible VO list
VOs = set()
if queueDict.get('VO', ''):
VOs = set([
q.strip()
for q in queueDict.get('VO', '').split(',') if q
])
if vo not in VOs:
VOs.add(vo)
VOs = list(VOs)
newVOs = ','.join(VOs)
addToChangeSet((queueSection, 'VO', '', newVOs),
changeSet)
return S_OK(changeSet)
def getSiteUpdates(vo, bdiiInfo=None, log=None):
""" Get all the necessary updates for the already defined sites and CEs
"""
def addToChangeSet(entry, changeSet):
""" Inner function to update changeSet with entry (a tuple)
:param tuple entry: entry to add to changeSet
:param set changeSet: set collecting stuff to change
"""
_section, _option, value, new_value = entry
if new_value and new_value != value:
changeSet.add(entry)
if log is None:
log = gLogger
ceBdiiDict = bdiiInfo
if bdiiInfo is None:
result = getBdiiCEInfo(vo)
if not result['OK']:
return result
ceBdiiDict = result['Value']
changeSet = set()
for site in ceBdiiDict:
result = getDIRACSiteName(site)
if not result['OK']:
continue
siteNames = result['Value']
for siteName in siteNames:
siteSection = cfgPath('/Resources', 'Sites', siteName.split('.')[0], siteName)
result = gConfig.getOptionsDict(siteSection)
if not result['OK']:
continue
siteDict = result['Value']
# Current CS values
coor = siteDict.get('Coordinates', 'Unknown')
mail = siteDict.get('Mail', 'Unknown').replace(' ', '')
description = siteDict.get('Description', 'Unknown')
description = description.replace(' ,', ',')
longitude = ceBdiiDict[site].get('GlueSiteLongitude', '').strip()
latitude = ceBdiiDict[site].get('GlueSiteLatitude', '').strip()
# Current BDII value
newcoor = ''
if longitude and latitude:
newcoor = "%s:%s" % (longitude, latitude)
newmail = ceBdiiDict[site].get('GlueSiteSysAdminContact', '').replace('mailto:', '').strip()
newdescription = ceBdiiDict[site].get('GlueSiteDescription', '').strip()
newdescription = ", ".join([line.strip() for line in newdescription.split(",")])
# Adding site data to the changes list
addToChangeSet((siteSection, 'Coordinates', coor, newcoor), changeSet)
addToChangeSet((siteSection, 'Mail', mail, newmail), changeSet)
addToChangeSet((siteSection, 'Description', description, newdescription), changeSet)
ces = gConfig.getValue(cfgPath(siteSection, 'CE'), [])
for ce in ces:
ceSection = cfgPath(siteSection, 'CEs', ce)
ceDict = {}
result = gConfig.getOptionsDict(ceSection)
if result['OK']:
ceDict = result['Value']
else:
if ceBdiiDict[site]['CEs'].get(ce, None):
log.notice("Adding new CE", "%s to site %s/%s" % (ce, siteName, site))
ceInfo = ceBdiiDict[site]['CEs'].get(ce, None)
if ceInfo is None:
ceType = ceDict.get('CEType', '')
continue
# Current CS CE info
arch = ceDict.get('architecture', 'Unknown')
OS = ceDict.get('OS', 'Unknown')
si00 = ceDict.get('SI00', 'Unknown')
ceType = ceDict.get('CEType', 'Unknown')
ram = ceDict.get('MaxRAM', 'Unknown')
submissionMode = ceDict.get('SubmissionMode', 'Unknown')
# Current BDII CE info
newarch = ceBdiiDict[site]['CEs'][ce].get('GlueHostArchitecturePlatformType', '').strip()
systemName = ceInfo.get('GlueHostOperatingSystemName', '').strip()
systemVersion = ceInfo.get('GlueHostOperatingSystemVersion', '').strip()
systemRelease = ceInfo.get('GlueHostOperatingSystemRelease', '').strip()
newOS = ''
if systemName and systemVersion and systemRelease:
newOS = '_'.join((systemName, systemVersion, systemRelease))
newsi00 = ceInfo.get('GlueHostBenchmarkSI00', '').strip()
newCEType = 'Unknown'
for queue in ceInfo['Queues']:
queueDict = ceInfo['Queues'][queue]
newCEType = queueDict.get('GlueCEImplementationName', '').strip()
if newCEType:
break
if newCEType == 'ARC-CE':
newCEType = 'ARC'
newSubmissionMode = None
if newCEType in ['ARC', 'CREAM']:
newSubmissionMode = "Direct"
newRAM = ceInfo.get('GlueHostMainMemoryRAMSize', '').strip()
# Protect from unreasonable values
if newRAM and int(newRAM) > 150000:
newRAM = ''
# Adding CE data to the change list
addToChangeSet((ceSection, 'architecture', arch, newarch), changeSet)
addToChangeSet((ceSection, 'OS', OS, newOS), changeSet)
addToChangeSet((ceSection, 'SI00', si00, newsi00), changeSet)
addToChangeSet((ceSection, 'CEType', ceType, newCEType), changeSet)
addToChangeSet((ceSection, 'MaxRAM', ram, newRAM), changeSet)
if submissionMode == "Unknown" and newSubmissionMode:
addToChangeSet((ceSection, 'SubmissionMode', submissionMode, newSubmissionMode), changeSet)
queues = ceInfo['Queues'].keys()
for queue in queues:
queueInfo = ceInfo['Queues'][queue]
queueStatus = queueInfo['GlueCEStateStatus']
queueSection = cfgPath(ceSection, 'Queues', queue)
queueDict = {}
result = gConfig.getOptionsDict(queueSection)
if result['OK']:
queueDict = result['Value']
else:
if queueStatus.lower() == "production":
log.notice("Adding new queue", "%s to CE %s" % (queue, ce))
else:
continue
# Current CS queue info
maxCPUTime = queueDict.get('maxCPUTime', 'Unknown')
si00 = queueDict.get('SI00', 'Unknown')
maxTotalJobs = queueDict.get('MaxTotalJobs', 'Unknown')
# Current BDII queue info
newMaxCPUTime = queueInfo.get('GlueCEPolicyMaxCPUTime', '')
if newMaxCPUTime == "4" * len(newMaxCPUTime) or newMaxCPUTime == "9" * len(newMaxCPUTime):
newMaxCPUTime = ''
wallTime = queueInfo.get('GlueCEPolicyMaxWallClockTime', '')
if wallTime == "4" * len(wallTime) or wallTime == "9" * len(wallTime):
wallTime = ''
if wallTime and int(wallTime) > 0:
if not newMaxCPUTime:
newMaxCPUTime = str(int(0.8 * int(wallTime)))
else:
if int(wallTime) <= int(newMaxCPUTime):
newMaxCPUTime = str(int(0.8 * int(wallTime)))
newSI00 = ''
caps = queueInfo.get('GlueCECapability', [])
if isinstance(caps, basestring):
caps = [caps]
for cap in caps:
if 'CPUScalingReferenceSI00' in cap:
newSI00 = cap.split('=')[-1]
# Adding queue info to the CS
addToChangeSet((queueSection, 'maxCPUTime', maxCPUTime, newMaxCPUTime), changeSet)
addToChangeSet((queueSection, 'SI00', si00, newSI00), changeSet)
if maxTotalJobs == "Unknown":
newTotalJobs = min(1000, int(int(queueInfo.get('GlueCEInfoTotalCPUs', 0)) / 2))
newWaitingJobs = max(2, int(newTotalJobs * 0.1))
newTotalJobs = str(newTotalJobs)
newWaitingJobs = str(newWaitingJobs)
addToChangeSet((queueSection, 'MaxTotalJobs', '', newTotalJobs), changeSet)
addToChangeSet((queueSection, 'MaxWaitingJobs', '', newWaitingJobs), changeSet)
# Updating eligible VO list
VOs = set()
if queueDict.get('VO', ''):
VOs = set([q.strip() for q in queueDict.get('VO', '').split(',') if q])
if vo not in VOs:
VOs.add(vo)
VOs = list(VOs)
newVOs = ','.join(VOs)
addToChangeSet((queueSection, 'VO', '', newVOs), changeSet)
return S_OK(changeSet)
def getSiteUpdates(vo, bdiiInfo=None, log=None, glue2=True):
""" Get all the necessary updates for the already defined sites and CEs
:param str vo: VO name
:param dict bdiiInfo: information from DBII
:param object log: logger
:result: S_OK(set)/S_ERROR()
"""
def addToChangeSet(entry, changeSet):
""" Inner function to update changeSet with entry (a tuple)
:param tuple entry: entry to add to changeSet
:param set changeSet: set collecting stuff to change
"""
_section, _option, value, new_value = entry
if new_value and new_value != value:
changeSet.add(entry)
if log is None:
log = gLogger
ceBdiiDict = bdiiInfo
if bdiiInfo is None:
result = getBdiiCEInfo(vo, glue2=glue2)
if not result['OK']:
return result
ceBdiiDict = result['Value']
changeSet = set()
for site in ceBdiiDict:
result = getDIRACSiteName(site)
if not result['OK']:
continue
siteNames = result['Value']
for siteName in siteNames:
siteSection = cfgPath('/Resources', 'Sites',
siteName.split('.')[0], siteName)
result = gConfig.getOptionsDict(siteSection)
if not result['OK']:
continue
siteDict = result['Value']
# Current CS values
coor = siteDict.get('Coordinates', 'Unknown')
mail = siteDict.get('Mail', 'Unknown').replace(' ', '')
description = siteDict.get('Description', 'Unknown')
description = description.replace(' ,', ',')
longitude = ceBdiiDict[site].get('GlueSiteLongitude', '').strip()
latitude = ceBdiiDict[site].get('GlueSiteLatitude', '').strip()
# Current BDII value
newcoor = ''
if longitude and latitude:
newcoor = "%s:%s" % (longitude, latitude)
newmail = ceBdiiDict[site].get('GlueSiteSysAdminContact',
'').replace('mailto:', '').strip()
newdescription = ceBdiiDict[site].get('GlueSiteDescription',
'').strip()
newdescription = ", ".join(
[line.strip() for line in newdescription.split(",")])
# Adding site data to the changes list
addToChangeSet((siteSection, 'Coordinates', coor, newcoor),
changeSet)
addToChangeSet((siteSection, 'Mail', mail, newmail), changeSet)
addToChangeSet(
(siteSection, 'Description', description, newdescription),
changeSet)
ces = gConfig.getSections(cfgPath(siteSection, 'CEs'))
for ce in ces.get('Value', []):
ceSection = cfgPath(siteSection, 'CEs', ce)
ceDict = {}
result = gConfig.getOptionsDict(ceSection)
if result['OK']:
ceDict = result['Value']
else:
if ceBdiiDict[site]['CEs'].get(ce, None):
log.notice("Adding new CE",
"%s to site %s/%s" % (ce, siteName, site))
ceInfo = ceBdiiDict[site]['CEs'].get(ce, None)
if ceInfo is None:
ceType = ceDict.get('CEType', '')
continue
# Current CS CE info
arch = ceDict.get('architecture', 'Unknown')
OS = ceDict.get('OS', 'Unknown')
si00 = ceDict.get('SI00', 'Unknown')
ceType = ceDict.get('CEType', 'Unknown')
ram = ceDict.get('MaxRAM', 'Unknown')
submissionMode = ceDict.get('SubmissionMode', 'Unknown')
# Current BDII CE info
newarch = ceBdiiDict[site]['CEs'][ce].get(
'GlueHostArchitecturePlatformType', '').strip()
systemName = ceInfo.get('GlueHostOperatingSystemName',
'').strip()
systemVersion = ceInfo.get('GlueHostOperatingSystemVersion',
'').strip()
systemRelease = ceInfo.get('GlueHostOperatingSystemRelease',
'').strip()
newOS = ''
if systemName and systemVersion and systemRelease:
newOS = '_'.join(
(systemName, systemVersion, systemRelease))
newsi00 = ceInfo.get('GlueHostBenchmarkSI00', '').strip()
newCEType = 'Unknown'
for queue in ceInfo['Queues']:
queueDict = ceInfo['Queues'][queue]
newCEType = queueDict.get('GlueCEImplementationName',
'').strip()
if newCEType:
break
if newCEType == 'ARC-CE':
newCEType = 'ARC'
newSubmissionMode = None
if newCEType in ['ARC', 'CREAM']:
newSubmissionMode = "Direct"
newRAM = ceInfo.get('GlueHostMainMemoryRAMSize', '').strip()
# Protect from unreasonable values
if newRAM and int(newRAM) > 150000:
newRAM = ''
# Adding CE data to the change list
addToChangeSet((ceSection, 'architecture', arch, newarch),
changeSet)
addToChangeSet((ceSection, 'OS', OS, newOS), changeSet)
addToChangeSet((ceSection, 'SI00', si00, newsi00), changeSet)
addToChangeSet((ceSection, 'CEType', ceType, newCEType),
changeSet)
addToChangeSet((ceSection, 'MaxRAM', ram, newRAM), changeSet)
if submissionMode == "Unknown" and newSubmissionMode:
addToChangeSet((ceSection, 'SubmissionMode',
submissionMode, newSubmissionMode),
changeSet)
for queue, queueInfo in ceInfo['Queues'].items():
queueStatus = queueInfo['GlueCEStateStatus']
queueSection = cfgPath(ceSection, 'Queues', queue)
queueDict = {}
result = gConfig.getOptionsDict(queueSection)
if result['OK']:
queueDict = result['Value']
else:
if queueStatus.lower() == "production":
log.notice("Adding new queue",
"%s to CE %s" % (queue, ce))
else:
continue
# Current CS queue info
maxCPUTime = queueDict.get('maxCPUTime', 'Unknown')
si00 = queueDict.get('SI00', 'Unknown')
maxTotalJobs = queueDict.get('MaxTotalJobs', 'Unknown')
# Current BDII queue info
newMaxCPUTime = queueInfo.get('GlueCEPolicyMaxCPUTime', '')
if newMaxCPUTime == "4" * len(
newMaxCPUTime) or newMaxCPUTime == "9" * len(
newMaxCPUTime):
newMaxCPUTime = ''
wallTime = queueInfo.get('GlueCEPolicyMaxWallClockTime',
'')
if wallTime == "4" * len(
wallTime) or wallTime == "9" * len(wallTime):
wallTime = ''
if wallTime and int(wallTime) > 0:
if not newMaxCPUTime:
newMaxCPUTime = str(int(0.8 * int(wallTime)))
else:
if int(wallTime) <= int(newMaxCPUTime):
newMaxCPUTime = str(int(0.8 * int(wallTime)))
newSI00 = ''
caps = queueInfo.get('GlueCECapability', [])
if isinstance(caps, six.string_types):
caps = [caps]
for cap in caps:
if 'CPUScalingReferenceSI00' in cap:
newSI00 = cap.split('=')[-1]
# tags, processors, localCEType
tag = queueDict.get('Tag', '')
# LocalCEType can be empty (equivalent to "InProcess")
# or "Pool", "Singularity", but also "Pool/Singularity"
localCEType = queueDict.get('LocalCEType', '')
try:
localCEType_inner = localCEType.split('/')[1]
except IndexError:
localCEType_inner = ''
numberOfProcessors = int(
queueDict.get('NumberOfProcessors', 0))
newNOP = int(queueInfo.get('NumberOfProcessors', 1))
# Adding queue info to the CS
addToChangeSet((queueSection, 'maxCPUTime', maxCPUTime,
newMaxCPUTime), changeSet)
addToChangeSet((queueSection, 'SI00', si00, newSI00),
changeSet)
if newNOP != numberOfProcessors:
addToChangeSet((queueSection, 'NumberOfProcessors',
numberOfProcessors, newNOP), changeSet)
if newNOP > 1:
# if larger than one, add MultiProcessor to site tags, and LocalCEType=Pool
newTag = ','.join(
sorted(
set(tag.split(',')).union(
{'MultiProcessor'}))).strip(',')
addToChangeSet((queueSection, 'Tag', tag, newTag),
changeSet)
if localCEType_inner:
newLocalCEType = 'Pool/' + localCEType_inner
else:
newLocalCEType = 'Pool'
addToChangeSet((queueSection, 'LocalCEType',
localCEType, newLocalCEType),
changeSet)
else:
# if not larger than one, drop MultiProcessor Tag.
# Here we do not change the LocalCEType as Pool CE would still be perfectly valid.
newTag = ','.join(
sorted(
set(tag.split(',')).difference(
{'MultiProcessor'}))).strip(',')
changeSet.add((queueSection, 'Tag', tag, newTag))
if maxTotalJobs == "Unknown":
newTotalJobs = min(
1000,
int(
int(queueInfo.get('GlueCEInfoTotalCPUs', 0)) /
2))
newWaitingJobs = max(2, int(newTotalJobs * 0.1))
newTotalJobs = str(newTotalJobs)
newWaitingJobs = str(newWaitingJobs)
addToChangeSet(
(queueSection, 'MaxTotalJobs', '', newTotalJobs),
changeSet)
addToChangeSet((queueSection, 'MaxWaitingJobs', '',
newWaitingJobs), changeSet)
# Updating eligible VO list
VOs = set()
if queueDict.get('VO', ''):
VOs = set([
q.strip()
for q in queueDict.get('VO', '').split(',') if q
])
if vo not in VOs:
VOs.add(vo)
VOs = list(VOs)
newVOs = ','.join(VOs)
addToChangeSet((queueSection, 'VO', '', newVOs),
changeSet)
return S_OK(changeSet)
