def getGridSRMs( vo, bdiiInfo = None, srmBlackList = None, unUsed = False ):
result = ldapService( serviceType = 'SRM', vo = vo )
if not result['OK']:
return result
srmBdiiDict = result['Value']
knownSRMs = set()
if srmBlackList is not None:
knownSRMs = knownSRMs.union( set( srmBlackList ) )
siteSRMDict = {}
for srm in srmBdiiDict:
endPoint = srm.get( 'GlueServiceEndpoint', '')
srmHost = ''
if endPoint:
result = pfnparse( endPoint )
if not result['OK']:
continue
srmHost = result['Value']['Host']
if not srmHost:
continue
if srmHost in knownSRMs:
continue
if unUsed:
result = getDIRACSesForSRM( srmHost )
if not result['OK']:
return result
diracSEs = result['Value']
if diracSEs:
# If it is a known SRM and only new SRMs are requested, continue
continue
site = srm.get( 'GlueForeignKey', '' ).replace( 'GlueSiteUniqueID=', '' )
siteSRMDict.setdefault( site, {} )
siteSRMDict[site][srmHost] = srm
if bdiiInfo is None:
result = getBdiiSEInfo( vo )
if not result['OK']:
return result
seBdiiDict = result['Value']
else:
seBdiiDict = bdiiInfo
srmSeDict = {}
for site in siteSRMDict:
srms = siteSRMDict[site].keys()
for srm in srms:
if seBdiiDict.get( site, {} ).get( 'SEs', {} ).get( srm, {} ):
srmSeDict.setdefault( site, {} )
srmSeDict[site].setdefault( srm, {} )
srmSeDict[site][srm]['SRM'] = siteSRMDict[site][srm]
srmSeDict[site][srm]['SE'] = seBdiiDict[site]['SEs'][srm]
return S_OK( srmSeDict )
def getGridSRMs( vo, bdiiInfo = None, srmBlackList = None, unUsed = False ):
result = ldapService( serviceType = 'SRM', vo = vo )
if not result['OK']:
return result
srmBdiiDict = result['Value']
knownSRMs = set()
if srmBlackList is not None:
knownSRMs = knownSRMs.union( set( srmBlackList ) )
siteSRMDict = {}
for srm in srmBdiiDict:
endPoint = srm.get( 'GlueServiceEndpoint', '')
srmHost = ''
if endPoint:
result = pfnparse( endPoint )
if not result['OK']:
continue
srmHost = result['Value']['Host']
if not srmHost:
continue
if srmHost in knownSRMs:
continue
if unUsed:
result = getDIRACSesForSRM( srmHost )
if not result['OK']:
return result
diracSEs = result['Value']
if diracSEs:
# If it is a known SRM and only new SRMs are requested, continue
continue
site = srm.get( 'GlueForeignKey', '' ).replace( 'GlueSiteUniqueID=', '' )
siteSRMDict.setdefault( site, {} )
siteSRMDict[site][srmHost] = srm
if bdiiInfo is None:
result = getBdiiSEInfo( vo )
if not result['OK']:
return result
seBdiiDict = result['Value']
else:
seBdiiDict = bdiiInfo
srmSeDict = {}
for site in siteSRMDict:
srms = siteSRMDict[site].keys()
for srm in srms:
if seBdiiDict.get( site, {} ).get( 'SEs', {} ).get( srm, {} ):
srmSeDict.setdefault( site, {} )
srmSeDict[site].setdefault( srm, {} )
srmSeDict[site][srm]['SRM'] = siteSRMDict[site][srm]
srmSeDict[site][srm]['SE'] = seBdiiDict[site]['SEs'][srm]
return S_OK( srmSeDict )
def __infoFromCE(self):
sitesSection = cfgPath('Resources', 'Sites')
result = gConfig.getSections(sitesSection)
if not result['OK']:
return
grids = result['Value']
changed = False
body = ""
for grid in grids:
gridSection = cfgPath(sitesSection, grid)
result = gConfig.getSections(gridSection)
if not result['OK']:
return
sites = result['Value']
for site in sites:
# if site[-2:]!='ru':
# continue
siteSection = cfgPath(gridSection, site)
opt = gConfig.getOptionsDict(siteSection)['Value']
name = opt.get('Name', '')
if name:
coor = opt.get('Coordinates', 'Unknown')
mail = opt.get('Mail', 'Unknown')
result = ldapSite(name)
if not result['OK']:
self.log.warn("BDII site %s: %s" %
(name, result['Message']))
result = self.__checkAlternativeBDIISite(
ldapSite, name)
if result['OK']:
bdiisites = result['Value']
if len(bdiisites) == 0:
self.log.warn(name, "Error in bdii: leng = 0")
else:
if not len(bdiisites) == 1:
self.log.warn(
name, "Warning in bdii: leng = %d" %
len(bdiisites))
bdiisite = bdiisites[0]
try:
longitude = bdiisite['GlueSiteLongitude']
latitude = bdiisite['GlueSiteLatitude']
newcoor = "%s:%s" % (longitude, latitude)
except:
self.log.warn("Error in bdii coor")
newcoor = "Unknown"
try:
newmail = bdiisite[
'GlueSiteSysAdminContact'].split(
":")[-1].strip()
except:
self.log.warn("Error in bdii mail")
newmail = "Unknown"
self.log.debug("%s %s %s" %
(name, newcoor, newmail))
if newcoor != coor:
self.log.info("%s" % (name),
"%s -> %s" % (coor, newcoor))
if coor == 'Unknown':
self.csAPI.setOption(
cfgPath(siteSection, 'Coordinates'),
newcoor)
else:
self.csAPI.modifyValue(
cfgPath(siteSection, 'Coordinates'),
newcoor)
changed = True
if newmail != mail:
self.log.info("%s" % (name),
"%s -> %s" % (mail, newmail))
if mail == 'Unknown':
self.csAPI.setOption(
cfgPath(siteSection, 'Mail'), newmail)
else:
self.csAPI.modifyValue(
cfgPath(siteSection, 'Mail'), newmail)
changed = True
celist = List.fromChar(opt.get('CE', ''))
if not celist:
self.log.warn(site, 'Empty site list')
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in celist:
ceSection = cfgPath(siteSection, 'CEs', ce)
result = gConfig.getOptionsDict(ceSection)
if not result['OK']:
self.log.debug("Section CE", result['Message'])
wnTmpDir = 'Unknown'
arch = 'Unknown'
os = 'Unknown'
si00 = 'Unknown'
pilot = 'Unknown'
cetype = 'Unknown'
else:
ceopt = result['Value']
wnTmpDir = ceopt.get('wnTmpDir', 'Unknown')
arch = ceopt.get('architecture', 'Unknown')
os = ceopt.get('OS', 'Unknown')
si00 = ceopt.get('SI00', 'Unknown')
pilot = ceopt.get('Pilot', 'Unknown')
cetype = ceopt.get('CEType', 'Unknown')
result = ldapCE(ce)
if not result['OK']:
self.log.warn('Error in bdii for %s' % ce,
result['Message'])
result = self.__checkAlternativeBDIISite(ldapCE, ce)
continue
try:
bdiice = result['Value'][0]
except:
self.log.warn('Error in bdii for %s' % ce, result)
bdiice = None
if bdiice:
try:
newwnTmpDir = bdiice['GlueSubClusterWNTmpDir']
except:
newwnTmpDir = 'Unknown'
if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown':
section = cfgPath(ceSection, 'wnTmpDir')
self.log.info(section, " -> ".join(
(wnTmpDir, newwnTmpDir)))
if wnTmpDir == 'Unknown':
self.csAPI.setOption(section, newwnTmpDir)
else:
self.csAPI.modifyValue(section, newwnTmpDir)
changed = True
try:
newarch = bdiice[
'GlueHostArchitecturePlatformType']
except:
newarch = 'Unknown'
if arch != newarch and newarch != 'Unknown':
section = cfgPath(ceSection, 'architecture')
self.log.info(section, " -> ".join(
(arch, newarch)))
if arch == 'Unknown':
self.csAPI.setOption(section, newarch)
else:
self.csAPI.modifyValue(section, newarch)
changed = True
try:
newos = '_'.join(
(bdiice['GlueHostOperatingSystemName'],
bdiice['GlueHostOperatingSystemVersion'],
bdiice['GlueHostOperatingSystemRelease']))
except:
newos = 'Unknown'
if os != newos and newos != 'Unknown':
section = cfgPath(ceSection, 'OS')
self.log.info(section, " -> ".join((os, newos)))
if os == 'Unknown':
self.csAPI.setOption(section, newos)
else:
self.csAPI.modifyValue(section, newos)
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % (
os, newos, ce, site)
try:
newsi00 = bdiice['GlueHostBenchmarkSI00']
except:
newsi00 = 'Unknown'
if si00 != newsi00 and newsi00 != 'Unknown':
section = cfgPath(ceSection, 'SI00')
self.log.info(section, " -> ".join(
(si00, newsi00)))
if si00 == 'Unknown':
self.csAPI.setOption(section, newsi00)
else:
self.csAPI.modifyValue(section, newsi00)
changed = True
try:
rte = bdiice[
'GlueHostApplicationSoftwareRunTimeEnvironment']
if self.voName == 'lhcb':
if 'VO-lhcb-pilot' in rte:
newpilot = 'True'
else:
newpilot = 'False'
else:
newpilot = 'Unknown'
except:
newpilot = 'Unknown'
if pilot != newpilot and newpilot != 'Unknown':
section = cfgPath(ceSection, 'Pilot')
self.log.info(section, " -> ".join(
(pilot, newpilot)))
if pilot == 'Unknown':
self.csAPI.setOption(section, newpilot)
else:
self.csAPI.modifyValue(section, newpilot)
changed = True
result = ldapService(ce)
if not result['OK']:
result = self.__checkAlternativeBDIISite(
ldapService, ce)
if result['OK'] and result['Value']:
services = result['Value']
newcetype = 'LCG'
for service in services:
if service['GlueServiceType'].count('CREAM'):
newcetype = "CREAM"
else:
newcetype = 'Unknown'
if cetype != newcetype and newcetype != 'Unknown':
section = cfgPath(ceSection, 'CEType')
self.log.info(section, " -> ".join(
(cetype, newcetype)))
if cetype == 'Unknown':
self.csAPI.setOption(section, newcetype)
else:
self.csAPI.modifyValue(section, newcetype)
changed = True
result = ldapCEState(ce, vo=self.voName) #getBDIICEVOView
if not result['OK']:
self.log.warn('Error in bdii for queue %s' % ce,
result['Message'])
result = self.__checkAlternativeBDIISite(
ldapCEState, ce, self.voName)
continue
try:
queues = result['Value']
except:
self.log.warn('Error in bdii for queue %s' % ce,
result['Massage'])
continue
for queue in queues:
try:
queueName = queue['GlueCEUniqueID'].split('/')[-1]
except:
self.log.warn('error in queuename ', queue)
continue
try:
newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime']
except:
newmaxCPUTime = None
newsi00 = None
try:
caps = queue['GlueCECapability']
if type(caps) == type(''):
caps = [caps]
for cap in caps:
if cap.count('CPUScalingReferenceSI00'):
newsi00 = cap.split('=')[-1]
except:
newsi00 = None
queueSection = cfgPath(ceSection, 'Queues', queueName)
result = gConfig.getOptionsDict(queueSection)
if not result['OK']:
self.log.warn("Section Queues", result['Message'])
maxCPUTime = 'Unknown'
si00 = 'Unknown'
else:
queueopt = result['Value']
maxCPUTime = queueopt.get('maxCPUTime', 'Unknown')
si00 = queueopt.get('SI00', 'Unknown')
if newmaxCPUTime and (maxCPUTime != newmaxCPUTime):
section = cfgPath(queueSection, 'maxCPUTime')
self.log.info(
section, " -> ".join(
(maxCPUTime, newmaxCPUTime)))
if maxCPUTime == 'Unknown':
self.csAPI.setOption(section, newmaxCPUTime)
else:
self.csAPI.modifyValue(section, newmaxCPUTime)
changed = True
if newsi00 and (si00 != newsi00):
section = cfgPath(queueSection, 'SI00')
self.log.info(section, " -> ".join(
(si00, newsi00)))
if si00 == 'Unknown':
self.csAPI.setOption(section, newsi00)
else:
self.csAPI.modifyValue(section, newsi00)
changed = True
if changed:
self.log.info(body)
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
return self.csAPI.commitChanges(sortUsers=False)
else:
self.log.info("No changes found")
return S_OK()
def getBDIIService(self, ce, host=None):
"""
Get information about ce from BDII at host
"""
return ldapService(ce, host=host)
def getBDIIService( self, ce, host = None ): """Get information about ce from BDII at host """ return ldapService( ce, host = host )
def __infoFromCE(self):
sitesSection = cfgPath("Resources", "Sites")
result = gConfig.getSections(sitesSection)
if not result["OK"]:
return
grids = result["Value"]
changed = False
body = ""
for grid in grids:
gridSection = cfgPath(sitesSection, grid)
result = gConfig.getSections(gridSection)
if not result["OK"]:
return
sites = result["Value"]
for site in sites:
# if site[-2:]!='ru':
# continue
siteSection = cfgPath(gridSection, site)
opt = gConfig.getOptionsDict(siteSection)["Value"]
name = opt.get("Name", "")
if name:
coor = opt.get("Coordinates", "Unknown")
mail = opt.get("Mail", "Unknown")
result = ldapSite(name)
if not result["OK"]:
self.log.warn("BDII site %s: %s" % (name, result["Message"]))
result = self.__checkAlternativeBDIISite(ldapSite, name)
if result["OK"]:
bdiisites = result["Value"]
if len(bdiisites) == 0:
self.log.warn(name, "Error in bdii: leng = 0")
else:
if not len(bdiisites) == 1:
self.log.warn(name, "Warning in bdii: leng = %d" % len(bdiisites))
bdiisite = bdiisites[0]
try:
longitude = bdiisite["GlueSiteLongitude"]
latitude = bdiisite["GlueSiteLatitude"]
newcoor = "%s:%s" % (longitude, latitude)
except:
self.log.warn("Error in bdii coor")
newcoor = "Unknown"
try:
newmail = bdiisite["GlueSiteSysAdminContact"].split(":")[-1].strip()
except:
self.log.warn("Error in bdii mail")
newmail = "Unknown"
self.log.debug("%s %s %s" % (name, newcoor, newmail))
if newcoor != coor:
self.log.info("%s" % (name), "%s -> %s" % (coor, newcoor))
if coor == "Unknown":
self.csAPI.setOption(cfgPath(siteSection, "Coordinates"), newcoor)
else:
self.csAPI.modifyValue(cfgPath(siteSection, "Coordinates"), newcoor)
changed = True
if newmail != mail:
self.log.info("%s" % (name), "%s -> %s" % (mail, newmail))
if mail == "Unknown":
self.csAPI.setOption(cfgPath(siteSection, "Mail"), newmail)
else:
self.csAPI.modifyValue(cfgPath(siteSection, "Mail"), newmail)
changed = True
celist = List.fromChar(opt.get("CE", ""))
if not celist:
self.log.warn(site, "Empty site list")
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in celist:
ceSection = cfgPath(siteSection, "CEs", ce)
result = gConfig.getOptionsDict(ceSection)
if not result["OK"]:
self.log.debug("Section CE", result["Message"])
wnTmpDir = "Unknown"
arch = "Unknown"
os = "Unknown"
si00 = "Unknown"
pilot = "Unknown"
cetype = "Unknown"
else:
ceopt = result["Value"]
wnTmpDir = ceopt.get("wnTmpDir", "Unknown")
arch = ceopt.get("architecture", "Unknown")
os = ceopt.get("OS", "Unknown")
si00 = ceopt.get("SI00", "Unknown")
pilot = ceopt.get("Pilot", "Unknown")
cetype = ceopt.get("CEType", "Unknown")
result = ldapCE(ce)
if not result["OK"]:
self.log.warn("Error in bdii for %s" % ce, result["Message"])
result = self.__checkAlternativeBDIISite(ldapCE, ce)
continue
try:
bdiice = result["Value"][0]
except:
self.log.warn("Error in bdii for %s" % ce, result)
bdiice = None
if bdiice:
try:
newwnTmpDir = bdiice["GlueSubClusterWNTmpDir"]
except:
newwnTmpDir = "Unknown"
if wnTmpDir != newwnTmpDir and newwnTmpDir != "Unknown":
section = cfgPath(ceSection, "wnTmpDir")
self.log.info(section, " -> ".join((wnTmpDir, newwnTmpDir)))
if wnTmpDir == "Unknown":
self.csAPI.setOption(section, newwnTmpDir)
else:
self.csAPI.modifyValue(section, newwnTmpDir)
changed = True
try:
newarch = bdiice["GlueHostArchitecturePlatformType"]
except:
newarch = "Unknown"
if arch != newarch and newarch != "Unknown":
section = cfgPath(ceSection, "architecture")
self.log.info(section, " -> ".join((arch, newarch)))
if arch == "Unknown":
self.csAPI.setOption(section, newarch)
else:
self.csAPI.modifyValue(section, newarch)
changed = True
try:
newos = "_".join(
(
bdiice["GlueHostOperatingSystemName"],
bdiice["GlueHostOperatingSystemVersion"],
bdiice["GlueHostOperatingSystemRelease"],
)
)
except:
newos = "Unknown"
if os != newos and newos != "Unknown":
section = cfgPath(ceSection, "OS")
self.log.info(section, " -> ".join((os, newos)))
if os == "Unknown":
self.csAPI.setOption(section, newos)
else:
self.csAPI.modifyValue(section, newos)
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % (os, newos, ce, site)
try:
newsi00 = bdiice["GlueHostBenchmarkSI00"]
except:
newsi00 = "Unknown"
if si00 != newsi00 and newsi00 != "Unknown":
section = cfgPath(ceSection, "SI00")
self.log.info(section, " -> ".join((si00, newsi00)))
if si00 == "Unknown":
self.csAPI.setOption(section, newsi00)
else:
self.csAPI.modifyValue(section, newsi00)
changed = True
try:
rte = bdiice["GlueHostApplicationSoftwareRunTimeEnvironment"]
if self.voName.lower() == "lhcb":
if "VO-lhcb-pilot" in rte:
newpilot = "True"
else:
newpilot = "False"
else:
newpilot = "Unknown"
except:
newpilot = "Unknown"
if pilot != newpilot and newpilot != "Unknown":
section = cfgPath(ceSection, "Pilot")
self.log.info(section, " -> ".join((pilot, newpilot)))
if pilot == "Unknown":
self.csAPI.setOption(section, newpilot)
else:
self.csAPI.modifyValue(section, newpilot)
changed = True
result = ldapService(ce)
if not result["OK"]:
result = self.__checkAlternativeBDIISite(ldapService, ce)
if result["OK"] and result["Value"]:
services = result["Value"]
newcetype = "LCG"
for service in services:
if service["GlueServiceType"].count("CREAM"):
newcetype = "CREAM"
else:
newcetype = "Unknown"
if cetype != newcetype and newcetype != "Unknown":
section = cfgPath(ceSection, "CEType")
self.log.info(section, " -> ".join((cetype, newcetype)))
if cetype == "Unknown":
self.csAPI.setOption(section, newcetype)
else:
self.csAPI.modifyValue(section, newcetype)
changed = True
result = ldapCEState(ce, vo=self.voName) # getBDIICEVOView
if not result["OK"]:
self.log.warn("Error in bdii for queue %s" % ce, result["Message"])
result = self.__checkAlternativeBDIISite(ldapCEState, ce, self.voName)
continue
try:
queues = result["Value"]
except:
self.log.warn("Error in bdii for queue %s" % ce, result["Massage"])
continue
for queue in queues:
try:
queueName = queue["GlueCEUniqueID"].split("/")[-1]
except:
self.log.warn("error in queuename ", queue)
continue
try:
newmaxCPUTime = queue["GlueCEPolicyMaxCPUTime"]
except:
newmaxCPUTime = None
newsi00 = None
try:
caps = queue["GlueCECapability"]
if type(caps) == type(""):
caps = [caps]
for cap in caps:
if cap.count("CPUScalingReferenceSI00"):
newsi00 = cap.split("=")[-1]
except:
newsi00 = None
queueSection = cfgPath(ceSection, "Queues", queueName)
result = gConfig.getOptionsDict(queueSection)
if not result["OK"]:
self.log.warn("Section Queues", result["Message"])
maxCPUTime = "Unknown"
si00 = "Unknown"
else:
queueopt = result["Value"]
maxCPUTime = queueopt.get("maxCPUTime", "Unknown")
si00 = queueopt.get("SI00", "Unknown")
if newmaxCPUTime and (maxCPUTime != newmaxCPUTime):
section = cfgPath(queueSection, "maxCPUTime")
self.log.info(section, " -> ".join((maxCPUTime, newmaxCPUTime)))
if maxCPUTime == "Unknown":
self.csAPI.setOption(section, newmaxCPUTime)
else:
self.csAPI.modifyValue(section, newmaxCPUTime)
changed = True
if newsi00 and (si00 != newsi00):
section = cfgPath(queueSection, "SI00")
self.log.info(section, " -> ".join((si00, newsi00)))
if si00 == "Unknown":
self.csAPI.setOption(section, newsi00)
else:
self.csAPI.modifyValue(section, newsi00)
changed = True
if changed:
self.log.info(body)
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False)
return self.csAPI.commit()
else:
self.log.info("No changes found")
return S_OK()
def _infoFromCE( self ):
sitesSection = cfgPath( 'Resources', 'Sites' )
result = gConfig.getSections( sitesSection )
if not result['OK']:
return
grids = result['Value']
for grid in grids:
gridSection = cfgPath( sitesSection, grid )
result = gConfig.getSections( gridSection )
if not result['OK']:
return
sites = result['Value']
changed = False
body = ""
for site in sites:
# if site[-2:]!='ru':
# continue
siteSection = cfgPath( gridSection, site )
opt = gConfig.getOptionsDict( siteSection )['Value']
name = opt.get( 'Name', '' )
if name:
coor = opt.get( 'Coordinates', 'Unknown' )
mail = opt.get( 'Mail', 'Unknown' )
result = ldapSite( name )
if not result['OK']:
self.log.warn( "BDII site %s: %s" % (name, result['Message'] ))
result = self._checkAlternativeBDIISite( ldapSite, name )
if result['OK']:
bdiisites = result['Value']
if len( bdiisites ) == 0:
self.log.warn( name, "Error in bdii: leng = 0" )
else:
if not len( bdiisites ) == 1:
self.log.warn( name, "Warning in bdii: leng = %d" % len( bdiisites ) )
bdiisite = bdiisites[0]
try:
longitude = bdiisite['GlueSiteLongitude']
latitude = bdiisite['GlueSiteLatitude']
newcoor = "%s:%s" % ( longitude, latitude )
except:
self.log.warn( "Error in bdii coor" )
newcoor = "Unknown"
try:
newmail = bdiisite['GlueSiteSysAdminContact'].split( ":" )[-1].strip()
except:
self.log.warn( "Error in bdii mail" )
newmail = "Unknown"
self.log.debug( "%s %s %s" % ( name, newcoor, newmail ) )
if newcoor != coor:
self.log.info( "%s" % ( name ), "%s -> %s" % ( coor, newcoor ) )
if coor == 'Unknown':
self.csAPI.setOption( cfgPath( siteSection, 'Coordinates' ), newcoor )
else:
self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newcoor )
changed = True
if newmail != mail:
self.log.info( "%s" % ( name ), "%s -> %s" % ( mail, newmail ) )
if mail == 'Unknown':
self.csAPI.setOption( cfgPath( siteSection, 'Mail' ), newmail )
else:
self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newmail )
changed = True
celist = List.fromChar( opt.get( 'CE', '' ) )
if not celist:
self.log.warn( site, 'Empty site list' )
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in celist:
ceSection = cfgPath( siteSection, 'CEs', ce )
result = gConfig.getOptionsDict( ceSection )
if not result['OK']:
self.log.debug( "Section CE", result['Message'] )
wnTmpDir = 'Unknown'
arch = 'Unknown'
os = 'Unknown'
si00 = 'Unknown'
pilot = 'Unknown'
cetype = 'Unknown'
else:
ceopt = result['Value']
wnTmpDir = ceopt.get( 'wnTmpDir', 'Unknown' )
arch = ceopt.get( 'architecture', 'Unknown' )
os = ceopt.get( 'OS', 'Unknown' )
si00 = ceopt.get( 'SI00', 'Unknown' )
pilot = ceopt.get( 'Pilot', 'Unknown' )
cetype = ceopt.get( 'CEType', 'Unknown' )
result = ldapCE( ce )
if not result['OK']:
self.log.warn( 'Error in bdii for %s' % ce, result['Message'] )
result = self._checkAlternativeBDIISite( ldapCE, ce )
continue
try:
bdiice = result['Value'][0]
except:
self.log.warn( 'Error in bdii for %s' % ce, result )
bdiice = None
if bdiice:
try:
newwnTmpDir = bdiice['GlueSubClusterWNTmpDir']
except:
newwnTmpDir = 'Unknown'
if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown':
section = cfgPath( ceSection, 'wnTmpDir' )
self.log.info( section, " -> ".join( ( wnTmpDir, newwnTmpDir ) ) )
if wnTmpDir == 'Unknown':
self.csAPI.setOption( section, newwnTmpDir )
else:
self.csAPI.modifyValue( section, newwnTmpDir )
changed = True
try:
newarch = bdiice['GlueHostArchitecturePlatformType']
except:
newarch = 'Unknown'
if arch != newarch and newarch != 'Unknown':
section = cfgPath( ceSection, 'architecture' )
self.log.info( section, " -> ".join( ( arch, newarch ) ) )
if arch == 'Unknown':
self.csAPI.setOption( section, newarch )
else:
self.csAPI.modifyValue( section, newarch )
changed = True
try:
newos = '_'.join( ( bdiice['GlueHostOperatingSystemName'], bdiice['GlueHostOperatingSystemVersion'], bdiice['GlueHostOperatingSystemRelease'] ) )
except:
newos = 'Unknown'
if os != newos and newos != 'Unknown':
section = cfgPath( ceSection, 'OS' )
self.log.info( section, " -> ".join( ( os, newos ) ) )
if os == 'Unknown':
self.csAPI.setOption( section, newos )
else:
self.csAPI.modifyValue( section, newos )
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newos, ce, site )
try:
newsi00 = bdiice['GlueHostBenchmarkSI00']
except:
newsi00 = 'Unknown'
if si00 != newsi00 and newsi00 != 'Unknown':
section = cfgPath( ceSection, 'SI00' )
self.log.info( section, " -> ".join( ( si00, newsi00 ) ) )
if si00 == 'Unknown':
self.csAPI.setOption( section, newsi00 )
else:
self.csAPI.modifyValue( section, newsi00 )
changed = True
try:
rte = bdiice['GlueHostApplicationSoftwareRunTimeEnvironment']
if self.vo == 'lhcb':
if 'VO-lhcb-pilot' in rte:
newpilot = 'True'
else:
newpilot = 'False'
else:
newpilot = 'Unknown'
except:
newpilot = 'Unknown'
if pilot != newpilot and newpilot != 'Unknown':
section = cfgPath( ceSection, 'Pilot' )
self.log.info( section, " -> ".join( ( pilot, newpilot ) ) )
if pilot == 'Unknown':
self.csAPI.setOption( section, newpilot )
else:
self.csAPI.modifyValue( section, newpilot )
changed = True
result = ldapService( ce )
if not result['OK'] :
result = self._checkAlternativeBDIISite( ldapService, ce )
if result['OK']:
services = result['Value']
newcetype = 'LCG'
for service in services:
if service['GlueServiceType'].count( 'CREAM' ):
newcetype = "CREAM"
else:
newcetype = 'Unknown'
if cetype != newcetype and newcetype != 'Unknown':
section = cfgPath( ceSection, 'CEType' )
self.log.info( section, " -> ".join( ( cetype, newcetype ) ) )
if cetype == 'Unknown':
self.csAPI.setOption( section, newcetype )
else:
self.csAPI.modifyValue( section, newcetype )
changed = True
result = ldapCEState( ce, vo = self.vo ) #getBDIICEVOView
if not result['OK']:
self.log.warn( 'Error in bdii for queue %s' % ce, result['Message'] )
result = self._checkAlternativeBDIISite( ldapCEState, ce, self.vo )
continue
try:
queues = result['Value']
except:
self.log.warn( 'Error in bdii for queue %s' % ce, result['Massage'] )
continue
for queue in queues:
try:
queueName = queue['GlueCEUniqueID'].split( '/' )[-1]
except:
self.log.warn( 'error in queuename ', queue )
continue
try:
newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime']
except:
newmaxCPUTime = None
newsi00 = None
try:
caps = queue['GlueCECapability']
if type( caps ) == type( '' ):
caps = [caps]
for cap in caps:
if cap.count( 'CPUScalingReferenceSI00' ):
newsi00 = cap.split( '=' )[-1]
except:
newsi00 = None
queueSection = cfgPath( ceSection, 'Queues', queueName )
result = gConfig.getOptionsDict( queueSection )
if not result['OK']:
self.log.warn( "Section Queues", result['Message'] )
maxCPUTime = 'Unknown'
si00 = 'Unknown'
else:
queueopt = result['Value']
maxCPUTime = queueopt.get( 'maxCPUTime', 'Unknown' )
si00 = queueopt.get( 'SI00', 'Unknown' )
if newmaxCPUTime and ( maxCPUTime != newmaxCPUTime ):
section = cfgPath( queueSection, 'maxCPUTime' )
self.log.info( section, " -> ".join( ( maxCPUTime, newmaxCPUTime ) ) )
if maxCPUTime == 'Unknown':
self.csAPI.setOption( section, newmaxCPUTime )
else:
self.csAPI.modifyValue( section, newmaxCPUTime )
changed = True
if newsi00 and ( si00 != newsi00 ):
section = cfgPath( queueSection, 'SI00' )
self.log.info( section, " -> ".join( ( si00, newsi00 ) ) )
if si00 == 'Unknown':
self.csAPI.setOption( section, newsi00 )
else:
self.csAPI.modifyValue( section, newsi00 )
changed = True
if False and changed:
self.log.info( body )
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
return self.csAPI.commitChanges( sortUsers = False )
else:
self.log.info( "No changes found" )
return S_OK()
def getGridSRMs(vo, bdiiInfo=None, srmBlackList=None, unUsed=False):
""" Get all the SRMs available for a given VO
:param str vo: VO name
:param dict bdiiInfo: information from BDII
:param list srmBlackList: SRMs from black list
:param bool unUsed: unused
:return: S_OK(dict)/S_ERROR()
"""
result = ldapService(serviceType='SRM', vo=vo)
if not result['OK']:
return result
srmBdiiDict = result['Value']
knownSRMs = set()
if srmBlackList is not None:
knownSRMs = knownSRMs.union(set(srmBlackList))
siteSRMDict = {}
for srm in srmBdiiDict:
srm = dict(srm)
endPoint = srm.get('GlueServiceEndpoint', '')
srmHost = ''
if endPoint:
srmHost = urlparse(endPoint).hostname
if not srmHost:
continue
if srmHost in knownSRMs:
continue
if unUsed:
result = getDIRACSesForHostName(srmHost)
if not result['OK']:
return result
diracSEs = result['Value']
if diracSEs:
# If it is a known SRM and only new SRMs are requested, continue
continue
site = srm.get('GlueForeignKey', '').replace('GlueSiteUniqueID=', '')
siteSRMDict.setdefault(site, {})
siteSRMDict[site][srmHost] = srm
if bdiiInfo is None:
result = getBdiiSEInfo(vo)
if not result['OK']:
return result
seBdiiDict = dict(result['Value'])
else:
seBdiiDict = dict(bdiiInfo)
srmSeDict = {}
for site in siteSRMDict:
srms = siteSRMDict[site].keys()
for srm in srms:
if seBdiiDict.get(site, {}).get('SEs', {}).get(srm, {}):
srmSeDict.setdefault(site, {})
srmSeDict[site].setdefault(srm, {})
srmSeDict[site][srm]['SRM'] = siteSRMDict[site][srm]
srmSeDict[site][srm]['SE'] = seBdiiDict[site]['SEs'][srm]
return S_OK(srmSeDict)