def getGridSRMs( vo, bdiiInfo = None, srmBlackList = None, unUsed = False ): result = ldapService( serviceType = 'SRM', vo = vo ) if not result['OK']: return result srmBdiiDict = result['Value'] knownSRMs = set() if srmBlackList is not None: knownSRMs = knownSRMs.union( set( srmBlackList ) ) siteSRMDict = {} for srm in srmBdiiDict: endPoint = srm.get( 'GlueServiceEndpoint', '') srmHost = '' if endPoint: result = pfnparse( endPoint ) if not result['OK']: continue srmHost = result['Value']['Host'] if not srmHost: continue if srmHost in knownSRMs: continue if unUsed: result = getDIRACSesForSRM( srmHost ) if not result['OK']: return result diracSEs = result['Value'] if diracSEs: # If it is a known SRM and only new SRMs are requested, continue continue site = srm.get( 'GlueForeignKey', '' ).replace( 'GlueSiteUniqueID=', '' ) siteSRMDict.setdefault( site, {} ) siteSRMDict[site][srmHost] = srm if bdiiInfo is None: result = getBdiiSEInfo( vo ) if not result['OK']: return result seBdiiDict = result['Value'] else: seBdiiDict = bdiiInfo srmSeDict = {} for site in siteSRMDict: srms = siteSRMDict[site].keys() for srm in srms: if seBdiiDict.get( site, {} ).get( 'SEs', {} ).get( srm, {} ): srmSeDict.setdefault( site, {} ) srmSeDict[site].setdefault( srm, {} ) srmSeDict[site][srm]['SRM'] = siteSRMDict[site][srm] srmSeDict[site][srm]['SE'] = seBdiiDict[site]['SEs'][srm] return S_OK( srmSeDict )
def __infoFromCE(self): sitesSection = cfgPath('Resources', 'Sites') result = gConfig.getSections(sitesSection) if not result['OK']: return grids = result['Value'] changed = False body = "" for grid in grids: gridSection = cfgPath(sitesSection, grid) result = gConfig.getSections(gridSection) if not result['OK']: return sites = result['Value'] for site in sites: # if site[-2:]!='ru': # continue siteSection = cfgPath(gridSection, site) opt = gConfig.getOptionsDict(siteSection)['Value'] name = opt.get('Name', '') if name: coor = opt.get('Coordinates', 'Unknown') mail = opt.get('Mail', 'Unknown') result = ldapSite(name) if not result['OK']: self.log.warn("BDII site %s: %s" % (name, result['Message'])) result = self.__checkAlternativeBDIISite( ldapSite, name) if result['OK']: bdiisites = result['Value'] if len(bdiisites) == 0: self.log.warn(name, "Error in bdii: leng = 0") else: if not len(bdiisites) == 1: self.log.warn( name, "Warning in bdii: leng = %d" % len(bdiisites)) bdiisite = bdiisites[0] try: longitude = bdiisite['GlueSiteLongitude'] latitude = bdiisite['GlueSiteLatitude'] newcoor = "%s:%s" % (longitude, latitude) except: self.log.warn("Error in bdii coor") newcoor = "Unknown" try: newmail = bdiisite[ 'GlueSiteSysAdminContact'].split( ":")[-1].strip() except: self.log.warn("Error in bdii mail") newmail = "Unknown" self.log.debug("%s %s %s" % (name, newcoor, newmail)) if newcoor != coor: self.log.info("%s" % (name), "%s -> %s" % (coor, newcoor)) if coor == 'Unknown': self.csAPI.setOption( cfgPath(siteSection, 'Coordinates'), newcoor) else: self.csAPI.modifyValue( cfgPath(siteSection, 'Coordinates'), newcoor) changed = True if newmail != mail: self.log.info("%s" % (name), "%s -> %s" % (mail, newmail)) if mail == 'Unknown': self.csAPI.setOption( cfgPath(siteSection, 'Mail'), newmail) else: self.csAPI.modifyValue( cfgPath(siteSection, 'Mail'), newmail) changed = True celist = List.fromChar(opt.get('CE', '')) if not celist: self.log.warn(site, 'Empty site list') continue # result = gConfig.getSections( cfgPath( siteSection,'CEs' ) # if not result['OK']: # self.log.debug( "Section CEs:", result['Message'] ) for ce in celist: ceSection = cfgPath(siteSection, 'CEs', ce) result = gConfig.getOptionsDict(ceSection) if not result['OK']: self.log.debug("Section CE", result['Message']) wnTmpDir = 'Unknown' arch = 'Unknown' os = 'Unknown' si00 = 'Unknown' pilot = 'Unknown' cetype = 'Unknown' else: ceopt = result['Value'] wnTmpDir = ceopt.get('wnTmpDir', 'Unknown') arch = ceopt.get('architecture', 'Unknown') os = ceopt.get('OS', 'Unknown') si00 = ceopt.get('SI00', 'Unknown') pilot = ceopt.get('Pilot', 'Unknown') cetype = ceopt.get('CEType', 'Unknown') result = ldapCE(ce) if not result['OK']: self.log.warn('Error in bdii for %s' % ce, result['Message']) result = self.__checkAlternativeBDIISite(ldapCE, ce) continue try: bdiice = result['Value'][0] except: self.log.warn('Error in bdii for %s' % ce, result) bdiice = None if bdiice: try: newwnTmpDir = bdiice['GlueSubClusterWNTmpDir'] except: newwnTmpDir = 'Unknown' if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown': section = cfgPath(ceSection, 'wnTmpDir') self.log.info(section, " -> ".join( (wnTmpDir, newwnTmpDir))) if wnTmpDir == 'Unknown': self.csAPI.setOption(section, newwnTmpDir) else: self.csAPI.modifyValue(section, newwnTmpDir) changed = True try: newarch = bdiice[ 'GlueHostArchitecturePlatformType'] except: newarch = 'Unknown' if arch != newarch and newarch != 'Unknown': section = cfgPath(ceSection, 'architecture') self.log.info(section, " -> ".join( (arch, newarch))) if arch == 'Unknown': self.csAPI.setOption(section, newarch) else: self.csAPI.modifyValue(section, newarch) changed = True try: newos = '_'.join( (bdiice['GlueHostOperatingSystemName'], bdiice['GlueHostOperatingSystemVersion'], bdiice['GlueHostOperatingSystemRelease'])) except: newos = 'Unknown' if os != newos and newos != 'Unknown': section = cfgPath(ceSection, 'OS') self.log.info(section, " -> ".join((os, newos))) if os == 'Unknown': self.csAPI.setOption(section, newos) else: self.csAPI.modifyValue(section, newos) changed = True body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newos, ce, site) try: newsi00 = bdiice['GlueHostBenchmarkSI00'] except: newsi00 = 'Unknown' if si00 != newsi00 and newsi00 != 'Unknown': section = cfgPath(ceSection, 'SI00') self.log.info(section, " -> ".join( (si00, newsi00))) if si00 == 'Unknown': self.csAPI.setOption(section, newsi00) else: self.csAPI.modifyValue(section, newsi00) changed = True try: rte = bdiice[ 'GlueHostApplicationSoftwareRunTimeEnvironment'] if self.voName == 'lhcb': if 'VO-lhcb-pilot' in rte: newpilot = 'True' else: newpilot = 'False' else: newpilot = 'Unknown' except: newpilot = 'Unknown' if pilot != newpilot and newpilot != 'Unknown': section = cfgPath(ceSection, 'Pilot') self.log.info(section, " -> ".join( (pilot, newpilot))) if pilot == 'Unknown': self.csAPI.setOption(section, newpilot) else: self.csAPI.modifyValue(section, newpilot) changed = True result = ldapService(ce) if not result['OK']: result = self.__checkAlternativeBDIISite( ldapService, ce) if result['OK'] and result['Value']: services = result['Value'] newcetype = 'LCG' for service in services: if service['GlueServiceType'].count('CREAM'): newcetype = "CREAM" else: newcetype = 'Unknown' if cetype != newcetype and newcetype != 'Unknown': section = cfgPath(ceSection, 'CEType') self.log.info(section, " -> ".join( (cetype, newcetype))) if cetype == 'Unknown': self.csAPI.setOption(section, newcetype) else: self.csAPI.modifyValue(section, newcetype) changed = True result = ldapCEState(ce, vo=self.voName) #getBDIICEVOView if not result['OK']: self.log.warn('Error in bdii for queue %s' % ce, result['Message']) result = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName) continue try: queues = result['Value'] except: self.log.warn('Error in bdii for queue %s' % ce, result['Massage']) continue for queue in queues: try: queueName = queue['GlueCEUniqueID'].split('/')[-1] except: self.log.warn('error in queuename ', queue) continue try: newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime'] except: newmaxCPUTime = None newsi00 = None try: caps = queue['GlueCECapability'] if type(caps) == type(''): caps = [caps] for cap in caps: if cap.count('CPUScalingReferenceSI00'): newsi00 = cap.split('=')[-1] except: newsi00 = None queueSection = cfgPath(ceSection, 'Queues', queueName) result = gConfig.getOptionsDict(queueSection) if not result['OK']: self.log.warn("Section Queues", result['Message']) maxCPUTime = 'Unknown' si00 = 'Unknown' else: queueopt = result['Value'] maxCPUTime = queueopt.get('maxCPUTime', 'Unknown') si00 = queueopt.get('SI00', 'Unknown') if newmaxCPUTime and (maxCPUTime != newmaxCPUTime): section = cfgPath(queueSection, 'maxCPUTime') self.log.info( section, " -> ".join( (maxCPUTime, newmaxCPUTime))) if maxCPUTime == 'Unknown': self.csAPI.setOption(section, newmaxCPUTime) else: self.csAPI.modifyValue(section, newmaxCPUTime) changed = True if newsi00 and (si00 != newsi00): section = cfgPath(queueSection, 'SI00') self.log.info(section, " -> ".join( (si00, newsi00))) if si00 == 'Unknown': self.csAPI.setOption(section, newsi00) else: self.csAPI.modifyValue(section, newsi00) changed = True if changed: self.log.info(body) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False) return self.csAPI.commitChanges(sortUsers=False) else: self.log.info("No changes found") return S_OK()
def getBDIIService(self, ce, host=None): """ Get information about ce from BDII at host """ return ldapService(ce, host=host)
def getBDIIService( self, ce, host = None ): """Get information about ce from BDII at host """ return ldapService( ce, host = host )
def __infoFromCE(self): sitesSection = cfgPath("Resources", "Sites") result = gConfig.getSections(sitesSection) if not result["OK"]: return grids = result["Value"] changed = False body = "" for grid in grids: gridSection = cfgPath(sitesSection, grid) result = gConfig.getSections(gridSection) if not result["OK"]: return sites = result["Value"] for site in sites: # if site[-2:]!='ru': # continue siteSection = cfgPath(gridSection, site) opt = gConfig.getOptionsDict(siteSection)["Value"] name = opt.get("Name", "") if name: coor = opt.get("Coordinates", "Unknown") mail = opt.get("Mail", "Unknown") result = ldapSite(name) if not result["OK"]: self.log.warn("BDII site %s: %s" % (name, result["Message"])) result = self.__checkAlternativeBDIISite(ldapSite, name) if result["OK"]: bdiisites = result["Value"] if len(bdiisites) == 0: self.log.warn(name, "Error in bdii: leng = 0") else: if not len(bdiisites) == 1: self.log.warn(name, "Warning in bdii: leng = %d" % len(bdiisites)) bdiisite = bdiisites[0] try: longitude = bdiisite["GlueSiteLongitude"] latitude = bdiisite["GlueSiteLatitude"] newcoor = "%s:%s" % (longitude, latitude) except: self.log.warn("Error in bdii coor") newcoor = "Unknown" try: newmail = bdiisite["GlueSiteSysAdminContact"].split(":")[-1].strip() except: self.log.warn("Error in bdii mail") newmail = "Unknown" self.log.debug("%s %s %s" % (name, newcoor, newmail)) if newcoor != coor: self.log.info("%s" % (name), "%s -> %s" % (coor, newcoor)) if coor == "Unknown": self.csAPI.setOption(cfgPath(siteSection, "Coordinates"), newcoor) else: self.csAPI.modifyValue(cfgPath(siteSection, "Coordinates"), newcoor) changed = True if newmail != mail: self.log.info("%s" % (name), "%s -> %s" % (mail, newmail)) if mail == "Unknown": self.csAPI.setOption(cfgPath(siteSection, "Mail"), newmail) else: self.csAPI.modifyValue(cfgPath(siteSection, "Mail"), newmail) changed = True celist = List.fromChar(opt.get("CE", "")) if not celist: self.log.warn(site, "Empty site list") continue # result = gConfig.getSections( cfgPath( siteSection,'CEs' ) # if not result['OK']: # self.log.debug( "Section CEs:", result['Message'] ) for ce in celist: ceSection = cfgPath(siteSection, "CEs", ce) result = gConfig.getOptionsDict(ceSection) if not result["OK"]: self.log.debug("Section CE", result["Message"]) wnTmpDir = "Unknown" arch = "Unknown" os = "Unknown" si00 = "Unknown" pilot = "Unknown" cetype = "Unknown" else: ceopt = result["Value"] wnTmpDir = ceopt.get("wnTmpDir", "Unknown") arch = ceopt.get("architecture", "Unknown") os = ceopt.get("OS", "Unknown") si00 = ceopt.get("SI00", "Unknown") pilot = ceopt.get("Pilot", "Unknown") cetype = ceopt.get("CEType", "Unknown") result = ldapCE(ce) if not result["OK"]: self.log.warn("Error in bdii for %s" % ce, result["Message"]) result = self.__checkAlternativeBDIISite(ldapCE, ce) continue try: bdiice = result["Value"][0] except: self.log.warn("Error in bdii for %s" % ce, result) bdiice = None if bdiice: try: newwnTmpDir = bdiice["GlueSubClusterWNTmpDir"] except: newwnTmpDir = "Unknown" if wnTmpDir != newwnTmpDir and newwnTmpDir != "Unknown": section = cfgPath(ceSection, "wnTmpDir") self.log.info(section, " -> ".join((wnTmpDir, newwnTmpDir))) if wnTmpDir == "Unknown": self.csAPI.setOption(section, newwnTmpDir) else: self.csAPI.modifyValue(section, newwnTmpDir) changed = True try: newarch = bdiice["GlueHostArchitecturePlatformType"] except: newarch = "Unknown" if arch != newarch and newarch != "Unknown": section = cfgPath(ceSection, "architecture") self.log.info(section, " -> ".join((arch, newarch))) if arch == "Unknown": self.csAPI.setOption(section, newarch) else: self.csAPI.modifyValue(section, newarch) changed = True try: newos = "_".join( ( bdiice["GlueHostOperatingSystemName"], bdiice["GlueHostOperatingSystemVersion"], bdiice["GlueHostOperatingSystemRelease"], ) ) except: newos = "Unknown" if os != newos and newos != "Unknown": section = cfgPath(ceSection, "OS") self.log.info(section, " -> ".join((os, newos))) if os == "Unknown": self.csAPI.setOption(section, newos) else: self.csAPI.modifyValue(section, newos) changed = True body = body + "OS was changed %s -> %s for %s at %s\n" % (os, newos, ce, site) try: newsi00 = bdiice["GlueHostBenchmarkSI00"] except: newsi00 = "Unknown" if si00 != newsi00 and newsi00 != "Unknown": section = cfgPath(ceSection, "SI00") self.log.info(section, " -> ".join((si00, newsi00))) if si00 == "Unknown": self.csAPI.setOption(section, newsi00) else: self.csAPI.modifyValue(section, newsi00) changed = True try: rte = bdiice["GlueHostApplicationSoftwareRunTimeEnvironment"] if self.voName.lower() == "lhcb": if "VO-lhcb-pilot" in rte: newpilot = "True" else: newpilot = "False" else: newpilot = "Unknown" except: newpilot = "Unknown" if pilot != newpilot and newpilot != "Unknown": section = cfgPath(ceSection, "Pilot") self.log.info(section, " -> ".join((pilot, newpilot))) if pilot == "Unknown": self.csAPI.setOption(section, newpilot) else: self.csAPI.modifyValue(section, newpilot) changed = True result = ldapService(ce) if not result["OK"]: result = self.__checkAlternativeBDIISite(ldapService, ce) if result["OK"] and result["Value"]: services = result["Value"] newcetype = "LCG" for service in services: if service["GlueServiceType"].count("CREAM"): newcetype = "CREAM" else: newcetype = "Unknown" if cetype != newcetype and newcetype != "Unknown": section = cfgPath(ceSection, "CEType") self.log.info(section, " -> ".join((cetype, newcetype))) if cetype == "Unknown": self.csAPI.setOption(section, newcetype) else: self.csAPI.modifyValue(section, newcetype) changed = True result = ldapCEState(ce, vo=self.voName) # getBDIICEVOView if not result["OK"]: self.log.warn("Error in bdii for queue %s" % ce, result["Message"]) result = self.__checkAlternativeBDIISite(ldapCEState, ce, self.voName) continue try: queues = result["Value"] except: self.log.warn("Error in bdii for queue %s" % ce, result["Massage"]) continue for queue in queues: try: queueName = queue["GlueCEUniqueID"].split("/")[-1] except: self.log.warn("error in queuename ", queue) continue try: newmaxCPUTime = queue["GlueCEPolicyMaxCPUTime"] except: newmaxCPUTime = None newsi00 = None try: caps = queue["GlueCECapability"] if type(caps) == type(""): caps = [caps] for cap in caps: if cap.count("CPUScalingReferenceSI00"): newsi00 = cap.split("=")[-1] except: newsi00 = None queueSection = cfgPath(ceSection, "Queues", queueName) result = gConfig.getOptionsDict(queueSection) if not result["OK"]: self.log.warn("Section Queues", result["Message"]) maxCPUTime = "Unknown" si00 = "Unknown" else: queueopt = result["Value"] maxCPUTime = queueopt.get("maxCPUTime", "Unknown") si00 = queueopt.get("SI00", "Unknown") if newmaxCPUTime and (maxCPUTime != newmaxCPUTime): section = cfgPath(queueSection, "maxCPUTime") self.log.info(section, " -> ".join((maxCPUTime, newmaxCPUTime))) if maxCPUTime == "Unknown": self.csAPI.setOption(section, newmaxCPUTime) else: self.csAPI.modifyValue(section, newmaxCPUTime) changed = True if newsi00 and (si00 != newsi00): section = cfgPath(queueSection, "SI00") self.log.info(section, " -> ".join((si00, newsi00))) if si00 == "Unknown": self.csAPI.setOption(section, newsi00) else: self.csAPI.modifyValue(section, newsi00) changed = True if changed: self.log.info(body) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False) return self.csAPI.commit() else: self.log.info("No changes found") return S_OK()
def _infoFromCE( self ): sitesSection = cfgPath( 'Resources', 'Sites' ) result = gConfig.getSections( sitesSection ) if not result['OK']: return grids = result['Value'] for grid in grids: gridSection = cfgPath( sitesSection, grid ) result = gConfig.getSections( gridSection ) if not result['OK']: return sites = result['Value'] changed = False body = "" for site in sites: # if site[-2:]!='ru': # continue siteSection = cfgPath( gridSection, site ) opt = gConfig.getOptionsDict( siteSection )['Value'] name = opt.get( 'Name', '' ) if name: coor = opt.get( 'Coordinates', 'Unknown' ) mail = opt.get( 'Mail', 'Unknown' ) result = ldapSite( name ) if not result['OK']: self.log.warn( "BDII site %s: %s" % (name, result['Message'] )) result = self._checkAlternativeBDIISite( ldapSite, name ) if result['OK']: bdiisites = result['Value'] if len( bdiisites ) == 0: self.log.warn( name, "Error in bdii: leng = 0" ) else: if not len( bdiisites ) == 1: self.log.warn( name, "Warning in bdii: leng = %d" % len( bdiisites ) ) bdiisite = bdiisites[0] try: longitude = bdiisite['GlueSiteLongitude'] latitude = bdiisite['GlueSiteLatitude'] newcoor = "%s:%s" % ( longitude, latitude ) except: self.log.warn( "Error in bdii coor" ) newcoor = "Unknown" try: newmail = bdiisite['GlueSiteSysAdminContact'].split( ":" )[-1].strip() except: self.log.warn( "Error in bdii mail" ) newmail = "Unknown" self.log.debug( "%s %s %s" % ( name, newcoor, newmail ) ) if newcoor != coor: self.log.info( "%s" % ( name ), "%s -> %s" % ( coor, newcoor ) ) if coor == 'Unknown': self.csAPI.setOption( cfgPath( siteSection, 'Coordinates' ), newcoor ) else: self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newcoor ) changed = True if newmail != mail: self.log.info( "%s" % ( name ), "%s -> %s" % ( mail, newmail ) ) if mail == 'Unknown': self.csAPI.setOption( cfgPath( siteSection, 'Mail' ), newmail ) else: self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newmail ) changed = True celist = List.fromChar( opt.get( 'CE', '' ) ) if not celist: self.log.warn( site, 'Empty site list' ) continue # result = gConfig.getSections( cfgPath( siteSection,'CEs' ) # if not result['OK']: # self.log.debug( "Section CEs:", result['Message'] ) for ce in celist: ceSection = cfgPath( siteSection, 'CEs', ce ) result = gConfig.getOptionsDict( ceSection ) if not result['OK']: self.log.debug( "Section CE", result['Message'] ) wnTmpDir = 'Unknown' arch = 'Unknown' os = 'Unknown' si00 = 'Unknown' pilot = 'Unknown' cetype = 'Unknown' else: ceopt = result['Value'] wnTmpDir = ceopt.get( 'wnTmpDir', 'Unknown' ) arch = ceopt.get( 'architecture', 'Unknown' ) os = ceopt.get( 'OS', 'Unknown' ) si00 = ceopt.get( 'SI00', 'Unknown' ) pilot = ceopt.get( 'Pilot', 'Unknown' ) cetype = ceopt.get( 'CEType', 'Unknown' ) result = ldapCE( ce ) if not result['OK']: self.log.warn( 'Error in bdii for %s' % ce, result['Message'] ) result = self._checkAlternativeBDIISite( ldapCE, ce ) continue try: bdiice = result['Value'][0] except: self.log.warn( 'Error in bdii for %s' % ce, result ) bdiice = None if bdiice: try: newwnTmpDir = bdiice['GlueSubClusterWNTmpDir'] except: newwnTmpDir = 'Unknown' if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown': section = cfgPath( ceSection, 'wnTmpDir' ) self.log.info( section, " -> ".join( ( wnTmpDir, newwnTmpDir ) ) ) if wnTmpDir == 'Unknown': self.csAPI.setOption( section, newwnTmpDir ) else: self.csAPI.modifyValue( section, newwnTmpDir ) changed = True try: newarch = bdiice['GlueHostArchitecturePlatformType'] except: newarch = 'Unknown' if arch != newarch and newarch != 'Unknown': section = cfgPath( ceSection, 'architecture' ) self.log.info( section, " -> ".join( ( arch, newarch ) ) ) if arch == 'Unknown': self.csAPI.setOption( section, newarch ) else: self.csAPI.modifyValue( section, newarch ) changed = True try: newos = '_'.join( ( bdiice['GlueHostOperatingSystemName'], bdiice['GlueHostOperatingSystemVersion'], bdiice['GlueHostOperatingSystemRelease'] ) ) except: newos = 'Unknown' if os != newos and newos != 'Unknown': section = cfgPath( ceSection, 'OS' ) self.log.info( section, " -> ".join( ( os, newos ) ) ) if os == 'Unknown': self.csAPI.setOption( section, newos ) else: self.csAPI.modifyValue( section, newos ) changed = True body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newos, ce, site ) try: newsi00 = bdiice['GlueHostBenchmarkSI00'] except: newsi00 = 'Unknown' if si00 != newsi00 and newsi00 != 'Unknown': section = cfgPath( ceSection, 'SI00' ) self.log.info( section, " -> ".join( ( si00, newsi00 ) ) ) if si00 == 'Unknown': self.csAPI.setOption( section, newsi00 ) else: self.csAPI.modifyValue( section, newsi00 ) changed = True try: rte = bdiice['GlueHostApplicationSoftwareRunTimeEnvironment'] if self.vo == 'lhcb': if 'VO-lhcb-pilot' in rte: newpilot = 'True' else: newpilot = 'False' else: newpilot = 'Unknown' except: newpilot = 'Unknown' if pilot != newpilot and newpilot != 'Unknown': section = cfgPath( ceSection, 'Pilot' ) self.log.info( section, " -> ".join( ( pilot, newpilot ) ) ) if pilot == 'Unknown': self.csAPI.setOption( section, newpilot ) else: self.csAPI.modifyValue( section, newpilot ) changed = True result = ldapService( ce ) if not result['OK'] : result = self._checkAlternativeBDIISite( ldapService, ce ) if result['OK']: services = result['Value'] newcetype = 'LCG' for service in services: if service['GlueServiceType'].count( 'CREAM' ): newcetype = "CREAM" else: newcetype = 'Unknown' if cetype != newcetype and newcetype != 'Unknown': section = cfgPath( ceSection, 'CEType' ) self.log.info( section, " -> ".join( ( cetype, newcetype ) ) ) if cetype == 'Unknown': self.csAPI.setOption( section, newcetype ) else: self.csAPI.modifyValue( section, newcetype ) changed = True result = ldapCEState( ce, vo = self.vo ) #getBDIICEVOView if not result['OK']: self.log.warn( 'Error in bdii for queue %s' % ce, result['Message'] ) result = self._checkAlternativeBDIISite( ldapCEState, ce, self.vo ) continue try: queues = result['Value'] except: self.log.warn( 'Error in bdii for queue %s' % ce, result['Massage'] ) continue for queue in queues: try: queueName = queue['GlueCEUniqueID'].split( '/' )[-1] except: self.log.warn( 'error in queuename ', queue ) continue try: newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime'] except: newmaxCPUTime = None newsi00 = None try: caps = queue['GlueCECapability'] if type( caps ) == type( '' ): caps = [caps] for cap in caps: if cap.count( 'CPUScalingReferenceSI00' ): newsi00 = cap.split( '=' )[-1] except: newsi00 = None queueSection = cfgPath( ceSection, 'Queues', queueName ) result = gConfig.getOptionsDict( queueSection ) if not result['OK']: self.log.warn( "Section Queues", result['Message'] ) maxCPUTime = 'Unknown' si00 = 'Unknown' else: queueopt = result['Value'] maxCPUTime = queueopt.get( 'maxCPUTime', 'Unknown' ) si00 = queueopt.get( 'SI00', 'Unknown' ) if newmaxCPUTime and ( maxCPUTime != newmaxCPUTime ): section = cfgPath( queueSection, 'maxCPUTime' ) self.log.info( section, " -> ".join( ( maxCPUTime, newmaxCPUTime ) ) ) if maxCPUTime == 'Unknown': self.csAPI.setOption( section, newmaxCPUTime ) else: self.csAPI.modifyValue( section, newmaxCPUTime ) changed = True if newsi00 and ( si00 != newsi00 ): section = cfgPath( queueSection, 'SI00' ) self.log.info( section, " -> ".join( ( si00, newsi00 ) ) ) if si00 == 'Unknown': self.csAPI.setOption( section, newsi00 ) else: self.csAPI.modifyValue( section, newsi00 ) changed = True if False and changed: self.log.info( body ) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False ) return self.csAPI.commitChanges( sortUsers = False ) else: self.log.info( "No changes found" ) return S_OK()
def getGridSRMs(vo, bdiiInfo=None, srmBlackList=None, unUsed=False): """ Get all the SRMs available for a given VO :param str vo: VO name :param dict bdiiInfo: information from BDII :param list srmBlackList: SRMs from black list :param bool unUsed: unused :return: S_OK(dict)/S_ERROR() """ result = ldapService(serviceType='SRM', vo=vo) if not result['OK']: return result srmBdiiDict = result['Value'] knownSRMs = set() if srmBlackList is not None: knownSRMs = knownSRMs.union(set(srmBlackList)) siteSRMDict = {} for srm in srmBdiiDict: srm = dict(srm) endPoint = srm.get('GlueServiceEndpoint', '') srmHost = '' if endPoint: srmHost = urlparse(endPoint).hostname if not srmHost: continue if srmHost in knownSRMs: continue if unUsed: result = getDIRACSesForHostName(srmHost) if not result['OK']: return result diracSEs = result['Value'] if diracSEs: # If it is a known SRM and only new SRMs are requested, continue continue site = srm.get('GlueForeignKey', '').replace('GlueSiteUniqueID=', '') siteSRMDict.setdefault(site, {}) siteSRMDict[site][srmHost] = srm if bdiiInfo is None: result = getBdiiSEInfo(vo) if not result['OK']: return result seBdiiDict = dict(result['Value']) else: seBdiiDict = dict(bdiiInfo) srmSeDict = {} for site in siteSRMDict: srms = siteSRMDict[site].keys() for srm in srms: if seBdiiDict.get(site, {}).get('SEs', {}).get(srm, {}): srmSeDict.setdefault(site, {}) srmSeDict[site].setdefault(srm, {}) srmSeDict[site][srm]['SRM'] = siteSRMDict[site][srm] srmSeDict[site][srm]['SE'] = seBdiiDict[site]['SEs'][srm] return S_OK(srmSeDict)