def __infoFromCE(self): sitesSection = cfgPath('Resources', 'Sites') result = gConfig.getSections(sitesSection) if not result['OK']: return grids = result['Value'] changed = False body = "" for grid in grids: gridSection = cfgPath(sitesSection, grid) result = gConfig.getSections(gridSection) if not result['OK']: return sites = result['Value'] for site in sites: # if site[-2:]!='ru': # continue siteSection = cfgPath(gridSection, site) opt = gConfig.getOptionsDict(siteSection)['Value'] name = opt.get('Name', '') if name: coor = opt.get('Coordinates', 'Unknown') mail = opt.get('Mail', 'Unknown') result = ldapSite(name) if not result['OK']: self.log.warn("BDII site %s: %s" % (name, result['Message'])) result = self.__checkAlternativeBDIISite( ldapSite, name) if result['OK']: bdiisites = result['Value'] if len(bdiisites) == 0: self.log.warn(name, "Error in bdii: leng = 0") else: if not len(bdiisites) == 1: self.log.warn( name, "Warning in bdii: leng = %d" % len(bdiisites)) bdiisite = bdiisites[0] try: longitude = bdiisite['GlueSiteLongitude'] latitude = bdiisite['GlueSiteLatitude'] newcoor = "%s:%s" % (longitude, latitude) except: self.log.warn("Error in bdii coor") newcoor = "Unknown" try: newmail = bdiisite[ 'GlueSiteSysAdminContact'].split( ":")[-1].strip() except: self.log.warn("Error in bdii mail") newmail = "Unknown" self.log.debug("%s %s %s" % (name, newcoor, newmail)) if newcoor != coor: self.log.info("%s" % (name), "%s -> %s" % (coor, newcoor)) if coor == 'Unknown': self.csAPI.setOption( cfgPath(siteSection, 'Coordinates'), newcoor) else: self.csAPI.modifyValue( cfgPath(siteSection, 'Coordinates'), newcoor) changed = True if newmail != mail: self.log.info("%s" % (name), "%s -> %s" % (mail, newmail)) if mail == 'Unknown': self.csAPI.setOption( cfgPath(siteSection, 'Mail'), newmail) else: self.csAPI.modifyValue( cfgPath(siteSection, 'Mail'), newmail) changed = True celist = List.fromChar(opt.get('CE', '')) if not celist: self.log.warn(site, 'Empty site list') continue # result = gConfig.getSections( cfgPath( siteSection,'CEs' ) # if not result['OK']: # self.log.debug( "Section CEs:", result['Message'] ) for ce in celist: ceSection = cfgPath(siteSection, 'CEs', ce) result = gConfig.getOptionsDict(ceSection) if not result['OK']: self.log.debug("Section CE", result['Message']) wnTmpDir = 'Unknown' arch = 'Unknown' os = 'Unknown' si00 = 'Unknown' pilot = 'Unknown' cetype = 'Unknown' else: ceopt = result['Value'] wnTmpDir = ceopt.get('wnTmpDir', 'Unknown') arch = ceopt.get('architecture', 'Unknown') os = ceopt.get('OS', 'Unknown') si00 = ceopt.get('SI00', 'Unknown') pilot = ceopt.get('Pilot', 'Unknown') cetype = ceopt.get('CEType', 'Unknown') result = ldapCE(ce) if not result['OK']: self.log.warn('Error in bdii for %s' % ce, result['Message']) result = self.__checkAlternativeBDIISite(ldapCE, ce) continue try: bdiice = result['Value'][0] except: self.log.warn('Error in bdii for %s' % ce, result) bdiice = None if bdiice: try: newwnTmpDir = bdiice['GlueSubClusterWNTmpDir'] except: newwnTmpDir = 'Unknown' if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown': section = cfgPath(ceSection, 'wnTmpDir') self.log.info(section, " -> ".join( (wnTmpDir, newwnTmpDir))) if wnTmpDir == 'Unknown': self.csAPI.setOption(section, newwnTmpDir) else: self.csAPI.modifyValue(section, newwnTmpDir) changed = True try: newarch = bdiice[ 'GlueHostArchitecturePlatformType'] except: newarch = 'Unknown' if arch != newarch and newarch != 'Unknown': section = cfgPath(ceSection, 'architecture') self.log.info(section, " -> ".join( (arch, newarch))) if arch == 'Unknown': self.csAPI.setOption(section, newarch) else: self.csAPI.modifyValue(section, newarch) changed = True try: newos = '_'.join( (bdiice['GlueHostOperatingSystemName'], bdiice['GlueHostOperatingSystemVersion'], bdiice['GlueHostOperatingSystemRelease'])) except: newos = 'Unknown' if os != newos and newos != 'Unknown': section = cfgPath(ceSection, 'OS') self.log.info(section, " -> ".join((os, newos))) if os == 'Unknown': self.csAPI.setOption(section, newos) else: self.csAPI.modifyValue(section, newos) changed = True body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newos, ce, site) try: newsi00 = bdiice['GlueHostBenchmarkSI00'] except: newsi00 = 'Unknown' if si00 != newsi00 and newsi00 != 'Unknown': section = cfgPath(ceSection, 'SI00') self.log.info(section, " -> ".join( (si00, newsi00))) if si00 == 'Unknown': self.csAPI.setOption(section, newsi00) else: self.csAPI.modifyValue(section, newsi00) changed = True try: rte = bdiice[ 'GlueHostApplicationSoftwareRunTimeEnvironment'] if self.voName == 'lhcb': if 'VO-lhcb-pilot' in rte: newpilot = 'True' else: newpilot = 'False' else: newpilot = 'Unknown' except: newpilot = 'Unknown' if pilot != newpilot and newpilot != 'Unknown': section = cfgPath(ceSection, 'Pilot') self.log.info(section, " -> ".join( (pilot, newpilot))) if pilot == 'Unknown': self.csAPI.setOption(section, newpilot) else: self.csAPI.modifyValue(section, newpilot) changed = True result = ldapService(ce) if not result['OK']: result = self.__checkAlternativeBDIISite( ldapService, ce) if result['OK'] and result['Value']: services = result['Value'] newcetype = 'LCG' for service in services: if service['GlueServiceType'].count('CREAM'): newcetype = "CREAM" else: newcetype = 'Unknown' if cetype != newcetype and newcetype != 'Unknown': section = cfgPath(ceSection, 'CEType') self.log.info(section, " -> ".join( (cetype, newcetype))) if cetype == 'Unknown': self.csAPI.setOption(section, newcetype) else: self.csAPI.modifyValue(section, newcetype) changed = True result = ldapCEState(ce, vo=self.voName) #getBDIICEVOView if not result['OK']: self.log.warn('Error in bdii for queue %s' % ce, result['Message']) result = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName) continue try: queues = result['Value'] except: self.log.warn('Error in bdii for queue %s' % ce, result['Massage']) continue for queue in queues: try: queueName = queue['GlueCEUniqueID'].split('/')[-1] except: self.log.warn('error in queuename ', queue) continue try: newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime'] except: newmaxCPUTime = None newsi00 = None try: caps = queue['GlueCECapability'] if type(caps) == type(''): caps = [caps] for cap in caps: if cap.count('CPUScalingReferenceSI00'): newsi00 = cap.split('=')[-1] except: newsi00 = None queueSection = cfgPath(ceSection, 'Queues', queueName) result = gConfig.getOptionsDict(queueSection) if not result['OK']: self.log.warn("Section Queues", result['Message']) maxCPUTime = 'Unknown' si00 = 'Unknown' else: queueopt = result['Value'] maxCPUTime = queueopt.get('maxCPUTime', 'Unknown') si00 = queueopt.get('SI00', 'Unknown') if newmaxCPUTime and (maxCPUTime != newmaxCPUTime): section = cfgPath(queueSection, 'maxCPUTime') self.log.info( section, " -> ".join( (maxCPUTime, newmaxCPUTime))) if maxCPUTime == 'Unknown': self.csAPI.setOption(section, newmaxCPUTime) else: self.csAPI.modifyValue(section, newmaxCPUTime) changed = True if newsi00 and (si00 != newsi00): section = cfgPath(queueSection, 'SI00') self.log.info(section, " -> ".join( (si00, newsi00))) if si00 == 'Unknown': self.csAPI.setOption(section, newsi00) else: self.csAPI.modifyValue(section, newsi00) changed = True if changed: self.log.info(body) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False) return self.csAPI.commitChanges(sortUsers=False) else: self.log.info("No changes found") return S_OK()
def __infoFromCE( self ): sitesSection = cfgPath( 'Resources', 'Sites' ) result = gConfig.getSections( sitesSection ) if not result['OK']: return grids = result['Value'] changed = False body = "" for grid in grids: gridSection = cfgPath( sitesSection, grid ) result = gConfig.getSections( gridSection ) if not result['OK']: return sites = result['Value'] for site in sites: siteSection = cfgPath( gridSection, site ) opt = gConfig.getOptionsDict( siteSection )['Value'] name = opt.get( 'Name', '' ) if name: coor = opt.get( 'Coordinates', 'Unknown' ) mail = opt.get( 'Mail', 'Unknown' ) result = ldapSite( name ) if not result['OK']: self.log.warn( "BDII site %s: %s" % ( name, result['Message'] ) ) result = self.__checkAlternativeBDIISite( ldapSite, name ) if result['OK']: bdiisites = result['Value'] if len( bdiisites ) == 0: self.log.warn( name, "Error in bdii: leng = 0" ) else: if not len( bdiisites ) == 1: self.log.warn( name, "Warning in bdii: leng = %d" % len( bdiisites ) ) bdiisite = bdiisites[0] try: longitude = bdiisite['GlueSiteLongitude'] latitude = bdiisite['GlueSiteLatitude'] newcoor = "%s:%s" % ( longitude, latitude ) except: self.log.warn( "Error in bdii coor" ) newcoor = "Unknown" try: newmail = bdiisite['GlueSiteSysAdminContact'].split( ":" )[-1].strip() except: self.log.warn( "Error in bdii mail" ) newmail = "Unknown" self.log.debug( "%s %s %s" % ( name, newcoor, newmail ) ) if newcoor != coor: self.log.info( "%s" % ( name ), "%s -> %s" % ( coor, newcoor ) ) if coor == 'Unknown': self.csAPI.setOption( cfgPath( siteSection, 'Coordinates' ), newcoor ) else: self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newcoor ) changed = True if newmail != mail: self.log.info( "%s" % ( name ), "%s -> %s" % ( mail, newmail ) ) if mail == 'Unknown': self.csAPI.setOption( cfgPath( siteSection, 'Mail' ), newmail ) else: self.csAPI.modifyValue( cfgPath( siteSection, 'Mail' ), newmail ) changed = True celist = List.fromChar( opt.get( 'CE', '' ) ) if not celist: self.log.warn( site, 'Empty site list' ) continue # result = gConfig.getSections( cfgPath( siteSection,'CEs' ) # if not result['OK']: # self.log.debug( "Section CEs:", result['Message'] ) for ce in celist: ceSection = cfgPath( siteSection, 'CEs', ce ) result = gConfig.getOptionsDict( ceSection ) if not result['OK']: self.log.debug( "Section CE", result['Message'] ) wnTmpDir = 'Unknown' arch = 'Unknown' os = 'Unknown' si00 = 'Unknown' pilot = 'Unknown' cetype = 'Unknown' else: ceopt = result['Value'] wnTmpDir = ceopt.get( 'wnTmpDir', 'Unknown' ) arch = ceopt.get( 'architecture', 'Unknown' ) os = ceopt.get( 'OS', 'Unknown' ) si00 = ceopt.get( 'SI00', 'Unknown' ) pilot = ceopt.get( 'Pilot', 'Unknown' ) cetype = ceopt.get( 'CEType', 'Unknown' ) result = ldapCE( ce ) if not result['OK']: self.log.warn( 'Error in bdii for %s' % ce, result['Message'] ) result = self.__checkAlternativeBDIISite( ldapCE, ce ) continue try: bdiice = result['Value'][0] except: self.log.warn( 'Error in bdii for %s' % ce, result ) bdiice = None if bdiice: try: newwnTmpDir = bdiice['GlueSubClusterWNTmpDir'] except: newwnTmpDir = 'Unknown' if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown': section = cfgPath( ceSection, 'wnTmpDir' ) self.log.info( section, " -> ".join( ( wnTmpDir, newwnTmpDir ) ) ) if wnTmpDir == 'Unknown': self.csAPI.setOption( section, newwnTmpDir ) else: self.csAPI.modifyValue( section, newwnTmpDir ) changed = True try: newarch = bdiice['GlueHostArchitecturePlatformType'] except: newarch = 'Unknown' if arch != newarch and newarch != 'Unknown': section = cfgPath( ceSection, 'architecture' ) self.log.info( section, " -> ".join( ( arch, newarch ) ) ) if arch == 'Unknown': self.csAPI.setOption( section, newarch ) else: self.csAPI.modifyValue( section, newarch ) changed = True try: newos = '_'.join( ( bdiice['GlueHostOperatingSystemName'], bdiice['GlueHostOperatingSystemVersion'], bdiice['GlueHostOperatingSystemRelease'] ) ) except: newos = 'Unknown' if os != newos and newos != 'Unknown': section = cfgPath( ceSection, 'OS' ) self.log.info( section, " -> ".join( ( os, newos ) ) ) if os == 'Unknown': self.csAPI.setOption( section, newos ) else: self.csAPI.modifyValue( section, newos ) changed = True body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newos, ce, site ) try: newsi00 = bdiice['GlueHostBenchmarkSI00'] except: newsi00 = 'Unknown' if si00 != newsi00 and newsi00 != 'Unknown': section = cfgPath( ceSection, 'SI00' ) self.log.info( section, " -> ".join( ( si00, newsi00 ) ) ) if si00 == 'Unknown': self.csAPI.setOption( section, newsi00 ) else: self.csAPI.modifyValue( section, newsi00 ) changed = True try: rte = bdiice['GlueHostApplicationSoftwareRunTimeEnvironment'] if self.voName.lower() == 'lhcb': if 'VO-lhcb-pilot' in rte: newpilot = 'True' else: newpilot = 'False' else: newpilot = 'Unknown' except: newpilot = 'Unknown' if pilot != newpilot and newpilot != 'Unknown': section = cfgPath( ceSection, 'Pilot' ) self.log.info( section, " -> ".join( ( pilot, newpilot ) ) ) if pilot == 'Unknown': self.csAPI.setOption( section, newpilot ) else: self.csAPI.modifyValue( section, newpilot ) changed = True result = ldapCEState( ce, vo = self.voName ) #getBDIICEVOView if not result['OK']: self.log.warn( 'Error in bdii for queue %s' % ce, result['Message'] ) result = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName ) continue try: queues = result['Value'] except: self.log.warn( 'Error in bdii for queue %s' % ce, result['Massage'] ) continue newcetype = 'Unknown' for queue in queues: try: queuetype = queue['GlueCEImplementationName'] except: queuetype = 'Unknown' if newcetype == 'Unknown': newcetype = queuetype else: if queuetype != newcetype: self.log.warn( 'Error in bdii for ce %s ' % ce, 'different cetypes %s %s' % ( newcetype, queuetype ) ) if newcetype=='ARC-CE': newcetype = 'ARC' if cetype != newcetype and newcetype != 'Unknown': section = cfgPath( ceSection, 'CEType' ) self.log.info( section, " -> ".join( ( cetype, newcetype ) ) ) if cetype == 'Unknown': self.csAPI.setOption( section, newcetype ) else: self.csAPI.modifyValue( section, newcetype ) changed = True for queue in queues: try: queueName = queue['GlueCEUniqueID'].split( '/' )[-1] except: self.log.warn( 'error in queuename ', queue ) continue try: newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime'] except: newmaxCPUTime = None newsi00 = None try: caps = queue['GlueCECapability'] if type( caps ) == type( '' ): caps = [caps] for cap in caps: if cap.count( 'CPUScalingReferenceSI00' ): newsi00 = cap.split( '=' )[-1] except: newsi00 = None queueSection = cfgPath( ceSection, 'Queues', queueName ) result = gConfig.getOptionsDict( queueSection ) if not result['OK']: self.log.warn( "Section Queues", result['Message'] ) maxCPUTime = 'Unknown' si00 = 'Unknown' else: queueopt = result['Value'] maxCPUTime = queueopt.get( 'maxCPUTime', 'Unknown' ) si00 = queueopt.get( 'SI00', 'Unknown' ) if newmaxCPUTime and ( maxCPUTime != newmaxCPUTime ): section = cfgPath( queueSection, 'maxCPUTime' ) self.log.info( section, " -> ".join( ( maxCPUTime, newmaxCPUTime ) ) ) if maxCPUTime == 'Unknown': self.csAPI.setOption( section, newmaxCPUTime ) else: self.csAPI.modifyValue( section, newmaxCPUTime ) changed = True if newsi00 and ( si00 != newsi00 ): section = cfgPath( queueSection, 'SI00' ) self.log.info( section, " -> ".join( ( si00, newsi00 ) ) ) if si00 == 'Unknown': self.csAPI.setOption( section, newsi00 ) else: self.csAPI.modifyValue( section, newsi00 ) changed = True if changed: self.log.info( body ) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False ) return self.csAPI.commit() else: self.log.info( "No changes found" ) return S_OK()
def getBDIISite(self, site, host=None): """ Get information about site from BDII at host """ return ldapSite(site, host=host)
def __infoFromCE(self): sitesSection = cfgPath('Resources', 'Sites') result = gConfig.getSections(sitesSection) if not result['OK']: return grids = result['Value'] changed = False body = "" for grid in grids: gridSection = cfgPath(sitesSection, grid) result = gConfig.getSections(gridSection) if not result['OK']: return sites = result['Value'] for site in sites: siteSection = cfgPath(gridSection, site) opt = gConfig.getOptionsDict(siteSection)['Value'] name = opt.get('Name', '') if name: coor = opt.get('Coordinates', 'Unknown') mail = opt.get('Mail', 'Unknown') result = ldapSite(name) if not result['OK']: self.log.warn("BDII site %s: %s" % (name, result['Message'])) result = self.__checkAlternativeBDIISite( ldapSite, name) if result['OK']: bdiiSites = result['Value'] if len(bdiiSites) == 0: self.log.warn(name, "Error in BDII: leng = 0") else: if not len(bdiiSites) == 1: self.log.warn( name, "Warning in BDII: leng = %d" % len(bdiiSites)) bdiiSite = bdiiSites[0] try: longitude = bdiiSite['GlueSiteLongitude'] latitude = bdiiSite['GlueSiteLatitude'] newcoor = "%s:%s" % (longitude, latitude) except: self.log.warn("Error in BDII coordinates") newcoor = "Unknown" try: newmail = bdiiSite[ 'GlueSiteSysAdminContact'].split( ":")[-1].strip() except: self.log.warn("Error in BDII mail") newmail = "Unknown" self.log.debug("%s %s %s" % (name, newcoor, newmail)) if newcoor != coor: self.log.info("%s" % (name), "%s -> %s" % (coor, newcoor)) if coor == 'Unknown': self.csAPI.setOption( cfgPath(siteSection, 'Coordinates'), newcoor) else: self.csAPI.modifyValue( cfgPath(siteSection, 'Coordinates'), newcoor) changed = True if newmail != mail: self.log.info("%s" % (name), "%s -> %s" % (mail, newmail)) if mail == 'Unknown': self.csAPI.setOption( cfgPath(siteSection, 'Mail'), newmail) else: self.csAPI.modifyValue( cfgPath(siteSection, 'Mail'), newmail) changed = True ceList = List.fromChar(opt.get('CE', '')) if not ceList: self.log.warn(site, 'Empty site list') continue # result = gConfig.getSections( cfgPath( siteSection,'CEs' ) # if not result['OK']: # self.log.debug( "Section CEs:", result['Message'] ) for ce in ceList: ceSection = cfgPath(siteSection, 'CEs', ce) result = gConfig.getOptionsDict(ceSection) if not result['OK']: self.log.debug("Section CE", result['Message']) wnTmpDir = 'Unknown' arch = 'Unknown' os = 'Unknown' si00 = 'Unknown' pilot = 'Unknown' ceType = 'Unknown' else: ceopt = result['Value'] wnTmpDir = ceopt.get('wnTmpDir', 'Unknown') arch = ceopt.get('architecture', 'Unknown') os = ceopt.get('OS', 'Unknown') si00 = ceopt.get('SI00', 'Unknown') pilot = ceopt.get('Pilot', 'Unknown') ceType = ceopt.get('CEType', 'Unknown') result = ldapCE(ce) if not result['OK']: self.log.warn('Error in BDII for %s' % ce, result['Message']) result = self.__checkAlternativeBDIISite(ldapCE, ce) continue try: bdiiCE = result['Value'][0] except: self.log.warn('Error in BDII for %s' % ce, result) bdiiCE = None if bdiiCE: try: newWNTmpDir = bdiiCE['GlueSubClusterWNTmpDir'] except: newWNTmpDir = 'Unknown' if wnTmpDir != newWNTmpDir and newWNTmpDir != 'Unknown': section = cfgPath(ceSection, 'wnTmpDir') self.log.info(section, " -> ".join( (wnTmpDir, newWNTmpDir))) if wnTmpDir == 'Unknown': self.csAPI.setOption(section, newWNTmpDir) else: self.csAPI.modifyValue(section, newWNTmpDir) changed = True try: newArch = bdiiCE[ 'GlueHostArchitecturePlatformType'] except: newArch = 'Unknown' if arch != newArch and newArch != 'Unknown': section = cfgPath(ceSection, 'architecture') self.log.info(section, " -> ".join( (arch, newArch))) if arch == 'Unknown': self.csAPI.setOption(section, newArch) else: self.csAPI.modifyValue(section, newArch) changed = True try: newOS = '_'.join( (bdiiCE['GlueHostOperatingSystemName'], bdiiCE['GlueHostOperatingSystemVersion'], bdiiCE['GlueHostOperatingSystemRelease'])) except: newOS = 'Unknown' if os != newOS and newOS != 'Unknown': section = cfgPath(ceSection, 'OS') self.log.info(section, " -> ".join((os, newOS))) if os == 'Unknown': self.csAPI.setOption(section, newOS) else: self.csAPI.modifyValue(section, newOS) changed = True body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newOS, ce, site) try: newSI00 = bdiiCE['GlueHostBenchmarkSI00'] except: newSI00 = 'Unknown' if si00 != newSI00 and newSI00 != 'Unknown': section = cfgPath(ceSection, 'SI00') self.log.info(section, " -> ".join( (si00, newSI00))) if si00 == 'Unknown': self.csAPI.setOption(section, newSI00) else: self.csAPI.modifyValue(section, newSI00) changed = True try: rte = bdiiCE[ 'GlueHostApplicationSoftwareRunTimeEnvironment'] for vo in self.voName: if vo.lower() == 'lhcb': if 'VO-lhcb-pilot' in rte: newPilot = 'True' else: newPilot = 'False' else: newPilot = 'Unknown' except: newPilot = 'Unknown' if pilot != newPilot and newPilot != 'Unknown': section = cfgPath(ceSection, 'Pilot') self.log.info(section, " -> ".join( (pilot, newPilot))) if pilot == 'Unknown': self.csAPI.setOption(section, newPilot) else: self.csAPI.modifyValue(section, newPilot) changed = True newVO = '' for vo in self.voName: result = ldapCEState(ce, vo) #getBDIICEVOView if not result['OK']: self.log.warn('Error in BDII for queue %s' % ce, result['Message']) result = self.__checkAlternativeBDIISite( ldapCEState, ce, vo) continue try: queues = result['Value'] except: self.log.warn('Error in BDII for queue %s' % ce, result['Massage']) continue newCEType = 'Unknown' for queue in queues: try: queueType = queue['GlueCEImplementationName'] except: queueType = 'Unknown' if newCEType == 'Unknown': newCEType = queueType else: if queueType != newCEType: self.log.warn( 'Error in BDII for CE %s ' % ce, 'different CE types %s %s' % (newCEType, queueType)) if newCEType == 'ARC-CE': newCEType = 'ARC' if ceType != newCEType and newCEType != 'Unknown': section = cfgPath(ceSection, 'CEType') self.log.info(section, " -> ".join( (ceType, newCEType))) if ceType == 'Unknown': self.csAPI.setOption(section, newCEType) else: self.csAPI.modifyValue(section, newCEType) changed = True for queue in queues: try: queueName = queue['GlueCEUniqueID'].split( '/')[-1] except: self.log.warn('Error in queueName ', queue) continue try: newMaxCPUTime = queue['GlueCEPolicyMaxCPUTime'] except: newMaxCPUTime = None newSI00 = None try: caps = queue['GlueCECapability'] if type(caps) == type(''): caps = [caps] for cap in caps: if cap.count('CPUScalingReferenceSI00'): newSI00 = cap.split('=')[-1] except: newSI00 = None queueSection = cfgPath(ceSection, 'Queues', queueName) result = gConfig.getOptionsDict(queueSection) if not result['OK']: self.log.warn("Section Queues", result['Message']) maxCPUTime = 'Unknown' si00 = 'Unknown' allowedVOs = [''] else: queueOpt = result['Value'] maxCPUTime = queueOpt.get( 'maxCPUTime', 'Unknown') si00 = queueOpt.get('SI00', 'Unknown') if newVO == '': # Remember previous iteration, if none - read from conf allowedVOs = queueOpt.get('VO', '').split(",") else: # Else use newVO, as it can contain changes, which aren't in conf yet allowedVOs = newVO.split(",") if newMaxCPUTime and (maxCPUTime != newMaxCPUTime): section = cfgPath(queueSection, 'maxCPUTime') self.log.info( section, " -> ".join( (maxCPUTime, newMaxCPUTime))) if maxCPUTime == 'Unknown': self.csAPI.setOption( section, newMaxCPUTime) else: self.csAPI.modifyValue( section, newMaxCPUTime) changed = True if newSI00 and (si00 != newSI00): section = cfgPath(queueSection, 'SI00') self.log.info(section, " -> ".join( (si00, newSI00))) if si00 == 'Unknown': self.csAPI.setOption(section, newSI00) else: self.csAPI.modifyValue(section, newSI00) changed = True modifyVO = True # Flag saying if we need VO option to change newVO = '' if allowedVOs != ['']: for allowedVO in allowedVOs: allowedVO = allowedVO.strip( ) # Get rid of spaces newVO += allowedVO if allowedVO == vo: # Current VO has been already in list newVO = '' modifyVO = False # Don't change anything break # Skip next 'if', proceed to next VO newVO += ', ' if modifyVO: section = cfgPath(queueSection, 'VO') newVO += vo self.log.info( section, " -> ".join( ('%s' % allowedVOs, newVO))) if allowedVOs == ['']: self.csAPI.setOption(section, newVO) else: self.csAPI.modifyValue(section, newVO) changed = True if changed: self.log.info(body) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False) return self.csAPI.commit() else: self.log.info("No changes found") return S_OK()
def getBDIISite( self, site, host = None ): """Get information about site from BDII at host """ return ldapSite( site, host = host )
def __infoFromCE(self): sitesSection = cfgPath("Resources", "Sites") result = gConfig.getSections(sitesSection) if not result["OK"]: return grids = result["Value"] changed = False body = "" for grid in grids: gridSection = cfgPath(sitesSection, grid) result = gConfig.getSections(gridSection) if not result["OK"]: return sites = result["Value"] for site in sites: # if site[-2:]!='ru': # continue siteSection = cfgPath(gridSection, site) opt = gConfig.getOptionsDict(siteSection)["Value"] name = opt.get("Name", "") if name: coor = opt.get("Coordinates", "Unknown") mail = opt.get("Mail", "Unknown") result = ldapSite(name) if not result["OK"]: self.log.warn("BDII site %s: %s" % (name, result["Message"])) result = self.__checkAlternativeBDIISite(ldapSite, name) if result["OK"]: bdiisites = result["Value"] if len(bdiisites) == 0: self.log.warn(name, "Error in bdii: leng = 0") else: if not len(bdiisites) == 1: self.log.warn(name, "Warning in bdii: leng = %d" % len(bdiisites)) bdiisite = bdiisites[0] try: longitude = bdiisite["GlueSiteLongitude"] latitude = bdiisite["GlueSiteLatitude"] newcoor = "%s:%s" % (longitude, latitude) except: self.log.warn("Error in bdii coor") newcoor = "Unknown" try: newmail = bdiisite["GlueSiteSysAdminContact"].split(":")[-1].strip() except: self.log.warn("Error in bdii mail") newmail = "Unknown" self.log.debug("%s %s %s" % (name, newcoor, newmail)) if newcoor != coor: self.log.info("%s" % (name), "%s -> %s" % (coor, newcoor)) if coor == "Unknown": self.csAPI.setOption(cfgPath(siteSection, "Coordinates"), newcoor) else: self.csAPI.modifyValue(cfgPath(siteSection, "Coordinates"), newcoor) changed = True if newmail != mail: self.log.info("%s" % (name), "%s -> %s" % (mail, newmail)) if mail == "Unknown": self.csAPI.setOption(cfgPath(siteSection, "Mail"), newmail) else: self.csAPI.modifyValue(cfgPath(siteSection, "Mail"), newmail) changed = True celist = List.fromChar(opt.get("CE", "")) if not celist: self.log.warn(site, "Empty site list") continue # result = gConfig.getSections( cfgPath( siteSection,'CEs' ) # if not result['OK']: # self.log.debug( "Section CEs:", result['Message'] ) for ce in celist: ceSection = cfgPath(siteSection, "CEs", ce) result = gConfig.getOptionsDict(ceSection) if not result["OK"]: self.log.debug("Section CE", result["Message"]) wnTmpDir = "Unknown" arch = "Unknown" os = "Unknown" si00 = "Unknown" pilot = "Unknown" cetype = "Unknown" else: ceopt = result["Value"] wnTmpDir = ceopt.get("wnTmpDir", "Unknown") arch = ceopt.get("architecture", "Unknown") os = ceopt.get("OS", "Unknown") si00 = ceopt.get("SI00", "Unknown") pilot = ceopt.get("Pilot", "Unknown") cetype = ceopt.get("CEType", "Unknown") result = ldapCE(ce) if not result["OK"]: self.log.warn("Error in bdii for %s" % ce, result["Message"]) result = self.__checkAlternativeBDIISite(ldapCE, ce) continue try: bdiice = result["Value"][0] except: self.log.warn("Error in bdii for %s" % ce, result) bdiice = None if bdiice: try: newwnTmpDir = bdiice["GlueSubClusterWNTmpDir"] except: newwnTmpDir = "Unknown" if wnTmpDir != newwnTmpDir and newwnTmpDir != "Unknown": section = cfgPath(ceSection, "wnTmpDir") self.log.info(section, " -> ".join((wnTmpDir, newwnTmpDir))) if wnTmpDir == "Unknown": self.csAPI.setOption(section, newwnTmpDir) else: self.csAPI.modifyValue(section, newwnTmpDir) changed = True try: newarch = bdiice["GlueHostArchitecturePlatformType"] except: newarch = "Unknown" if arch != newarch and newarch != "Unknown": section = cfgPath(ceSection, "architecture") self.log.info(section, " -> ".join((arch, newarch))) if arch == "Unknown": self.csAPI.setOption(section, newarch) else: self.csAPI.modifyValue(section, newarch) changed = True try: newos = "_".join( ( bdiice["GlueHostOperatingSystemName"], bdiice["GlueHostOperatingSystemVersion"], bdiice["GlueHostOperatingSystemRelease"], ) ) except: newos = "Unknown" if os != newos and newos != "Unknown": section = cfgPath(ceSection, "OS") self.log.info(section, " -> ".join((os, newos))) if os == "Unknown": self.csAPI.setOption(section, newos) else: self.csAPI.modifyValue(section, newos) changed = True body = body + "OS was changed %s -> %s for %s at %s\n" % (os, newos, ce, site) try: newsi00 = bdiice["GlueHostBenchmarkSI00"] except: newsi00 = "Unknown" if si00 != newsi00 and newsi00 != "Unknown": section = cfgPath(ceSection, "SI00") self.log.info(section, " -> ".join((si00, newsi00))) if si00 == "Unknown": self.csAPI.setOption(section, newsi00) else: self.csAPI.modifyValue(section, newsi00) changed = True try: rte = bdiice["GlueHostApplicationSoftwareRunTimeEnvironment"] if self.voName.lower() == "lhcb": if "VO-lhcb-pilot" in rte: newpilot = "True" else: newpilot = "False" else: newpilot = "Unknown" except: newpilot = "Unknown" if pilot != newpilot and newpilot != "Unknown": section = cfgPath(ceSection, "Pilot") self.log.info(section, " -> ".join((pilot, newpilot))) if pilot == "Unknown": self.csAPI.setOption(section, newpilot) else: self.csAPI.modifyValue(section, newpilot) changed = True result = ldapService(ce) if not result["OK"]: result = self.__checkAlternativeBDIISite(ldapService, ce) if result["OK"] and result["Value"]: services = result["Value"] newcetype = "LCG" for service in services: if service["GlueServiceType"].count("CREAM"): newcetype = "CREAM" else: newcetype = "Unknown" if cetype != newcetype and newcetype != "Unknown": section = cfgPath(ceSection, "CEType") self.log.info(section, " -> ".join((cetype, newcetype))) if cetype == "Unknown": self.csAPI.setOption(section, newcetype) else: self.csAPI.modifyValue(section, newcetype) changed = True result = ldapCEState(ce, vo=self.voName) # getBDIICEVOView if not result["OK"]: self.log.warn("Error in bdii for queue %s" % ce, result["Message"]) result = self.__checkAlternativeBDIISite(ldapCEState, ce, self.voName) continue try: queues = result["Value"] except: self.log.warn("Error in bdii for queue %s" % ce, result["Massage"]) continue for queue in queues: try: queueName = queue["GlueCEUniqueID"].split("/")[-1] except: self.log.warn("error in queuename ", queue) continue try: newmaxCPUTime = queue["GlueCEPolicyMaxCPUTime"] except: newmaxCPUTime = None newsi00 = None try: caps = queue["GlueCECapability"] if type(caps) == type(""): caps = [caps] for cap in caps: if cap.count("CPUScalingReferenceSI00"): newsi00 = cap.split("=")[-1] except: newsi00 = None queueSection = cfgPath(ceSection, "Queues", queueName) result = gConfig.getOptionsDict(queueSection) if not result["OK"]: self.log.warn("Section Queues", result["Message"]) maxCPUTime = "Unknown" si00 = "Unknown" else: queueopt = result["Value"] maxCPUTime = queueopt.get("maxCPUTime", "Unknown") si00 = queueopt.get("SI00", "Unknown") if newmaxCPUTime and (maxCPUTime != newmaxCPUTime): section = cfgPath(queueSection, "maxCPUTime") self.log.info(section, " -> ".join((maxCPUTime, newmaxCPUTime))) if maxCPUTime == "Unknown": self.csAPI.setOption(section, newmaxCPUTime) else: self.csAPI.modifyValue(section, newmaxCPUTime) changed = True if newsi00 and (si00 != newsi00): section = cfgPath(queueSection, "SI00") self.log.info(section, " -> ".join((si00, newsi00))) if si00 == "Unknown": self.csAPI.setOption(section, newsi00) else: self.csAPI.modifyValue(section, newsi00) changed = True if changed: self.log.info(body) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False) return self.csAPI.commit() else: self.log.info("No changes found") return S_OK()
def __infoFromCE( self ): sitesSection = cfgPath( 'Resources', 'Sites' ) result = gConfig.getSections( sitesSection ) if not result['OK']: return grids = result['Value'] changed = False body = "" for grid in grids: gridSection = cfgPath( sitesSection, grid ) result = gConfig.getSections( gridSection ) if not result['OK']: return sites = result['Value'] for site in sites: siteSection = cfgPath( gridSection, site ) opt = gConfig.getOptionsDict( siteSection )['Value'] name = opt.get( 'Name', '' ) if name: coor = opt.get( 'Coordinates', 'Unknown' ) mail = opt.get( 'Mail', 'Unknown' ) result = ldapSite( name ) if not result['OK']: self.log.warn( "BDII site %s: %s" % ( name, result['Message'] ) ) result = self.__checkAlternativeBDIISite( ldapSite, name ) if result['OK']: bdiiSites = result['Value'] if len( bdiiSites ) == 0: self.log.warn( name, "Error in BDII: leng = 0" ) else: if not len( bdiiSites ) == 1: self.log.warn( name, "Warning in BDII: leng = %d" % len( bdiiSites ) ) bdiiSite = bdiiSites[0] try: longitude = bdiiSite['GlueSiteLongitude'] latitude = bdiiSite['GlueSiteLatitude'] newcoor = "%s:%s" % ( longitude, latitude ) except: self.log.warn( "Error in BDII coordinates" ) newcoor = "Unknown" try: newmail = bdiiSite['GlueSiteSysAdminContact'].split( ":" )[-1].strip() except: self.log.warn( "Error in BDII mail" ) newmail = "Unknown" self.log.debug( "%s %s %s" % ( name, newcoor, newmail ) ) if newcoor != coor: self.log.info( "%s" % ( name ), "%s -> %s" % ( coor, newcoor ) ) if coor == 'Unknown': self.csAPI.setOption( cfgPath( siteSection, 'Coordinates' ), newcoor ) else: self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newcoor ) changed = True if newmail != mail: self.log.info( "%s" % ( name ), "%s -> %s" % ( mail, newmail ) ) if mail == 'Unknown': self.csAPI.setOption( cfgPath( siteSection, 'Mail' ), newmail ) else: self.csAPI.modifyValue( cfgPath( siteSection, 'Mail' ), newmail ) changed = True ceList = List.fromChar( opt.get( 'CE', '' ) ) if not ceList: self.log.warn( site, 'Empty site list' ) continue # result = gConfig.getSections( cfgPath( siteSection,'CEs' ) # if not result['OK']: # self.log.debug( "Section CEs:", result['Message'] ) for ce in ceList: ceSection = cfgPath( siteSection, 'CEs', ce ) result = gConfig.getOptionsDict( ceSection ) if not result['OK']: self.log.debug( "Section CE", result['Message'] ) wnTmpDir = 'Unknown' arch = 'Unknown' os = 'Unknown' si00 = 'Unknown' pilot = 'Unknown' ceType = 'Unknown' else: ceopt = result['Value'] wnTmpDir = ceopt.get( 'wnTmpDir', 'Unknown' ) arch = ceopt.get( 'architecture', 'Unknown' ) os = ceopt.get( 'OS', 'Unknown' ) si00 = ceopt.get( 'SI00', 'Unknown' ) pilot = ceopt.get( 'Pilot', 'Unknown' ) ceType = ceopt.get( 'CEType', 'Unknown' ) result = ldapCE( ce ) if not result['OK']: self.log.warn( 'Error in BDII for %s' % ce, result['Message'] ) result = self.__checkAlternativeBDIISite( ldapCE, ce ) continue try: bdiiCE = result['Value'][0] except: self.log.warn( 'Error in BDII for %s' % ce, result ) bdiiCE = None if bdiiCE: try: newWNTmpDir = bdiiCE['GlueSubClusterWNTmpDir'] except: newWNTmpDir = 'Unknown' if wnTmpDir != newWNTmpDir and newWNTmpDir != 'Unknown': section = cfgPath( ceSection, 'wnTmpDir' ) self.log.info( section, " -> ".join( ( wnTmpDir, newWNTmpDir ) ) ) if wnTmpDir == 'Unknown': self.csAPI.setOption( section, newWNTmpDir ) else: self.csAPI.modifyValue( section, newWNTmpDir ) changed = True try: newArch = bdiiCE['GlueHostArchitecturePlatformType'] except: newArch = 'Unknown' if arch != newArch and newArch != 'Unknown': section = cfgPath( ceSection, 'architecture' ) self.log.info( section, " -> ".join( ( arch, newArch ) ) ) if arch == 'Unknown': self.csAPI.setOption( section, newArch ) else: self.csAPI.modifyValue( section, newArch ) changed = True try: newOS = '_'.join( ( bdiiCE['GlueHostOperatingSystemName'], bdiiCE['GlueHostOperatingSystemVersion'], bdiiCE['GlueHostOperatingSystemRelease'] ) ) except: newOS = 'Unknown' if os != newOS and newOS != 'Unknown': section = cfgPath( ceSection, 'OS' ) self.log.info( section, " -> ".join( ( os, newOS ) ) ) if os == 'Unknown': self.csAPI.setOption( section, newOS ) else: self.csAPI.modifyValue( section, newOS ) changed = True body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newOS, ce, site ) try: newSI00 = bdiiCE['GlueHostBenchmarkSI00'] except: newSI00 = 'Unknown' if si00 != newSI00 and newSI00 != 'Unknown': section = cfgPath( ceSection, 'SI00' ) self.log.info( section, " -> ".join( ( si00, newSI00 ) ) ) if si00 == 'Unknown': self.csAPI.setOption( section, newSI00 ) else: self.csAPI.modifyValue( section, newSI00 ) changed = True try: rte = bdiiCE['GlueHostApplicationSoftwareRunTimeEnvironment'] for vo in self.voName: if vo.lower() == 'lhcb': if 'VO-lhcb-pilot' in rte: newPilot = 'True' else: newPilot = 'False' else: newPilot = 'Unknown' except: newPilot = 'Unknown' if pilot != newPilot and newPilot != 'Unknown': section = cfgPath( ceSection, 'Pilot' ) self.log.info( section, " -> ".join( ( pilot, newPilot ) ) ) if pilot == 'Unknown': self.csAPI.setOption( section, newPilot ) else: self.csAPI.modifyValue( section, newPilot ) changed = True newVO = '' for vo in self.voName: result = ldapCEState( ce, vo ) #getBDIICEVOView if not result['OK']: self.log.warn( 'Error in BDII for queue %s' % ce, result['Message'] ) result = self.__checkAlternativeBDIISite( ldapCEState, ce, vo ) continue try: queues = result['Value'] except: self.log.warn( 'Error in BDII for queue %s' % ce, result['Massage'] ) continue newCEType = 'Unknown' for queue in queues: try: queueType = queue['GlueCEImplementationName'] except: queueType = 'Unknown' if newCEType == 'Unknown': newCEType = queueType else: if queueType != newCEType: self.log.warn( 'Error in BDII for CE %s ' % ce, 'different CE types %s %s' % ( newCEType, queueType ) ) if newCEType=='ARC-CE': newCEType = 'ARC' if ceType != newCEType and newCEType != 'Unknown': section = cfgPath( ceSection, 'CEType' ) self.log.info( section, " -> ".join( ( ceType, newCEType ) ) ) if ceType == 'Unknown': self.csAPI.setOption( section, newCEType ) else: self.csAPI.modifyValue( section, newCEType ) changed = True for queue in queues: try: queueName = queue['GlueCEUniqueID'].split( '/' )[-1] except: self.log.warn( 'Error in queueName ', queue ) continue try: newMaxCPUTime = queue['GlueCEPolicyMaxCPUTime'] except: newMaxCPUTime = None newSI00 = None try: caps = queue['GlueCECapability'] if type( caps ) == type( '' ): caps = [caps] for cap in caps: if cap.count( 'CPUScalingReferenceSI00' ): newSI00 = cap.split( '=' )[-1] except: newSI00 = None queueSection = cfgPath( ceSection, 'Queues', queueName ) result = gConfig.getOptionsDict( queueSection ) if not result['OK']: self.log.warn( "Section Queues", result['Message'] ) maxCPUTime = 'Unknown' si00 = 'Unknown' allowedVOs = [''] else: queueOpt = result['Value'] maxCPUTime = queueOpt.get( 'maxCPUTime', 'Unknown' ) si00 = queueOpt.get( 'SI00', 'Unknown' ) if newVO == '': # Remember previous iteration, if none - read from conf allowedVOs = queueOpt.get( 'VO', '' ).split( "," ) else: # Else use newVO, as it can contain changes, which aren't in conf yet allowedVOs = newVO.split( "," ) if newMaxCPUTime and ( maxCPUTime != newMaxCPUTime ): section = cfgPath( queueSection, 'maxCPUTime' ) self.log.info( section, " -> ".join( ( maxCPUTime, newMaxCPUTime ) ) ) if maxCPUTime == 'Unknown': self.csAPI.setOption( section, newMaxCPUTime ) else: self.csAPI.modifyValue( section, newMaxCPUTime ) changed = True if newSI00 and ( si00 != newSI00 ): section = cfgPath( queueSection, 'SI00' ) self.log.info( section, " -> ".join( ( si00, newSI00 ) ) ) if si00 == 'Unknown': self.csAPI.setOption( section, newSI00 ) else: self.csAPI.modifyValue( section, newSI00 ) changed = True modifyVO = True # Flag saying if we need VO option to change newVO = '' if allowedVOs != ['']: for allowedVO in allowedVOs: allowedVO = allowedVO.strip() # Get rid of spaces newVO += allowedVO if allowedVO == vo: # Current VO has been already in list newVO = '' modifyVO = False # Don't change anything break # Skip next 'if', proceed to next VO newVO += ', ' if modifyVO: section = cfgPath( queueSection, 'VO' ) newVO += vo self.log.info( section, " -> ".join( ( '%s' % allowedVOs, newVO ) ) ) if allowedVOs == ['']: self.csAPI.setOption( section, newVO ) else: self.csAPI.modifyValue( section, newVO ) changed = True if changed: self.log.info( body ) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False ) return self.csAPI.commit() else: self.log.info( "No changes found" ) return S_OK()
def __infoFromCE(self): sitesSection = cfgPath("Resources", "Sites") result = gConfig.getSections(sitesSection) if not result["OK"]: return grids = result["Value"] changed = False body = "" for grid in grids: gridSection = cfgPath(sitesSection, grid) result = gConfig.getSections(gridSection) if not result["OK"]: return sites = result["Value"] for site in sites: siteSection = cfgPath(gridSection, site) opt = gConfig.getOptionsDict(siteSection)["Value"] name = opt.get("Name", "") if name: coor = opt.get("Coordinates", "Unknown") mail = opt.get("Mail", "Unknown") result = ldapSite(name) if not result["OK"]: self.log.warn("BDII site %s: %s" % (name, result["Message"])) result = self.__checkAlternativeBDIISite(ldapSite, name) if result["OK"]: bdiiSites = result["Value"] if len(bdiiSites) == 0: self.log.warn(name, "Error in BDII: leng = 0") else: if not len(bdiiSites) == 1: self.log.warn(name, "Warning in BDII: leng = %d" % len(bdiiSites)) bdiiSite = bdiiSites[0] try: longitude = bdiiSite["GlueSiteLongitude"] latitude = bdiiSite["GlueSiteLatitude"] newcoor = "%s:%s" % (longitude, latitude) except: self.log.warn("Error in BDII coordinates") newcoor = "Unknown" try: newmail = bdiiSite["GlueSiteSysAdminContact"].split(":")[-1].strip() except: self.log.warn("Error in BDII mail") newmail = "Unknown" self.log.debug("%s %s %s" % (name, newcoor, newmail)) if newcoor != coor: self.log.info("%s" % (name), "%s -> %s" % (coor, newcoor)) if coor == "Unknown": self.csAPI.setOption(cfgPath(siteSection, "Coordinates"), newcoor) else: self.csAPI.modifyValue(cfgPath(siteSection, "Coordinates"), newcoor) changed = True if newmail != mail: self.log.info("%s" % (name), "%s -> %s" % (mail, newmail)) if mail == "Unknown": self.csAPI.setOption(cfgPath(siteSection, "Mail"), newmail) else: self.csAPI.modifyValue(cfgPath(siteSection, "Mail"), newmail) changed = True ceList = List.fromChar(opt.get("CE", "")) if not ceList: self.log.warn(site, "Empty site list") continue # result = gConfig.getSections( cfgPath( siteSection,'CEs' ) # if not result['OK']: # self.log.debug( "Section CEs:", result['Message'] ) for ce in ceList: ceSection = cfgPath(siteSection, "CEs", ce) result = gConfig.getOptionsDict(ceSection) if not result["OK"]: self.log.debug("Section CE", result["Message"]) wnTmpDir = "Unknown" arch = "Unknown" os = "Unknown" si00 = "Unknown" pilot = "Unknown" ceType = "Unknown" else: ceopt = result["Value"] wnTmpDir = ceopt.get("wnTmpDir", "Unknown") arch = ceopt.get("architecture", "Unknown") os = ceopt.get("OS", "Unknown") si00 = ceopt.get("SI00", "Unknown") pilot = ceopt.get("Pilot", "Unknown") ceType = ceopt.get("CEType", "Unknown") result = ldapCE(ce) if not result["OK"]: self.log.warn("Error in BDII for %s" % ce, result["Message"]) result = self.__checkAlternativeBDIISite(ldapCE, ce) continue try: bdiiCE = result["Value"][0] except: self.log.warn("Error in BDII for %s" % ce, result) bdiiCE = None if bdiiCE: try: newWNTmpDir = bdiiCE["GlueSubClusterWNTmpDir"] except: newWNTmpDir = "Unknown" if wnTmpDir != newWNTmpDir and newWNTmpDir != "Unknown": section = cfgPath(ceSection, "wnTmpDir") self.log.info(section, " -> ".join((wnTmpDir, newWNTmpDir))) if wnTmpDir == "Unknown": self.csAPI.setOption(section, newWNTmpDir) else: self.csAPI.modifyValue(section, newWNTmpDir) changed = True try: newArch = bdiiCE["GlueHostArchitecturePlatformType"] except: newArch = "Unknown" if arch != newArch and newArch != "Unknown": section = cfgPath(ceSection, "architecture") self.log.info(section, " -> ".join((arch, newArch))) if arch == "Unknown": self.csAPI.setOption(section, newArch) else: self.csAPI.modifyValue(section, newArch) changed = True try: newOS = "_".join( ( bdiiCE["GlueHostOperatingSystemName"], bdiiCE["GlueHostOperatingSystemVersion"], bdiiCE["GlueHostOperatingSystemRelease"], ) ) except: newOS = "Unknown" if os != newOS and newOS != "Unknown": section = cfgPath(ceSection, "OS") self.log.info(section, " -> ".join((os, newOS))) if os == "Unknown": self.csAPI.setOption(section, newOS) else: self.csAPI.modifyValue(section, newOS) changed = True body = body + "OS was changed %s -> %s for %s at %s\n" % (os, newOS, ce, site) try: newSI00 = bdiiCE["GlueHostBenchmarkSI00"] except: newSI00 = "Unknown" if si00 != newSI00 and newSI00 != "Unknown": section = cfgPath(ceSection, "SI00") self.log.info(section, " -> ".join((si00, newSI00))) if si00 == "Unknown": self.csAPI.setOption(section, newSI00) else: self.csAPI.modifyValue(section, newSI00) changed = True try: rte = bdiiCE["GlueHostApplicationSoftwareRunTimeEnvironment"] for vo in self.voName: if vo.lower() == "lhcb": if "VO-lhcb-pilot" in rte: newPilot = "True" else: newPilot = "False" else: newPilot = "Unknown" except: newPilot = "Unknown" if pilot != newPilot and newPilot != "Unknown": section = cfgPath(ceSection, "Pilot") self.log.info(section, " -> ".join((pilot, newPilot))) if pilot == "Unknown": self.csAPI.setOption(section, newPilot) else: self.csAPI.modifyValue(section, newPilot) changed = True newVO = "" for vo in self.voName: result = ldapCEState(ce, vo) # getBDIICEVOView if not result["OK"]: self.log.warn("Error in BDII for queue %s" % ce, result["Message"]) result = self.__checkAlternativeBDIISite(ldapCEState, ce, vo) continue try: queues = result["Value"] except: self.log.warn("Error in BDII for queue %s" % ce, result["Massage"]) continue newCEType = "Unknown" for queue in queues: try: queueType = queue["GlueCEImplementationName"] except: queueType = "Unknown" if newCEType == "Unknown": newCEType = queueType else: if queueType != newCEType: self.log.warn( "Error in BDII for CE %s " % ce, "different CE types %s %s" % (newCEType, queueType), ) if newCEType == "ARC-CE": newCEType = "ARC" if ceType != newCEType and newCEType != "Unknown": section = cfgPath(ceSection, "CEType") self.log.info(section, " -> ".join((ceType, newCEType))) if ceType == "Unknown": self.csAPI.setOption(section, newCEType) else: self.csAPI.modifyValue(section, newCEType) changed = True for queue in queues: try: queueName = queue["GlueCEUniqueID"].split("/")[-1] except: self.log.warn("Error in queueName ", queue) continue try: newMaxCPUTime = queue["GlueCEPolicyMaxCPUTime"] except: newMaxCPUTime = None newSI00 = None try: caps = queue["GlueCECapability"] if type(caps) == type(""): caps = [caps] for cap in caps: if cap.count("CPUScalingReferenceSI00"): newSI00 = cap.split("=")[-1] except: newSI00 = None queueSection = cfgPath(ceSection, "Queues", queueName) result = gConfig.getOptionsDict(queueSection) if not result["OK"]: self.log.warn("Section Queues", result["Message"]) maxCPUTime = "Unknown" si00 = "Unknown" allowedVOs = [""] else: queueOpt = result["Value"] maxCPUTime = queueOpt.get("maxCPUTime", "Unknown") si00 = queueOpt.get("SI00", "Unknown") if newVO == "": # Remember previous iteration, if none - read from conf allowedVOs = queueOpt.get("VO", "").split(",") else: # Else use newVO, as it can contain changes, which aren't in conf yet allowedVOs = newVO.split(",") if newMaxCPUTime and (maxCPUTime != newMaxCPUTime): section = cfgPath(queueSection, "maxCPUTime") self.log.info(section, " -> ".join((maxCPUTime, newMaxCPUTime))) if maxCPUTime == "Unknown": self.csAPI.setOption(section, newMaxCPUTime) else: self.csAPI.modifyValue(section, newMaxCPUTime) changed = True if newSI00 and (si00 != newSI00): section = cfgPath(queueSection, "SI00") self.log.info(section, " -> ".join((si00, newSI00))) if si00 == "Unknown": self.csAPI.setOption(section, newSI00) else: self.csAPI.modifyValue(section, newSI00) changed = True modifyVO = True # Flag saying if we need VO option to change newVO = "" if allowedVOs != [""]: for allowedVO in allowedVOs: allowedVO = allowedVO.strip() # Get rid of spaces newVO += allowedVO if allowedVO == vo: # Current VO has been already in list newVO = "" modifyVO = False # Don't change anything break # Skip next 'if', proceed to next VO newVO += ", " if modifyVO: section = cfgPath(queueSection, "VO") newVO += vo self.log.info(section, " -> ".join(("%s" % allowedVOs, newVO))) if allowedVOs == [""]: self.csAPI.setOption(section, newVO) else: self.csAPI.modifyValue(section, newVO) changed = True if changed: self.log.info(body) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False) return self.csAPI.commit() else: self.log.info("No changes found") return S_OK()