def getVOMSServerInfo( requestedVO = '' ): """ Get information on VOMS servers for the given VO or for all of them """ vomsDict = {} # For backward compatibility check the VOMS section first result = gConfig.getSections( '%s/VOMS/Servers' % gBaseRegistrySection ) if result['OK']: voNames = result['Value'] for vo in voNames: if requestedVO and vo != requestedVO: continue vomsDict.setdefault( vo, {} ) vomsDict[vo]['VOMSName'] = vo result = gConfig.getSections( '%s/VOMS/Servers/%s' % (gBaseRegistrySection, vo) ) if result['OK']: serverList = result['Value'] vomsDict[vo].setdefault( "Servers", {} ) for server in serverList: DN = gConfig.getValue( '%s/VOMS/Servers/%s/%s/DN' % (gBaseRegistrySection, vo, server), '' ) CA = gConfig.getValue( '%s/VOMS/Servers/%s/%s/CA' % (gBaseRegistrySection, vo, server), '' ) port = gConfig.getValue( '%s/VOMS/Servers/%s/%s/Port' % (gBaseRegistrySection, vo, server), 0 ) vomsDict[vo]['Servers'].setdefault( server, {} ) vomsDict[vo]['Servers'][server]['DN'] = DN vomsDict[vo]['Servers'][server]['CA'] = CA vomsDict[vo]['Servers'][server]['Port'] = port result = getVOs() if result['OK']: voNames = result['Value'] for vo in voNames: if requestedVO and vo != requestedVO: continue vomsName = getVOOption( vo, 'VOMSName', '' ) if not vomsName: continue vomsDict.setdefault( vo, {} ) vomsDict[vo]['VOMSName'] = getVOOption( vo, 'VOMSName', '' ) result = gConfig.getSections( '%s/VO/%s/VOMSServers' % (gBaseRegistrySection, vo) ) if result['OK']: serverList = result['Value'] vomsDict[vo].setdefault( "Servers", {} ) for server in serverList: vomsDict[vo]['Servers'].setdefault( server, {} ) DN = gConfig.getValue( '%s/VO/%s/VOMSServers/%s/DN' % (gBaseRegistrySection, vo, server), '' ) CA = gConfig.getValue( '%s/VO/%s/VOMSServers/%s/CA' % (gBaseRegistrySection, vo, server), '' ) port = gConfig.getValue( '%s/VO/%s/VOMSServers/%s/Port' % (gBaseRegistrySection, vo, server), 0 ) vomsDict[vo]['Servers'][server]['DN'] = DN vomsDict[vo]['Servers'][server]['CA'] = CA vomsDict[vo]['Servers'][server]['Port'] = port return S_OK( vomsDict )
def getVOMSServerInfo( requestedVO = '' ): """ Get information on VOMS servers for the given VO or for all of them """ vomsDict = {} # For backward compatibility check the VOMS section first result = gConfig.getSections( '%s/VOMS/Servers' % gBaseRegistrySection ) if result['OK']: voNames = result['Value'] for vo in voNames: if requestedVO and vo != requestedVO: continue vomsDict.setdefault( vo, {} ) vomsDict[vo]['VOMSName'] = vo result = gConfig.getSections( '%s/VOMS/Servers/%s' % (gBaseRegistrySection, vo) ) if result['OK']: serverList = result['Value'] vomsDict[vo].setdefault( "Servers", {} ) for server in serverList: DN = gConfig.getValue( '%s/VOMS/Servers/%s/%s/DN' % (gBaseRegistrySection, vo, server), '' ) CA = gConfig.getValue( '%s/VOMS/Servers/%s/%s/CA' % (gBaseRegistrySection, vo, server), '' ) port = gConfig.getValue( '%s/VOMS/Servers/%s/%s/Port' % (gBaseRegistrySection, vo, server), 0 ) vomsDict[vo]['Servers'].setdefault( server, {} ) vomsDict[vo]['Servers'][server]['DN'] = DN vomsDict[vo]['Servers'][server]['CA'] = CA vomsDict[vo]['Servers'][server]['Port'] = port result = getVOs() if result['OK']: voNames = result['Value'] for vo in voNames: if requestedVO and vo != requestedVO: continue vomsName = getVOOption( vo, 'VOMSName', '' ) if not vomsName: continue vomsDict.setdefault( vo, {} ) vomsDict[vo]['VOMSName'] = getVOOption( vo, 'VOMSName', '' ) result = gConfig.getSections( '%s/VO/%s/VOMSServers' % (gBaseRegistrySection, vo) ) if result['OK']: serverList = result['Value'] vomsDict[vo].setdefault( "Servers", {} ) for server in serverList: vomsDict[vo]['Servers'].setdefault( server, {} ) DN = gConfig.getValue( '%s/VO/%s/VOMSServers/%s/DN' % (gBaseRegistrySection, vo, server), '' ) CA = gConfig.getValue( '%s/VO/%s/VOMSServers/%s/CA' % (gBaseRegistrySection, vo, server), '' ) port = gConfig.getValue( '%s/VO/%s/VOMSServers/%s/Port' % (gBaseRegistrySection, vo, server), 0 ) vomsDict[vo]['Servers'][server]['DN'] = DN vomsDict[vo]['Servers'][server]['CA'] = CA vomsDict[vo]['Servers'][server]['Port'] = port return S_OK( vomsDict )
def getSchemaSections(self, path=""): retDict = gConfig.getSections( "%s/Schema/%s" % (self.webSection, path), listOrdered=True) if retDict['OK']: return retDict['Value'] else: return []
def getAllGroups(): """ Get all groups :return: list """ result = gConfig.getSections("%s/Groups" % gBaseRegistrySection) return result['Value'] if result['OK'] else []
def getVOs(): """ Get all the configured VOs """ voName = getVO() if voName: return S_OK([voName]) return gConfig.getSections('%s/VO' % gBaseRegistrySection)
def getVOs(): """ Get all the configured VOs """ voName = getVO() if voName: return S_OK([voName]) return gConfig.getSections('%s/VO' % gBaseRegistrySection)
def initializeParameters( self ): """ Initialize the CE parameters after they are collected from various sources """ # Collect global defaults first result = gConfig.getSections( '/Resources/Computing' ) if result['OK'] and result['Value']: ceSections = result['Value'] if 'CEDefaults' in ceSections: result = self.__getCEParameters( '/Resources/Computing/CEDefaults' ) #can be overwritten by other sections if not result['OK']: self.log.warn( result['Message'] ) if self.ceName in ceSections: result = self.__getCEParameters( '/Resources/Computing/%s' % self.ceName ) if not result['OK']: self.log.warn( result['Message'] ) # Get local CE configuration localConfigDict = getCEConfigDict( self.ceName ) self.ceParameters.update( localConfigDict ) result = self.__getSiteParameters() if not result['OK']: self.log.warn( result['Message'] ) result = self.__getResourceRequirements() if not result['OK']: self.log.warn( result['Message'] ) self._addCEConfigDefaults()
def initializeParameters(self): """ Initialize the CE parameters after they are collected from various sources """ # Collect global defaults first result = gConfig.getSections("/Resources/Computing") if result["OK"] and result["Value"]: ceSections = result["Value"] if "CEDefaults" in ceSections: result = self.__getCEParameters( "/Resources/Computing/CEDefaults" ) # can be overwritten by other sections if not result["OK"]: self.log.warn(result["Message"]) if self.ceName in ceSections: result = self.__getCEParameters("/Resources/Computing/%s" % self.ceName) if not result["OK"]: self.log.warn(result["Message"]) # Get local CE configuration localConfigDict = getCEConfigDict(self.ceName) self.ceParameters.update(localConfigDict) result = self.__getSiteParameters() if not result["OK"]: self.log.warn(result["Message"]) self._addCEConfigDefaults()
def initializeParameters( self ): """ Initialize the CE parameters after they are collected from various sources """ # Collect global defaults first result = gConfig.getSections( '/Resources/Computing' ) if result['OK'] and result['Value']: ceSections = result['Value'] if 'CEDefaults' in ceSections: result = self.__getCEParameters( '/Resources/Computing/CEDefaults' ) #can be overwritten by other sections if not result['OK']: self.log.warn( result['Message'] ) if self.ceName in ceSections: result = self.__getCEParameters( '/Resources/Computing/%s' % self.ceName ) if not result['OK']: self.log.warn( result['Message'] ) # Get local CE configuration localConfigDict = getCEConfigDict( self.ceName ) self.ceParameters.update( localConfigDict ) result = self.__getSiteParameters() if not result['OK']: self.log.warn( result['Message'] ) self._addCEConfigDefaults()
def getAllUsers(): """ Get all users :return: list """ result = gConfig.getSections("%s/Users" % gBaseRegistrySection) return result['Value'] if result['OK'] else []
def getVOs(): """ Get all the configured VOs :return: S_OK(list)/S_ERROR() """ voName = getVO() return S_OK([voName]) if voName else gConfig.getSections( '%s/VO' % gBaseRegistrySection)
def getGroupsWithVOMSAttribute( vomsAttr ): retVal = gConfig.getSections( "%s/Groups" % ( gBaseSecuritySection ) ) if not retVal[ 'OK' ]: return [] groups = [] for group in retVal[ 'Value' ]: if vomsAttr == gConfig.getValue( "%s/Groups/%s/VOMSRole" % ( gBaseSecuritySection, group ), "" ): groups.append( group ) return groups
def getHostnameForDN( dn ): retVal = gConfig.getSections( "%s/Hosts" % gBaseSecuritySection ) if not retVal[ 'OK' ]: return retVal hostList = retVal[ 'Value' ] for hostname in hostList: if dn in gConfig.getValue( "%s/Hosts/%s/DN" % ( gBaseSecuritySection, hostname ), [] ): return S_OK( hostname ) return S_ERROR( "No hostname found for dn %s" % dn )
def getGroupsForVO(vo): """ Get groups for VO :param str vo: DIRAC VO name :return: S_OK(list)/S_ERROR() """ if getVO(): return gConfig.getSections("%s/Groups" % gBaseRegistrySection) return __getGroupsWithAttr('VO', vo)
def getUsernameForID(ID, usersList=False): if not usersList: retVal = gConfig.getSections("%s/Users" % gBaseRegistrySection) if not retVal['OK']: return retVal usersList = retVal['Value'] for username in usersList: if ID in gConfig.getValue("%s/Users/%s/ID" % (gBaseRegistrySection, username), []): return S_OK(username) return S_ERROR("No username found for ID %s" % ID)
def getUsernameForID(ID, usersList=False): if not usersList: retVal = gConfig.getSections("%s/Users" % gBaseRegistrySection) if not retVal['OK']: return retVal usersList = retVal['Value'] for username in usersList: if ID in gConfig.getValue("%s/Users/%s/ID" % (gBaseRegistrySection, username), []): return S_OK(username) return S_ERROR("No username found for ID %s" % ID)
def getUsernameForDN( dn, usersList = False ): if not usersList: retVal = gConfig.getSections( "%s/Users" % gBaseSecuritySection ) if not retVal[ 'OK' ]: return retVal usersList = retVal[ 'Value' ] for username in usersList: if dn in gConfig.getValue( "%s/Users/%s/DN" % ( gBaseSecuritySection, username ), [] ): return S_OK( username ) return S_ERROR( "No username found for dn %s" % dn )
def getSites( grids = None ): if grids == None: grids = ['LCG'] if isinstance(grids, basestring): grids = [grids] sites = [] for grid in grids: s = gConfig.getSections('%s/Sites/%s' %( g_BaseResourcesSection, grid ), True) if not s['OK']: return s sites = sites + s['Value'] return S_OK(sites)
def getGroupsForVO(vo): """ Get groups for VO :param str vo: DIRAC VO name :return: S_OK(list)/S_ERROR() """ if getVO(): # tries to get default VO in /DIRAC/VirtualOrganization return gConfig.getSections("%s/Groups" % gBaseRegistrySection) if not vo: return S_ERROR("No VO requested") return __getGroupsWithAttr('VO', vo)
def getGroupsForUser( username ): retVal = gConfig.getSections( "%s/Groups" % gBaseSecuritySection ) if not retVal[ 'OK' ]: return retVal groupsList = retVal[ 'Value' ] userGroups = [] for group in groupsList: if username in gConfig.getValue( "%s/Groups/%s/Users" % ( gBaseSecuritySection, group ), [] ): userGroups.append( group ) if not userGroups: return S_ERROR( "No groups found for user %s" % username ) userGroups.sort() return S_OK( userGroups )
def getGroupsForUser(username): retVal = gConfig.getSections("%s/Groups" % gBaseSecuritySection) if not retVal["OK"]: return retVal groupsList = retVal["Value"] userGroups = [] for group in groupsList: if username in gConfig.getValue("%s/Groups/%s/Users" % (gBaseSecuritySection, group), []): userGroups.append(group) if not userGroups: return S_ERROR("No groups found for user %s" % username) userGroups.sort() return S_OK(userGroups)
def __getGroupsWithAttr( attrName, value ): retVal = gConfig.getSections( "%s/Groups" % gBaseSecuritySection ) if not retVal[ 'OK' ]: return retVal groupsList = retVal[ 'Value' ] groups = [] for group in groupsList: if value in gConfig.getValue( "%s/Groups/%s/%s" % ( gBaseSecuritySection, group, attrName ), [] ): groups.append( group ) if not groups: return S_ERROR( "No groups found for %s=%s" % ( attrName,value ) ) groups.sort() return S_OK( groups )
def getSites(grids=None): if grids == None: grids = ['LCG'] if isinstance(grids, basestring): grids = [grids] sites = [] for grid in grids: s = gConfig.getSections('%s/Sites/%s' % (g_BaseResourcesSection, grid), True) if not s['OK']: return s sites = sites + s['Value'] return S_OK(sites)
def getGroupsWithVOMSAttribute(vomsAttr): """ Search groups with VOMS attribute :param str vomsAttr: VOMS attribute :return: list """ groups = [] for group in gConfig.getSections("%s/Groups" % (gBaseRegistrySection)).get( 'Value', []): if vomsAttr == gConfig.getValue( "%s/Groups/%s/VOMSRole" % (gBaseRegistrySection, group), ""): groups.append(group) return groups
def getStorageElements( hostName = None ): SEs = gConfig.getSections('%s/StorageElements' %g_BaseResourcesSection) if not SEs['OK']: return SEs SEs = SEs['Value'] if hostName != None: removeSEs = [] if isinstance(hostName, basestring): hostName = [hostName] for SE in SEs: host = gConfig.getValue('%s/StorageElements/%s/AccessProtocol.1/Host' %(g_BaseResourcesSection, SE) ) if host not in hostName: removeSEs.append(SE) for SE in removeSEs: SEs.remove(SE) return S_OK(SEs)
def getHostnameForDN(dn): """ Search host name for host DN :param str dn: host DN :return: S_OK()/S_ERROR() """ result = gConfig.getSections("%s/Hosts" % gBaseRegistrySection) if not result['OK']: return result hostList = result['Value'] for hostname in hostList: if dn in gConfig.getValue( "%s/Hosts/%s/DN" % (gBaseRegistrySection, hostname), []): return S_OK(hostname) return S_ERROR("No hostname found for dn %s" % dn)
def getUsernameForDN(dn, usersList=None): """ Find DIRAC user for DN :param str dn: user DN :param list usersList: list of possible users :return: S_OK(str)/S_ERROR() """ if not usersList: result = gConfig.getSections("%s/Users" % gBaseRegistrySection) if not result['OK']: return result usersList = result['Value'] for username in usersList: if dn in gConfig.getValue( "%s/Users/%s/DN" % (gBaseRegistrySection, username), []): return S_OK(username) return S_ERROR("No username found for dn %s" % dn)
def getStorageElements(hostName=None): SEs = gConfig.getSections('%s/StorageElements' % g_BaseResourcesSection) if not SEs['OK']: return SEs SEs = SEs['Value'] if hostName != None: removeSEs = [] if isinstance(hostName, basestring): hostName = [hostName] for SE in SEs: host = gConfig.getValue( '%s/StorageElements/%s/AccessProtocol.1/Host' % (g_BaseResourcesSection, SE)) if host not in hostName: removeSEs.append(SE) for SE in removeSEs: SEs.remove(SE) return S_OK(SEs)
def getUsernameForID(ID, usersList=None): """ Get DIRAC user name by ID :param str ID: user ID :param list usersList: list of DIRAC user names :return: S_OK(str)/S_ERROR() """ if not usersList: result = gConfig.getSections("%s/Users" % gBaseRegistrySection) if not result['OK']: return result usersList = result['Value'] for username in usersList: if ID in gConfig.getValue( "%s/Users/%s/ID" % (gBaseRegistrySection, username), []): return S_OK(username) return S_ERROR("No username found for ID %s" % ID)
def __getGroupsWithAttr(attrName, value): """ Get all possible groups with some attribute :param str attrName: attribute name :param str value: attribute value :return: S_OK(list)/S_ERROR() -- contain list of groups """ result = gConfig.getSections("%s/Groups" % gBaseRegistrySection) if not result['OK']: return result groupsList = result['Value'] groups = [] for group in groupsList: if value in gConfig.getValue( "%s/Groups/%s/%s" % (gBaseRegistrySection, group, attrName), []): groups.append(group) groups.sort() return S_OK(groups) if groups else S_ERROR("No groups found for %s=%s" % (attrName, value))
def getDNProperty(userDN, value, defaultValue=None): """ Get property from DNProperties section by user DN :param str userDN: user DN :param str value: option that need to get :param defaultValue: default value :return: S_OK()/S_ERROR() -- str or list that contain option value """ result = getUsernameForDN(userDN) if not result['OK']: return result pathDNProperties = "%s/Users/%s/DNProperties" % (gBaseRegistrySection, result['Value']) result = gConfig.getSections(pathDNProperties) if result['OK']: for section in result['Value']: if userDN == gConfig.getValue("%s/%s/DN" % (pathDNProperties, section)): return S_OK( gConfig.getValue( "%s/%s/%s" % (pathDNProperties, section, value), defaultValue)) return S_OK(defaultValue)
def getLFCSites(): lfcL = gConfig.getSections('%s/FileCatalogs/LcgFileCatalogCombined' %g_BaseResourcesSection, True) return lfcL
def getHosts(): return gConfig.getSections('%s/Hosts' % gBaseRegistrySection)
def getAllGroups(): retVal = gConfig.getSections("%s/Groups" % gBaseRegistrySection) if not retVal['OK']: return [] return retVal['Value']
def getAllUsers(): retVal = gConfig.getSections( "%s/Users" % gBaseSecuritySection ) if not retVal[ 'OK' ]: return [] return retVal[ 'Value' ]
def getHosts(): return gConfig.getSections( '%s/Hosts' % gBaseSecuritySection )
def getSetups( self ): setupsList = gConfig.getSections( "/DIRAC/Setups" ) if not setupsList[ 'OK' ]: return [] return setupsList[ 'Value' ]
def getHosts(): """ Get all hosts :return: S_OK()/S_ERROR() """ return gConfig.getSections('%s/Hosts' % gBaseRegistrySection)
def getHosts(): return gConfig.getSections('%s/Hosts' % gBaseRegistrySection)
def getAllGroups(): retVal = gConfig.getSections("%s/Groups" % gBaseRegistrySection) if not retVal['OK']: return [] return retVal['Value']
def getHosts(): return gConfig.getSections('%s/Hosts' % gBaseSecuritySection)
def getGroupsForVO(vo): if getVO(): return gConfig.getSections("%s/Groups" % gBaseRegistrySection) return __getGroupsWithAttr('VO', vo)
def getVOMSEndpoints(): voms_S = gConfig.getSections("%s/VOMS/Servers/lhcb/" %(g_BaseRegistrySection) ,'') return voms_S
def web_getData(self): siteMaxJobs = {} #Calculate Max pledges for Cluster and GRIDs for dir in gConfig.getSections('/Resources/Sites')['Value']: for site in gConfig.getSections('/Resources/Sites/'+dir)['Value']: if 'CEs' in gConfig.getSections('/Resources/Sites/'+dir+'/'+site)['Value']: for ce in gConfig.getSections('/Resources/Sites/'+dir+'/'+site+'/CEs')['Value']: for queue in gConfig.getSections('/Resources/Sites/'+dir+'/'+site+'/CEs/'+ce+'/Queues')['Value']: siteMaxJobs[site] = siteMaxJobs.get(site, 0) + int(gConfig.getValue('/Resources/Sites/'+dir+'/'+site+'/CEs/'+ce+'/Queues/'+queue+'/MaxTotalJobs')) #Calculate Max pledges for clouds for vm in gConfig.getSections('/Resources/VirtualMachines/CloudEndpoints')['Value']: site = gConfig.getValue('/Resources/VirtualMachines/CloudEndpoints/'+vm+'/siteName') limit = gConfig.getValue('/Resources/VirtualMachines/CloudEndpoints/'+vm+'/maxEndpointInstances') if site is not None and limit is not None: siteMaxJobs[site] = int(limit) pprint(siteMaxJobs) for site in siteMaxJobs: siteMaxJobs[site] = {'max': siteMaxJobs[site], 'current':0} #Get current Running jobs import MySQLdb #@UnresolvedImport db = MySQLdb.connect(host="diracdb2.ihep.ac.cn", # your host, usually localhost user="******", # your username passwd="###DIRAC_DB_PASS###", # your password db="JobDB") cur = db.cursor() cur.execute("select Site, count(*) from Jobs where Status='Running' group by Site") for row in cur.fetchall() : if row[0] not in siteMaxJobs: siteMaxJobs[row[0]] = {'max':0} siteMaxJobs[row[0]]['current'] = row[1] #Get Weekly Done and Failed jobs for site in siteMaxJobs: siteMaxJobs[site]['week_done'] = 0 siteMaxJobs[site]['week_failed'] = 0 cur.execute("select Site, count(*), min(SubmissionTime), max(SubmissionTime) from Jobs where (Status='Done') and SubmissionTime>DATE_SUB(NOW(), INTERVAL 168 HOUR) group by Site") for row in cur.fetchall(): if row[0] not in siteMaxJobs: siteMaxJobs[row[0]] = {'max':0, 'current':0, 'week_done':0, 'week_failed':0} siteMaxJobs[row[0]]['week_done'] = row[1] cur.execute("select Site, count(*), min(SubmissionTime), max(SubmissionTime) from Jobs where (Status='Failed') and SubmissionTime>DATE_SUB(NOW(), INTERVAL 168 HOUR) group by Site") for row in cur.fetchall(): if row[0] not in siteMaxJobs: siteMaxJobs[row[0]] = {'max':0, 'current':0, 'week_done':0, 'week_failed':0} siteMaxJobs[row[0]]['week_failed'] = row[1] cur.close() res = [] for site in siteMaxJobs: res.append({'site':site, 'max':siteMaxJobs[site]['max'], 'current':siteMaxJobs[site]['current'], 'week_done':siteMaxJobs[site]['week_done'], 'week_failed':siteMaxJobs[site]['week_failed']}) #from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB #db = JobDB() #cmd = 'SELECT count(*) FROM Jobs' #res = db._query( cmd ) #print res print siteMaxJobs self.write({"result":res})
def getSchemaSections( self, path = "" ): retDict = gConfig.getSections( "%s/Schema/%s" % ( self.webSection, path ), listOrdered = True ) if retDict[ 'OK' ]: return retDict[ 'Value' ] else: return []
def getGroupsForVO( vo ): if getVO(): return gConfig.getSections( "%s/Groups" % gBaseSecuritySection ) return __getGroupsWithAttr( 'VO', vo )