class Publisher: """ Class Publisher is in charge of getting dispersed information, to be published on the web. """ ############################################################################# def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None, WMSAdminIn = None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rmDB = ResourceManagementDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool( 2, 5 ) self.lockObj = threading.RLock() self.infoForPanel_res = {} ############################################################################# def getInfo(self, granularity, name, useNewRes = False): """ Standard method to get all the info to be published This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`) with 2-5 threads. The threaded method is :meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel` :params: :attr:`granularity`: string - a ValidRes :attr:`name`: string - name of the Validres :attr:`useNewRes`: boolean. When set to true, will get new results, otherwise it will get cached results (where available). """ if granularity not in ValidRes: raise InvalidRes, Utils.where(self, self.getInfo) self.infoForPanel_res = {} status = None formerStatus = None siteType = None serviceType = None resourceType = None if granularity in ('Resource', 'Resources'): try: resourceType = self.rsDB.getMonitoredsList('Resource', ['ResourceType'], resourceName = name)[0][0] except IndexError: return "%s does not exist!" %name if granularity in ('StorageElement', 'StorageElements'): try: siteType = self.rsDB.getMonitoredsList('StorageElement', ['SiteType'], storageElementName = name)[0][0] except IndexError: return "%s does not exist!" %name paramNames = ['Type', 'Group', 'Name', 'Policy', 'DIRAC Status', 'RSS Status', 'Reason', 'Description'] infoToGet = self.ig.getInfoToApply(('view_info', ), granularity, status = status, formerStatus = formerStatus, siteType = siteType, serviceType = serviceType, resourceType = resourceType, useNewRes = useNewRes)[0]['Panels'] infoToGet_res = {} recordsList = [] infosForPolicy = {} for panel in infoToGet.keys(): (granularityForPanel, nameForPanel) = self.__getNameForPanel(granularity, name, panel) if not self._resExist(granularityForPanel, nameForPanel): # completeInfoForPanel_res = None continue #take composite RSS result for name nameStatus_res = self._getStatus(nameForPanel, panel) recordBase = [None, None, None, None, None, None, None, None] recordBase[1] = panel.replace('_Panel', '') recordBase[2] = nameForPanel #nameForPanel try: recordBase[4] = nameStatus_res[nameForPanel]['DIRACStatus'] #DIRAC Status except: pass recordBase[5] = nameStatus_res[nameForPanel]['RSSStatus'] #RSS Status record = copy.deepcopy(recordBase) record[0] = 'ResultsForResource' recordsList.append(record) #take info that goes into the panel infoForPanel = infoToGet[panel] for info in infoForPanel: self.threadPool.generateJobAndQueueIt(self.getInfoForPanel, args = (info, granularityForPanel, nameForPanel) ) self.threadPool.processAllResults() for policy in [x.keys()[0] for x in infoForPanel]: record = copy.deepcopy(recordBase) record[0] = 'SpecificInformation' record[3] = policy #policyName record[4] = None #DIRAC Status record[5] = self.infoForPanel_res[policy]['Status'] #RSS status for the policy record[6] = self.infoForPanel_res[policy]['Reason'] #Reason record[7] = self.infoForPanel_res[policy]['desc'] #Description recordsList.append(record) infosForPolicy[policy] = self.infoForPanel_res[policy]['infos'] infoToGet_res['TotalRecords'] = len(recordsList) infoToGet_res['ParameterNames'] = paramNames infoToGet_res['Records'] = recordsList infoToGet_res['Extras'] = infosForPolicy return infoToGet_res ############################################################################# def getInfoForPanel(self, info, granularityForPanel, nameForPanel): #get single RSS policy results policyResToGet = info.keys()[0] pol_res = self.rmDB.getPolicyRes(nameForPanel, policyResToGet) if pol_res != []: pol_res_dict = {'Status' : pol_res[0], 'Reason' : pol_res[1]} else: pol_res_dict = {'Status' : 'Unknown', 'Reason' : 'Unknown'} self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet] = pol_res_dict finally: self.lockObj.release() #get policy description desc = self._getPolicyDesc(policyResToGet) #get other info othersInfo = info.values()[0] if not isinstance(othersInfo, list): othersInfo = [othersInfo] info_res = {} for oi in othersInfo: format_ = oi.keys()[0] what = oi.values()[0] info_bit_got = self._getInfo(granularityForPanel, nameForPanel, format_, what) info_res[format_] = info_bit_got self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet]['infos'] = info_res self.infoForPanel_res[policyResToGet]['desc'] = desc finally: self.lockObj.release() ############################################################################# def _getStatus(self, name, panel): #get RSS status RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1] #get DIRAC status if panel in ('Site_Panel', 'SE_Panel'): if panel == 'Site_Panel': DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name) if DIRACStatus['OK']: DIRACStatus = DIRACStatus['Value'][name].pop()[0] else: raise RSSException, Utils.where(self, self._getStatus) elif panel == 'SE_Panel': ra = getStorageElementStatus(name, 'ReadAccess')['Value'] wa = getStorageElementStatus(name, 'WriteAccess')['Value'] DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa} status = { name : { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } } else: status = { name : { 'RSSStatus': RSSStatus} } return status ############################################################################# def _getInfo(self, granularity, name, format_, what): if format_ == 'RSS': info_bit_got = self._getInfoFromRSSDB(name, what) else: if isinstance(what, dict): command = what['CommandIn'] extraArgs = what['args'] else: command = what extraArgs = None info_bit_got = self.cc.commandInvocation(granularity, name, None, None, command, extraArgs) try: info_bit_got = info_bit_got['Result'] except: pass return info_bit_got ############################################################################# def _getInfoFromRSSDB(self, name, what): paramsL = ['Status'] siteName = None serviceName = None resourceName = None storageElementName = None serviceType = None gridSiteName = None if what == 'ServiceOfSite': gran = 'Service' paramsL.insert(0, 'ServiceName') paramsL.append('Reason') siteName = name elif what == 'ResOfCompService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorEl': gran = 'StorageElements' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') storageElementName = name elif what == 'StorageElementsOfSite': gran = 'StorageElements' paramsL.insert(0, 'StorageElementName') paramsL.append('Reason') if '@' in name: DIRACsiteName = name.split('@').pop() else: DIRACsiteName = name gridSiteName = getGOCSiteName(DIRACsiteName) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'Site_Panel': gran = 'Site' paramsL.insert(0, 'SiteName') siteName = name elif what == 'Service_Computing_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_Storage_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VO-BOX_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VOMS_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Resource_Panel': gran = 'Resource' paramsL.insert(0, 'ResourceName') resourceName = name elif what == 'SE_Panel': gran = 'StorageElement' paramsL.insert(0, 'StorageElementName') storageElementName = name info_bit_got = self.rsDB.getMonitoredsList(gran, paramsList = paramsL, siteName = siteName, serviceName = serviceName, serviceType = serviceType, resourceName = resourceName, storageElementName = storageElementName, gridSiteName = gridSiteName) return info_bit_got ############################################################################# def _getPolicyDesc(self, policyName): return self.configModule.Policies[policyName]['Description'] ############################################################################# def __getNameForPanel(self, granularity, name, panel): if granularity in ('Site', 'Sites'): if panel == 'Service_Computing_Panel': granularity = 'Service' name = 'Computing@' + name elif panel == 'Service_Storage_Panel': granularity = 'Service' name = 'Storage@' + name elif panel == 'OtherServices_Panel': granularity = 'Service' name = 'OtherS@' + name elif panel == 'Service_VOMS_Panel': granularity = 'Service' name = 'VOMS@' + name elif panel == 'Service_VO-BOX_Panel': granularity = 'Service' name = 'VO-BOX@' + name # else: # granularity = granularity # name = name # else: # granularity = granularity # name = name return (granularity, name) ############################################################################# def _resExist(self, granularity, name): siteName = None serviceName = None resourceName = None storageElementName = None if granularity in ('Site', 'Sites'): siteName = name elif granularity in ('Service', 'Services'): serviceName = name elif granularity in ('Resource', 'Resources'): resourceName = name elif granularity in ('StorageElement', 'StorageElements'): storageElementName = name res = self.rsDB.getMonitoredsList(granularity, siteName = siteName, serviceName = serviceName, resourceName = resourceName, storageElementName = storageElementName) if res == []: return False else: return True
class Synchronizer: ############################################################################# def __init__( self, rsDBin = None, rmDBin = None ): self.rsDB = rsDBin self.rmDB = rmDBin if self.rsDB == None and self.rmDB == None: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.GOCDBClient = GOCDBClient() ############################################################################# # def sync(self, thingsToSync = None, fake_param = None): def sync( self, _a, _b ): """ :params: :attr:`thingsToSync`: list of things to sync """ thingsToSync = ['Utils', 'Sites', 'VOBOX', 'Resources', 'StorageElements', 'RegistryUsers'] gLogger.info( "!!! Sync DB content with CS content for %s !!!" % ( ' '.join( x for x in thingsToSync ) ) ) for thing in thingsToSync: getattr( self, '_sync' + thing )() return S_OK() ############################################################################# def _syncUtils( self ): """ Sync DB content with what is in :mod:`DIRAC.ResourceStatusSystem.Utilities.Utils` """ statusIn = self.rsDB.getStatusList() #delete status not more in Utils for stIn in statusIn: if stIn not in ValidStatus: self.rsDB.removeStatus( stIn ) #Add new status for s in ValidStatus: if s not in statusIn: self.rsDB.addStatus( s ) for g in ( 'Site', 'Service', 'Resource' ): typeIn = self.rsDB.getTypesList( g ) if g == 'Site': typesList = ValidSiteType elif g == 'Service': typesList = ValidServiceType if g == 'Resource': typesList = ValidResourceType #delete types not more in Utils for tIn in typeIn: if tIn not in typesList: self.rsDB.removeType( g, tIn ) #Add new types for t in typesList: if t not in typeIn: self.rsDB.addType( g, t ) ############################################################################# def _syncSites( self ): """ Sync DB content with sites that are in the CS """ # sites in the DB now sitesIn = self.rsDB.getMonitoredsList( 'Site', paramsList = ['SiteName'] ) sitesIn = [s[0] for s in sitesIn] # sites in CS now sitesList = getSites()['Value'] try: sitesList.remove( 'LCG.Dummy.ch' ) except ValueError: pass # remove sites from the DB not more in the CS for site in sitesIn: if site not in sitesList: self.rsDB.removeSite( site ) # add to DB what is in CS now and wasn't before for site in sitesList: if site not in sitesIn: # DIRAC Tier tier = getSiteTier( site )['Value'][0] if tier == 0 or tier == '0': t = 'T0' elif tier == 1 or tier == '1': t = 'T1' elif tier == 3 or tier == '3': t = 'T3' else: t = 'T2' #Grid Name of the site gridSiteName = getGOCSiteName( site ) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] #Grid Tier (with a workaround!) DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName ) if not DIRACSitesOfGridSites['OK']: raise RSSException, DIRACSitesOfGridSites['Message'] DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value'] if len( DIRACSitesOfGridSites ) == 1: gt = t else: gt = self.__getGOCTier( DIRACSitesOfGridSites ) self.rsDB.addOrModifySite( site, t, gridSiteName, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) self.rsDB.addOrModifyGridSite( gridSiteName, gt ) sitesIn.append( site ) ############################################################################# def _syncVOBOX( self ): """ Sync DB content with VOBoxes """ # services in the DB now servicesIn = self.rsDB.getMonitoredsList( 'Service', paramsList = ['ServiceName'] ) servicesIn = [s[0] for s in servicesIn] for site in ['LCG.CNAF.it', 'LCG.IN2P3.fr', 'LCG.PIC.es', 'LCG.RAL.uk', 'LCG.GRIDKA.de', 'LCG.NIKHEF.nl']: service = 'VO-BOX@' + site if service not in servicesIn: self.rsDB.addOrModifyService( service, 'VO-BOX', site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) ############################################################################# def _syncResources( self ): # resources in the DB now resourcesIn = self.rsDB.getMonitoredsList( 'Resource', paramsList = ['ResourceName'] ) resourcesIn = [r[0] for r in resourcesIn] # services in the DB now servicesIn = self.rsDB.getMonitoredsList( 'Service', paramsList = ['ServiceName'] ) servicesIn = [s[0] for s in servicesIn] # Site-CE mapping in CS now siteCE = getSiteCEMapping( 'LCG' )['Value'] # Site-SE mapping in CS now siteSE = getSiteSEMapping( 'LCG' )['Value'] # CEs in CS now CEList = [] for i in siteCE.values(): for ce in i: if ce is None: continue CEList.append( ce ) # SEs in CS now SEList = [] for i in siteSE.values(): for x in i: SEList.append( x ) # SE Nodes in CS now SENodeList = [] for SE in SEList: node = getSENodes( SE )['Value'][0] if node is None: continue if node not in SENodeList: SENodeList.append( node ) # LFC Nodes in CS now LFCNodeList_L = [] LFCNodeList_C = [] for site in getLFCSites()['Value']: for readable in ( 'ReadOnly', 'ReadWrite' ): LFCNode = getLFCNode( site, readable )['Value'] if LFCNode is None or LFCNode == []: continue LFCNode = LFCNode[0] if readable == 'ReadWrite': if LFCNode not in LFCNodeList_C: LFCNodeList_C.append( LFCNode ) elif readable == 'ReadOnly': if LFCNode not in LFCNodeList_L: LFCNodeList_L.append( LFCNode ) # FTS Nodes in CS now FTSNodeList = [] sitesWithFTS = getFTSSites() for site in sitesWithFTS['Value']: fts = getFTSEndpoint( site )['Value'] if fts is None or fts == []: continue fts = fts[0] if fts not in FTSNodeList: FTSNodeList.append( fts ) # VOMS Nodes in CS now VOMSNodeList = getVOMSEndpoints()['Value'] # complete list of resources in CS now resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList # list of services in CS now (to be done) servicesList = [] #remove resources no more in the CS for res in resourcesIn: if res not in resourcesList: self.rsDB.removeResource( res ) self.rsDB.removeStorageElement( resourceName = res ) # add to DB what is in CS now and wasn't before # CEs for site in siteCE.keys(): if site == 'LCG.Dummy.ch': continue for ce in siteCE[site]: if ce is None: continue siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', ce ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( ce )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue serviceType = 'Computing' service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if ce not in resourcesIn: CEType = getCEType( site, ce )['Value'] ceType = 'CE' if CEType == 'CREAM': ceType = 'CREAMCE' self.rsDB.addOrModifyResource( ce, ceType, serviceType, site, siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( ce ) # SRMs for srm in SENodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( srm )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if srm not in resourcesIn and srm is not None: self.rsDB.addOrModifyResource( srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( srm ) # LFC_C for lfc in LFCNodeList_C: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( lfc )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if lfc not in resourcesIn and lfc is not None: self.rsDB.addOrModifyResource( lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( lfc ) # LFC_L for lfc in LFCNodeList_L: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( lfc )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if lfc not in resourcesIn and lfc is not None: self.rsDB.addOrModifyResource( lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( lfc ) # FTSs for fts in FTSNodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', fts ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( fts )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if fts not in resourcesIn and fts is not None: self.rsDB.addOrModifyResource( fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( fts ) # VOMSs for voms in VOMSNodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', voms ) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex( voms )[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName ) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName( siteInGOCDB ) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] site = siteInDIRAC['Value'] serviceType = 'VOMS' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append( service ) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) servicesIn.append( service ) if voms not in resourcesIn and voms is not None: self.rsDB.addOrModifyResource( voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ) ) resourcesIn.append( voms ) #remove services no more in the CS for ser in servicesIn: if ser not in servicesList: serType = ser.split( '@' )[0] if serType != 'VO-BOX': self.rsDB.removeService( ser ) self.rsDB.removeResource( serviceName = ser ) site = ser.split( '@' )[1] if serType == 'Storage': self.rsDB.removeStorageElement( siteName = site ) ############################################################################# def _syncStorageElements( self ): # Get StorageElements from the CS SEs = getStorageElements() if not SEs['OK']: raise RSSException, SEs['Message'] SEs = SEs['Value'] for access in ( 'Read', 'Write' ): storageElementsIn = self.rsDB.getMonitoredsList( 'StorageElement' + access, paramsList = [ 'StorageElementName' ] ) try: storageElementsIn = [ x[ 0 ] for x in storageElementsIn ] except IndexError: pass #remove storageElements no more in the CS for se in storageElementsIn: if se not in SEs: self.rsDB.removeStorageElement( storageElementName = se, resourceName = None, access = access ) #Add new storage Elements for SE in SEs: srm = getSENodes( SE )[ 'Value' ][ 0 ] if srm == None: continue siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm ) if not siteInGOCDB[ 'OK' ]: raise RSSException, siteInGOCDB[ 'Message' ] if siteInGOCDB[ 'Value' ] == []: continue siteInGOCDB = siteInGOCDB[ 'Value' ][ 0 ][ 'SITENAME' ] if SE not in storageElementsIn: self.rsDB.addOrModifyStorageElement( SE, srm, siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ), access = access ) storageElementsIn.append( SE ) ############################################################################# def __getGOCTier( self, sitesList ): gridTier = 3 for site in sitesList: tier = getSiteTier( site )['Value'][0] if tier == 0 or tier == '0': tn = 0 elif tier == 1 or tier == '1': tn = 1 elif tier == 3 or tier == '3': tn = 3 else: tn = 2 if tn < gridTier: gridTier = tn if gridTier == 0: gt = 'T0' elif gridTier == 1: gt = 'T1' elif gridTier == 3: gt = 'T3' else: gt = 'T2' return gt ############################################################################# def _syncRegistryUsers(self): from DIRAC.ResourceStatusSystem.Utilities import CS users = CS.getTypedDictRootedAt("Users", root= "/Registry") for u in users: if type(users[u]['DN']) == list: users[u]['DN'] = users[u]['DN'][0] if type(users[u]['Email']) == list: users[u]['Email'] = users[u]['Email'][0] users[u]['DN'] = users[u]['DN'].split('=')[-1] self.rmDB.registryAddUser(u, users[u]['DN'].lower(), users[u]['Email'].lower())
class Publisher: """ Class Publisher is in charge of getting dispersed information, to be published on the web. """ ############################################################################# def __init__(self, VOExtension, rsDBIn=None, commandCallerIn=None, infoGetterIn=None, WMSAdminIn=None): """ Standard constructor :params: :attr:`VOExtension`: string, VO Extension (e.g. 'LHCb') :attr:`rsDBIn`: optional ResourceStatusDB object (see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`) :attr:`commandCallerIn`: optional CommandCaller object (see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`) :attr:`infoGetterIn`: optional InfoGetter object (see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`) :attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin (see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`) """ self.configModule = __import__( VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations", globals(), locals(), ['*']) if rsDBIn is not None: self.rsDB = rsDBIn else: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB self.rsDB = ResourceStatusDB() if commandCallerIn is not None: self.cc = commandCallerIn else: from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller self.cc = CommandCaller() if infoGetterIn is not None: self.ig = infoGetterIn else: from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter self.ig = InfoGetter(VOExtension) if WMSAdminIn is not None: self.WMSAdmin = WMSAdminIn else: from DIRAC.Core.DISET.RPCClient import RPCClient self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator") self.threadPool = ThreadPool(2, 5) self.lockObj = threading.RLock() self.infoForPanel_res = {} ############################################################################# def getInfo(self, granularity, name, useNewRes=False): """ Standard method to get all the info to be published This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`) with 2-5 threads. The threaded method is :meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel` :params: :attr:`granularity`: string - a ValidRes :attr:`name`: string - name of the Validres :attr:`useNewRes`: boolean. When set to true, will get new results, otherwise it will get cached results (where available). """ if granularity not in ValidRes: raise InvalidRes, where(self, self.getInfo) self.infoForPanel_res = {} status = None formerStatus = None siteType = None serviceType = None resourceType = None if granularity in ('Resource', 'Resources'): try: resourceType = self.rsDB.getMonitoredsList( 'Resource', ['ResourceType'], resourceName=name)[0][0] except IndexError: return "%s does not exist!" % name if granularity in ('StorageElement', 'StorageElements'): try: siteType = self.rsDB.getMonitoredsList( 'StorageElement', ['SiteType'], storageElementName=name)[0][0] except IndexError: return "%s does not exist!" % name paramNames = [ 'Type', 'Group', 'Name', 'Policy', 'DIRAC Status', 'RSS Status', 'Reason', 'Description' ] infoToGet = self.ig.getInfoToApply(('view_info', ), granularity, status=status, formerStatus=formerStatus, siteType=siteType, serviceType=serviceType, resourceType=resourceType, useNewRes=useNewRes)[0]['Panels'] infoToGet_res = {} recordsList = [] infosForPolicy = {} for panel in infoToGet.keys(): (granularityForPanel, nameForPanel) = self.__getNameForPanel(granularity, name, panel) if not self._resExist(granularityForPanel, nameForPanel): # completeInfoForPanel_res = None continue #take composite RSS result for name nameStatus_res = self._getStatus(nameForPanel, panel) recordBase = [None, None, None, None, None, None, None, None] recordBase[1] = panel.replace('_Panel', '') recordBase[2] = nameForPanel #nameForPanel try: recordBase[4] = nameStatus_res[nameForPanel][ 'DIRACStatus'] #DIRAC Status except: pass recordBase[5] = nameStatus_res[nameForPanel][ 'RSSStatus'] #RSS Status record = copy.deepcopy(recordBase) record[0] = 'ResultsForResource' recordsList.append(record) #take info that goes into the panel infoForPanel = infoToGet[panel] for info in infoForPanel: self.threadPool.generateJobAndQueueIt( self.getInfoForPanel, args=(info, granularityForPanel, nameForPanel)) self.threadPool.processAllResults() for policy in [x.keys()[0] for x in infoForPanel]: record = copy.deepcopy(recordBase) record[0] = 'SpecificInformation' record[3] = policy #policyName record[4] = None #DIRAC Status record[5] = self.infoForPanel_res[policy][ 'Status'] #RSS status for the policy record[6] = self.infoForPanel_res[policy]['Reason'] #Reason record[7] = self.infoForPanel_res[policy]['desc'] #Description recordsList.append(record) infosForPolicy[policy] = self.infoForPanel_res[policy]['infos'] infoToGet_res['TotalRecords'] = len(recordsList) infoToGet_res['ParameterNames'] = paramNames infoToGet_res['Records'] = recordsList infoToGet_res['Extras'] = infosForPolicy return infoToGet_res ############################################################################# def getInfoForPanel(self, info, granularityForPanel, nameForPanel): #get single RSS policy results policyResToGet = info.keys()[0] pol_res = self.rsDB.getPolicyRes(nameForPanel, policyResToGet) if pol_res != []: pol_res_dict = {'Status': pol_res[0], 'Reason': pol_res[1]} else: pol_res_dict = {'Status': 'Unknown', 'Reason': 'Unknown'} self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet] = pol_res_dict finally: self.lockObj.release() #get policy description desc = self._getPolicyDesc(policyResToGet) #get other info othersInfo = info.values()[0] if not isinstance(othersInfo, list): othersInfo = [othersInfo] info_res = {} for oi in othersInfo: format = oi.keys()[0] what = oi.values()[0] info_bit_got = self._getInfo(granularityForPanel, nameForPanel, format, what) info_res[format] = info_bit_got self.lockObj.acquire() try: self.infoForPanel_res[policyResToGet]['infos'] = info_res self.infoForPanel_res[policyResToGet]['desc'] = desc finally: self.lockObj.release() ############################################################################# def _getStatus(self, name, panel): #get RSS status RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1] #get DIRAC status if panel in ('Site_Panel', 'SE_Panel'): if panel == 'Site_Panel': DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name) if DIRACStatus['OK']: DIRACStatus = DIRACStatus['Value'][name].pop()[0] else: raise RSSException, where(self, self._getStatus) elif panel == 'SE_Panel': ra = getStorageElementStatus(name, 'ReadAccess')['Value'] wa = getStorageElementStatus(name, 'WriteAccess')['Value'] DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa} status = { name: { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } } else: status = {name: {'RSSStatus': RSSStatus}} return status ############################################################################# def _getInfo(self, granularity, name, format, what): if format == 'RSS': info_bit_got = self._getInfoFromRSSDB(name, what) else: if isinstance(what, dict): command = what['CommandIn'] extraArgs = what['args'] else: command = what extraArgs = None info_bit_got = self.cc.commandInvocation(granularity, name, None, None, command, extraArgs) try: info_bit_got = info_bit_got['Result'] except: pass return info_bit_got ############################################################################# def _getInfoFromRSSDB(self, name, what): paramsL = ['Status'] siteName = None serviceName = None resourceName = None storageElementName = None serviceType = None gridSiteName = None if what == 'ServiceOfSite': gran = 'Service' paramsL.insert(0, 'ServiceName') paramsL.append('Reason') siteName = name elif what == 'ResOfCompService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorService': gran = 'Resources' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') serviceType = name.split('@')[0] gridSiteName = getGOCSiteName(name.split('@')[1]) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'ResOfStorEl': gran = 'StorageElements' paramsL.insert(0, 'ResourceName') paramsL.append('Reason') storageElementName = name elif what == 'StorageElementsOfSite': gran = 'StorageElements' paramsL.insert(0, 'StorageElementName') paramsL.append('Reason') if '@' in name: DIRACsiteName = name.split('@').pop() else: DIRACsiteName = name gridSiteName = getGOCSiteName(DIRACsiteName) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] elif what == 'Site_Panel': gran = 'Site' paramsL.insert(0, 'SiteName') siteName = name elif what == 'Service_Computing_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_Storage_Panel': gran = 'Service' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VO-BOX_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Service_VOMS_Panel': gran = 'Services' paramsL.insert(0, 'ServiceName') serviceName = name elif what == 'Resource_Panel': gran = 'Resource' paramsL.insert(0, 'ResourceName') resourceName = name elif what == 'SE_Panel': gran = 'StorageElement' paramsL.insert(0, 'StorageElementName') storageElementName = name info_bit_got = self.rsDB.getMonitoredsList( gran, paramsList=paramsL, siteName=siteName, serviceName=serviceName, serviceType=serviceType, resourceName=resourceName, storageElementName=storageElementName, gridSiteName=gridSiteName) return info_bit_got ############################################################################# def _getPolicyDesc(self, policyName): return self.configModule.Policies[policyName]['Description'] ############################################################################# def __getNameForPanel(self, granularity, name, panel): if granularity in ('Site', 'Sites'): if panel == 'Service_Computing_Panel': granularity = 'Service' name = 'Computing@' + name elif panel == 'Service_Storage_Panel': granularity = 'Service' name = 'Storage@' + name elif panel == 'OtherServices_Panel': granularity = 'Service' name = 'OtherS@' + name elif panel == 'Service_VOMS_Panel': granularity = 'Service' name = 'VOMS@' + name elif panel == 'Service_VO-BOX_Panel': granularity = 'Service' name = 'VO-BOX@' + name # else: # granularity = granularity # name = name # else: # granularity = granularity # name = name return (granularity, name) ############################################################################# def _resExist(self, granularity, name): siteName = None serviceName = None resourceName = None storageElementName = None if granularity in ('Site', 'Sites'): siteName = name elif granularity in ('Service', 'Services'): serviceName = name elif granularity in ('Resource', 'Resources'): resourceName = name elif granularity in ('StorageElement', 'StorageElements'): storageElementName = name res = self.rsDB.getMonitoredsList( granularity, siteName=siteName, serviceName=serviceName, resourceName=resourceName, storageElementName=storageElementName) if res == []: return False else: return True
class Synchronizer: ############################################################################# def __init__(self, rsDBin=None, rmDBin=None): self.rsDB = rsDBin self.rmDB = rmDBin if self.rsDB == None and self.rmDB == None: from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.GOCDBClient = GOCDBClient() ############################################################################# # def sync(self, thingsToSync = None, fake_param = None): def sync(self, _a, _b): """ :params: :attr:`thingsToSync`: list of things to sync """ thingsToSync = [ 'Utils', 'Sites', 'VOBOX', 'Resources', 'StorageElements', 'RegistryUsers' ] gLogger.info("!!! Sync DB content with CS content for %s !!!" % (' '.join(x for x in thingsToSync))) for thing in thingsToSync: getattr(self, '_sync' + thing)() return S_OK() ############################################################################# def _syncUtils(self): """ Sync DB content with what is in :mod:`DIRAC.ResourceStatusSystem.Utilities.Utils` """ statusIn = self.rsDB.getStatusList() #delete status not more in Utils for stIn in statusIn: if stIn not in ValidStatus: self.rsDB.removeStatus(stIn) #Add new status for s in ValidStatus: if s not in statusIn: self.rsDB.addStatus(s) for g in ('Site', 'Service', 'Resource'): typeIn = self.rsDB.getTypesList(g) if g == 'Site': typesList = ValidSiteType elif g == 'Service': typesList = ValidServiceType if g == 'Resource': typesList = ValidResourceType #delete types not more in Utils for tIn in typeIn: if tIn not in typesList: self.rsDB.removeType(g, tIn) #Add new types for t in typesList: if t not in typeIn: self.rsDB.addType(g, t) ############################################################################# def _syncSites(self): """ Sync DB content with sites that are in the CS """ # sites in the DB now sitesIn = self.rsDB.getMonitoredsList('Site', paramsList=['SiteName']) sitesIn = [s[0] for s in sitesIn] # sites in CS now sitesList = getSites()['Value'] try: sitesList.remove('LCG.Dummy.ch') except ValueError: pass # remove sites from the DB not more in the CS for site in sitesIn: if site not in sitesList: self.rsDB.removeSite(site) # add to DB what is in CS now and wasn't before for site in sitesList: if site not in sitesIn: # DIRAC Tier tier = getSiteTier(site)['Value'][0] if tier == 0 or tier == '0': t = 'T0' elif tier == 1 or tier == '1': t = 'T1' elif tier == 3 or tier == '3': t = 'T3' else: t = 'T2' #Grid Name of the site gridSiteName = getGOCSiteName(site) if not gridSiteName['OK']: raise RSSException, gridSiteName['Message'] gridSiteName = gridSiteName['Value'] #Grid Tier (with a workaround!) DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName) if not DIRACSitesOfGridSites['OK']: raise RSSException, DIRACSitesOfGridSites['Message'] DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value'] if len(DIRACSitesOfGridSites) == 1: gt = t else: gt = self.__getGOCTier(DIRACSitesOfGridSites) self.rsDB.addOrModifySite( site, t, gridSiteName, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) self.rsDB.addOrModifyGridSite(gridSiteName, gt) sitesIn.append(site) ############################################################################# def _syncVOBOX(self): """ Sync DB content with VOBoxes """ # services in the DB now servicesIn = self.rsDB.getMonitoredsList('Service', paramsList=['ServiceName']) servicesIn = [s[0] for s in servicesIn] for site in [ 'LCG.CNAF.it', 'LCG.IN2P3.fr', 'LCG.PIC.es', 'LCG.RAL.uk', 'LCG.GRIDKA.de', 'LCG.NIKHEF.nl' ]: service = 'VO-BOX@' + site if service not in servicesIn: self.rsDB.addOrModifyService( service, 'VO-BOX', site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) ############################################################################# def _syncResources(self): # resources in the DB now resourcesIn = self.rsDB.getMonitoredsList('Resource', paramsList=['ResourceName']) resourcesIn = [r[0] for r in resourcesIn] # services in the DB now servicesIn = self.rsDB.getMonitoredsList('Service', paramsList=['ServiceName']) servicesIn = [s[0] for s in servicesIn] # Site-CE mapping in CS now siteCE = getSiteCEMapping('LCG')['Value'] # Site-SE mapping in CS now siteSE = getSiteSEMapping('LCG')['Value'] # CEs in CS now CEList = [] for i in siteCE.values(): for ce in i: if ce is None: continue CEList.append(ce) # SEs in CS now SEList = [] for i in siteSE.values(): for x in i: SEList.append(x) # SE Nodes in CS now SENodeList = [] for SE in SEList: node = getSENodes(SE)['Value'][0] if node is None: continue if node not in SENodeList: SENodeList.append(node) # LFC Nodes in CS now LFCNodeList_L = [] LFCNodeList_C = [] for site in getLFCSites()['Value']: for readable in ('ReadOnly', 'ReadWrite'): LFCNode = getLFCNode(site, readable)['Value'] if LFCNode is None or LFCNode == []: continue LFCNode = LFCNode[0] if readable == 'ReadWrite': if LFCNode not in LFCNodeList_C: LFCNodeList_C.append(LFCNode) elif readable == 'ReadOnly': if LFCNode not in LFCNodeList_L: LFCNodeList_L.append(LFCNode) # FTS Nodes in CS now FTSNodeList = [] sitesWithFTS = getFTSSites() for site in sitesWithFTS['Value']: fts = getFTSEndpoint(site)['Value'] if fts is None or fts == []: continue fts = fts[0] if fts not in FTSNodeList: FTSNodeList.append(fts) # VOMS Nodes in CS now VOMSNodeList = getVOMSEndpoints()['Value'] # complete list of resources in CS now resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList # list of services in CS now (to be done) servicesList = [] #remove resources no more in the CS for res in resourcesIn: if res not in resourcesList: self.rsDB.removeResource(res) self.rsDB.removeStorageElement(resourceName=res) # add to DB what is in CS now and wasn't before # CEs for site in siteCE.keys(): if site == 'LCG.Dummy.ch': continue for ce in siteCE[site]: if ce is None: continue siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', ce) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(ce)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue serviceType = 'Computing' service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if ce not in resourcesIn: CEType = getCEType(site, ce)['Value'] ceType = 'CE' if CEType == 'CREAM': ceType = 'CREAMCE' self.rsDB.addOrModifyResource( ce, ceType, serviceType, site, siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(ce) # SRMs for srm in SENodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(srm)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if srm not in resourcesIn and srm is not None: self.rsDB.addOrModifyResource( srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(srm) # LFC_C for lfc in LFCNodeList_C: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(lfc)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if lfc not in resourcesIn and lfc is not None: self.rsDB.addOrModifyResource( lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(lfc) # LFC_L for lfc in LFCNodeList_L: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(lfc)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if lfc not in resourcesIn and lfc is not None: self.rsDB.addOrModifyResource( lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(lfc) # FTSs for fts in FTSNodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', fts) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(fts)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] sites = siteInDIRAC['Value'] serviceType = 'Storage' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if fts not in resourcesIn and fts is not None: self.rsDB.addOrModifyResource( fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(fts) # VOMSs for voms in VOMSNodeList: siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', voms) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: trueName = socket.gethostbyname_ex(voms)[0] siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName) try: siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] except IndexError: continue siteInDIRAC = getDIRACSiteName(siteInGOCDB) if not siteInDIRAC['OK']: raise RSSException, siteInDIRAC['Message'] site = siteInDIRAC['Value'] serviceType = 'VOMS' for site in sites: service = serviceType + '@' + site if service not in servicesList: servicesList.append(service) if service not in servicesIn: self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) servicesIn.append(service) if voms not in resourcesIn and voms is not None: self.rsDB.addOrModifyResource( voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59)) resourcesIn.append(voms) #remove services no more in the CS for ser in servicesIn: if ser not in servicesList: serType = ser.split('@')[0] if serType != 'VO-BOX': self.rsDB.removeService(ser) self.rsDB.removeResource(serviceName=ser) site = ser.split('@')[1] if serType == 'Storage': self.rsDB.removeStorageElement(siteName=site) ############################################################################# def _syncStorageElements(self): # Get StorageElements from the CS SEs = getStorageElements() if not SEs['OK']: raise RSSException, SEs['Message'] SEs = SEs['Value'] for access in ('Read', 'Write'): storageElementsIn = self.rsDB.getMonitoredsList( 'StorageElement' + access, paramsList=['StorageElementName']) try: storageElementsIn = [x[0] for x in storageElementsIn] except IndexError: pass #remove storageElements no more in the CS for se in storageElementsIn: if se not in SEs: self.rsDB.removeStorageElement(storageElementName=se, resourceName=None, access=access) #Add new storage Elements for SE in SEs: srm = getSENodes(SE)['Value'][0] if srm == None: continue siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm) if not siteInGOCDB['OK']: raise RSSException, siteInGOCDB['Message'] if siteInGOCDB['Value'] == []: continue siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME'] if SE not in storageElementsIn: self.rsDB.addOrModifyStorageElement( SE, srm, siteInGOCDB, 'Active', 'init', datetime.datetime.utcnow().replace(microsecond=0), 'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59), access=access) storageElementsIn.append(SE) ############################################################################# def __getGOCTier(self, sitesList): gridTier = 3 for site in sitesList: tier = getSiteTier(site)['Value'][0] if tier == 0 or tier == '0': tn = 0 elif tier == 1 or tier == '1': tn = 1 elif tier == 3 or tier == '3': tn = 3 else: tn = 2 if tn < gridTier: gridTier = tn if gridTier == 0: gt = 'T0' elif gridTier == 1: gt = 'T1' elif gridTier == 3: gt = 'T3' else: gt = 'T2' return gt ############################################################################# def _syncRegistryUsers(self): from DIRAC.ResourceStatusSystem.Utilities import CS users = CS.getTypedDictRootedAt("Users", root="/Registry") for u in users: if type(users[u]['DN']) == list: users[u]['DN'] = users[u]['DN'][0] if type(users[u]['Email']) == list: users[u]['Email'] = users[u]['Email'][0] users[u]['DN'] = users[u]['DN'].split('=')[-1] self.rmDB.registryAddUser(u, users[u]['DN'].lower(), users[u]['Email'].lower())