class Publisher:
"""
Class Publisher is in charge of getting dispersed information, to be published on the web.
"""
#############################################################################
def __init__(self, VOExtension, rsDBIn = None, commandCallerIn = None, infoGetterIn = None,
WMSAdminIn = None):
"""
Standard constructor
:params:
:attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')
:attr:`rsDBIn`: optional ResourceStatusDB object
(see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)
:attr:`commandCallerIn`: optional CommandCaller object
(see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)
:attr:`infoGetterIn`: optional InfoGetter object
(see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)
:attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
(see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
"""
self.configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations", VOExtension)
if rsDBIn is not None:
self.rsDB = rsDBIn
else:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
self.rmDB = ResourceManagementDB()
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
if infoGetterIn is not None:
self.ig = infoGetterIn
else:
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
self.ig = InfoGetter(VOExtension)
if WMSAdminIn is not None:
self.WMSAdmin = WMSAdminIn
else:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
self.threadPool = ThreadPool( 2, 5 )
self.lockObj = threading.RLock()
self.infoForPanel_res = {}
#############################################################################
def getInfo(self, granularity, name, useNewRes = False):
"""
Standard method to get all the info to be published
This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`)
with 2-5 threads. The threaded method is
:meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel`
:params:
:attr:`granularity`: string - a ValidRes
:attr:`name`: string - name of the Validres
:attr:`useNewRes`: boolean. When set to true, will get new results,
otherwise it will get cached results (where available).
"""
if granularity not in ValidRes:
raise InvalidRes, Utils.where(self, self.getInfo)
self.infoForPanel_res = {}
status = None
formerStatus = None
siteType = None
serviceType = None
resourceType = None
if granularity in ('Resource', 'Resources'):
try:
resourceType = self.rsDB.getMonitoredsList('Resource', ['ResourceType'],
resourceName = name)[0][0]
except IndexError:
return "%s does not exist!" %name
if granularity in ('StorageElement', 'StorageElements'):
try:
siteType = self.rsDB.getMonitoredsList('StorageElement', ['SiteType'],
storageElementName = name)[0][0]
except IndexError:
return "%s does not exist!" %name
paramNames = ['Type', 'Group', 'Name', 'Policy', 'DIRAC Status',
'RSS Status', 'Reason', 'Description']
infoToGet = self.ig.getInfoToApply(('view_info', ), granularity, status = status,
formerStatus = formerStatus, siteType = siteType,
serviceType = serviceType, resourceType = resourceType,
useNewRes = useNewRes)[0]['Panels']
infoToGet_res = {}
recordsList = []
infosForPolicy = {}
for panel in infoToGet.keys():
(granularityForPanel, nameForPanel) = self.__getNameForPanel(granularity, name, panel)
if not self._resExist(granularityForPanel, nameForPanel):
# completeInfoForPanel_res = None
continue
#take composite RSS result for name
nameStatus_res = self._getStatus(nameForPanel, panel)
recordBase = [None, None, None, None, None, None, None, None]
recordBase[1] = panel.replace('_Panel', '')
recordBase[2] = nameForPanel #nameForPanel
try:
recordBase[4] = nameStatus_res[nameForPanel]['DIRACStatus'] #DIRAC Status
except:
pass
recordBase[5] = nameStatus_res[nameForPanel]['RSSStatus'] #RSS Status
record = copy.deepcopy(recordBase)
record[0] = 'ResultsForResource'
recordsList.append(record)
#take info that goes into the panel
infoForPanel = infoToGet[panel]
for info in infoForPanel:
self.threadPool.generateJobAndQueueIt(self.getInfoForPanel,
args = (info, granularityForPanel, nameForPanel) )
self.threadPool.processAllResults()
for policy in [x.keys()[0] for x in infoForPanel]:
record = copy.deepcopy(recordBase)
record[0] = 'SpecificInformation'
record[3] = policy #policyName
record[4] = None #DIRAC Status
record[5] = self.infoForPanel_res[policy]['Status'] #RSS status for the policy
record[6] = self.infoForPanel_res[policy]['Reason'] #Reason
record[7] = self.infoForPanel_res[policy]['desc'] #Description
recordsList.append(record)
infosForPolicy[policy] = self.infoForPanel_res[policy]['infos']
infoToGet_res['TotalRecords'] = len(recordsList)
infoToGet_res['ParameterNames'] = paramNames
infoToGet_res['Records'] = recordsList
infoToGet_res['Extras'] = infosForPolicy
return infoToGet_res
#############################################################################
def getInfoForPanel(self, info, granularityForPanel, nameForPanel):
#get single RSS policy results
policyResToGet = info.keys()[0]
pol_res = self.rmDB.getPolicyRes(nameForPanel, policyResToGet)
if pol_res != []:
pol_res_dict = {'Status' : pol_res[0], 'Reason' : pol_res[1]}
else:
pol_res_dict = {'Status' : 'Unknown', 'Reason' : 'Unknown'}
self.lockObj.acquire()
try:
self.infoForPanel_res[policyResToGet] = pol_res_dict
finally:
self.lockObj.release()
#get policy description
desc = self._getPolicyDesc(policyResToGet)
#get other info
othersInfo = info.values()[0]
if not isinstance(othersInfo, list):
othersInfo = [othersInfo]
info_res = {}
for oi in othersInfo:
format_ = oi.keys()[0]
what = oi.values()[0]
info_bit_got = self._getInfo(granularityForPanel, nameForPanel, format_, what)
info_res[format_] = info_bit_got
self.lockObj.acquire()
try:
self.infoForPanel_res[policyResToGet]['infos'] = info_res
self.infoForPanel_res[policyResToGet]['desc'] = desc
finally:
self.lockObj.release()
#############################################################################
def _getStatus(self, name, panel):
#get RSS status
RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1]
#get DIRAC status
if panel in ('Site_Panel', 'SE_Panel'):
if panel == 'Site_Panel':
DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name)
if DIRACStatus['OK']:
DIRACStatus = DIRACStatus['Value'][name].pop()[0]
else:
raise RSSException, Utils.where(self, self._getStatus)
elif panel == 'SE_Panel':
ra = getStorageElementStatus(name, 'ReadAccess')['Value']
wa = getStorageElementStatus(name, 'WriteAccess')['Value']
DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa}
status = { name : { 'RSSStatus': RSSStatus, 'DIRACStatus': DIRACStatus } }
else:
status = { name : { 'RSSStatus': RSSStatus} }
return status
#############################################################################
def _getInfo(self, granularity, name, format_, what):
if format_ == 'RSS':
info_bit_got = self._getInfoFromRSSDB(name, what)
else:
if isinstance(what, dict):
command = what['CommandIn']
extraArgs = what['args']
else:
command = what
extraArgs = None
info_bit_got = self.cc.commandInvocation(granularity, name, None,
None, command, extraArgs)
try:
info_bit_got = info_bit_got['Result']
except:
pass
return info_bit_got
#############################################################################
def _getInfoFromRSSDB(self, name, what):
paramsL = ['Status']
siteName = None
serviceName = None
resourceName = None
storageElementName = None
serviceType = None
gridSiteName = None
if what == 'ServiceOfSite':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
paramsL.append('Reason')
siteName = name
elif what == 'ResOfCompService':
gran = 'Resources'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
serviceType = name.split('@')[0]
gridSiteName = getGOCSiteName(name.split('@')[1])
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'ResOfStorService':
gran = 'Resources'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
serviceType = name.split('@')[0]
gridSiteName = getGOCSiteName(name.split('@')[1])
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'ResOfStorEl':
gran = 'StorageElements'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
storageElementName = name
elif what == 'StorageElementsOfSite':
gran = 'StorageElements'
paramsL.insert(0, 'StorageElementName')
paramsL.append('Reason')
if '@' in name:
DIRACsiteName = name.split('@').pop()
else:
DIRACsiteName = name
gridSiteName = getGOCSiteName(DIRACsiteName)
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'Site_Panel':
gran = 'Site'
paramsL.insert(0, 'SiteName')
siteName = name
elif what == 'Service_Computing_Panel':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_Storage_Panel':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_VO-BOX_Panel':
gran = 'Services'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_VOMS_Panel':
gran = 'Services'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Resource_Panel':
gran = 'Resource'
paramsL.insert(0, 'ResourceName')
resourceName = name
elif what == 'SE_Panel':
gran = 'StorageElement'
paramsL.insert(0, 'StorageElementName')
storageElementName = name
info_bit_got = self.rsDB.getMonitoredsList(gran, paramsList = paramsL, siteName = siteName,
serviceName = serviceName, serviceType = serviceType,
resourceName = resourceName,
storageElementName = storageElementName,
gridSiteName = gridSiteName)
return info_bit_got
#############################################################################
def _getPolicyDesc(self, policyName):
return self.configModule.Policies[policyName]['Description']
#############################################################################
def __getNameForPanel(self, granularity, name, panel):
if granularity in ('Site', 'Sites'):
if panel == 'Service_Computing_Panel':
granularity = 'Service'
name = 'Computing@' + name
elif panel == 'Service_Storage_Panel':
granularity = 'Service'
name = 'Storage@' + name
elif panel == 'OtherServices_Panel':
granularity = 'Service'
name = 'OtherS@' + name
elif panel == 'Service_VOMS_Panel':
granularity = 'Service'
name = 'VOMS@' + name
elif panel == 'Service_VO-BOX_Panel':
granularity = 'Service'
name = 'VO-BOX@' + name
# else:
# granularity = granularity
# name = name
# else:
# granularity = granularity
# name = name
return (granularity, name)
#############################################################################
def _resExist(self, granularity, name):
siteName = None
serviceName = None
resourceName = None
storageElementName = None
if granularity in ('Site', 'Sites'):
siteName = name
elif granularity in ('Service', 'Services'):
serviceName = name
elif granularity in ('Resource', 'Resources'):
resourceName = name
elif granularity in ('StorageElement', 'StorageElements'):
storageElementName = name
res = self.rsDB.getMonitoredsList(granularity, siteName = siteName,
serviceName = serviceName, resourceName = resourceName,
storageElementName = storageElementName)
if res == []:
return False
else:
return True
class Synchronizer:
#############################################################################
def __init__( self, rsDBin = None, rmDBin = None ):
self.rsDB = rsDBin
self.rmDB = rmDBin
if self.rsDB == None and self.rmDB == None:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.GOCDBClient = GOCDBClient()
#############################################################################
# def sync(self, thingsToSync = None, fake_param = None):
def sync( self, _a, _b ):
"""
:params:
:attr:`thingsToSync`: list of things to sync
"""
thingsToSync = ['Utils', 'Sites', 'VOBOX', 'Resources', 'StorageElements', 'RegistryUsers']
gLogger.info( "!!! Sync DB content with CS content for %s !!!" % ( ' '.join( x for x in thingsToSync ) ) )
for thing in thingsToSync:
getattr( self, '_sync' + thing )()
return S_OK()
#############################################################################
def _syncUtils( self ):
"""
Sync DB content with what is in :mod:`DIRAC.ResourceStatusSystem.Utilities.Utils`
"""
statusIn = self.rsDB.getStatusList()
#delete status not more in Utils
for stIn in statusIn:
if stIn not in ValidStatus:
self.rsDB.removeStatus( stIn )
#Add new status
for s in ValidStatus:
if s not in statusIn:
self.rsDB.addStatus( s )
for g in ( 'Site', 'Service', 'Resource' ):
typeIn = self.rsDB.getTypesList( g )
if g == 'Site':
typesList = ValidSiteType
elif g == 'Service':
typesList = ValidServiceType
if g == 'Resource':
typesList = ValidResourceType
#delete types not more in Utils
for tIn in typeIn:
if tIn not in typesList:
self.rsDB.removeType( g, tIn )
#Add new types
for t in typesList:
if t not in typeIn:
self.rsDB.addType( g, t )
#############################################################################
def _syncSites( self ):
"""
Sync DB content with sites that are in the CS
"""
# sites in the DB now
sitesIn = self.rsDB.getMonitoredsList( 'Site', paramsList = ['SiteName'] )
sitesIn = [s[0] for s in sitesIn]
# sites in CS now
sitesList = getSites()['Value']
try:
sitesList.remove( 'LCG.Dummy.ch' )
except ValueError:
pass
# remove sites from the DB not more in the CS
for site in sitesIn:
if site not in sitesList:
self.rsDB.removeSite( site )
# add to DB what is in CS now and wasn't before
for site in sitesList:
if site not in sitesIn:
# DIRAC Tier
tier = getSiteTier( site )['Value'][0]
if tier == 0 or tier == '0':
t = 'T0'
elif tier == 1 or tier == '1':
t = 'T1'
elif tier == 3 or tier == '3':
t = 'T3'
else:
t = 'T2'
#Grid Name of the site
gridSiteName = getGOCSiteName( site )
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
#Grid Tier (with a workaround!)
DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName )
if not DIRACSitesOfGridSites['OK']:
raise RSSException, DIRACSitesOfGridSites['Message']
DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']
if len( DIRACSitesOfGridSites ) == 1:
gt = t
else:
gt = self.__getGOCTier( DIRACSitesOfGridSites )
self.rsDB.addOrModifySite( site, t, gridSiteName, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
self.rsDB.addOrModifyGridSite( gridSiteName, gt )
sitesIn.append( site )
#############################################################################
def _syncVOBOX( self ):
"""
Sync DB content with VOBoxes
"""
# services in the DB now
servicesIn = self.rsDB.getMonitoredsList( 'Service', paramsList = ['ServiceName'] )
servicesIn = [s[0] for s in servicesIn]
for site in ['LCG.CNAF.it', 'LCG.IN2P3.fr', 'LCG.PIC.es',
'LCG.RAL.uk', 'LCG.GRIDKA.de', 'LCG.NIKHEF.nl']:
service = 'VO-BOX@' + site
if service not in servicesIn:
self.rsDB.addOrModifyService( service, 'VO-BOX', site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
#############################################################################
def _syncResources( self ):
# resources in the DB now
resourcesIn = self.rsDB.getMonitoredsList( 'Resource', paramsList = ['ResourceName'] )
resourcesIn = [r[0] for r in resourcesIn]
# services in the DB now
servicesIn = self.rsDB.getMonitoredsList( 'Service', paramsList = ['ServiceName'] )
servicesIn = [s[0] for s in servicesIn]
# Site-CE mapping in CS now
siteCE = getSiteCEMapping( 'LCG' )['Value']
# Site-SE mapping in CS now
siteSE = getSiteSEMapping( 'LCG' )['Value']
# CEs in CS now
CEList = []
for i in siteCE.values():
for ce in i:
if ce is None:
continue
CEList.append( ce )
# SEs in CS now
SEList = []
for i in siteSE.values():
for x in i:
SEList.append( x )
# SE Nodes in CS now
SENodeList = []
for SE in SEList:
node = getSENodes( SE )['Value'][0]
if node is None:
continue
if node not in SENodeList:
SENodeList.append( node )
# LFC Nodes in CS now
LFCNodeList_L = []
LFCNodeList_C = []
for site in getLFCSites()['Value']:
for readable in ( 'ReadOnly', 'ReadWrite' ):
LFCNode = getLFCNode( site, readable )['Value']
if LFCNode is None or LFCNode == []:
continue
LFCNode = LFCNode[0]
if readable == 'ReadWrite':
if LFCNode not in LFCNodeList_C:
LFCNodeList_C.append( LFCNode )
elif readable == 'ReadOnly':
if LFCNode not in LFCNodeList_L:
LFCNodeList_L.append( LFCNode )
# FTS Nodes in CS now
FTSNodeList = []
sitesWithFTS = getFTSSites()
for site in sitesWithFTS['Value']:
fts = getFTSEndpoint( site )['Value']
if fts is None or fts == []:
continue
fts = fts[0]
if fts not in FTSNodeList:
FTSNodeList.append( fts )
# VOMS Nodes in CS now
VOMSNodeList = getVOMSEndpoints()['Value']
# complete list of resources in CS now
resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList
# list of services in CS now (to be done)
servicesList = []
#remove resources no more in the CS
for res in resourcesIn:
if res not in resourcesList:
self.rsDB.removeResource( res )
self.rsDB.removeStorageElement( resourceName = res )
# add to DB what is in CS now and wasn't before
# CEs
for site in siteCE.keys():
if site == 'LCG.Dummy.ch':
continue
for ce in siteCE[site]:
if ce is None:
continue
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', ce )
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex( ce )[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
serviceType = 'Computing'
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append( service )
if service not in servicesIn:
self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
servicesIn.append( service )
if ce not in resourcesIn:
CEType = getCEType( site, ce )['Value']
ceType = 'CE'
if CEType == 'CREAM':
ceType = 'CREAMCE'
self.rsDB.addOrModifyResource( ce, ceType, serviceType, site, siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
resourcesIn.append( ce )
# SRMs
for srm in SENodeList:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex( srm )[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName( siteInGOCDB )
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append( service )
if service not in servicesIn:
self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
servicesIn.append( service )
if srm not in resourcesIn and srm is not None:
self.rsDB.addOrModifyResource( srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
resourcesIn.append( srm )
# LFC_C
for lfc in LFCNodeList_C:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc )
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex( lfc )[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName( siteInGOCDB )
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append( service )
if service not in servicesIn:
self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
servicesIn.append( service )
if lfc not in resourcesIn and lfc is not None:
self.rsDB.addOrModifyResource( lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
resourcesIn.append( lfc )
# LFC_L
for lfc in LFCNodeList_L:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc )
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex( lfc )[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName( siteInGOCDB )
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append( service )
if service not in servicesIn:
self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
servicesIn.append( service )
if lfc not in resourcesIn and lfc is not None:
self.rsDB.addOrModifyResource( lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
resourcesIn.append( lfc )
# FTSs
for fts in FTSNodeList:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', fts )
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex( fts )[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName( siteInGOCDB )
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append( service )
if service not in servicesIn:
self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
servicesIn.append( service )
if fts not in resourcesIn and fts is not None:
self.rsDB.addOrModifyResource( fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
resourcesIn.append( fts )
# VOMSs
for voms in VOMSNodeList:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', voms )
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex( voms )[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName( siteInGOCDB )
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
site = siteInDIRAC['Value']
serviceType = 'VOMS'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append( service )
if service not in servicesIn:
self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
servicesIn.append( service )
if voms not in resourcesIn and voms is not None:
self.rsDB.addOrModifyResource( voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
resourcesIn.append( voms )
#remove services no more in the CS
for ser in servicesIn:
if ser not in servicesList:
serType = ser.split( '@' )[0]
if serType != 'VO-BOX':
self.rsDB.removeService( ser )
self.rsDB.removeResource( serviceName = ser )
site = ser.split( '@' )[1]
if serType == 'Storage':
self.rsDB.removeStorageElement( siteName = site )
#############################################################################
def _syncStorageElements( self ):
# Get StorageElements from the CS
SEs = getStorageElements()
if not SEs['OK']:
raise RSSException, SEs['Message']
SEs = SEs['Value']
for access in ( 'Read', 'Write' ):
storageElementsIn = self.rsDB.getMonitoredsList( 'StorageElement' + access,
paramsList = [ 'StorageElementName' ] )
try:
storageElementsIn = [ x[ 0 ] for x in storageElementsIn ]
except IndexError:
pass
#remove storageElements no more in the CS
for se in storageElementsIn:
if se not in SEs:
self.rsDB.removeStorageElement( storageElementName = se, resourceName = None, access = access )
#Add new storage Elements
for SE in SEs:
srm = getSENodes( SE )[ 'Value' ][ 0 ]
if srm == None:
continue
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )
if not siteInGOCDB[ 'OK' ]:
raise RSSException, siteInGOCDB[ 'Message' ]
if siteInGOCDB[ 'Value' ] == []:
continue
siteInGOCDB = siteInGOCDB[ 'Value' ][ 0 ][ 'SITENAME' ]
if SE not in storageElementsIn:
self.rsDB.addOrModifyStorageElement( SE, srm, siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ),
'RS_SVC', datetime.datetime( 9999, 12, 31, 23, 59, 59 ),
access = access )
storageElementsIn.append( SE )
#############################################################################
def __getGOCTier( self, sitesList ):
gridTier = 3
for site in sitesList:
tier = getSiteTier( site )['Value'][0]
if tier == 0 or tier == '0':
tn = 0
elif tier == 1 or tier == '1':
tn = 1
elif tier == 3 or tier == '3':
tn = 3
else:
tn = 2
if tn < gridTier:
gridTier = tn
if gridTier == 0:
gt = 'T0'
elif gridTier == 1:
gt = 'T1'
elif gridTier == 3:
gt = 'T3'
else:
gt = 'T2'
return gt
#############################################################################
def _syncRegistryUsers(self):
from DIRAC.ResourceStatusSystem.Utilities import CS
users = CS.getTypedDictRootedAt("Users", root= "/Registry")
for u in users:
if type(users[u]['DN']) == list:
users[u]['DN'] = users[u]['DN'][0]
if type(users[u]['Email']) == list:
users[u]['Email'] = users[u]['Email'][0]
users[u]['DN'] = users[u]['DN'].split('=')[-1]
self.rmDB.registryAddUser(u, users[u]['DN'].lower(), users[u]['Email'].lower())
class Publisher:
"""
Class Publisher is in charge of getting dispersed information, to be published on the web.
"""
#############################################################################
def __init__(self,
VOExtension,
rsDBIn=None,
commandCallerIn=None,
infoGetterIn=None,
WMSAdminIn=None):
"""
Standard constructor
:params:
:attr:`VOExtension`: string, VO Extension (e.g. 'LHCb')
:attr:`rsDBIn`: optional ResourceStatusDB object
(see :class: `DIRAC.ResourceStatusSystem.DB.ResourceStatusDB.ResourceStatusDB`)
:attr:`commandCallerIn`: optional CommandCaller object
(see :class: `DIRAC.ResourceStatusSystem.Command.CommandCaller.CommandCaller`)
:attr:`infoGetterIn`: optional InfoGetter object
(see :class: `DIRAC.ResourceStatusSystem.Utilities.InfoGetter.InfoGetter`)
:attr:`WMSAdminIn`: optional RPCClient object for WMSAdmin
(see :class: `DIRAC.Core.DISET.RPCClient.RPCClient`)
"""
self.configModule = __import__(
VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations",
globals(), locals(), ['*'])
if rsDBIn is not None:
self.rsDB = rsDBIn
else:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
if commandCallerIn is not None:
self.cc = commandCallerIn
else:
from DIRAC.ResourceStatusSystem.Command.CommandCaller import CommandCaller
self.cc = CommandCaller()
if infoGetterIn is not None:
self.ig = infoGetterIn
else:
from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import InfoGetter
self.ig = InfoGetter(VOExtension)
if WMSAdminIn is not None:
self.WMSAdmin = WMSAdminIn
else:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.WMSAdmin = RPCClient("WorkloadManagement/WMSAdministrator")
self.threadPool = ThreadPool(2, 5)
self.lockObj = threading.RLock()
self.infoForPanel_res = {}
#############################################################################
def getInfo(self, granularity, name, useNewRes=False):
"""
Standard method to get all the info to be published
This method uses a ThreadPool (:class:`DIRAC.Core.Utilities.ThreadPool.ThreadPool`)
with 2-5 threads. The threaded method is
:meth:`DIRAC.ResourceStatusSystem.Utilities.Publisher.Publisher.getInfoForPanel`
:params:
:attr:`granularity`: string - a ValidRes
:attr:`name`: string - name of the Validres
:attr:`useNewRes`: boolean. When set to true, will get new results,
otherwise it will get cached results (where available).
"""
if granularity not in ValidRes:
raise InvalidRes, where(self, self.getInfo)
self.infoForPanel_res = {}
status = None
formerStatus = None
siteType = None
serviceType = None
resourceType = None
if granularity in ('Resource', 'Resources'):
try:
resourceType = self.rsDB.getMonitoredsList(
'Resource', ['ResourceType'], resourceName=name)[0][0]
except IndexError:
return "%s does not exist!" % name
if granularity in ('StorageElement', 'StorageElements'):
try:
siteType = self.rsDB.getMonitoredsList(
'StorageElement', ['SiteType'],
storageElementName=name)[0][0]
except IndexError:
return "%s does not exist!" % name
paramNames = [
'Type', 'Group', 'Name', 'Policy', 'DIRAC Status', 'RSS Status',
'Reason', 'Description'
]
infoToGet = self.ig.getInfoToApply(('view_info', ),
granularity,
status=status,
formerStatus=formerStatus,
siteType=siteType,
serviceType=serviceType,
resourceType=resourceType,
useNewRes=useNewRes)[0]['Panels']
infoToGet_res = {}
recordsList = []
infosForPolicy = {}
for panel in infoToGet.keys():
(granularityForPanel,
nameForPanel) = self.__getNameForPanel(granularity, name, panel)
if not self._resExist(granularityForPanel, nameForPanel):
# completeInfoForPanel_res = None
continue
#take composite RSS result for name
nameStatus_res = self._getStatus(nameForPanel, panel)
recordBase = [None, None, None, None, None, None, None, None]
recordBase[1] = panel.replace('_Panel', '')
recordBase[2] = nameForPanel #nameForPanel
try:
recordBase[4] = nameStatus_res[nameForPanel][
'DIRACStatus'] #DIRAC Status
except:
pass
recordBase[5] = nameStatus_res[nameForPanel][
'RSSStatus'] #RSS Status
record = copy.deepcopy(recordBase)
record[0] = 'ResultsForResource'
recordsList.append(record)
#take info that goes into the panel
infoForPanel = infoToGet[panel]
for info in infoForPanel:
self.threadPool.generateJobAndQueueIt(
self.getInfoForPanel,
args=(info, granularityForPanel, nameForPanel))
self.threadPool.processAllResults()
for policy in [x.keys()[0] for x in infoForPanel]:
record = copy.deepcopy(recordBase)
record[0] = 'SpecificInformation'
record[3] = policy #policyName
record[4] = None #DIRAC Status
record[5] = self.infoForPanel_res[policy][
'Status'] #RSS status for the policy
record[6] = self.infoForPanel_res[policy]['Reason'] #Reason
record[7] = self.infoForPanel_res[policy]['desc'] #Description
recordsList.append(record)
infosForPolicy[policy] = self.infoForPanel_res[policy]['infos']
infoToGet_res['TotalRecords'] = len(recordsList)
infoToGet_res['ParameterNames'] = paramNames
infoToGet_res['Records'] = recordsList
infoToGet_res['Extras'] = infosForPolicy
return infoToGet_res
#############################################################################
def getInfoForPanel(self, info, granularityForPanel, nameForPanel):
#get single RSS policy results
policyResToGet = info.keys()[0]
pol_res = self.rsDB.getPolicyRes(nameForPanel, policyResToGet)
if pol_res != []:
pol_res_dict = {'Status': pol_res[0], 'Reason': pol_res[1]}
else:
pol_res_dict = {'Status': 'Unknown', 'Reason': 'Unknown'}
self.lockObj.acquire()
try:
self.infoForPanel_res[policyResToGet] = pol_res_dict
finally:
self.lockObj.release()
#get policy description
desc = self._getPolicyDesc(policyResToGet)
#get other info
othersInfo = info.values()[0]
if not isinstance(othersInfo, list):
othersInfo = [othersInfo]
info_res = {}
for oi in othersInfo:
format = oi.keys()[0]
what = oi.values()[0]
info_bit_got = self._getInfo(granularityForPanel, nameForPanel,
format, what)
info_res[format] = info_bit_got
self.lockObj.acquire()
try:
self.infoForPanel_res[policyResToGet]['infos'] = info_res
self.infoForPanel_res[policyResToGet]['desc'] = desc
finally:
self.lockObj.release()
#############################################################################
def _getStatus(self, name, panel):
#get RSS status
RSSStatus = self._getInfoFromRSSDB(name, panel)[0][1]
#get DIRAC status
if panel in ('Site_Panel', 'SE_Panel'):
if panel == 'Site_Panel':
DIRACStatus = self.WMSAdmin.getSiteMaskLogging(name)
if DIRACStatus['OK']:
DIRACStatus = DIRACStatus['Value'][name].pop()[0]
else:
raise RSSException, where(self, self._getStatus)
elif panel == 'SE_Panel':
ra = getStorageElementStatus(name, 'ReadAccess')['Value']
wa = getStorageElementStatus(name, 'WriteAccess')['Value']
DIRACStatus = {'ReadAccess': ra, 'WriteAccess': wa}
status = {
name: {
'RSSStatus': RSSStatus,
'DIRACStatus': DIRACStatus
}
}
else:
status = {name: {'RSSStatus': RSSStatus}}
return status
#############################################################################
def _getInfo(self, granularity, name, format, what):
if format == 'RSS':
info_bit_got = self._getInfoFromRSSDB(name, what)
else:
if isinstance(what, dict):
command = what['CommandIn']
extraArgs = what['args']
else:
command = what
extraArgs = None
info_bit_got = self.cc.commandInvocation(granularity, name, None,
None, command, extraArgs)
try:
info_bit_got = info_bit_got['Result']
except:
pass
return info_bit_got
#############################################################################
def _getInfoFromRSSDB(self, name, what):
paramsL = ['Status']
siteName = None
serviceName = None
resourceName = None
storageElementName = None
serviceType = None
gridSiteName = None
if what == 'ServiceOfSite':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
paramsL.append('Reason')
siteName = name
elif what == 'ResOfCompService':
gran = 'Resources'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
serviceType = name.split('@')[0]
gridSiteName = getGOCSiteName(name.split('@')[1])
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'ResOfStorService':
gran = 'Resources'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
serviceType = name.split('@')[0]
gridSiteName = getGOCSiteName(name.split('@')[1])
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'ResOfStorEl':
gran = 'StorageElements'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
storageElementName = name
elif what == 'StorageElementsOfSite':
gran = 'StorageElements'
paramsL.insert(0, 'StorageElementName')
paramsL.append('Reason')
if '@' in name:
DIRACsiteName = name.split('@').pop()
else:
DIRACsiteName = name
gridSiteName = getGOCSiteName(DIRACsiteName)
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'Site_Panel':
gran = 'Site'
paramsL.insert(0, 'SiteName')
siteName = name
elif what == 'Service_Computing_Panel':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_Storage_Panel':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_VO-BOX_Panel':
gran = 'Services'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_VOMS_Panel':
gran = 'Services'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Resource_Panel':
gran = 'Resource'
paramsL.insert(0, 'ResourceName')
resourceName = name
elif what == 'SE_Panel':
gran = 'StorageElement'
paramsL.insert(0, 'StorageElementName')
storageElementName = name
info_bit_got = self.rsDB.getMonitoredsList(
gran,
paramsList=paramsL,
siteName=siteName,
serviceName=serviceName,
serviceType=serviceType,
resourceName=resourceName,
storageElementName=storageElementName,
gridSiteName=gridSiteName)
return info_bit_got
#############################################################################
def _getPolicyDesc(self, policyName):
return self.configModule.Policies[policyName]['Description']
#############################################################################
def __getNameForPanel(self, granularity, name, panel):
if granularity in ('Site', 'Sites'):
if panel == 'Service_Computing_Panel':
granularity = 'Service'
name = 'Computing@' + name
elif panel == 'Service_Storage_Panel':
granularity = 'Service'
name = 'Storage@' + name
elif panel == 'OtherServices_Panel':
granularity = 'Service'
name = 'OtherS@' + name
elif panel == 'Service_VOMS_Panel':
granularity = 'Service'
name = 'VOMS@' + name
elif panel == 'Service_VO-BOX_Panel':
granularity = 'Service'
name = 'VO-BOX@' + name
# else:
# granularity = granularity
# name = name
# else:
# granularity = granularity
# name = name
return (granularity, name)
#############################################################################
def _resExist(self, granularity, name):
siteName = None
serviceName = None
resourceName = None
storageElementName = None
if granularity in ('Site', 'Sites'):
siteName = name
elif granularity in ('Service', 'Services'):
serviceName = name
elif granularity in ('Resource', 'Resources'):
resourceName = name
elif granularity in ('StorageElement', 'StorageElements'):
storageElementName = name
res = self.rsDB.getMonitoredsList(
granularity,
siteName=siteName,
serviceName=serviceName,
resourceName=resourceName,
storageElementName=storageElementName)
if res == []:
return False
else:
return True
class Synchronizer:
#############################################################################
def __init__(self, rsDBin=None, rmDBin=None):
self.rsDB = rsDBin
self.rmDB = rmDBin
if self.rsDB == None and self.rmDB == None:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.GOCDBClient = GOCDBClient()
#############################################################################
# def sync(self, thingsToSync = None, fake_param = None):
def sync(self, _a, _b):
"""
:params:
:attr:`thingsToSync`: list of things to sync
"""
thingsToSync = [
'Utils', 'Sites', 'VOBOX', 'Resources', 'StorageElements',
'RegistryUsers'
]
gLogger.info("!!! Sync DB content with CS content for %s !!!" %
(' '.join(x for x in thingsToSync)))
for thing in thingsToSync:
getattr(self, '_sync' + thing)()
return S_OK()
#############################################################################
def _syncUtils(self):
"""
Sync DB content with what is in :mod:`DIRAC.ResourceStatusSystem.Utilities.Utils`
"""
statusIn = self.rsDB.getStatusList()
#delete status not more in Utils
for stIn in statusIn:
if stIn not in ValidStatus:
self.rsDB.removeStatus(stIn)
#Add new status
for s in ValidStatus:
if s not in statusIn:
self.rsDB.addStatus(s)
for g in ('Site', 'Service', 'Resource'):
typeIn = self.rsDB.getTypesList(g)
if g == 'Site':
typesList = ValidSiteType
elif g == 'Service':
typesList = ValidServiceType
if g == 'Resource':
typesList = ValidResourceType
#delete types not more in Utils
for tIn in typeIn:
if tIn not in typesList:
self.rsDB.removeType(g, tIn)
#Add new types
for t in typesList:
if t not in typeIn:
self.rsDB.addType(g, t)
#############################################################################
def _syncSites(self):
"""
Sync DB content with sites that are in the CS
"""
# sites in the DB now
sitesIn = self.rsDB.getMonitoredsList('Site', paramsList=['SiteName'])
sitesIn = [s[0] for s in sitesIn]
# sites in CS now
sitesList = getSites()['Value']
try:
sitesList.remove('LCG.Dummy.ch')
except ValueError:
pass
# remove sites from the DB not more in the CS
for site in sitesIn:
if site not in sitesList:
self.rsDB.removeSite(site)
# add to DB what is in CS now and wasn't before
for site in sitesList:
if site not in sitesIn:
# DIRAC Tier
tier = getSiteTier(site)['Value'][0]
if tier == 0 or tier == '0':
t = 'T0'
elif tier == 1 or tier == '1':
t = 'T1'
elif tier == 3 or tier == '3':
t = 'T3'
else:
t = 'T2'
#Grid Name of the site
gridSiteName = getGOCSiteName(site)
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
#Grid Tier (with a workaround!)
DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName)
if not DIRACSitesOfGridSites['OK']:
raise RSSException, DIRACSitesOfGridSites['Message']
DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']
if len(DIRACSitesOfGridSites) == 1:
gt = t
else:
gt = self.__getGOCTier(DIRACSitesOfGridSites)
self.rsDB.addOrModifySite(
site, t, gridSiteName, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
self.rsDB.addOrModifyGridSite(gridSiteName, gt)
sitesIn.append(site)
#############################################################################
def _syncVOBOX(self):
"""
Sync DB content with VOBoxes
"""
# services in the DB now
servicesIn = self.rsDB.getMonitoredsList('Service',
paramsList=['ServiceName'])
servicesIn = [s[0] for s in servicesIn]
for site in [
'LCG.CNAF.it', 'LCG.IN2P3.fr', 'LCG.PIC.es', 'LCG.RAL.uk',
'LCG.GRIDKA.de', 'LCG.NIKHEF.nl'
]:
service = 'VO-BOX@' + site
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, 'VO-BOX', site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
#############################################################################
def _syncResources(self):
# resources in the DB now
resourcesIn = self.rsDB.getMonitoredsList('Resource',
paramsList=['ResourceName'])
resourcesIn = [r[0] for r in resourcesIn]
# services in the DB now
servicesIn = self.rsDB.getMonitoredsList('Service',
paramsList=['ServiceName'])
servicesIn = [s[0] for s in servicesIn]
# Site-CE mapping in CS now
siteCE = getSiteCEMapping('LCG')['Value']
# Site-SE mapping in CS now
siteSE = getSiteSEMapping('LCG')['Value']
# CEs in CS now
CEList = []
for i in siteCE.values():
for ce in i:
if ce is None:
continue
CEList.append(ce)
# SEs in CS now
SEList = []
for i in siteSE.values():
for x in i:
SEList.append(x)
# SE Nodes in CS now
SENodeList = []
for SE in SEList:
node = getSENodes(SE)['Value'][0]
if node is None:
continue
if node not in SENodeList:
SENodeList.append(node)
# LFC Nodes in CS now
LFCNodeList_L = []
LFCNodeList_C = []
for site in getLFCSites()['Value']:
for readable in ('ReadOnly', 'ReadWrite'):
LFCNode = getLFCNode(site, readable)['Value']
if LFCNode is None or LFCNode == []:
continue
LFCNode = LFCNode[0]
if readable == 'ReadWrite':
if LFCNode not in LFCNodeList_C:
LFCNodeList_C.append(LFCNode)
elif readable == 'ReadOnly':
if LFCNode not in LFCNodeList_L:
LFCNodeList_L.append(LFCNode)
# FTS Nodes in CS now
FTSNodeList = []
sitesWithFTS = getFTSSites()
for site in sitesWithFTS['Value']:
fts = getFTSEndpoint(site)['Value']
if fts is None or fts == []:
continue
fts = fts[0]
if fts not in FTSNodeList:
FTSNodeList.append(fts)
# VOMS Nodes in CS now
VOMSNodeList = getVOMSEndpoints()['Value']
# complete list of resources in CS now
resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList
# list of services in CS now (to be done)
servicesList = []
#remove resources no more in the CS
for res in resourcesIn:
if res not in resourcesList:
self.rsDB.removeResource(res)
self.rsDB.removeStorageElement(resourceName=res)
# add to DB what is in CS now and wasn't before
# CEs
for site in siteCE.keys():
if site == 'LCG.Dummy.ch':
continue
for ce in siteCE[site]:
if ce is None:
continue
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', ce)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex(ce)[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', trueName)
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
serviceType = 'Computing'
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append(service)
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
servicesIn.append(service)
if ce not in resourcesIn:
CEType = getCEType(site, ce)['Value']
ceType = 'CE'
if CEType == 'CREAM':
ceType = 'CREAMCE'
self.rsDB.addOrModifyResource(
ce, ceType, serviceType, site, siteInGOCDB, 'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
resourcesIn.append(ce)
# SRMs
for srm in SENodeList:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', srm)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex(srm)[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', trueName)
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName(siteInGOCDB)
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append(service)
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
servicesIn.append(service)
if srm not in resourcesIn and srm is not None:
self.rsDB.addOrModifyResource(
srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
resourcesIn.append(srm)
# LFC_C
for lfc in LFCNodeList_C:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', lfc)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex(lfc)[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', trueName)
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName(siteInGOCDB)
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append(service)
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
servicesIn.append(service)
if lfc not in resourcesIn and lfc is not None:
self.rsDB.addOrModifyResource(
lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
resourcesIn.append(lfc)
# LFC_L
for lfc in LFCNodeList_L:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', lfc)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex(lfc)[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', trueName)
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName(siteInGOCDB)
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append(service)
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
servicesIn.append(service)
if lfc not in resourcesIn and lfc is not None:
self.rsDB.addOrModifyResource(
lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
resourcesIn.append(lfc)
# FTSs
for fts in FTSNodeList:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', fts)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex(fts)[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', trueName)
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName(siteInGOCDB)
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append(service)
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
servicesIn.append(service)
if fts not in resourcesIn and fts is not None:
self.rsDB.addOrModifyResource(
fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
resourcesIn.append(fts)
# VOMSs
for voms in VOMSNodeList:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', voms)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex(voms)[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', trueName)
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName(siteInGOCDB)
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
site = siteInDIRAC['Value']
serviceType = 'VOMS'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append(service)
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
servicesIn.append(service)
if voms not in resourcesIn and voms is not None:
self.rsDB.addOrModifyResource(
voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
resourcesIn.append(voms)
#remove services no more in the CS
for ser in servicesIn:
if ser not in servicesList:
serType = ser.split('@')[0]
if serType != 'VO-BOX':
self.rsDB.removeService(ser)
self.rsDB.removeResource(serviceName=ser)
site = ser.split('@')[1]
if serType == 'Storage':
self.rsDB.removeStorageElement(siteName=site)
#############################################################################
def _syncStorageElements(self):
# Get StorageElements from the CS
SEs = getStorageElements()
if not SEs['OK']:
raise RSSException, SEs['Message']
SEs = SEs['Value']
for access in ('Read', 'Write'):
storageElementsIn = self.rsDB.getMonitoredsList(
'StorageElement' + access, paramsList=['StorageElementName'])
try:
storageElementsIn = [x[0] for x in storageElementsIn]
except IndexError:
pass
#remove storageElements no more in the CS
for se in storageElementsIn:
if se not in SEs:
self.rsDB.removeStorageElement(storageElementName=se,
resourceName=None,
access=access)
#Add new storage Elements
for SE in SEs:
srm = getSENodes(SE)['Value'][0]
if srm == None:
continue
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', srm)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
continue
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
if SE not in storageElementsIn:
self.rsDB.addOrModifyStorageElement(
SE,
srm,
siteInGOCDB,
'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC',
datetime.datetime(9999, 12, 31, 23, 59, 59),
access=access)
storageElementsIn.append(SE)
#############################################################################
def __getGOCTier(self, sitesList):
gridTier = 3
for site in sitesList:
tier = getSiteTier(site)['Value'][0]
if tier == 0 or tier == '0':
tn = 0
elif tier == 1 or tier == '1':
tn = 1
elif tier == 3 or tier == '3':
tn = 3
else:
tn = 2
if tn < gridTier:
gridTier = tn
if gridTier == 0:
gt = 'T0'
elif gridTier == 1:
gt = 'T1'
elif gridTier == 3:
gt = 'T3'
else:
gt = 'T2'
return gt
#############################################################################
def _syncRegistryUsers(self):
from DIRAC.ResourceStatusSystem.Utilities import CS
users = CS.getTypedDictRootedAt("Users", root="/Registry")
for u in users:
if type(users[u]['DN']) == list:
users[u]['DN'] = users[u]['DN'][0]
if type(users[u]['Email']) == list:
users[u]['Email'] = users[u]['Email'][0]
users[u]['DN'] = users[u]['DN'].split('=')[-1]
self.rmDB.registryAddUser(u, users[u]['DN'].lower(),
users[u]['Email'].lower())