def callClient(args, clientIn = None):
if clientIn is not None:
c = clientIn
else:
# use standard GGUSTickets Client
from DIRAC.Core.LCG.GGUSTicketsClient import GGUSTicketsClient
c = GGUSTicketsClient()
name = args[1]
name = getGOCSiteName(name)
if not name['OK']:
raise RSSException, name['Message']
name = name['Value']
try:
openTickets = c.getTicketsList(name)
if not openTickets['OK']:
return 'Unknown'
return openTickets['Value']
except urllib2.URLError:
gLogger.error("GGUSTicketsClient timed out for " + name)
return 'Unknown'
except:
gLogger.exception("Exception when calling GGUSTicketsClient for " + name)
return 'Unknown'
def getJobsEff(self, granularity, name, periods):
"""
Return job stats of entity in args for periods
:Parameters:
`granularity`
string - should be a ValidRes
`name`
should be the name of the ValidRes
`periods`
list - contains the periods to consider in the query
:return:
{
'JobsEff': X (0-1)'
}
"""
if granularity == "Site":
entity = getGOCSiteName(name)
if not entity["OK"]:
raise RSSException, entity["Message"]
entity = entity["Value"]
_granularity = "Site"
else:
entity = name
_granularity = "GridCE"
def getPilotsStats(self, granularity, name, periods):
"""
Return pilot stats
:Parameters:
`granularity`
string - should be a ValidRes
`name`
string - should be the name of the ValidRes
`periods`
list - contains the periods to consider in the query
:return:
{
'MeanProcessedPilots': X'
'LastProcessedPilots': X'
}
"""
if granularity.capitalize() not in ValidRes:
raise InvalidRes, where(self, self.getPilotsStats)
if granularity == 'Site':
entity = getGOCSiteName(name)
if not entity['OK']:
raise RSSException, entity['Message']
entity = entity['Value']
_granularity = 'Site'
else:
entity = name
_granularity = 'GridCE'
def getPilotsStats(self, granularity, name, periods):
"""
Return pilot stats
:Parameters:
`granularity`
string - should be a ValidRes
`name`
string - should be the name of the ValidRes
`periods`
list - contains the periods to consider in the query
:return:
{
'MeanProcessedPilots': X'
'LastProcessedPilots': X'
}
"""
if granularity.capitalize() not in ValidRes:
raise InvalidRes, where(self, self.getPilotsStats)
if granularity == 'Site':
entity = getGOCSiteName(name)
if not entity['OK']:
raise RSSException, entity['Message']
entity = entity['Value']
_granularity = 'Site'
else:
entity = name
_granularity = 'GridCE'
def _syncSites(self):
"""
Sync DB content with sites that are in the CS
"""
# sites in the DB now
sitesIn = self.rsDB.getMonitoredsList('Site', paramsList=['SiteName'])
sitesIn = [s[0] for s in sitesIn]
# sites in CS now
sitesList = getSites()['Value']
try:
sitesList.remove('LCG.Dummy.ch')
except ValueError:
pass
# remove sites from the DB not more in the CS
for site in sitesIn:
if site not in sitesList:
self.rsDB.removeSite(site)
# add to DB what is in CS now and wasn't before
for site in sitesList:
if site not in sitesIn:
# DIRAC Tier
tier = getSiteTier(site)['Value'][0]
if tier == 0 or tier == '0':
t = 'T0'
elif tier == 1 or tier == '1':
t = 'T1'
elif tier == 3 or tier == '3':
t = 'T3'
else:
t = 'T2'
#Grid Name of the site
gridSiteName = getGOCSiteName(site)
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
#Grid Tier (with a workaround!)
DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName)
if not DIRACSitesOfGridSites['OK']:
raise RSSException, DIRACSitesOfGridSites['Message']
DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']
if len(DIRACSitesOfGridSites) == 1:
gt = t
else:
gt = self.__getGOCTier(DIRACSitesOfGridSites)
self.rsDB.addOrModifySite(
site, t, gridSiteName, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
self.rsDB.addOrModifyGridSite(gridSiteName, gt)
sitesIn.append(site)
def getJobsEff(self, granularity, name, periods):
"""
Return job stats of entity in args for periods
:Parameters:
`granularity`
string - should be a ValidRes
`name`
should be the name of the ValidRes
`periods`
list - contains the periods to consider in the query
:return:
{
'JobsEff': X (0-1)'
}
"""
if granularity == 'Site':
entity = getGOCSiteName(name)
if not entity['OK']:
raise RSSException, entity['Message']
entity = entity['Value']
_granularity = 'Site'
else:
entity = name
_granularity = 'GridCE'
def _syncSites( self ):
"""
Sync DB content with sites that are in the CS
"""
# sites in the DB now
sitesIn = self.rsDB.getMonitoredsList( 'Site', paramsList = ['SiteName'] )
sitesIn = [s[0] for s in sitesIn]
# sites in CS now
sitesList = getSites()['Value']
try:
sitesList.remove( 'LCG.Dummy.ch' )
except ValueError:
pass
# remove sites from the DB not more in the CS
for site in sitesIn:
if site not in sitesList:
self.rsDB.removeSite( site )
# add to DB what is in CS now and wasn't before
for site in sitesList:
if site not in sitesIn:
# DIRAC Tier
tier = getSiteTier( site )['Value'][0]
if tier == 0 or tier == '0':
t = 'T0'
elif tier == 1 or tier == '1':
t = 'T1'
elif tier == 3 or tier == '3':
t = 'T3'
else:
t = 'T2'
#Grid Name of the site
gridSiteName = getGOCSiteName( site )
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
#Grid Tier (with a workaround!)
DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName )
if not DIRACSitesOfGridSites['OK']:
raise RSSException, DIRACSitesOfGridSites['Message']
DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']
if len( DIRACSitesOfGridSites ) == 1:
gt = t
else:
gt = self.__getGOCTier( DIRACSitesOfGridSites )
self.rsDB.addOrModifySite( site, t, gridSiteName, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
self.rsDB.addOrModifyGridSite( gridSiteName, gt )
sitesIn.append( site )
def _prepareCommand( self ):
'''
DowntimeCommand requires four arguments:
- name : <str>
- element : Site / Resource
- elementType: <str>
If the elements are Site(s), we need to get their GOCDB names. They may
not have, so we ignore them if they do not have.
'''
if 'name' not in self.args:
return S_ERROR( '"name" not found in self.args' )
elementName = self.args[ 'name' ]
if 'element' not in self.args:
return S_ERROR( '"element" not found in self.args' )
element = self.args[ 'element' ]
if 'elementType' not in self.args:
return S_ERROR( '"elementType" not found in self.args' )
elementType = self.args[ 'elementType' ]
if not element in [ 'Site', 'Resource' ]:
return S_ERROR( 'element is not Site nor Resource' )
hours = None
if 'hours' in self.args:
hours = self.args[ 'hours' ]
gocdbServiceType = None
# Transform DIRAC site names into GOCDB topics
if element == 'Site':
gocSite = getGOCSiteName( elementName )
if not gocSite[ 'OK' ]:
return gocSite
elementName = gocSite[ 'Value' ]
# The DIRAC se names mean nothing on the grid, but their hosts do mean.
elif elementType == 'StorageElement':
# We need to distinguish if it's tape or disk
if getStorageElementOptions( elementName )['Value']['TapeSE']:
gocdbServiceType = "srm"
elif getStorageElementOptions( elementName )['Value']['DiskSE']:
gocdbServiceType = "srm.nearline"
seHost = CSHelpers.getSEHost( elementName )
if not seHost:
return S_ERROR( 'No seHost for %s' % elementName )
elementName = seHost
return S_OK( ( element, elementName, hours, gocdbServiceType ) )
def _prepareCommand( self ):
'''
DowntimeCommand requires four arguments:
- name : <str>
- element : Site / Resource
- elementType: <str>
If the elements are Site(s), we need to get their GOCDB names. They may
not have, so we ignore them if they do not have.
'''
if 'name' not in self.args:
return S_ERROR( '"name" not found in self.args' )
elementName = self.args[ 'name' ]
if 'element' not in self.args:
return S_ERROR( '"element" not found in self.args' )
element = self.args[ 'element' ]
if 'elementType' not in self.args:
return S_ERROR( '"elementType" not found in self.args' )
elementType = self.args[ 'elementType' ]
if not element in [ 'Site', 'Resource' ]:
return S_ERROR( 'element is not Site nor Resource' )
hours = None
if 'hours' in self.args:
hours = self.args[ 'hours' ]
gocdbServiceType = None
# Transform DIRAC site names into GOCDB topics
if element == 'Site':
gocSite = getGOCSiteName( elementName )
if not gocSite[ 'OK' ]:
return gocSite
elementName = gocSite[ 'Value' ]
# The DIRAC se names mean nothing on the grid, but their hosts do mean.
elif elementType == 'StorageElement':
# We need to distinguish if it's tape or disk
if getStorageElementOptions( elementName )['Value']['TapeSE']:
gocdbServiceType = "srm.nearline"
elif getStorageElementOptions( elementName )['Value']['DiskSE']:
gocdbServiceType = "srm"
seHost = CSHelpers.getSEHost( elementName )
if not seHost:
return S_ERROR( 'No seHost for %s' % elementName )
elementName = seHost
return S_OK( ( element, elementName, hours, gocdbServiceType ) )
def _prepareCommand( self ):
'''
GGUSTicketsCommand requires one arguments:
- elementName : <str>
GGUSTickets are associated with gocDB names, so we have to transform the
diracSiteName into a gocSiteName.
'''
if 'name' not in self.args:
return S_ERROR( '"name" not found in self.args' )
name = self.args[ 'name' ]
return getGOCSiteName( name )
def _prepareCommand(self):
'''
GGUSTicketsCommand requires one arguments:
- elementName : <str>
GGUSTickets are associated with gocDB names, so we have to transform the
diracSiteName into a gocSiteName.
'''
if 'name' not in self.args:
return S_ERROR('"name" not found in self.args')
name = self.args['name']
return getGOCSiteName(name)
def _prepareCommand(self):
"""
DowntimeCommand requires three arguments:
- name : <str>
- element : Site / Resource
- elementType: <str>
If the elements are Site(s), we need to get their GOCDB names. They may
not have, so we ignore them if they do not have.
"""
if "name" not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args["name"]
if "element" not in self.args:
return S_ERROR('"element" not found in self.args')
element = self.args["element"]
if "elementType" not in self.args:
return S_ERROR('"elementType" not found in self.args')
elementType = self.args["elementType"]
if not element in ["Site", "Resource"]:
return S_ERROR("element is not Site nor Resource")
hours = None
if "hours" in self.args:
hours = self.args["hours"]
# Transform DIRAC site names into GOCDB topics
if element == "Site":
gocSite = getGOCSiteName(elementName)
if not gocSite["OK"]:
return gocSite
elementName = gocSite["Value"]
# The DIRAC se names mean nothing on the grid, but their hosts do mean.
elif elementType == "StorageElement":
seHost = CSHelpers.getSEHost(elementName)
if not seHost:
return S_ERROR("No seHost for %s" % elementName)
elementName = seHost
return S_OK((element, elementName, hours))
def getGOCSites(diracSites=None):
if diracSites is None:
diracSites = getSites()
if not diracSites['OK']:
return diracSites
diracSites = diracSites['Value']
gocSites = []
for diracSite in diracSites:
gocSite = getGOCSiteName(diracSite)
if not gocSite['OK']:
continue
gocSites.append(gocSite['Value'])
return S_OK(list(set(gocSites)))
def doCommand(self):
"""
Return getStatus from GOC DB Client
:attr:`args`:
- args[0]: string: should be a ValidElement
- args[1]: string: should be the name of the ValidElement
- args[2]: string: optional, number of hours in which
the down time is starting
"""
timeFormat = "%Y-%m-%d %H:%M"
super(GOCDBStatus_Command, self).doCommand()
self.APIs = initAPIs(self.__APIs__, self.APIs)
try:
granularity = self.args[0]
name = self.args[1]
if len(self.args) > 2:
hours = self.args[2]
else:
hours = None
if granularity == 'Site':
name = getGOCSiteName(name)['Value']
res = self.APIs['GOCDBClient'].getStatus(granularity, name, None,
hours)
if not res['OK']:
return {'Result': res}
res = res['Value']
if res is None or res == {}:
return {'Result': S_OK({'DT': None})}
DT_dict_result = {}
now = datetime.utcnow().replace(microsecond=0, second=0)
if len(res) > 1:
#there's more than one DT
resDT = None
for dt_ID in res:
#looking for an ongoing one
startSTR = res[dt_ID]['FORMATED_START_DATE']
start_datetime = datetime.strptime(startSTR, timeFormat)
if start_datetime < now:
resDT = res[dt_ID]
break
#if I'm here, there's no OnGoing DT
if resDT is None:
resDT = res[res.keys()[0]]
res = resDT
else:
res = res[res.keys()[0]]
DT_dict_result['DT'] = res['SEVERITY']
DT_dict_result['EndDate'] = res['FORMATED_END_DATE']
startSTR = res['FORMATED_START_DATE']
start_datetime = datetime.strptime(startSTR, timeFormat)
if start_datetime > now:
diff = convertTime(start_datetime - now, 'hours')
DT_dict_result['DT'] = DT_dict_result['DT'] + " in " + str(
diff) + ' hours'
res = S_OK(DT_dict_result)
except Exception, e:
_msg = '%s (%s): %s' % (self.__class__.__name__, self.args, e)
gLogger.exception(_msg)
return {'Result': S_ERROR(_msg)}
def doCommand(self):
"""
Return getStatus from GOC DB Client
:attr:`args`:
- args[0]: string: should be a ValidRes
- args[1]: string: should be the name of the ValidRes
- args[2]: string: optional, number of hours in which
the down time is starting
"""
super(GOCDBStatus_Command, self).doCommand()
if self.client is None:
# use standard GOC DB Client
from DIRAC.Core.LCG.GOCDBClient import GOCDBClient
self.client = GOCDBClient()
granularity = self.args[0]
name = self.args[1]
try:
hours = self.args[2]
except IndexError:
hours = None
if granularity in ('Site', 'Sites'):
name = getGOCSiteName(name)
if not name['OK']:
raise RSSException, name['Message']
name = name['Value']
try:
res = self.client.getStatus(granularity, name, None, hours,
self.timeout)
if not res['OK']:
return {'Result': 'Unknown'}
res = res['Value']
if res is None or res == {}:
return {'Result': {'DT': None}}
DT_dict_result = {}
now = datetime.datetime.utcnow().replace(microsecond=0, second=0)
if len(res) > 1:
#there's more than one DT
for dt_ID in res:
#looking for an ongoing one
startSTR = res[dt_ID]['FORMATED_START_DATE']
start_datetime = datetime.datetime(
*time.strptime(startSTR, "%Y-%m-%d %H:%M")[0:5])
if start_datetime < now:
resDT = res[dt_ID]
break
try:
resDT
except:
#if I'm here, there's no OnGoing DT
resDT = res[res.keys()[0]]
res = resDT
else:
res = res[res.keys()[0]]
DT_dict_result['DT'] = res['SEVERITY']
DT_dict_result['EndDate'] = res['FORMATED_END_DATE']
startSTR = res['FORMATED_START_DATE']
start_datetime = datetime.datetime(
*time.strptime(startSTR, "%Y-%m-%d %H:%M")[0:5])
if start_datetime > now:
diff = convertTime(start_datetime - now, 'hours')
DT_dict_result['DT'] = DT_dict_result['DT'] + " in " + str(
diff) + ' hours'
return {'Result': DT_dict_result}
except urllib2.URLError:
gLogger.error("GOCDB timed out for " + granularity + " " + name)
return {'Result': 'Unknown'}
except:
gLogger.exception("Exception when calling GOCDBClient for " +
granularity + " " + name)
return {'Result': 'Unknown'}
def _prepareCommand( self ):
'''
DowntimeCommand requires four arguments:
- name : <str>
- element : Site / Resource
- elementType: <str>
If the elements are Site(s), we need to get their GOCDB names. They may
not have, so we ignore them if they do not have.
'''
if 'name' not in self.args:
return S_ERROR( '"name" not found in self.args' )
elementName = self.args[ 'name' ]
if 'element' not in self.args:
return S_ERROR( '"element" not found in self.args' )
element = self.args[ 'element' ]
if 'elementType' not in self.args:
return S_ERROR( '"elementType" not found in self.args' )
elementType = self.args[ 'elementType' ]
if not element in [ 'Site', 'Resource' ]:
return S_ERROR( 'element is neither Site nor Resource' )
hours = None
if 'hours' in self.args:
hours = self.args[ 'hours' ]
gocdbServiceType = None
# Transform DIRAC site names into GOCDB topics
if element == 'Site':
gocSite = getGOCSiteName( elementName )
if not gocSite[ 'OK' ]:
return gocSite
elementName = gocSite[ 'Value' ]
# The DIRAC se names mean nothing on the grid, but their hosts do mean.
elif elementType == 'StorageElement':
# We need to distinguish if it's tape or disk
seOptions = getStorageElementOptions( elementName )
if not seOptions['OK']:
return seOptions
if seOptions['Value'].get( 'TapeSE' ):
gocdbServiceType = "srm.nearline"
elif seOptions['Value'].get( 'DiskSE' ):
gocdbServiceType = "srm"
seHost = CSHelpers.getSEHost( elementName )
if not seHost['OK']:
return seHost
seHost = seHost['Value']
if not seHost:
return S_ERROR( 'No seHost for %s' % elementName )
elementName = seHost
elif elementType == 'FTS' or elementType == 'FTS3':
gocdbServiceType = 'FTS'
try:
#WARNING: this method presupposes that the server is an FTS3 type
elementName = getGOCFTSName(elementName)
except:
return S_ERROR( 'No FTS3 server specified in dirac.cfg (see Resources/FTSEndpoints)' )
return S_OK( ( element, elementName, hours, gocdbServiceType ) )
def callClient(name, clientIn):
name = getGOCSiteName(name)['Value']
openTickets = clientIn.getTicketsList(name)
return openTickets
def _getInfoFromRSSDB(self, name, what):
paramsL = ['Status']
siteName = None
serviceName = None
resourceName = None
storageElementName = None
serviceType = None
gridSiteName = None
if what == 'ServiceOfSite':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
paramsL.append('Reason')
siteName = name
elif what == 'ResOfCompService':
gran = 'Resources'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
serviceType = name.split('@')[0]
gridSiteName = getGOCSiteName(name.split('@')[1])
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'ResOfStorService':
gran = 'Resources'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
serviceType = name.split('@')[0]
gridSiteName = getGOCSiteName(name.split('@')[1])
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'ResOfStorEl':
gran = 'StorageElements'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
storageElementName = name
elif what == 'StorageElementsOfSite':
gran = 'StorageElements'
paramsL.insert(0, 'StorageElementName')
paramsL.append('Reason')
if '@' in name:
DIRACsiteName = name.split('@').pop()
else:
DIRACsiteName = name
gridSiteName = getGOCSiteName(DIRACsiteName)
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'Site_Panel':
gran = 'Site'
paramsL.insert(0, 'SiteName')
siteName = name
elif what == 'Service_Computing_Panel':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_Storage_Panel':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_VO-BOX_Panel':
gran = 'Services'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_VOMS_Panel':
gran = 'Services'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Resource_Panel':
gran = 'Resource'
paramsL.insert(0, 'ResourceName')
resourceName = name
elif what == 'SE_Panel':
gran = 'StorageElement'
paramsL.insert(0, 'StorageElementName')
storageElementName = name
info_bit_got = self.rsDB.getMonitoredsList(
gran,
paramsList=paramsL,
siteName=siteName,
serviceName=serviceName,
serviceType=serviceType,
resourceName=resourceName,
storageElementName=storageElementName,
gridSiteName=gridSiteName)
return info_bit_got
def callClient( name, clientIn ):
name = getGOCSiteName( name )[ 'Value' ]
openTickets = clientIn.getTicketsList( name )
return openTickets
def doCommand(self, sites=None):
"""
Returns downtimes information for all the sites in input.
:params:
:attr:`sites`: list of site names (when not given, take every site)
:returns:
{'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK',
'StartDate': 'aDate', ...} ... }
"""
if self.client is None:
from DIRAC.Core.LCG.GOCDBClient import GOCDBClient
self.client = GOCDBClient()
if sites is None:
# from DIRAC.Core.DISET.RPCClient import RPCClient
RPC = RPCClient("ResourceStatus/ResourceStatus")
GOC_sites = RPC.getGridSitesList()
if not GOC_sites['OK']:
raise RSSException, where(
self, self.doCommand) + " " + sites['Message']
else:
GOC_sites = GOC_sites['Value']
else:
GOC_sites = [getGOCSiteName(x)['Value'] for x in sites]
try:
res = self.client.getStatus('Site', GOC_sites, None, 120)
except:
gLogger.exception("Exception when calling GOCDBClient.")
return {}
if not res['OK']:
raise RSSException, where(self,
self.doCommand) + " " + res['Message']
else:
res = res['Value']
if res == None:
return {}
resToReturn = {}
for dt_ID in res:
try:
dt = {}
dt['ID'] = dt_ID
dt['StartDate'] = res[dt_ID]['FORMATED_START_DATE']
dt['EndDate'] = res[dt_ID]['FORMATED_END_DATE']
dt['Severity'] = res[dt_ID]['SEVERITY']
dt['Description'] = res[dt_ID]['DESCRIPTION'].replace('\'', '')
dt['Link'] = res[dt_ID]['GOCDB_PORTAL_URL']
DIRACnames = getDIRACSiteName(res[dt_ID]['SITENAME'])
if not DIRACnames['OK']:
raise RSSException, DIRACnames['Message']
DIRACnames = DIRACnames['Value']
for DIRACname in DIRACnames:
resToReturn[dt_ID.split()[0] + ' ' + DIRACname] = dt
except KeyError:
continue
return resToReturn
def _getInfo( self, requestParams ):
gLogger.info( requestParams )
if not requestParams[ 'name' ]:
gLogger.warn( 'No name given' )
self.finish( { 'success': 'false', 'error': 'We need a Site Name to generate an Overview' } )
elementName = requestParams[ 'name' ][ 0 ]
pub = RPCClient( 'ResourceStatus/Publisher' )
elementStatuses = pub.getElementStatuses( 'Site',
str( elementName ),
None,
'all',
None,
None )
if not elementStatuses[ 'OK' ]:
gLogger.error( elementStatuses[ 'Message' ] )
self.finish( { 'success': 'false', 'error': 'Error getting ElementStatus information' } )
if not elementStatuses[ 'Value' ]:
gLogger.error( 'element "%s" not found' % elementName )
self.finish( { 'success' : 'false', 'error' : 'element "%s" not found' % elementName } )
elementStatus = [ dict( zip( elementStatuses[ 'Columns' ], element ) ) for element in elementStatuses[ 'Value' ] ][ 0 ]
elementStatus[ 'DateEffective' ] = str( elementStatus[ 'DateEffective' ] )
elementStatus[ 'LastCheckTime' ] = str( elementStatus[ 'LastCheckTime' ] )
elementStatus[ 'TokenExpiration' ] = str( elementStatus[ 'TokenExpiration' ] )
gocdb_name = getGOCSiteName( elementName )
if not gocdb_name[ 'OK' ]:
gLogger.error( gocdb_name[ 'Message' ] )
elementStatus[ 'GOCDB' ] = ""
gocdb_name = ''
else:
gocdb_name = gocdb_name[ 'Value' ]
elementStatus[ 'GOCDB' ] = '<a href="https://goc.egi.eu/portal/index.php?Page_Type=Submit_Search&SearchString=%s" target="_blank">%s</a>' % ( gocdb_name, gocdb_name )
dirac_names = getDIRACSiteName( gocdb_name )
if not dirac_names[ 'OK' ]:
gLogger.error( dirac_names[ 'Message' ] )
dirac_names = []
else:
elementStatus[ 'GOCDB' ] += "("
for i in dirac_names['Value']:
elementStatus[ 'GOCDB' ] += "%s " % i
elementStatus[ 'GOCDB' ] += ")"
elementStatus["GGUS"] = '<a href="https://ggus.eu/ws/ticket_search.php?show_columns_check[]=REQUEST_ID&show_columns_check[]=TICKET_TYPE&show_columns_check[]=AFFECTED_VO&'
elementStatus["GGUS"] += 'show_columns_check[]=AFFECTED_SITE&show_columns_check[]=PRIORITY&show_columns_check[]=RESPONSIBLE_UNIT&show_columns_check[]=STATUS&show_columns_check[]=DATE_OF_CREATION&'
elementStatus["GGUS"] += 'show_columns_check[]=LAST_UPDATE&show_columns_check[]=TYPE_OF_PROBLEM&show_columns_check[]=SUBJECT&ticket=&supportunit=all&su_hierarchy=all&vo=all&user=&keyword=&involvedsupporter=&assignto=&'
elementStatus["GGUS"] += 'affectedsite=%s&specattrib=0&status=open&priority=all&typeofproblem=all&ticketcategory=&mouarea=&technology_provider=&date_type=creation+date&radiotf=1&timeframe=any&from_date=&to_date=&' % gocdb_name
elementStatus["GGUS"] += 'untouched_date=&orderticketsby=GHD_INT_REQUEST_ID&orderhow=descending" target="_blank"> %s tickets</a>' % gocdb_name
convertName = {'CERN-PROD':'CERN',
'INFN-T1':'CNAF',
'FZK-LCG2':'GridKa',
'IN2P3-CC':'IN2P3',
'NIKHEF-ELPROD':'NIKHEF',
'pic':'PIC',
'RAL-LCG2':'RAL',
'SARA-MATRIX':'SARA'}
elog = convertName.get( gocdb_name, "" );
elementStatus['Elog'] = '<a href="https://lblogbook.cern.ch/Operations/?Site=^' + elog + '%24&mode=summary" target="_blank">' + elog + '</a>'
return { 'success' : 'true', 'result' : elementStatus, 'total' : len( elementStatus ) }
def _syncSites( self ):
"""
Sync DB content with sites that are in the CS
"""
def getGOCTier(sitesList):
return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)]))
# sites in the DB now
#sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite())))
sites = self.rsClient.getSite()
if not sites[ 'OK' ]:
gLogger.error( sites[ 'Message' ] )
return sites
sitesDB = set( [ site[0] for site in sites[ 'Value' ] ] )
# sites in CS now
sitesCS = set( CS.getSites() )
gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" % (len(sitesCS), len(sitesDB)))
# remove sites and associated resources, services, and storage
# elements from the DB that are not in the CS:
for s in sitesDB - sitesCS:
gLogger.info("Purging Site %s (not in CS anymore)" % s)
self.__purge_site(s)
# add to DB what is missing
gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB))
for site in sitesCS - sitesDB:
siteType = site.split(".")[0]
# DIRAC Tier
tier = "T" + str(CS.getSiteTier( site ))
if siteType == "LCG":
# Grid Name of the site
#gridSiteName = Utils.unpack(getGOCSiteName(site))
gridSiteName = getGOCSiteName( site )
if not gridSiteName[ 'OK' ]:
gLogger.error( gridSiteName[ 'Message' ] )
return gridSiteName
gridSiteName = gridSiteName[ 'Value' ]
# Grid Tier (with a workaround!)
#DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName))
DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName )
if not DIRACSitesOfGridSites[ 'OK' ]:
gLogger.error( DIRACSitesOfGridSites[ 'Message' ] )
return DIRACSitesOfGridSites
DIRACSitesOfGridSites = DIRACSitesOfGridSites[ 'Value' ]
if len( DIRACSitesOfGridSites ) == 1:
gt = tier
else:
gt = getGOCTier( DIRACSitesOfGridSites )
#Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt)
res = self.rsClient.addOrModifyGridSite( gridSiteName, gt )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
#Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName )
res = self.rsClient.addOrModifySite( site, tier, gridSiteName )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
elif siteType == "DIRAC":
#Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" )
res = self.rsClient.addOrModifySite( site, tier, "NULL" )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
def _getInfoFromRSSDB(self, name, what):
paramsL = ['Status']
siteName = None
serviceName = None
resourceName = None
storageElementName = None
serviceType = None
gridSiteName = None
if what == 'ServiceOfSite':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
paramsL.append('Reason')
siteName = name
elif what == 'ResOfCompService':
gran = 'Resources'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
serviceType = name.split('@')[0]
gridSiteName = getGOCSiteName(name.split('@')[1])
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'ResOfStorService':
gran = 'Resources'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
serviceType = name.split('@')[0]
gridSiteName = getGOCSiteName(name.split('@')[1])
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'ResOfStorEl':
gran = 'StorageElements'
paramsL.insert(0, 'ResourceName')
paramsL.append('Reason')
storageElementName = name
elif what == 'StorageElementsOfSite':
gran = 'StorageElements'
paramsL.insert(0, 'StorageElementName')
paramsL.append('Reason')
if '@' in name:
DIRACsiteName = name.split('@').pop()
else:
DIRACsiteName = name
gridSiteName = getGOCSiteName(DIRACsiteName)
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
elif what == 'Site_Panel':
gran = 'Site'
paramsL.insert(0, 'SiteName')
siteName = name
elif what == 'Service_Computing_Panel':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_Storage_Panel':
gran = 'Service'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_VO-BOX_Panel':
gran = 'Services'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Service_VOMS_Panel':
gran = 'Services'
paramsL.insert(0, 'ServiceName')
serviceName = name
elif what == 'Resource_Panel':
gran = 'Resource'
paramsL.insert(0, 'ResourceName')
resourceName = name
elif what == 'SE_Panel':
gran = 'StorageElement'
paramsL.insert(0, 'StorageElementName')
storageElementName = name
info_bit_got = self.rsDB.getMonitoredsList(gran, paramsList = paramsL, siteName = siteName,
serviceName = serviceName, serviceType = serviceType,
resourceName = resourceName,
storageElementName = storageElementName,
gridSiteName = gridSiteName)
return info_bit_got
def _getInfo( self, requestParams ):
gLogger.info( requestParams )
if not requestParams[ 'name' ]:
gLogger.warn( 'No name given' )
self.finish( { 'success': 'false', 'error': 'We need a Site Name to generate an Overview' } )
elementName = requestParams[ 'name' ][ 0 ]
pub = RPCClient( 'ResourceStatus/Publisher' )
elementStatuses = pub.getElementStatuses( 'Site',
str( elementName ),
None,
'all',
None,
None )
if not elementStatuses[ 'OK' ]:
gLogger.error( elementStatuses[ 'Message' ] )
self.finish( { 'success': 'false', 'error': 'Error getting ElementStatus information' } )
if not elementStatuses[ 'Value' ]:
gLogger.error( 'element "%s" not found' % elementName )
self.finish( { 'success' : 'false', 'error' : 'element "%s" not found' % elementName } )
elementStatus = [ dict( zip( elementStatuses[ 'Columns' ], element ) ) for element in elementStatuses[ 'Value' ] ][ 0 ]
elementStatus[ 'DateEffective' ] = str( elementStatus[ 'DateEffective' ] )
elementStatus[ 'LastCheckTime' ] = str( elementStatus[ 'LastCheckTime' ] )
elementStatus[ 'TokenExpiration' ] = str( elementStatus[ 'TokenExpiration' ] )
gocdb_name = getGOCSiteName( elementName )
if not gocdb_name[ 'OK' ]:
gLogger.error( gocdb_name[ 'Message' ] )
elementStatus[ 'GOCDB' ] = ""
gocdb_name = ''
else:
gocdb_name = gocdb_name[ 'Value' ]
elementStatus[ 'GOCDB' ] = '<a href="https://goc.egi.eu/portal/index.php?Page_Type=Submit_Search&SearchString=%s" target="_blank">%s</a>' % ( gocdb_name, gocdb_name )
dirac_names = getDIRACSiteName( gocdb_name )
if not dirac_names[ 'OK' ]:
gLogger.error( dirac_names[ 'Message' ] )
dirac_names = []
else:
elementStatus[ 'GOCDB' ] += "("
for i in dirac_names['Value']:
elementStatus[ 'GOCDB' ] += "%s " % i
elementStatus[ 'GOCDB' ] += ")"
elementStatus["GGUS"] = '<a href="https://ggus.eu/ws/ticket_search.php?show_columns_check[]=REQUEST_ID&show_columns_check[]=TICKET_TYPE&show_columns_check[]=AFFECTED_VO&'
elementStatus["GGUS"] += 'show_columns_check[]=AFFECTED_SITE&show_columns_check[]=PRIORITY&show_columns_check[]=RESPONSIBLE_UNIT&show_columns_check[]=STATUS&show_columns_check[]=DATE_OF_CREATION&'
elementStatus["GGUS"] += 'show_columns_check[]=LAST_UPDATE&show_columns_check[]=TYPE_OF_PROBLEM&show_columns_check[]=SUBJECT&ticket=&supportunit=all&su_hierarchy=all&vo=all&user=&keyword=&involvedsupporter=&assignto=&'
elementStatus["GGUS"] += 'affectedsite=%s&specattrib=0&status=open&priority=all&typeofproblem=all&ticketcategory=&mouarea=&technology_provider=&date_type=creation+date&radiotf=1&timeframe=any&from_date=&to_date=&' % gocdb_name
elementStatus["GGUS"] += 'untouched_date=&orderticketsby=GHD_INT_REQUEST_ID&orderhow=descending" target="_blank"> %s tickets</a>' % gocdb_name
convertName = {'CERN-PROD':'CERN',
'INFN-T1':'CNAF',
'FZK-LCG2':'GridKa',
'IN2P3-CC':'IN2P3',
'NIKHEF-ELPROD':'NIKHEF',
'pic':'PIC',
'RAL-LCG2':'RAL',
'SARA-MATRIX':'SARA'}
elog = convertName.get( gocdb_name, "" );
elementStatus['Elog'] = '<a href="https://lblogbook.cern.ch/Operations/?Site=^' + elog + '%24&mode=summary" target="_blank">' + elog + '</a>'
return { 'success' : 'true', 'result' : elementStatus, 'total' : len( elementStatus ) }
def doCommand(self, rsClientIn=None):
"""
Return getStatus from SAM Results Client
:attr:`args`:
- args[0]: string: should be a ValidRes
- args[1]: string: should be the (DIRAC) name of the ValidRes
- args[2]: string: optional - should be the (DIRAC) site name of the ValidRes
- args[3]: list: list of tests
"""
super(SAMResults_Command, self).doCommand()
if self.client is None:
from DIRAC.Core.LCG.SAMResultsClient import SAMResultsClient
self.client = SAMResultsClient()
if rsClientIn is not None:
rsc = rsClientIn
else:
# use standard RS Client
from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient
rsc = ResourceStatusClient()
granularity = self.args[0]
name = self.args[1]
try:
siteName = self.args[2]
except IndexError:
siteName = None
if granularity in ('Site', 'Sites'):
siteName = getGOCSiteName(name)
if not siteName['OK']:
raise RSSException, siteName['Message']
siteName = siteName['Value']
elif granularity in ('Resource', 'Resources'):
if siteName is None:
siteName = rsc.getGridSiteName(granularity, name)
else:
siteName = getGOCSiteName(siteName)
if not siteName['OK']:
raise RSSException, siteName['Message']
siteName = siteName['Value']
else:
raise InvalidRes, where(self, self.doCommand)
try:
tests = self.args[3]
except IndexError:
tests = None
finally:
try:
res = self.client.getStatus(granularity, name, siteName, tests,
timeout = self.timeout)
if not res['OK']:
gLogger.error("There are no SAM tests for " + granularity + " " + name )
return {'Result':None}
except urllib2.URLError:
gLogger.error("SAM timed out for " + granularity + " " + name )
return {'Result':'Unknown'}
except httplib.BadStatusLine:
gLogger.error("httplib.BadStatusLine: could not read" + granularity + " " + name )
return {'Result':'Unknown'}
except:
gLogger.exception("Exception when calling SAMResultsClient for %s %s" %(granularity, name))
return {'Result':'Unknown'}
return {'Result':res['Value']}
def _prepareCommand(self):
'''
DowntimeCommand requires four arguments:
- name : <str>
- element : Site / Resource
- elementType: <str>
If the elements are Site(s), we need to get their GOCDB names. They may
not have, so we ignore them if they do not have.
'''
if 'name' not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args['name']
if 'element' not in self.args:
return S_ERROR('"element" not found in self.args')
element = self.args['element']
if 'elementType' not in self.args:
return S_ERROR('"elementType" not found in self.args')
elementType = self.args['elementType']
if element not in ['Site', 'Resource']:
return S_ERROR('element is neither Site nor Resource')
hours = None
if 'hours' in self.args:
hours = self.args['hours']
gOCDBServiceType = None
# Transform DIRAC site names into GOCDB topics
if element == 'Site':
gocSite = getGOCSiteName(elementName)
if not gocSite['OK']: # The site is most probably is not a grid site - not an issue, of course
pass # so, elementName remains unchanged
else:
elementName = gocSite['Value']
# The DIRAC se names mean nothing on the grid, but their hosts do mean.
elif elementType == 'StorageElement':
# We need to distinguish if it's tape or disk
try:
seOptions = StorageElement(elementName).options
except AttributeError: # Sometimes the SE can't be instantiated properly
self.log.error(
"Failure instantiating StorageElement object for %s" % elementName)
return S_ERROR("Failure instantiating StorageElement")
if 'SEType' in seOptions:
# Type should follow the convention TXDY
seType = seOptions['SEType']
diskSE = re.search('D[1-9]', seType) != None
tapeSE = re.search('T[1-9]', seType) != None
if tapeSE:
gOCDBServiceType = "srm.nearline"
elif diskSE:
gOCDBServiceType = "srm"
seHost = CSHelpers.getSEHost(elementName)
if not seHost['OK']:
return seHost
seHost = seHost['Value']
if not seHost:
return S_ERROR('No seHost for %s' % elementName)
elementName = seHost
elif elementType in ['FTS', 'FTS3']:
gOCDBServiceType = 'FTS'
# WARNING: this method presupposes that the server is an FTS3 type
gocSite = getGOCFTSName(elementName)
if not gocSite['OK']:
self.log.warn("%s not in Resources/FTSEndpoints/FTS3 ?" % elementName)
else:
elementName = gocSite['Value']
return S_OK((element, elementName, hours, gOCDBServiceType))
def doCommand(self, sites=None):
"""
Returns downtimes information for all the sites in input.
:params:
:attr:`sites`: list of site names (when not given, take every site)
:returns:
{'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK',
'StartDate': 'aDate', ...} ... }
"""
self.APIs = initAPIs(self.__APIs__, self.APIs)
try:
if sites is None:
GOC_sites = self.APIs['ResourceStatusClient'].getGridSite(
meta={'columns': 'GridSiteName'})
if not GOC_sites['OK']:
return {'Result': GOC_sites}
GOC_sites = [gs[0] for gs in GOC_sites['Value']]
else:
GOC_sites = [getGOCSiteName(x)['Value'] for x in sites]
resGOC = self.APIs['GOCDBClient'].getStatus(
'Site', GOC_sites, None, 120)
if not resGOC['OK']:
return {'Result': resGOC}
resGOC = resGOC['Value']
if resGOC == None:
resGOC = []
res = {}
for dt_ID in resGOC:
try:
dt = {}
dt['ID'] = dt_ID
dt['StartDate'] = resGOC[dt_ID]['FORMATED_START_DATE']
dt['EndDate'] = resGOC[dt_ID]['FORMATED_END_DATE']
dt['Severity'] = resGOC[dt_ID]['SEVERITY']
dt['Description'] = resGOC[dt_ID]['DESCRIPTION'].replace(
'\'', '')
dt['Link'] = resGOC[dt_ID]['GOCDB_PORTAL_URL']
DIRACnames = getDIRACSiteName(res[dt_ID]['SITENAME'])
if not DIRACnames['OK']:
return {'Result': DIRACnames}
for DIRACname in DIRACnames['Value']:
res[dt_ID.split()[0] + ' ' + DIRACname] = dt
except KeyError:
continue
res = S_OK(res)
except Exception, e:
_msg = '%s (%s): %s' % (self.__class__.__name__, self.args, e)
gLogger.exception(_msg)
return {'Result': S_ERROR(_msg)}
def _prepareCommand(self):
"""
DowntimeCommand requires four arguments:
- name : <str>
- element : Site / Resource
- elementType: <str>
If the elements are Site(s), we need to get their GOCDB names. They may
not have, so we ignore them if they do not have.
"""
if 'name' not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args['name']
if 'element' not in self.args:
return S_ERROR('"element" not found in self.args')
element = self.args['element']
if 'elementType' not in self.args:
return S_ERROR('"elementType" not found in self.args')
elementType = self.args['elementType']
if element not in ['Site', 'Resource']:
return S_ERROR('element is neither Site nor Resource')
hours = None
if 'hours' in self.args:
hours = self.args['hours']
gOCDBServiceType = None
# Transform DIRAC site names into GOCDB topics
if element == 'Site':
gocSite = getGOCSiteName(elementName)
if not gocSite[
'OK']: # The site is most probably is not a grid site - not an issue, of course
pass # so, elementName remains unchanged
else:
elementName = gocSite['Value']
# The DIRAC se names mean nothing on the grid, but their hosts do mean.
elif elementType == 'StorageElement':
# for SRM and SRM only, we need to distinguish if it's tape or disk
# if it's not SRM, then gOCDBServiceType will be None (and we'll use them all)
try:
se = StorageElement(elementName)
seOptions = se.options
seProtocols = set(se.localAccessProtocolList) | set(
se.localWriteProtocolList)
except AttributeError: # Sometimes the SE can't be instantiated properly
self.log.error("Failure instantiating StorageElement object",
elementName)
return S_ERROR("Failure instantiating StorageElement")
if 'SEType' in seOptions and 'srm' in seProtocols:
# Type should follow the convention TXDY
seType = seOptions['SEType']
diskSE = re.search('D[1-9]', seType) is not None
tapeSE = re.search('T[1-9]', seType) is not None
if tapeSE:
gOCDBServiceType = "srm.nearline"
elif diskSE:
gOCDBServiceType = "srm"
res = getSEHosts(elementName)
if not res['OK']:
return res
seHosts = res['Value']
if not seHosts:
return S_ERROR('No seHost(s) for %s' % elementName)
elementName = seHosts # in this case it will return a list, because there might be more than one host only
elif elementType in ['FTS', 'FTS3']:
gOCDBServiceType = 'FTS'
# WARNING: this method presupposes that the server is an FTS3 type
gocSite = getGOCFTSName(elementName)
if not gocSite['OK']:
self.log.warn("FTS not in Resources/FTSEndpoints/FTS3 ?",
elementName)
else:
elementName = gocSite['Value']
return S_OK((element, elementName, hours, gOCDBServiceType))
def doCommand(self):
"""
Return getStatus from GOC DB Client
:attr:`args`:
- args[0]: string: should be a ValidRes
- args[1]: string: should be the name of the ValidRes
- args[2]: string: optional, number of hours in which
the down time is starting
"""
super(GOCDBStatus_Command, self).doCommand()
if self.client is None:
# use standard GOC DB Client
from DIRAC.Core.LCG.GOCDBClient import GOCDBClient
self.client = GOCDBClient()
granularity = self.args[0]
name = self.args[1]
try:
hours = self.args[2]
except IndexError:
hours = None
if granularity in ('Site', 'Sites'):
name = getGOCSiteName(name)
if not name['OK']:
raise RSSException, name['Message']
name = name['Value']
try:
res = self.client.getStatus(granularity, name, None, hours, self.timeout)
if not res['OK']:
return {'Result':'Unknown'}
res = res['Value']
if res is None or res == {}:
return {'Result':{'DT':None}}
DT_dict_result = {}
now = datetime.datetime.utcnow().replace(microsecond = 0, second = 0)
if len(res) > 1:
#there's more than one DT
for dt_ID in res:
#looking for an ongoing one
startSTR = res[dt_ID]['FORMATED_START_DATE']
start_datetime = datetime.datetime( *time.strptime(startSTR, "%Y-%m-%d %H:%M")[0:5] )
if start_datetime < now:
resDT = res[dt_ID]
break
try:
resDT
except:
#if I'm here, there's no OnGoing DT
resDT = res[res.keys()[0]]
res = resDT
else:
res = res[res.keys()[0]]
DT_dict_result['DT'] = res['SEVERITY']
DT_dict_result['EndDate'] = res['FORMATED_END_DATE']
startSTR = res['FORMATED_START_DATE']
start_datetime = datetime.datetime( *time.strptime(startSTR, "%Y-%m-%d %H:%M")[0:5] )
if start_datetime > now:
diff = convertTime(start_datetime - now, 'hours')
DT_dict_result['DT'] = DT_dict_result['DT'] + " in " + str(diff) + ' hours'
return {'Result':DT_dict_result}
except urllib2.URLError:
gLogger.error("GOCDB timed out for " + granularity + " " + name )
return {'Result':'Unknown'}
except:
gLogger.exception("Exception when calling GOCDBClient for " + granularity + " " + name )
return {'Result':'Unknown'}
def doCommand( self, sites = None ):
"""
Returns downtimes information for all the sites in input.
:params:
:attr:`sites`: list of site names (when not given, take every site)
:returns:
{'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK',
'StartDate': 'aDate', ...} ... }
"""
self.APIs = initAPIs( self.__APIs__, self.APIs )
try:
if sites is None:
GOC_sites = self.APIs[ 'ResourceStatusClient' ].getGridSite( meta = { 'columns' : 'GridSiteName' })
if not GOC_sites['OK']:
return { 'Result' : GOC_sites }
GOC_sites = [ gs[0] for gs in GOC_sites['Value'] ]
else:
GOC_sites = [ getGOCSiteName( x )['Value'] for x in sites ]
resGOC = self.APIs[ 'GOCDBClient' ].getStatus( 'Site', GOC_sites, None, 120 )
if not resGOC['OK']:
return { 'Result' : resGOC }
resGOC = resGOC['Value']
if resGOC == None:
resGOC = []
res = {}
for dt_ID in resGOC:
try:
dt = {}
dt['ID'] = dt_ID
dt['StartDate'] = resGOC[dt_ID]['FORMATED_START_DATE']
dt['EndDate'] = resGOC[dt_ID]['FORMATED_END_DATE']
dt['Severity'] = resGOC[dt_ID]['SEVERITY']
dt['Description'] = resGOC[dt_ID]['DESCRIPTION'].replace( '\'', '' )
dt['Link'] = resGOC[dt_ID]['GOCDB_PORTAL_URL']
DIRACnames = getDIRACSiteName( res[dt_ID]['SITENAME'] )
if not DIRACnames['OK']:
return { 'Result' : DIRACnames }
for DIRACname in DIRACnames['Value']:
res[dt_ID.split()[0] + ' ' + DIRACname] = dt
except KeyError:
continue
res = S_OK( res )
except Exception, e:
_msg = '%s (%s): %s' % ( self.__class__.__name__, self.args, e )
gLogger.exception( _msg )
return { 'Result' : S_ERROR( _msg ) }
def doCommand(self):
"""
Return getStatus from GOC DB Client
:attr:`args`:
- args[0]: string: should be a ValidElement
- args[1]: string: should be the name of the ValidElement
- args[2]: string: optional, number of hours in which
the down time is starting
"""
timeFormat = "%Y-%m-%d %H:%M"
super(GOCDBStatus_Command, self).doCommand()
self.APIs = initAPIs( self.__APIs__, self.APIs )
try:
granularity = self.args[0]
name = self.args[1]
if len( self.args ) > 2:
hours = self.args[2]
else:
hours = None
if granularity == 'Site':
name = getGOCSiteName( name )[ 'Value' ]
res = self.APIs[ 'GOCDBClient' ].getStatus( granularity, name, None, hours )
if not res['OK']:
return { 'Result' : res }
res = res['Value']
if res is None or res == {}:
return { 'Result' : S_OK( { 'DT' : None } ) }
DT_dict_result = {}
now = datetime.utcnow().replace( microsecond = 0, second = 0 )
if len( res ) > 1:
#there's more than one DT
resDT = None
for dt_ID in res:
#looking for an ongoing one
startSTR = res[ dt_ID ][ 'FORMATED_START_DATE' ]
start_datetime = datetime.strptime( startSTR, timeFormat )
if start_datetime < now:
resDT = res[ dt_ID ]
break
#if I'm here, there's no OnGoing DT
if resDT is None:
resDT = res[res.keys()[0]]
res = resDT
else:
res = res[res.keys()[0]]
DT_dict_result['DT'] = res['SEVERITY']
DT_dict_result['EndDate'] = res['FORMATED_END_DATE']
startSTR = res['FORMATED_START_DATE']
start_datetime = datetime.strptime( startSTR, timeFormat )
if start_datetime > now:
diff = convertTime( start_datetime - now, 'hours' )
DT_dict_result[ 'DT' ] = DT_dict_result['DT'] + " in " + str( diff ) + ' hours'
res = S_OK( DT_dict_result )
except Exception, e:
_msg = '%s (%s): %s' % ( self.__class__.__name__, self.args, e )
gLogger.exception( _msg )
return { 'Result' : S_ERROR( _msg ) }
def _prepareCommand(self):
'''
DowntimeCommand requires four arguments:
- name : <str>
- element : Site / Resource
- elementType: <str>
If the elements are Site(s), we need to get their GOCDB names. They may
not have, so we ignore them if they do not have.
'''
if 'name' not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args['name']
if 'element' not in self.args:
return S_ERROR('"element" not found in self.args')
element = self.args['element']
if 'elementType' not in self.args:
return S_ERROR('"elementType" not found in self.args')
elementType = self.args['elementType']
if element not in ['Site', 'Resource']:
return S_ERROR('element is neither Site nor Resource')
hours = None
if 'hours' in self.args:
hours = self.args['hours']
gOCDBServiceType = None
# Transform DIRAC site names into GOCDB topics
if element == 'Site':
gocSite = getGOCSiteName(elementName)
if not gocSite[
'OK']: # The site is most probably is not a grid site - not an issue, of course
pass # so, elementName remains unchanged
else:
elementName = gocSite['Value']
# The DIRAC se names mean nothing on the grid, but their hosts do mean.
elif elementType == 'StorageElement':
# We need to distinguish if it's tape or disk
try:
seOptions = StorageElement(elementName).options
except AttributeError: # Sometimes the SE can't be instantiated properly
self.log.error(
"Failure instantiating StorageElement object for %s" %
elementName)
return S_ERROR("Failure instantiating StorageElement")
if 'SEType' in seOptions:
# Type should follow the convention TXDY
seType = seOptions['SEType']
diskSE = re.search('D[1-9]', seType) != None
tapeSE = re.search('T[1-9]', seType) != None
if tapeSE:
gOCDBServiceType = "srm.nearline"
elif diskSE:
gOCDBServiceType = "srm"
seHost = CSHelpers.getSEHost(elementName)
if not seHost['OK']:
return seHost
seHost = seHost['Value']
if not seHost:
return S_ERROR('No seHost for %s' % elementName)
elementName = seHost
elif elementType in ['FTS', 'FTS3']:
gOCDBServiceType = 'FTS'
# WARNING: this method presupposes that the server is an FTS3 type
gocSite = getGOCFTSName(elementName)
if not gocSite['OK']:
self.log.warn("%s not in Resources/FTSEndpoints/FTS3 ?" %
elementName)
else:
elementName = gocSite['Value']
return S_OK((element, elementName, hours, gOCDBServiceType))
def _prepareCommand(self):
"""
DowntimeCommand requires four arguments:
- name : <str>
- element : Site / Resource
- elementType: <str>
If the elements are Site(s), we need to get their GOCDB names. They may
not have, so we ignore them if they do not have.
"""
if "name" not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args["name"]
if "element" not in self.args:
return S_ERROR('"element" not found in self.args')
element = self.args["element"]
if "elementType" not in self.args:
return S_ERROR('"elementType" not found in self.args')
elementType = self.args["elementType"]
if not element in ["Site", "Resource"]:
return S_ERROR("element is neither Site nor Resource")
hours = None
if "hours" in self.args:
hours = self.args["hours"]
gocdbServiceType = None
# Transform DIRAC site names into GOCDB topics
if element == "Site":
gocSite = getGOCSiteName(elementName)
if not gocSite["OK"]:
return gocSite
elementName = gocSite["Value"]
# The DIRAC se names mean nothing on the grid, but their hosts do mean.
elif elementType == "StorageElement":
# We need to distinguish if it's tape or disk
seOptions = getStorageElementOptions(elementName)
if not seOptions["OK"]:
return seOptions
if seOptions["Value"].get("TapeSE"):
gocdbServiceType = "srm.nearline"
elif seOptions["Value"].get("DiskSE"):
gocdbServiceType = "srm"
seHost = CSHelpers.getSEHost(elementName)
if not seHost["OK"]:
return seHost
seHost = seHost["Value"]
if not seHost:
return S_ERROR("No seHost for %s" % elementName)
elementName = seHost
elif elementType in ["FTS", "FTS3"]:
gocdbServiceType = "FTS"
try:
# WARNING: this method presupposes that the server is an FTS3 type
elementName = getGOCFTSName(elementName)
except:
return S_ERROR("No FTS3 server specified in dirac.cfg (see Resources/FTSEndpoints)")
return S_OK((element, elementName, hours, gocdbServiceType))
def _prepareCommand(self):
'''
DowntimeCommand requires four arguments:
- name : <str>
- element : Site / Resource
- elementType: <str>
If the elements are Site(s), we need to get their GOCDB names. They may
not have, so we ignore them if they do not have.
'''
if 'name' not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args['name']
if 'element' not in self.args:
return S_ERROR('"element" not found in self.args')
element = self.args['element']
if 'elementType' not in self.args:
return S_ERROR('"elementType" not found in self.args')
elementType = self.args['elementType']
if not element in ['Site', 'Resource']:
return S_ERROR('element is neither Site nor Resource')
hours = None
if 'hours' in self.args:
hours = self.args['hours']
gOCDBServiceType = None
# Transform DIRAC site names into GOCDB topics
if element == 'Site':
gocSite = getGOCSiteName(elementName)
if not gocSite[
'OK']: # The site is most probably not a grid site - not an issue, of course
pass # so, elementName remains unchanged
else:
elementName = gocSite['Value']
# The DIRAC se names mean nothing on the grid, but their hosts do mean.
elif elementType == 'StorageElement':
# We need to distinguish if it's tape or disk
seOptions = getStorageElementOptions(elementName)
if not seOptions['OK']:
return seOptions
if seOptions['Value'].get('TapeSE'):
gOCDBServiceType = "srm.nearline"
elif seOptions['Value'].get('DiskSE'):
gOCDBServiceType = "srm"
seHost = CSHelpers.getSEHost(elementName)
if not seHost['OK']:
return seHost
seHost = seHost['Value']
if not seHost:
return S_ERROR('No seHost for %s' % elementName)
elementName = seHost
elif elementType in ['FTS', 'FTS3']:
gOCDBServiceType = 'FTS'
try:
#WARNING: this method presupposes that the server is an FTS3 type
elementName = getGOCFTSName(elementName)
except:
return S_ERROR(
'No FTS3 server specified in dirac.cfg (see Resources/FTSEndpoints)'
)
return S_OK((element, elementName, hours, gOCDBServiceType))
def doCommand( self, sites = None ):
"""
Returns downtimes information for all the sites in input.
:params:
:attr:`sites`: list of site names (when not given, take every site)
:returns:
{'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK',
'StartDate': 'aDate', ...} ... }
"""
if self.client is None:
from DIRAC.Core.LCG.GOCDBClient import GOCDBClient
self.client = GOCDBClient()
if sites is None:
# from DIRAC.Core.DISET.RPCClient import RPCClient
RPC = RPCClient( "ResourceStatus/ResourceStatus" )
GOC_sites = RPC.getGridSitesList()
if not GOC_sites['OK']:
raise RSSException, where( self, self.doCommand ) + " " + sites['Message']
else:
GOC_sites = GOC_sites['Value']
else:
GOC_sites = [getGOCSiteName( x )['Value'] for x in sites]
try:
res = self.client.getStatus( 'Site', GOC_sites, None, 120 )
except:
gLogger.exception( "Exception when calling GOCDBClient." )
return {}
if not res['OK']:
raise RSSException, where( self, self.doCommand ) + " " + res['Message']
else:
res = res['Value']
if res == None:
return {}
resToReturn = {}
for dt_ID in res:
try:
dt = {}
dt['ID'] = dt_ID
dt['StartDate'] = res[dt_ID]['FORMATED_START_DATE']
dt['EndDate'] = res[dt_ID]['FORMATED_END_DATE']
dt['Severity'] = res[dt_ID]['SEVERITY']
dt['Description'] = res[dt_ID]['DESCRIPTION'].replace( '\'', '' )
dt['Link'] = res[dt_ID]['GOCDB_PORTAL_URL']
DIRACnames = getDIRACSiteName( res[dt_ID]['SITENAME'] )
if not DIRACnames['OK']:
raise RSSException, DIRACnames['Message']
DIRACnames = DIRACnames['Value']
for DIRACname in DIRACnames:
resToReturn[dt_ID.split()[0] + ' ' + DIRACname] = dt
except KeyError:
continue
return resToReturn
def _syncSites(self):
"""
Sync DB content with sites that are in the CS
"""
def getGOCTier(sitesList):
return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)]))
# sites in the DB now
#sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite())))
sites = self.rsClient.getSite()
if not sites['OK']:
gLogger.error(sites['Message'])
return sites
sitesDB = set([site[0] for site in sites['Value']])
# sites in CS now
sitesCS = set(CS.getSites())
gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" %
(len(sitesCS), len(sitesDB)))
# remove sites and associated resources, services, and storage
# elements from the DB that are not in the CS:
for s in sitesDB - sitesCS:
gLogger.info("Purging Site %s (not in CS anymore)" % s)
self.__purge_site(s)
# add to DB what is missing
gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB))
for site in sitesCS - sitesDB:
siteType = site.split(".")[0]
# DIRAC Tier
tier = "T" + str(CS.getSiteTier(site))
if siteType == "LCG":
# Grid Name of the site
#gridSiteName = Utils.unpack(getGOCSiteName(site))
gridSiteName = getGOCSiteName(site)
if not gridSiteName['OK']:
gLogger.error(gridSiteName['Message'])
return gridSiteName
gridSiteName = gridSiteName['Value']
# Grid Tier (with a workaround!)
#DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName))
DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName)
if not DIRACSitesOfGridSites['OK']:
gLogger.error(DIRACSitesOfGridSites['Message'])
return DIRACSitesOfGridSites
DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']
if len(DIRACSitesOfGridSites) == 1:
gt = tier
else:
gt = getGOCTier(DIRACSitesOfGridSites)
#Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt)
res = self.rsClient.addOrModifyGridSite(gridSiteName, gt)
if not res['OK']:
gLogger.error(res['Message'])
return res
#Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName )
res = self.rsClient.addOrModifySite(site, tier, gridSiteName)
if not res['OK']:
gLogger.error(res['Message'])
return res
elif siteType == "DIRAC":
#Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" )
res = self.rsClient.addOrModifySite(site, tier, "NULL")
if not res['OK']:
gLogger.error(res['Message'])
return res
def doCommand(self, rsClientIn=None):
"""
Return getStatus from SAM Results Client
:attr:`args`:
- args[0]: string: should be a ValidRes
- args[1]: string: should be the (DIRAC) name of the ValidRes
- args[2]: string: optional - should be the (DIRAC) site name of the ValidRes
- args[3]: list: list of tests
"""
super(SAMResults_Command, self).doCommand()
if self.client is None:
from DIRAC.Core.LCG.SAMResultsClient import SAMResultsClient
self.client = SAMResultsClient()
if rsClientIn is not None:
rsc = rsClientIn
else:
# use standard RS Client
from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient
rsc = ResourceStatusClient()
granularity = self.args[0]
name = self.args[1]
try:
siteName = self.args[2]
except IndexError:
siteName = None
if granularity in ('Site', 'Sites'):
siteName = getGOCSiteName(name)
if not siteName['OK']:
raise RSSException, siteName['Message']
siteName = siteName['Value']
elif granularity in ('Resource', 'Resources'):
if siteName is None:
siteName = rsc.getGridSiteName(granularity, name)
else:
siteName = getGOCSiteName(siteName)
if not siteName['OK']:
raise RSSException, siteName['Message']
siteName = siteName['Value']
else:
raise InvalidRes, where(self, self.doCommand)
try:
tests = self.args[3]
except IndexError:
tests = None
finally:
try:
res = self.client.getStatus(granularity,
name,
siteName,
tests,
timeout=self.timeout)
if not res['OK']:
gLogger.error("There are no SAM tests for " + granularity +
" " + name)
return {'Result': None}
except urllib2.URLError:
gLogger.error("SAM timed out for " + granularity + " " + name)
return {'Result': 'Unknown'}
except httplib.BadStatusLine:
gLogger.error("httplib.BadStatusLine: could not read" +
granularity + " " + name)
return {'Result': 'Unknown'}
except:
gLogger.exception(
"Exception when calling SAMResultsClient for %s %s" %
(granularity, name))
return {'Result': 'Unknown'}
return {'Result': res['Value']}
