def doCommand(self):
"""
Returns transfer quality as it is cached
:attr:`args`:
- args[0]: string: should be a ValidRes
- args[1]: string should be the name of the ValidRes
:returns:
{'Result': None | a float between 0.0 and 100.0}
"""
super(TransferQualityCached_Command, self).doCommand()
if self.client is None:
from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient
self.client = ResourceManagementClient(timeout=self.timeout)
name = self.args[1]
try:
res = self.client.getCachedResult(name, 'TransferQualityEverySEs',
'TQ', 'NULL')
if res == []:
return {'Result': None}
except:
gLogger.exception(
"Exception when calling ResourceManagementClient for %s" %
(name))
return {'Result': 'Unknown'}
return {'Result': float(res[0])}
class PilotsEffSimpleCached_Command(Command):
def doCommand(self):
"""
Returns simple pilots efficiency
:attr:`args`:
- args[0]: string: should be a ValidRes
- args[1]: string should be the name of the ValidRes
returns:
{
'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad'
}
"""
super(PilotsEffSimpleCached_Command, self).doCommand()
client = self.client
if client is None:
from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient
self.client = ResourceStatusClient(timeout=self.timeout)
if self.args[0] in ('Service', 'Services'):
try:
name = self.client.getGeneralName(self.args[0], self.args[1],
'Site')['Value'][0]
except:
gLogger.error(
"PilotsEffSimpleCached_Command: can't get a general name for %s %s"
% (self.args[0], self.args[1]))
return {'Result': 'Unknown'}
granularity = 'Site'
elif self.args[0] in ('Site', 'Sites'):
name = self.args[1]
granularity = self.args[0]
else:
raise InvalidRes, where(self, self.doCommand)
try:
if client is None:
from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient
self.client = ResourceManagementClient(timeout=self.timeout)
res = self.client.getCachedResult(name,
'PilotsEffSimpleEverySites',
'PE_S', 'NULL')['Value']
if res == None:
return {'Result': 'Idle'}
if res == []:
return {'Result': 'Idle'}
except:
gLogger.exception(
"Exception when calling ResourceManagementClient for %s %s" %
(granularity, name))
return {'Result': 'Unknown'}
return {'Result': res[0]}
doCommand.__doc__ = Command.doCommand.__doc__ + doCommand.__doc__
def __init__(self, args=None, clients=None):
super(SpaceTokenOccupancyCommand, self).__init__(args, clients)
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(CachedPlotCommand, self).__init__(args, clients)
if "ResourceManagementClient" in self.apis:
self.rmClient = self.apis["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
def __init__( self, args = None, clients = None ):
super( TransferQualityFromCachedPlotCommand, self ).__init__( args, clients )
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def __init__(self, am):
super(SpaceTokenOccupancyTest, self).__init__(am)
self.xmlPath = rootPath + "/" + self.getAgentOption(
"webRoot") + self.getTestOption("dir")
self.rmClient = ResourceManagementClient()
mkDir(self.xmlPath)
self.generate_xml_and_dashboard()
class PilotsEffSimpleCached_Command(Command):
def doCommand(self):
"""
Returns simple pilots efficiency
:attr:`args`:
- args[0]: string: should be a ValidRes
- args[1]: string should be the name of the ValidRes
returns:
{
'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad'
}
"""
super(PilotsEffSimpleCached_Command, self).doCommand()
client = self.client
if client is None:
from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient
self.client = ResourceStatusClient(timeout = self.timeout)
if self.args[0] in ('Service', 'Services'):
try:
name = self.client.getGeneralName(self.args[0], self.args[1], 'Site')[0]
except:
gLogger.error("PilotsEffSimpleCached_Command: can't get a general name for %s %s" %(self.args[0], self.args[1]))
return {'Result':'Unknown'}
granularity = 'Site'
elif self.args[0] in ('Site', 'Sites'):
name = self.args[1]
granularity = self.args[0]
else:
raise InvalidRes, where(self, self.doCommand)
try:
if client is None:
from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient
self.client = ResourceManagementClient(timeout = self.timeout)
res = self.client.getCachedResult(name, 'PilotsEffSimpleEverySites', 'PE_S', 'NULL')
if res == None:
return {'Result':'Idle'}
if res == []:
return {'Result':'Idle'}
except:
gLogger.exception("Exception when calling ResourceManagementClient for %s %s" %(granularity, name))
return {'Result':'Unknown'}
return {'Result':res[0]}
doCommand.__doc__ = Command.doCommand.__doc__ + doCommand.__doc__
def __init__(self, rsClient=None, rmClient=None):
self.GOCDBClient = GOCDBClient()
self.rsClient = ResourceStatusClient(
) if rsClient == None else rsClient
self.rmClient = ResourceManagementClient(
) if rmClient == None else rmClient
self.synclist = [
'Sites', 'Resources', 'StorageElements', 'Services',
'RegistryUsers'
]
def __init__(self, granularity, name, status_type, pdp_decision, **kw):
ActionBase.__init__(self, granularity, name, status_type, pdp_decision,
**kw)
try:
self.rsClient = self.kw["Clients"]['ResourceStatusClient']
except KeyError:
self.rsClient = ResourceStatusClient()
try:
self.rmClient = self.kw["Clients"]['ResourceManagementClient']
except KeyError:
self.rmClient = ResourceManagementClient()
def initialize( self ):
self.am_setOption( 'shifterProxy', 'DataManager' )
self.rmClient = ResourceManagementClient()
self.commands[ 'Downtime' ] = [ { 'Downtime' : {} } ]
self.commands[ 'SpaceTokenOccupancy' ] = [ { 'SpaceTokenOccupancy' : {} } ]
self.commands[ 'Pilot' ] = [ { 'Pilot' : { 'timespan' : 1800 } },]
# { 'Pilot' : { 'timespan' : 86400 } },
# { 'Pilot' : { 'timespan' : 604800 } }]
#PilotsCommand
# self.commands[ 'Pilots' ] = [
# { 'PilotsWMS' : { 'element' : 'Site', 'siteName' : None } },
# { 'PilotsWMS' : { 'element' : 'Resource', 'siteName' : None } }
# ]
#FIXME: do not forget about hourly vs Always ...etc
#AccountingCacheCommand
# self.commands[ 'AccountingCache' ] = [
# {'SuccessfullJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'FailedJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'SuccessfullPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'SuccessfullPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'RunningJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
## {'RunningJobsBySiteSplitted' :{'hours' :168, 'plotType' :'Job' }},
## {'RunningJobsBySiteSplitted' :{'hours' :720, 'plotType' :'Job' }},
## {'RunningJobsBySiteSplitted' :{'hours' :8760, 'plotType' :'Job' }},
# ]
#VOBOXAvailability
# self.commands[ 'VOBOXAvailability' ] = [
# { 'VOBOXAvailability' : {} }
#
#Reuse clients for the commands
self.clients[ 'GOCDBClient' ] = GOCDBClient()
self.clients[ 'ReportGenerator' ] = RPCClient( 'Accounting/ReportGenerator' )
self.clients[ 'ReportsClient' ] = ReportsClient()
self.clients[ 'ResourceStatusClient' ] = ResourceStatusClient()
self.clients[ 'ResourceManagementClient' ] = ResourceManagementClient()
self.clients[ 'PilotsDB' ] = PilotAgentsDB()
self.clients[ 'WMSAdministrator' ] = RPCClient( 'WorkloadManagement/WMSAdministrator' )
self.cCaller = CommandCaller
return S_OK()
def __init__(self, args=None, clients=None):
super(JobsEffSimpleCachedCommand, self).__init__(args, clients)
if 'ResourceStatusClient' in self.apis:
self.rsClient = self.apis['ResourceStatusClient']
else:
self.rsClient = ResourceStatusClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(JobCommand, self).__init__(args, clients)
if 'JobDB' in self.apis:
self.jobDB = self.apis['JobDB']
else:
self.jobDB = JobDB()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(PilotCommand, self).__init__(args, clients)
if "Pilots" in self.apis:
self.pilots = self.apis["Pilots"]
else:
self.pilots = PilotManagerClient()
if "ResourceManagementClient" in self.apis:
self.rmClient = self.apis["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(PilotCommand, self).__init__(args, clients)
if 'Pilots' in self.apis:
self.pilots = self.apis['Pilots']
else:
self.pilots = PilotManagerClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(DowntimeCommand, self).__init__(args, clients)
if 'GOCDBClient' in self.apis:
self.gClient = self.apis['GOCDBClient']
else:
self.gClient = GOCDBClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(TransferCommand, self).__init__(args, clients)
if "ReportsClient" in self.apis:
self.rClient = self.apis["ReportsClient"]
else:
self.rClient = ReportsClient()
if "ResourceManagementClient" in self.apis:
self.rmClient = self.apis["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(GGUSTicketsCommand, self).__init__(args, clients)
if 'GGUSTicketsClient' in self.apis:
self.gClient = self.apis['GGUSTicketsClient']
else:
self.gClient = GGUSTicketsClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(JobCommand, self).__init__(args, clients)
if 'WMSAdministrator' in self.apis:
self.wmsAdmin = self.apis['WMSAdministrator']
else:
self.wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(JobCommand, self).__init__(args, clients)
if "WMSAdministrator" in self.apis:
self.wmsAdmin = self.apis["WMSAdministrator"]
else:
self.wmsAdmin = WMSAdministratorClient()
if "ResourceManagementClient" in self.apis:
self.rmClient = self.apis["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(PilotCommand, self).__init__(args, clients)
if 'WMSAdministrator' in self.apis:
self.wmsAdmin = self.apis['WMSAdministrator']
else:
self.wmsAdmin = WMSAdministratorClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def doCommand(self):
"""
Returns transfer quality from the plot cached in the accounting cache.
:attr:`args`:
- args[0]: string: should be a ValidRes
- args[1]: string should be the name of the ValidRes
:returns:
{'Result': None | a float between 0.0 and 100.0}
"""
super(TransferQualityFromCachedPlot_Command, self).doCommand()
if self.client is None:
from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient
self.client = ResourceManagementClient(timeout=self.timeout)
granularity = self.args[0]
name = self.args[1]
plotType = self.args[2]
plotName = self.args[3]
try:
res = self.client.getCachedAccountingResult(
name, plotType, plotName)
res = res['Value']
if res == []:
return {'Result': None}
res = eval(res[0])
s = 0
n = 0
try:
SE = res['data'].keys()[0]
except IndexError:
return {'Result': None}
n = n + len(res['data'][SE])
s = s + sum(res['data'][SE].values())
meanQuality = s / n
except:
gLogger.exception(
"Exception when calling ResourcePolicyClient for %s" % (name))
return {'Result': 'Unknown'}
return {'Result': meanQuality}
def __init__(self, args=None, clients=None):
super(GOCDBSyncCommand, self).__init__(args, clients)
if 'GOCDBClient' in self.apis:
self.gClient = self.apis['GOCDBClient']
else:
self.gClient = GOCDBClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
self.seenHostnames = set()
def __init__(self,
name,
decissionParams,
enforcementResult,
singlePolicyResults,
clients=None):
super(LogPolicyResultAction,
self).__init__(name, decissionParams, enforcementResult,
singlePolicyResults, clients)
if clients is not None and 'ResourceManagementClient' in clients:
self.rmClient = clients['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
class TransferQualityFromCachedPlotCommand(Command):
def __init__(self, args=None, clients=None):
super(TransferQualityFromCachedPlotCommand, self).__init__(args, clients)
if "ResourceManagementClient" in self.apis:
self.rmClient = self.apis["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
def doCommand(self):
"""
Returns transfer quality from the plot cached in the accounting cache.
:attr:`args`:
- args[0]: string: should be a ValidElement
- args[1]: string should be the name of the ValidElement
:returns:
{'Result': None | a float between 0.0 and 100.0}
"""
if "name" not in self.args:
return S_ERROR("Name no specified")
name = self.args["name"]
if "plotType" not in self.args:
return S_ERROR("plotType no specified")
plotType = self.args["plotType"]
if "plotName" not in self.args:
return S_ERROR("plotName no specified")
plotName = self.args["plotName"]
meta = {"columns": "Result"}
results = self.rmClient.selectAccountingCache(name=name, plotType=plotType, plotName=plotName, meta=meta)
if not results["OK"]:
return results
results = results["Value"]
if results == []:
results = None
else:
# FIXME: remove the eval from here !!
results = eval(results[0][0])
num, den = 0, 0
se = results["data"].keys()[0]
num = num + len(results["data"][se])
den = den + sum(results["data"][se].values())
meanQuality = den / num
results = meanQuality
return S_OK(results)
def __init__(self, granularity, name, status_type, pdp_decision, **kw):
ActionBase.__init__( self, granularity, name, status_type, pdp_decision, **kw )
try: self.rsClient = self.kw["Clients"][ 'ResourceStatusClient' ]
except KeyError: self.rsClient = ResourceStatusClient()
try: self.rmClient = self.kw["Clients"][ 'ResourceManagementClient' ]
except KeyError: self.rmClient = ResourceManagementClient()
def __init__(self, clients=None):
""" Constructor
examples:
>>> pep = PEP()
>>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } )
>>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } )
:Parameters:
**clients** - [ None, `dict` ]
dictionary with clients to be used in the commands issued by the policies.
If not defined, the commands will import them. It is a measure to avoid
opening the same connection every time a policy is evaluated.
"""
if clients is None:
clients = {}
# PEP uses internally two of the clients: ResourceStatusClient and ResouceManagementClient
if 'ResourceStatusClient' in clients:
self.rsClient = clients['ResourceStatusClient']
else:
self.rsClient = ResourceStatusClient()
if 'ResourceManagementClient' in clients:
self.rmClient = clients['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
self.clients = clients
# Pass to the PDP the clients that are going to be used on the Commands
self.pdp = PDP(clients)
def __init__( self, rsClient = None, rmClient = None ):
self.GOCDBClient = GOCDBClient()
self.rsClient = ResourceStatusClient() if rsClient == None else rsClient
self.rmClient = ResourceManagementClient() if rmClient == None else rmClient
self.synclist = [ 'Sites', 'Resources', 'StorageElements', 'Services', 'RegistryUsers' ]
class TransferQualityFromCachedPlotCommand(Command):
def __init__(self, args=None, clients=None):
super(TransferQualityFromCachedPlotCommand, self).__init__(args, clients)
if "ResourceManagementClient" in self.apis:
self.rmClient = self.apis["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
def doCommand(self):
"""
Returns transfer quality from the plot cached in the accounting cache.
:attr:`args`:
- args[0]: string: should be a ValidElement
- args[1]: string should be the name of the ValidElement
:returns:
{'Result': None | a float between 0.0 and 100.0}
"""
if "name" not in self.args:
return S_ERROR("Name no specified")
name = self.args["name"]
if "plotType" not in self.args:
return S_ERROR("plotType no specified")
plotType = self.args["plotType"]
if "plotName" not in self.args:
return S_ERROR("plotName no specified")
plotName = self.args["plotName"]
meta = {"columns": "Result"}
results = self.rmClient.selectAccountingCache(name=name, plotType=plotType, plotName=plotName, meta=meta)
if not results["OK"]:
return results
results = results["Value"]
if results == []:
results = None
else:
# FIXME: remove the eval from here !!
results = eval(results[0][0])
num, den = 0, 0
se = list(results["data"])[0]
num = num + len(results["data"][se])
den = den + sum(results["data"][se].values())
meanQuality = den / num
results = meanQuality
return S_OK(results)
def doCommand(self):
"""
Returns transfer quality as it is cached
:attr:`args`:
- args[0]: string: should be a ValidRes
- args[1]: string should be the name of the ValidRes
:returns:
{'Result': None | a float between 0.0 and 100.0}
"""
super(TransferQualityCached_Command, self).doCommand()
if self.client is None:
from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient
self.client = ResourceManagementClient(timeout = self.timeout)
name = self.args[1]
try:
res = self.client.getCachedResult(name, 'TransferQualityEverySEs', 'TQ', 'NULL')
if res == []:
return {'Result':None}
except:
gLogger.exception("Exception when calling ResourceManagementClient for %s" %(name))
return {'Result':'Unknown'}
return {'Result':float(res[0])}
def __init__(self, args=None, clients=None):
super(CachedPlotCommand, self).__init__(args, clients)
if "ResourceManagementClient" in self.apis:
self.rmClient = self.apis["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
def setUp( self ):
self.mockRSS = Mock()
self.RSCli = ResourceStatusClient( serviceIn = self.mockRSS )
self.RMCli = ResourceManagementClient( serviceIn = self.mockRSS )
self.PilotsCli = PilotsClient()
self.JobsCli = JobsClient()
def __init__( self, args = None, clients = None ):
super( SpaceTokenOccupancyCommand, self ).__init__( args, clients )
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def __init__( self, name, decissionParams, enforcementResult, singlePolicyResults,
clients = None ):
super( LogPolicyResultAction, self ).__init__( name, decissionParams, enforcementResult,
singlePolicyResults, clients )
if clients is not None and 'ResourceManagementClient' in clients:
self.rmClient = clients[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def initialize(self):
'''
TokenAgent initialization
'''
# Attribute defined outside __init__
# pylint: disable-msg=W0201
self.notifyHours = self.am_getOption('notifyHours', 10)
try:
self.rsClient = ResourceStatusClient()
self.rmClient = ResourceManagementClient()
self.noClient = NotificationClient()
return S_OK()
except Exception:
errorStr = "TokenAgent initialization"
self.log.exception(errorStr)
return S_ERROR(errorStr)
def initializePublisherHandler( _serviceInfo ):
'''
Handler initialization in the usual horrible way.
'''
global rsClient
rsClient = ResourceStatusClient()
global rmClient
rmClient = ResourceManagementClient()
return S_OK()
def __init__(self, args=None, clients=None):
super(TransferCommand, self).__init__(args, clients)
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis['ReportGenerator']
else:
self.rgClient = RPCClient('Accounting/ReportGenerator')
self.rClient.rpcClient = self.rgClient
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def initialize(self):
# Attribute defined outside __init__
# pylint: disable-msg=W0201
try:
self.rsClient = ResourceStatusClient()
self.rmClient = ResourceManagementClient()
validElements = RssConfiguration.getValidElements()
self.historyTables = ['%sHistory' % x for x in validElements]
return S_OK()
except Exception:
errorStr = "CacheCleanerAgent initialization"
self.log.exception(errorStr)
return S_ERROR(errorStr)
def doCommand(self):
"""
Returns transfer quality plot as it is cached in the accounting cache.
:attr:`args`:
- args[0]: string - should be a ValidRes
- args[1]: string - should be the name of the ValidRes
- args[2]: string - should be the plot type
- args[3]: string - should be the plot name
:returns:
a plot
"""
super(CachedPlot_Command, self).doCommand()
if self.client is None:
from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient
self.client = ResourceManagementClient(timeout=self.timeout)
granularity = self.args[0]
name = self.args[1]
plotType = self.args[2]
plotName = self.args[3]
if granularity == 'Service':
name = name.split('@')[1]
try:
res = self.client.getCachedAccountingResult(
name, plotType, plotName)
if res == []:
return {'Result': {'data': {}, 'granularity': 900}}
except:
gLogger.exception(
"Exception when calling ResourcePolicyClient for %s" % (name))
return {'Result': 'Unknown'}
return {'Result': eval(res[0])}
class TransferQualityFromCachedPlot_Command(Command):
def doCommand(self):
"""
Returns transfer quality from the plot cached in the accounting cache.
:attr:`args`:
- args[0]: string: should be a ValidRes
- args[1]: string should be the name of the ValidRes
:returns:
{'Result': None | a float between 0.0 and 100.0}
"""
super(TransferQualityFromCachedPlot_Command, self).doCommand()
if self.client is None:
from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient
self.client = ResourceManagementClient(timeout = self.timeout)
granularity = self.args[0]
name = self.args[1]
plotType = self.args[2]
plotName = self.args[3]
try:
res = self.client.getCachedAccountingResult(name, plotType, plotName)
res = res[ 'Value' ]
if res == []:
return {'Result':None}
res = eval(res[0])
s = 0
n = 0
try:
SE = res['data'].keys()[0]
except IndexError:
return {'Result':None}
n = n + len(res['data'][SE])
s = s + sum(res['data'][SE].values())
meanQuality = s/n
except:
gLogger.exception("Exception when calling ResourcePolicyClient for %s" %(name))
return {'Result':'Unknown'}
return {'Result':meanQuality}
doCommand.__doc__ = Command.doCommand.__doc__ + doCommand.__doc__
def __init__( self, args = None, clients = None ):
super( DowntimeCommand, self ).__init__( args, clients )
if 'GOCDBClient' in self.apis:
self.gClient = self.apis[ 'GOCDBClient' ]
else:
self.gClient = GOCDBClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(GGUSTicketsCommand, self).__init__(args, clients)
if "GGUSTicketsClient" in self.apis:
self.gClient = self.apis["GGUSTicketsClient"]
else:
self.gClient = GGUSTicketsClient()
if "ResourceManagementClient" in self.apis:
self.rmClient = self.apis["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
def __init__( self, args = None, clients = None ):
super( PilotCommand, self ).__init__( args, clients )
if 'WMSAdministrator' in self.apis:
self.wmsAdmin = self.apis[ 'WMSAdministrator' ]
else:
self.wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' )
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def __init__( self, args = None, clients = None ):
super( JobCommand, self ).__init__( args, clients )
if 'JobDB' in self.apis:
self.jobDB = self.apis[ 'JobDB' ]
else:
self.jobDB = JobDB()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
class CachedPlot_Command(Command):
def doCommand(self):
"""
Returns transfer quality plot as it is cached in the accounting cache.
:attr:`args`:
- args[0]: string - should be a ValidRes
- args[1]: string - should be the name of the ValidRes
- args[2]: string - should be the plot type
- args[3]: string - should be the plot name
:returns:
a plot
"""
super(CachedPlot_Command, self).doCommand()
if self.client is None:
from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient
self.client = ResourceManagementClient(timeout = self.timeout)
granularity = self.args[0]
name = self.args[1]
plotType = self.args[2]
plotName = self.args[3]
if granularity == 'Service':
name = name.split('@')[1]
try:
res = self.client.getCachedAccountingResult(name, plotType, plotName)
if res == []:
return {'Result':{'data':{}, 'granularity':900}}
except:
gLogger.exception("Exception when calling ResourcePolicyClient for %s" %(name))
return {'Result':'Unknown'}
return {'Result':eval(res[0])}
doCommand.__doc__ = Command.doCommand.__doc__ + doCommand.__doc__
def __init__( self, args = None, clients = None ):
super( TransferCommand, self ).__init__( args, clients )
if 'ReportsClient' in self.apis:
self.rClient = self.apis[ 'ReportsClient' ]
else:
self.rClient = ReportsClient()
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis[ 'ReportGenerator' ]
else:
self.rgClient = RPCClient( 'Accounting/ReportGenerator' )
self.rClient.rpcClient = self.rgClient
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def initialize( self ):
# Attribute defined outside __init__
# pylint: disable-msg=W0201
try:
self.rsClient = ResourceStatusClient()
self.rmClient = ResourceManagementClient()
validElements = RssConfiguration.getValidElements()
self.historyTables = [ '%sHistory' % x for x in validElements ]
return S_OK()
except Exception:
errorStr = "CacheCleanerAgent initialization"
self.log.exception( errorStr )
return S_ERROR( errorStr )
def initialize( self ):
'''
TokenAgent initialization
'''
# Attribute defined outside __init__
# pylint: disable-msg=W0201
self.notifyHours = self.am_getOption( 'notifyHours', 10 )
try:
self.rsClient = ResourceStatusClient()
self.rmClient = ResourceManagementClient()
self.noClient = NotificationClient()
return S_OK()
except Exception:
errorStr = "TokenAgent initialization"
self.log.exception( errorStr )
return S_ERROR( errorStr )
def __init__( self, args = None, clients = None ):
""" Constructor.
:Parameters:
**args** - [, `dict` ]
arguments to be passed to be used in the _prepareCommand method ( name and
timespan are the expected ones )
**clients - [, `dict` ]
clients from where information is fetched. Mainly used to avoid creating
new connections on agents looping over clients. ResourceManagementClient
and PilotsDB are most welcome.
"""
super( PilotCommand, self ).__init__( args, clients )
if 'PilotsDB' in self.apis:
self.pilotsDB = self.apis[ 'PilotsDB' ]
else:
self.pilotsDB = PilotAgentsDB()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
class DowntimeCommand( Command ):
'''
Downtime "master" Command.
'''
def __init__( self, args = None, clients = None ):
super( DowntimeCommand, self ).__init__( args, clients )
if 'GOCDBClient' in self.apis:
self.gClient = self.apis[ 'GOCDBClient' ]
else:
self.gClient = GOCDBClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def _storeCommand( self, result ):
'''
Stores the results of doNew method on the database.
'''
for dt in result:
resQuery = self.rmClient.addOrModifyDowntimeCache(
downtimeID = dt[ 'DowntimeID' ],
element = dt[ 'Element' ],
name = dt[ 'Name' ],
startDate = dt[ 'StartDate' ],
endDate = dt[ 'EndDate' ],
severity = dt[ 'Severity' ],
description = dt[ 'Description' ],
link = dt[ 'Link' ],
gocdbServiceType = dt[ 'GOCDBServiceType' ] )
return resQuery
def _cleanCommand( self, element, elementNames):
'''
Clear Cache from expired DT.
'''
resQuery = []
for elementName in elementNames:
#reading all the cache entries
result = self.rmClient.selectDowntimeCache(
element = element,
name = elementName
)
if not result[ 'OK' ]:
return result
uniformResult = [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ]
currentDate = datetime.utcnow()
if len(uniformResult) == 0:
return S_OK( None )
for dt in uniformResult:
if dt[ 'EndDate' ] < currentDate:
result = self.rmClient.deleteDowntimeCache (
downtimeID = dt[ 'DowntimeID' ]
)
resQuery.append(result)
return S_OK( resQuery )
def _prepareCommand( self ):
'''
DowntimeCommand requires four arguments:
- name : <str>
- element : Site / Resource
- elementType: <str>
If the elements are Site(s), we need to get their GOCDB names. They may
not have, so we ignore them if they do not have.
'''
if 'name' not in self.args:
return S_ERROR( '"name" not found in self.args' )
elementName = self.args[ 'name' ]
if 'element' not in self.args:
return S_ERROR( '"element" not found in self.args' )
element = self.args[ 'element' ]
if 'elementType' not in self.args:
return S_ERROR( '"elementType" not found in self.args' )
elementType = self.args[ 'elementType' ]
if not element in [ 'Site', 'Resource' ]:
return S_ERROR( 'element is neither Site nor Resource' )
hours = None
if 'hours' in self.args:
hours = self.args[ 'hours' ]
gocdbServiceType = None
# Transform DIRAC site names into GOCDB topics
if element == 'Site':
gocSite = getGOCSiteName( elementName )
if not gocSite[ 'OK' ]:
return gocSite
elementName = gocSite[ 'Value' ]
# The DIRAC se names mean nothing on the grid, but their hosts do mean.
elif elementType == 'StorageElement':
# We need to distinguish if it's tape or disk
if getStorageElementOptions( elementName )['Value']['TapeSE']:
gocdbServiceType = "srm.nearline"
elif getStorageElementOptions( elementName )['Value']['DiskSE']:
gocdbServiceType = "srm"
seHost = CSHelpers.getSEHost( elementName )
if not seHost:
return S_ERROR( 'No seHost for %s' % elementName )
elementName = seHost
elif elementType == 'FTS' or elementType == 'FTS3':
gocdbServiceType = 'FTS'
try:
#WARNING: this method presupposes that the server is an FTS3 type
elementName = getGOCFTSName(elementName)
except:
return S_ERROR( 'No FTS3 server specified in dirac.cfg (see Resources/FTSEndpoints)' )
return S_OK( ( element, elementName, hours, gocdbServiceType ) )
def doNew( self, masterParams = None ):
'''
Gets the parameters to run, either from the master method or from its
own arguments.
For every elementName, unless it is given a list, in which case it contacts
the gocdb client. The server is not very stable, so in case of failure tries
a second time.
If there are downtimes, are recorded and then returned.
'''
if masterParams is not None:
element, elementNames = masterParams
#translate DIRAC CS elementNames into GOCDB elementNames
translatedElementNames = []
for e in elementNames:
translatedElementNames.append(CSHelpers.getSEHost( e ))
elementNames = translatedElementNames
hours = None
elementName = None
gocdbServiceType = None
else:
params = self._prepareCommand()
if not params[ 'OK' ]:
return params
element, elementName, hours, gocdbServiceType = params[ 'Value' ]
elementNames = [ elementName ]
#WARNING: checking all the DT that are ongoing or starting in given <hours> from now
startDate = None
if hours is not None:
startDate = datetime.utcnow() + timedelta( hours = hours )
try:
results = self.gClient.getStatus( element, elementNames, startDate )
except urllib2.URLError:
try:
#Let's give it a second chance..
results = self.gClient.getStatus( element, elementNames, startDate )
except urllib2.URLError, e:
return S_ERROR( e )
if not results[ 'OK' ]:
return results
results = results[ 'Value' ]
if results is None:
return S_OK( None )
#cleaning the Cache
cleanRes = self._cleanCommand(element, elementNames)
if not cleanRes[ 'OK' ]:
return cleanRes
uniformResult = []
# Humanize the results into a dictionary, not the most optimal, but readable
for downtime, downDic in results.items():
dt = {}
if 'HOSTNAME' in downDic.keys():
dt[ 'Name' ] = downDic[ 'HOSTNAME' ]
elif 'SITENAME' in downDic.keys():
dt[ 'Name' ] = downDic[ 'SITENAME' ]
else:
return S_ERROR( "SITENAME or HOSTNAME are missing" )
if 'SERVICE_TYPE' in downDic.keys():
dt[ 'GOCDBServiceType' ] = downDic[ 'SERVICE_TYPE' ]
if gocdbServiceType:
gocdbST = gocdbServiceType.lower()
csST = downDic[ 'SERVICE_TYPE' ].lower()
if gocdbST != csST:
return S_ERROR( "SERVICE_TYPE mismatch between GOCDB (%s) and CS (%s) for %s" % (gocdbST, csST, dt[ 'Name' ]) )
else:
#WARNING: do we want None as default value?
dt[ 'GOCDBServiceType' ] = None
dt[ 'DowntimeID' ] = downtime
dt[ 'Element' ] = element
dt[ 'StartDate' ] = downDic[ 'FORMATED_START_DATE' ]
dt[ 'EndDate' ] = downDic[ 'FORMATED_END_DATE' ]
dt[ 'Severity' ] = downDic[ 'SEVERITY' ]
dt[ 'Description' ] = downDic[ 'DESCRIPTION' ].replace( '\'', '' )
dt[ 'Link' ] = downDic[ 'GOCDB_PORTAL_URL' ]
uniformResult.append( dt )
storeRes = self._storeCommand( uniformResult )
if not storeRes[ 'OK' ]:
return storeRes
return S_OK()
class AlarmAction( ActionBase ):
def __init__(self, granularity, name, status_type, pdp_decision, **kw):
ActionBase.__init__( self, granularity, name, status_type, pdp_decision, **kw )
try: self.rsClient = self.kw["Clients"][ 'ResourceStatusClient' ]
except KeyError: self.rsClient = ResourceStatusClient()
try: self.rmClient = self.kw["Clients"][ 'ResourceManagementClient' ]
except KeyError: self.rmClient = ResourceManagementClient()
def _getUsersToNotify(self):
groups = CS.getTypedDictRootedAtOperations("AssigneeGroups/" + CS.getSetup()).values()
concerned_groups = [g for g in groups if Utils.dictMatch(self.kw["Params"], g)]
return [{'Users':g['Users'],
'Notifications':g['Notifications']} for g in concerned_groups]
def run(self):
""" Do actions required to notify users.
Mandatory keyword arguments:
- Granularity
Optional keyword arguments:
- SiteType
- ServiceType
- ResourceType
"""
# Initializing variables
nc = NotificationClient()
# raise alarms, right now makes a simple notification
if 'Granularity' not in self.kw['Params'].keys():
raise ValueError, "You have to provide a argument Granularity = <desired_granularity>"
if self.new_status['Action']:
notif = "%s %s is perceived as" % (self.granularity, self.name)
notif = notif + " %s. Reason: %s." % (self.new_status['Status'], self.new_status['Reason'])
users_to_notify = self._getUsersToNotify()
for notif in users_to_notify:
for user in notif['Users']:
if 'Web' in notif['Notifications']:
gLogger.info("Sending web notification to user %s" % user)
nc.addNotificationForUser(user, notif)
if 'Mail' in notif['Notifications']:
gLogger.info("Sending mail notification to user %s" % user)
was = self.rsClient.getElementHistory( self.granularity,
elementName = self.name,
statusType = self.status_type,
meta = {"order": "DESC", 'limit' : 1,
"columns": ['Status', 'Reason', 'DateEffective']})#[0]
if not was[ 'OK' ]:
gLogger.error( was[ 'Message' ] )
continue
was = was[ 'Value' ][ 0 ]
mailMessage = """ ---TESTING---
--------------------------------------------------------------------------------
RSS changed the status of the following resource:
Granularity:\t%s
Name:\t\t%s
New status:\t%s
Reason:\t\t%s
Was:\t\t%s (%s) since %s
Setup:\t\t%s
If you think RSS took the wrong decision, please set the status manually:
Use: dirac-rss-set-status -g <granularity> -n <element_name> -s <desired_status> [-t status_type]
(if you omit the optional last part of the command, all status types are matched.)
This notification has been sent according to those parameters:
%s
""" % (self.granularity, self.name, self.new_status['Status'],
self.new_status['Reason'], was[0], was[1], was[2], CS.getSetup(), str(users_to_notify))
# Actually send the mail!
resUser = self.rmClient.getUserRegistryCache( user )
if not resUser[ 'OK' ]:
gLogger.error( resUser[ 'Message' ] )
continue
resUser = resUser[ 'Value' ][ 0 ][ 2 ]
nc.sendMail(resUser,
'[RSS][%s][%s] %s -> %s'
% (self.granularity, self.name, self.new_status['Status'], was[0]), mailMessage)
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class Synchronizer( object ):
def __init__( self, rsClient = None, rmClient = None ):
self.GOCDBClient = GOCDBClient()
self.rsClient = ResourceStatusClient() if rsClient == None else rsClient
self.rmClient = ResourceManagementClient() if rmClient == None else rmClient
self.synclist = [ 'Sites', 'Resources', 'StorageElements', 'Services', 'RegistryUsers' ]
################################################################################
def sync( self, _a, _b ):
"""
:params:
:attr:`thingsToSync`: list of things to sync
"""
gLogger.info( "!!! Sync DB content with CS content for %s !!!" % ( ", ".join(self.synclist) ) )
for thing in self.synclist:
getattr( self, '_sync' + thing )()
return S_OK()
################################################################################
def __purge_resource( self, resourceName ):
# Maybe remove attached SEs
#SEs = Utils.unpack(self.rsClient.getStorageElement(resourceName=resourceName))
SEs = self.rsClient.getStorageElement( resourceName = resourceName )
if not SEs[ 'OK' ]:
gLogger.error( SEs[ 'Message' ] )
return SEs
#Utils.unpack(self.rsClient.removeElement("StorageElement", [s[0] for s in SEs]))
SEs = [ se[0] for se in SEs ]
res = self.rsClient.removeElement( 'StorageElement', SEs )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
# Remove resource itself.
#Utils.unpack(self.rsClient.removeElement("Resource", resourceName))
res = self.rsClient.removeElement( 'Resource', resourceName )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
def __purge_site( self, siteName ):
# Remove associated resources and services
#resources = Utils.unpack(self.rsClient.getResource(siteName=siteName))
resources = self.rsClient.getResource( siteName = siteName )
if not resources[ 'OK' ]:
gLogger.error( resources[ 'Message' ] )
return resources
#services = Utils.unpack(self.rsClient.getService(siteName=siteName))
services = self.rsClient.getService( siteName = siteName )
if not services[ 'OK' ]:
gLogger.error( services[ 'Message' ] )
return services
#_ = [self.__purge_resource(r[0]) for r in resources]
for resource in resources:
res = self.__purge_resource( resource[ 0 ] )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
#Utils.unpack(self.rsClient.removeElement("Service", [s[0] for s in services]))
services = [ service[ 0 ] for service in services[ 'Value' ] ]
res = self.rsClient.removeElement( 'Service', services )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
# Remove site itself
#Utils.unpack(self.rsClient.removeElement("Site", siteName))
res = self.rsClient.removeElement( 'Site', siteName )
if not res[ 'OK' ]:
gLogger.info( res[ 'Message' ] )
return res
def _syncSites( self ):
"""
Sync DB content with sites that are in the CS
"""
def getGOCTier(sitesList):
return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)]))
# sites in the DB now
#sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite())))
sites = self.rsClient.getSite()
if not sites[ 'OK' ]:
gLogger.error( sites[ 'Message' ] )
return sites
sitesDB = set( [ site[0] for site in sites[ 'Value' ] ] )
# sites in CS now
sitesCS = set( CS.getSites() )
gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" % (len(sitesCS), len(sitesDB)))
# remove sites and associated resources, services, and storage
# elements from the DB that are not in the CS:
for s in sitesDB - sitesCS:
gLogger.info("Purging Site %s (not in CS anymore)" % s)
self.__purge_site(s)
# add to DB what is missing
gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB))
for site in sitesCS - sitesDB:
siteType = site.split(".")[0]
# DIRAC Tier
tier = "T" + str(CS.getSiteTier( site ))
if siteType == "LCG":
# Grid Name of the site
#gridSiteName = Utils.unpack(getGOCSiteName(site))
gridSiteName = getGOCSiteName( site )
if not gridSiteName[ 'OK' ]:
gLogger.error( gridSiteName[ 'Message' ] )
return gridSiteName
gridSiteName = gridSiteName[ 'Value' ]
# Grid Tier (with a workaround!)
#DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName))
DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName )
if not DIRACSitesOfGridSites[ 'OK' ]:
gLogger.error( DIRACSitesOfGridSites[ 'Message' ] )
return DIRACSitesOfGridSites
DIRACSitesOfGridSites = DIRACSitesOfGridSites[ 'Value' ]
if len( DIRACSitesOfGridSites ) == 1:
gt = tier
else:
gt = getGOCTier( DIRACSitesOfGridSites )
#Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt)
res = self.rsClient.addOrModifyGridSite( gridSiteName, gt )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
#Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName )
res = self.rsClient.addOrModifySite( site, tier, gridSiteName )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
elif siteType == "DIRAC":
#Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" )
res = self.rsClient.addOrModifySite( site, tier, "NULL" )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
################################################################################
# _syncResources HELPER functions
def __updateService(self, site, type_):
service = type_ + '@' + site
#Utils.protect2(self.rsClient.addOrModifyService, service, type_, site )
res = self.rsClient.addOrModifyService( service, type_, site )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
def __getServiceEndpointInfo(self, node):
#res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) )
res = self.GOCDBClient.getServiceEndpointInfo( 'hostname', node )
if res['OK']:
res = res[ 'Value' ]
else:
gLogger.warn( 'Error getting hostname info for %s' % node )
return []
if res == []:
#res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo('hostname', Utils.canonicalURL(node)) )
url = Utils.canonicalURL(node)
res = self.GOCDBClient.getServiceEndpointInfo('hostname', url )
if res['OK']:
res = res[ 'Value' ]
else:
gLogger.warn( 'Error getting canonical hostname info for %s' % node )
res = []
return res
def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site = "NULL"):
nodesToUpdate = NodeInCS - resourcesInDB
if len(nodesToUpdate) > 0:
gLogger.debug(str(NodeInCS))
gLogger.debug(str(nodesToUpdate))
# Update Service table
siteInGOCDB = [self.__getServiceEndpointInfo(node) for node in nodesToUpdate]
siteInGOCDB = Utils.list_sanitize(siteInGOCDB)
#sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB]
sites = []
for sInGOCDB in siteInGOCDB:
siteName = getDIRACSiteName( sInGOCDB[ 0 ][ 'SITENAME' ] )
if not siteName[ 'OK' ]:
gLogger.error( siteName[ 'Message' ] )
return siteName
sites.append( siteName[ 'Value' ] )
sites = Utils.list_sanitize( Utils.list_flatten( sites ) )
_ = [ self.__updateService(s, serviceType) for s in sites ]
# Update Resource table
for node in NodeInCS:
if serviceType == "Computing":
resourceType = CS.getCEType(site, node)
if node not in resourcesInDB and node is not None:
try:
siteInGOCDB = self.__getServiceEndpointInfo(node)[0]['SITENAME']
except IndexError: # No INFO in GOCDB: Node does not exist
gLogger.warn("Node %s is not in GOCDB!! Considering that it does not exists!" % node)
continue
assert(type(siteInGOCDB) == str)
#Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB )
res = self.rsClient.addOrModifyResource( node, resourceType, serviceType, site, siteInGOCDB )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
resourcesInDB.add( node )
################################################################################
def _syncResources( self ):
gLogger.info("Starting sync of Resources")
# resources in the DB now
#resourcesInDB = set((r[0] for r in Utils.unpack(self.rsClient.getResource())))
resources = self.rsClient.getResource()
if not resources[ 'OK' ]:
gLogger.error( resources[ 'Message' ] )
return resources
resourcesInDB = set( [ resource[ 0 ] for resource in resources[ 'Value' ] ] )
# Site-CE / Site-SE mapping in CS now
#CEinCS = Utils.unpack(getSiteCEMapping( 'LCG' ))
CEinCS = getSiteCEMapping( 'LCG' )
if not CEinCS[ 'OK' ]:
gLogger.error( CEinCS[ 'Message' ] )
return CEinCS
CEinCS = CEinCS[ 'Value' ]
# All CEs in CS now
CEInCS = Utils.set_sanitize([CE for celist in CEinCS.values() for CE in celist])
# All SE Nodes in CS now
SENodeInCS = set(CS.getSENodes())
# LFC Nodes in CS now
LFCNodeInCS_L = set(CS.getLFCNode(readable = "ReadOnly"))
LFCNodeInCS_C = set(CS.getLFCNode(readable = "ReadWrite"))
# FTS Nodes in CS now
FTSNodeInCS = set([v.split("/")[2][0:-5] for v
in CS.getTypedDictRootedAt(root="/Resources/FTSEndpoints").values()])
# VOMS Nodes in CS now
VOMSNodeInCS = set(CS.getVOMSEndpoints())
# complete list of resources in CS now
resourcesInCS = CEInCS | SENodeInCS | LFCNodeInCS_L | LFCNodeInCS_C | FTSNodeInCS | VOMSNodeInCS
gLogger.info(" %d resources in CS, %s resources in DB, updating %d resources" %
(len(resourcesInCS), len(resourcesInDB), len(resourcesInCS)-len(resourcesInDB)))
# Remove resources that are not in the CS anymore
for res in resourcesInDB - resourcesInCS:
gLogger.info("Purging resource %s. Reason: not in CS anywore." % res)
self.__purge_resource(res)
# Add to DB what is in CS now and wasn't before
# CEs
for site in CEinCS:
self.__syncNode(set(CEinCS[site]), resourcesInDB, "", "Computing", site)
# SRMs
self.__syncNode(SENodeInCS, resourcesInDB, "SE", "Storage")
# LFC_C
self.__syncNode(LFCNodeInCS_C, resourcesInDB, "LFC_C", "Storage")
# LFC_L
self.__syncNode(LFCNodeInCS_L, resourcesInDB, "LFC_L", "Storage")
# FTSs
self.__syncNode(FTSNodeInCS, resourcesInDB, "FTS", "Storage")
# VOMSs
self.__syncNode(VOMSNodeInCS, resourcesInDB, "VOMS", "VOMS")
################################################################################
def _syncStorageElements( self ):
# Get StorageElements from the CS and the DB
CSSEs = set(CS.getSEs())
#DBSEs = set((s[0] for s in Utils.unpack(self.rsClient.getStorageElement())))
ses = self.rsClient.getStorageElement()
if not ses[ 'OK' ]:
gLogger.error( ses[ 'Message' ] )
return ses
DBSEs = set( [ se[0] for se in ses[ 'Value' ] ] )
# Remove storageElements that are in DB but not in CS
for se in DBSEs - CSSEs:
#Utils.protect2(self.rsClient.removeElement, 'StorageElement', se )
res = self.rsClient.removeElement( 'StorageElement', se )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
# Add new storage elements
gLogger.info("Updating %d StorageElements in DB (%d on CS vs %d on DB)" % (len(CSSEs - DBSEs), len(CSSEs), len(DBSEs)))
for SE in CSSEs - DBSEs:
srm = CS.getSEHost( SE )
if not srm:
gLogger.warn("%s has no srm URL in CS!!!" % SE)
continue
#siteInGOCDB = Utils.unpack(self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm ))
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )
if siteInGOCDB[ 'OK' ]:
siteInGOCDB = siteInGOCDB[ 'Value' ]
else:
gLogger.error("Error getting hostname for %s from GOCDB!!!" % srm)
continue
if siteInGOCDB == []:
gLogger.warn("%s is not in GOCDB!!!" % srm)
continue
siteInGOCDB = siteInGOCDB[ 0 ][ 'SITENAME' ]
#Utils.protect2(self.rsClient.addOrModifyStorageElement, SE, srm, siteInGOCDB )
res = self.rsClient.addOrModifyStorageElement( SE, srm, siteInGOCDB )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
################################################################################
def _syncServices(self):
"""This function is in charge of cleaning the Service table in DB
in case of obsolescence."""
# services in the DB now
#servicesInDB = Utils.unpack(self.rsClient.getService())
servicesInDB = self.rsClient.getService()
if not servicesInDB[ 'OK' ]:
gLogger.error( servicesInDB[ 'Message' ] )
return servicesInDB
servicesInDB = servicesInDB[ 'Value' ]
for service_name, service_type, site_name in servicesInDB:
if not service_type in ["VO-BOX", "CondDB", "VOMS", "Storage"]:
#if Utils.unpack(self.rsClient.getResource(siteName=site_name, serviceType=service_type)) == []:
resource = self.rsClient.getResource( siteName = site_name, serviceType = service_type )
if not resource[ 'OK' ]:
gLogger.error( resource[ 'Message' ] )
return resource
if resource[ 'Value' ] == []:
gLogger.info("Deleting Service %s since it has no corresponding resources." % service_name)
#Utils.protect2(self.rsClient.removeElement, "Service", service_name)
res = self.rsClient.removeElement( "Service", service_name )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
elif service_type == "Storage":
res = self.rsClient.getSite( siteName = site_name, meta = { 'columns' : 'GridSiteName'} )
if res[ 'OK' ]:
res = res[ 'Value' ]
else:
res = []
if res:
if self.rsClient.getResource( gridSiteName = res[0], serviceType = service_type ) == []:
gLogger.info("Deleting Service %s since it has no corresponding resources." % service_name)
#Utils.protect2(self.rsClient.removeElement, "Service", service_name)
res = self.rsClient.removeElement( "Service", service_name )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
def _syncRegistryUsers(self):
users = CS.getTypedDictRootedAt("Users", root= "/Registry")
usersInCS = set(users.keys())
#usersInDB = set((u[0] for u in Utils.unpack(self.rmClient.getUserRegistryCache())))
usersInCache = self.rmClient.getUserRegistryCache()
if not usersInCache[ 'OK' ]:
gLogger.error( usersInCache[ 'Message' ] )
return usersInCache
usersInDB = set( [ userInCache[ 0 ] for userInCache in usersInCache[ 'Value' ] ] )
usersToAdd = usersInCS - usersInDB
usersToDel = usersInDB - usersInCS
gLogger.info("Updating Registry Users: + %d, - %d" % (len(usersToAdd), len(usersToDel)))
if len(usersToAdd) > 0:
gLogger.debug(str(usersToAdd))
if len(usersToDel) > 0:
gLogger.debug(str(usersToDel))
for u in usersToAdd:
if type(users[u]['DN']) == list:
users[u]['DN'] = users[u]['DN'][0]
if type(users[u]['Email']) == list:
users[u]['Email'] = users[u]['Email'][0]
users[u]['DN'] = users[u]['DN'].split('=')[-1]
#Utils.unpack(self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower()))
res = self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower() )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
for u in usersToDel:
#Utils.protect2(self.rmClient.deleteUserRegistryCache, u)
res = self.rmClient.deleteUserRegistryCache( u )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class TransferCommand( Command ):
'''
Transfer "master" Command
'''
def __init__( self, args = None, clients = None ):
super( TransferCommand, self ).__init__( args, clients )
if 'ReportsClient' in self.apis:
self.rClient = self.apis[ 'ReportsClient' ]
else:
self.rClient = ReportsClient()
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis[ 'ReportGenerator' ]
else:
self.rgClient = RPCClient( 'Accounting/ReportGenerator' )
self.rClient.rpcClient = self.rgClient
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def _storeCommand( self, results ):
'''
Stores the results of doNew method on the database.
'''
for result in results:
resQuery = self.rmClient.addOrModifyTransferCache( result[ 'SourceName' ],
result[ 'DestinationName' ],
result[ 'Metric' ],
result[ 'Value' ] )
if not resQuery[ 'OK' ]:
return resQuery
return S_OK()
def _prepareCommand( self ):
'''
TransferChannelCommand requires four arguments:
- hours : <int>
- direction : Source | Destination
- elementName : <str>
- metric : Quality | FailedTransfers
GGUSTickets are associated with gocDB names, so we have to transform the
diracSiteName into a gocSiteName.
'''
if not 'hours' in self.args:
return S_ERROR( 'Number of hours not specified' )
hours = self.args[ 'hours' ]
if not 'direction' in self.args:
return S_ERROR( 'direction is missing' )
direction = self.args[ 'direction' ]
if direction not in [ 'Source', 'Destination' ]:
return S_ERROR( 'direction is not Source nor Destination' )
if not 'name' in self.args:
return S_ERROR( '"name" is missing' )
name = self.args[ 'name' ]
if not 'metric' in self.args:
return S_ERROR( 'metric is missing' )
metric = self.args[ 'metric' ]
if metric not in [ 'Quality', 'FailedTransfers' ]:
return S_ERROR( 'metric is not Quality nor FailedTransfers' )
return S_OK( ( hours, name, direction, metric ) )
def doNew( self, masterParams = None ):
'''
Gets the parameters to run, either from the master method or from its
own arguments.
For every elementName ( cannot process bulk queries.. ) contacts the
accounting client. It reurns dictionaries like { 'X -> Y' : { id: 100%.. } }
If there are ggus tickets, are recorded and then returned.
'''
if masterParams is not None:
hours, name, direction, metric = masterParams
else:
params = self._prepareCommand()
if not params[ 'OK' ]:
return params
hours, name, direction, metric = params[ 'Value' ]
toD = datetime.utcnow()
fromD = toD - timedelta( hours = hours )
# dictionary with conditions for the accounting
transferDict = {
'OperationType' : 'putAndRegister',
direction : name
}
if metric == 'FailedTransfers':
transferDict[ 'FinalStatus' ] = [ 'Failed' ]
transferResults = self.rClient.getReport( 'DataOperation', metric, fromD,
toD, transferDict, 'Channel' )
if not transferResults[ 'OK' ]:
return transferResults
transferResults = transferResults[ 'Value' ]
if not 'data' in transferResults:
return S_ERROR( 'Missing data key' )
transferResults = transferResults[ 'data' ]
uniformResult = []
for channel, elementDict in transferResults.items():
try:
source, destination = channel.split( ' -> ' )
except ValueError:
continue
channelDict = {}
channelDict[ 'SourceName' ] = source
channelDict[ 'DestinationName' ] = destination
channelDict[ 'Metric' ] = metric
channelDict[ 'Value' ] = sum( elementDict.values() ) / len( elementDict.values() )
uniformResult.append( channelDict )
storeRes = self._storeCommand( uniformResult )
if not storeRes[ 'OK' ]:
return storeRes
# Compute mean of all transfer channels
value = 0
for channelDict in uniformResult:
value += channelDict[ 'Value' ]
if uniformResult:
value = float( value ) / len( uniformResult )
else:
value = None
return S_OK( { 'Mean' : value, 'Name' : name } )
def doCache( self ):
'''
Method that reads the cache table and tries to read from it. It will
return a list of dictionaries if there are results.
'''
params = self._prepareCommand()
if not params[ 'OK' ]:
return params
_hours, name, direction, metric = params[ 'Value' ]
sourceName, destinationName = None, None
if direction == 'Source':
sourceName = name
if direction == 'Destination':
destinationName = name
result = self.rmClient.selectTransferCache( sourceName, destinationName, metric )
if not result[ 'OK' ]:
return result
result = [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ]
# Compute mean of all transfer channels
value = 0
for channelDict in result:
value += channelDict[ 'Value' ]
if result:
value = float( value ) / len( result )
else:
value = None
return S_OK( { 'Mean' : value, 'Name' : name } )
def doMaster( self ):
'''
Master method, which looks little bit spaguetti code, sorry !
- It gets all Sites.
- It gets all StorageElements
As there is no bulk query, it compares with what we have on the database.
It queries a portion of them.
'''
sites = CSHelpers.getSites()
if not sites[ 'OK' ]:
return sites
sites = sites[ 'Value' ]
ses = CSHelpers.getStorageElements()
if not ses[ 'OK' ]:
return ses
ses = ses[ 'Value' ]
elementNames = sites + ses
# sourceQuery = self.rmClient.selectTransferCache( meta = { 'columns' : [ 'SourceName' ] } )
# if not sourceQuery[ 'OK' ]:
# return sourceQuery
# sourceQuery = [ element[0] for element in sourceQuery[ 'Value' ] ]
#
# sourceElementsToQuery = list( set( elementNames ).difference( set( sourceQuery ) ) )
gLogger.info( 'Processing %s' % ', '.join( elementNames ) )
for metric in [ 'Quality', 'FailedTransfers' ]:
for direction in [ 'Source', 'Destination' ]:
# 2 hours of window
result = self.doNew( ( 2, elementNames, direction, metric ) )
if not result[ 'OK' ]:
self.metrics[ 'failed' ].append( result )
return S_OK( self.metrics )
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class FreeDiskSpaceCommand(Command):
"""
Uses diskSpace method to get the free space
"""
def __init__(self, args=None, clients=None):
super(FreeDiskSpaceCommand, self).__init__(args, clients=clients)
self.rpc = None
self.rsClient = ResourceManagementClient()
def _prepareCommand(self):
"""
FreeDiskSpaceCommand requires one argument:
- name : <str>
"""
if "name" not in self.args:
return S_ERROR('"name" not found in self.args')
elementName = self.args["name"]
return S_OK(elementName)
def doNew(self, masterParams=None):
"""
Gets the total and the free disk space of a DIPS storage element that
is found in the CS and inserts the results in the SpaceTokenOccupancyCache table
of ResourceManagementDB database.
"""
if masterParams is not None:
elementName = masterParams
else:
elementName = self._prepareCommand()
if not elementName["OK"]:
return elementName
se = StorageElement(elementName)
elementURL = se.getStorageParameters(protocol="dips")
if elementURL["OK"]:
elementURL = se.getStorageParameters(protocol="dips")["Value"]["URLBase"]
else:
gLogger.verbose("Not a DIPS storage element, skipping...")
return S_OK()
self.rpc = RPCClient(elementURL, timeout=120)
free = self.rpc.getFreeDiskSpace("/")
if not free["OK"]:
return free
free = free["Value"]
total = self.rpc.getTotalDiskSpace("/")
if not total["OK"]:
return total
total = total["Value"]
if free and free < 1:
free = 1
if total and total < 1:
total = 1
result = self.rsClient.addOrModifySpaceTokenOccupancyCache(
endpoint=elementURL, lastCheckTime=datetime.utcnow(), free=free, total=total, token=elementName
)
if not result["OK"]:
return result
return S_OK()
def doCache(self):
"""
This is a method that gets the element's details from the spaceTokenOccupancy cache.
"""
elementName = self._prepareCommand()
if not elementName["OK"]:
return elementName
result = self.rsClient.selectSpaceTokenOccupancyCache(token=elementName)
if not result["OK"]:
return result
return S_OK(result)
def doMaster(self):
"""
This method calls the doNew method for each storage element
that exists in the CS.
"""
elements = CSHelpers.getStorageElements()
for name in elements["Value"]:
diskSpace = self.doNew(name)
if not diskSpace["OK"]:
gLogger.error("Unable to calculate free disk space")
continue
return S_OK()
class GOCDBSyncCommand(Command):
def __init__(self, args=None, clients=None):
super(GOCDBSyncCommand, self).__init__(args, clients)
if 'GOCDBClient' in self.apis:
self.gClient = self.apis['GOCDBClient']
else:
self.gClient = GOCDBClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
self.seenHostnames = set()
def doNew(self, masterParams=None):
"""
Gets the downtime IDs and dates of a given hostname from the local database and compares the results
with the remote database of GOCDB. If the downtime dates have been changed it updates the local database.
:param: `masterParams` - string
:return: S_OK / S_ERROR
"""
if masterParams:
hostname = masterParams
else:
return S_ERROR(errno.EINVAL, 'masterParams is not provided')
result = self.rmClient.selectDowntimeCache(name=hostname)
if not result['OK']:
return result
for downtimes in result['Value']:
localDBdict = {'DowntimeID': downtimes[3],
'FORMATED_START_DATE': downtimes[6].strftime('%Y-%m-%d %H:%M'),
'FORMATED_END_DATE': downtimes[7].strftime('%Y-%m-%d %H:%M')}
response = self.gClient.getHostnameDowntime(hostname, ongoing=True)
if not response['OK']:
return response
doc = minidom.parseString(response['Value'])
downtimeElements = doc.getElementsByTagName("DOWNTIME")
for dtElement in downtimeElements:
GOCDBdict = _parseSingleElement(dtElement, ['PRIMARY_KEY', 'ENDPOINT',
'FORMATED_START_DATE', 'FORMATED_END_DATE'])
localDowntimeID = localDBdict['DowntimeID']
GOCDBDowntimeID = GOCDBdict['PRIMARY_KEY'] + ' ' + GOCDBdict['ENDPOINT']
if localDowntimeID == GOCDBDowntimeID:
if localDBdict['FORMATED_START_DATE'] != GOCDBdict['FORMATED_START_DATE']:
result = self.rmClient.addOrModifyDowntimeCache(downtimeID=localDBdict['DowntimeID'],
startDate=GOCDBdict['FORMATED_START_DATE'])
gLogger.verbose("The start date of %s has been changed!" % downtimes[3])
if not result['OK']:
return result
if localDBdict['FORMATED_END_DATE'] != GOCDBdict['FORMATED_END_DATE']:
result = self.rmClient.addOrModifyDowntimeCache(downtimeID=localDBdict['DowntimeID'],
endDate=GOCDBdict['FORMATED_END_DATE'])
gLogger.verbose("The end date of %s has been changed!" % downtimes[3])
if not result['OK']:
return result
return S_OK()
def doCache(self):
return S_OK()
def doMaster(self):
"""
This method calls the doNew method for each hostname that exists
in the DowntimeCache table of the local database.
:return: S_OK / S_ERROR
"""
# Query DB for all downtimes
result = self.rmClient.selectDowntimeCache()
if not result['OK']:
return result
for data in result['Value']:
# If already processed don't do it again
if data[0] in self.seenHostnames:
continue
# data[0] contains the hostname
gLogger.verbose("Checking if the downtime of %s has been changed" % data[0])
result = self.doNew(data[0])
if not result['OK']:
return result
self.seenHostnames.add(data[0])
return S_OK()
def __init__(self, args=None, clients=None):
super(FreeDiskSpaceCommand, self).__init__(args, clients=clients)
self.rpc = None
self.rsClient = ResourceManagementClient()
class DowntimeCommand( Command ):
'''
Downtime "master" Command.
'''
def __init__( self, args = None, clients = None ):
super( DowntimeCommand, self ).__init__( args, clients )
if 'GOCDBClient' in self.apis:
self.gClient = self.apis[ 'GOCDBClient' ]
else:
self.gClient = GOCDBClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def _storeCommand( self, result ):
'''
Stores the results of doNew method on the database.
'''
for dt in result:
resQuery = self.rmClient.addOrModifyDowntimeCache( dt[ 'DowntimeID' ],
dt[ 'Element' ],
dt[ 'Name' ],
dt[ 'StartDate' ],
dt[ 'EndDate' ],
dt[ 'Severity' ],
dt[ 'Description' ],
dt[ 'Link' ] )
if not resQuery[ 'OK' ]:
return resQuery
return S_OK()
def _prepareCommand( self ):
'''
DowntimeCommand requires three arguments:
- name : <str>
- element : Site / Resource
- elementType: <str>
If the elements are Site(s), we need to get their GOCDB names. They may
not have, so we ignore them if they do not have.
'''
if 'name' not in self.args:
return S_ERROR( '"name" not found in self.args' )
elementName = self.args[ 'name' ]
if 'element' not in self.args:
return S_ERROR( '"element" not found in self.args' )
element = self.args[ 'element' ]
if 'elementType' not in self.args:
return S_ERROR( '"elementType" not found in self.args' )
elementType = self.args[ 'elementType' ]
if not element in [ 'Site', 'Resource' ]:
return S_ERROR( 'element is not Site nor Resource' )
hours = None
if 'hours' in self.args:
hours = self.args[ 'hours' ]
# Transform DIRAC site names into GOCDB topics
if element == 'Site':
gocSite = getGOCSiteName( elementName )
if not gocSite[ 'OK' ]:
return gocSite
elementName = gocSite[ 'Value' ]
# The DIRAC se names mean nothing on the grid, but their hosts do mean.
elif elementType == 'StorageElement':
seHost = CSHelpers.getSEHost( elementName )
if not seHost:
return S_ERROR( 'No seHost for %s' % elementName )
elementName = seHost
return S_OK( ( element, elementName, hours ) )
def doNew( self, masterParams = None ):
'''
Gets the parameters to run, either from the master method or from its
own arguments.
For every elementName, unless it is given a list, in which case it contacts
the gocdb client. The server is not very stable, so in case of failure tries
a second time.
If there are downtimes, are recorded and then returned.
'''
if masterParams is not None:
element, elementNames = masterParams
hours = None
elementName = None
else:
params = self._prepareCommand()
if not params[ 'OK' ]:
return params
element, elementName, hours = params[ 'Value' ]
elementNames = [ elementName ]
startDate = datetime.utcnow() - timedelta( days = 2 )
try:
results = self.gClient.getStatus( element, elementName, startDate, 120 )
except urllib2.URLError:
try:
#Let's give it a second chance..
results = self.gClient.getStatus( element, elementName, startDate, 120 )
except urllib2.URLError, e:
return S_ERROR( e )
if not results[ 'OK' ]:
return results
results = results[ 'Value' ]
if results is None:
return S_OK( None )
uniformResult = []
# Humanize the results into a dictionary, not the most optimal, but readable
for downtime, downDic in results.items():
dt = {}
if element == 'Resource':
dt[ 'Name' ] = downDic[ 'HOSTNAME' ]
else:
dt[ 'Name' ] = downDic[ 'SITENAME' ]
if not dt[ 'Name' ] in elementNames:
continue
dt[ 'DowntimeID' ] = downtime
dt[ 'Element' ] = element
dt[ 'StartDate' ] = downDic[ 'FORMATED_START_DATE' ]
dt[ 'EndDate' ] = downDic[ 'FORMATED_END_DATE' ]
dt[ 'Severity' ] = downDic[ 'SEVERITY' ]
dt[ 'Description' ] = downDic[ 'DESCRIPTION' ].replace( '\'', '' )
dt[ 'Link' ] = downDic[ 'GOCDB_PORTAL_URL' ]
uniformResult.append( dt )
storeRes = self._storeCommand( uniformResult )
if not storeRes[ 'OK' ]:
return storeRes
# We return only one downtime, if its ongoind at dtDate
dtDate = datetime.now()
if hours:
dtDate = dtDate + timedelta( hours = hours )
result = None
for dt in uniformResult:
if ( dt[ 'StartDate' ] < str( dtDate ) ) and ( dt[ 'EndDate' ] > str( dtDate ) ):
result = dt
break
return S_OK( result )
class DowntimeCommand( Command ):
'''
Downtime "master" Command.
'''
def __init__( self, args = None, clients = None ):
super( DowntimeCommand, self ).__init__( args, clients )
if 'GOCDBClient' in self.apis:
self.gClient = self.apis[ 'GOCDBClient' ]
else:
self.gClient = GOCDBClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def _storeCommand( self, result ):
'''
Stores the results of doNew method on the database.
'''
for dt in result:
resQuery = self.rmClient.addOrModifyDowntimeCache(
downtimeID = dt[ 'DowntimeID' ],
element = dt[ 'Element' ],
name = dt[ 'Name' ],
startDate = dt[ 'StartDate' ],
endDate = dt[ 'EndDate' ],
severity = dt[ 'Severity' ],
description = dt[ 'Description' ],
link = dt[ 'Link' ],
gocdbServiceType = dt[ 'GOCDBServiceType' ] )
if not resQuery[ 'OK' ]:
return resQuery
return S_OK()
def _prepareCommand( self ):
'''
DowntimeCommand requires four arguments:
- name : <str>
- element : Site / Resource
- elementType: <str>
If the elements are Site(s), we need to get their GOCDB names. They may
not have, so we ignore them if they do not have.
'''
if 'name' not in self.args:
return S_ERROR( '"name" not found in self.args' )
elementName = self.args[ 'name' ]
if 'element' not in self.args:
return S_ERROR( '"element" not found in self.args' )
element = self.args[ 'element' ]
if 'elementType' not in self.args:
return S_ERROR( '"elementType" not found in self.args' )
elementType = self.args[ 'elementType' ]
if not element in [ 'Site', 'Resource' ]:
return S_ERROR( 'element is not Site nor Resource' )
hours = None
if 'hours' in self.args:
hours = self.args[ 'hours' ]
gocdbServiceType = None
# Transform DIRAC site names into GOCDB topics
if element == 'Site':
gocSite = getGOCSiteName( elementName )
if not gocSite[ 'OK' ]:
return gocSite
elementName = gocSite[ 'Value' ]
# The DIRAC se names mean nothing on the grid, but their hosts do mean.
elif elementType == 'StorageElement':
# We need to distinguish if it's tape or disk
if getStorageElementOptions( elementName )['Value']['TapeSE']:
gocdbServiceType = "srm"
elif getStorageElementOptions( elementName )['Value']['DiskSE']:
gocdbServiceType = "srm.nearline"
seHost = CSHelpers.getSEHost( elementName )
if not seHost:
return S_ERROR( 'No seHost for %s' % elementName )
elementName = seHost
return S_OK( ( element, elementName, hours, gocdbServiceType ) )
def doNew( self, masterParams = None ):
'''
Gets the parameters to run, either from the master method or from its
own arguments.
For every elementName, unless it is given a list, in which case it contacts
the gocdb client. The server is not very stable, so in case of failure tries
a second time.
If there are downtimes, are recorded and then returned.
'''
if masterParams is not None:
element, elementNames = masterParams
hours = None
elementName = None
gocdbServiceType = None
else:
params = self._prepareCommand()
if not params[ 'OK' ]:
return params
element, elementName, hours, gocdbServiceType = params[ 'Value' ]
elementNames = [ elementName ]
startDate = datetime.utcnow() - timedelta( days = 14 )
try:
results = self.gClient.getStatus( element, elementName, startDate, 120 )
except urllib2.URLError:
try:
#Let's give it a second chance..
results = self.gClient.getStatus( element, elementName, startDate, 120 )
except urllib2.URLError, e:
return S_ERROR( e )
if not results[ 'OK' ]:
return results
results = results[ 'Value' ]
if results is None:
return S_OK( None )
uniformResult = []
# Humanize the results into a dictionary, not the most optimal, but readable
for downtime, downDic in results.items():
dt = {}
if gocdbServiceType and downDic[ 'SERVICE_TYPE' ]:
if gocdbServiceType.lower() != downDic[ 'SERVICE_TYPE' ].lower():
continue
if element == 'Resource':
dt[ 'Name' ] = downDic[ 'HOSTNAME' ]
else:
dt[ 'Name' ] = downDic[ 'SITENAME' ]
if not dt[ 'Name' ] in elementNames:
continue
dt[ 'DowntimeID' ] = downtime
dt[ 'Element' ] = element
dt[ 'StartDate' ] = downDic[ 'FORMATED_START_DATE' ]
dt[ 'EndDate' ] = downDic[ 'FORMATED_END_DATE' ]
dt[ 'Severity' ] = downDic[ 'SEVERITY' ]
dt[ 'Description' ] = downDic[ 'DESCRIPTION' ].replace( '\'', '' )
dt[ 'Link' ] = downDic[ 'GOCDB_PORTAL_URL' ]
try:
dt[ 'GOCDBServiceType' ] = downDic[ 'SERVICE_TYPE' ]
except KeyError:
# SERVICE_TYPE is not always defined
pass
uniformResult.append( dt )
storeRes = self._storeCommand( uniformResult )
if not storeRes[ 'OK' ]:
return storeRes
# We return only one downtime, if its ongoing at dtDate
startDate = datetime.utcnow()
if hours:
startDate = startDate + timedelta( hours = hours )
endDate = startDate
result = None
dtOutages = []
dtWarnings = []
for dt in uniformResult:
if ( dt[ 'StartDate' ] < str( startDate ) ) and ( dt[ 'EndDate' ] > str( endDate ) ):
if dt[ 'Severity' ] == 'Outage':
dtOutages.append( dt )
else:
dtWarnings.append( dt )
#In case many overlapping downtimes have been declared, the first one in
#severity and then time order will be selected. We want to get the latest one
#( they are sorted by insertion time )
if len( dtOutages ) > 0:
result = dtOutages[-1]
elif len( dtWarnings ) > 0:
result = dtWarnings[-1]
return S_OK( result )
class CachedPlotCommand( Command ):
def __init__( self, args = None, clients = None ):
super( CachedPlotCommand, self ).__init__( args, clients )
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def doCommand( self ):
"""
Returns transfer quality plot as it is cached in the accounting cache.
:attr:`args`:
- args[0]: string - should be a ValidElement
- args[1]: string - should be the name of the ValidElement
- args[2]: string - should be the plot type
- args[3]: string - should be the plot name
:returns:
a plot
"""
if not 'element' in self.args:
return S_ERROR( 'element no specified' )
element = self.args[ 'element' ]
if not 'name' in self.args:
return S_ERROR( 'Name no specified' )
name = self.args[ 'name' ]
if not 'plotType' in self.args:
return S_ERROR( 'plotType no specified' )
plotType = self.args[ 'plotType' ]
if not 'plotName' in self.args:
return S_ERROR( 'plotName no specified' )
plotName = self.args[ 'plotName' ]
#FIXME: we have no any longer Service granularity !
if element == 'Service':
name = name.split('@')[1]
meta = { 'columns' : 'Result' }
results = self.rmClient.selectAccountingCache( name = name, plotType = plotType,
plotName = plotName, meta = meta )
if not results[ 'OK' ]:
return results
results = results[ 'Value' ]
if results == []:
results = { 'data' : {}, 'granularity' : 900 }
else:
#FIXME: WTH is an eval doing here !!!!
results = eval( results[0] )
return results
