def setUp(self):
self.mockRSS = mock.MagicMock()
self.GOCCli = GOCDBClient()
# self.SAMCli = SAMResultsClient()
self.GGUSCli = GGUSTicketsClient()
def initialize(self):
""" Define the commands to be executed, and instantiate the clients that will be used.
"""
res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient',
'ResourceStatusClient')
if not res['OK']:
self.log.error('Failed to load ResourceStatusClient class: %s' % res['Message'])
return res
rsClass = res['Value']
res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient',
'ResourceManagementClient')
if not res['OK']:
self.log.error('Failed to load ResourceManagementClient class: %s' % res['Message'])
return res
rmClass = res['Value']
self.commands['Downtime'] = [{'Downtime': {}}]
self.commands['GOCDBSync'] = [{'GOCDBSync': {}}]
self.commands['FreeDiskSpace'] = [{'FreeDiskSpace': {}}]
# PilotsCommand
# self.commands[ 'Pilots' ] = [
# { 'PilotsWMS' : { 'element' : 'Site', 'siteName' : None } },
# { 'PilotsWMS' : { 'element' : 'Resource', 'siteName' : None } }
# ]
# FIXME: do not forget about hourly vs Always ...etc
# AccountingCacheCommand
# self.commands[ 'AccountingCache' ] = [
# {'SuccessfullJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'FailedJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'SuccessfullPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'SuccessfullPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'RunningJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :168, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :720, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :8760, 'plotType' :'Job' }},
# ]
# VOBOXAvailability
# self.commands[ 'VOBOXAvailability' ] = [
# { 'VOBOXAvailability' : {} }
#
# Reuse clients for the commands
self.clients['GOCDBClient'] = GOCDBClient()
self.clients['ReportsClient'] = ReportsClient()
self.clients['ResourceStatusClient'] = rsClass()
self.clients['ResourceManagementClient'] = rmClass()
self.clients['WMSAdministrator'] = WMSAdministratorClient()
self.clients['Pilots'] = PilotManagerClient()
self.cCaller = CommandCaller
return S_OK()
def initialize(self):
"""Run at the agent initialization (normally every 500 cycles)"""
# client to connect to GOCDB
self.GOCDBClient = GOCDBClient()
self.dryRun = self.am_getOption("DryRun", self.dryRun)
# API needed to update configuration stored by CS
self.csAPI = CSAPI()
return self.csAPI.initialize()
def __init__(self, rsDBin=None):
self.rsDB = rsDBin
if self.rsDB == None:
from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
self.rsDB = ResourceStatusDB()
self.GOCDBClient = GOCDBClient()
def setUp(self):
from DIRAC.Core.Base.Script import parseCommandLine
parseCommandLine()
self.mockRSS = Mock()
self.GOCCli = GOCDBClient()
self.SLSCli = SLSClient()
self.SAMCli = SAMResultsClient()
self.GGUSCli = GGUSTicketsClient()
def doCommand(self, resources=None):
"""
Returns downtimes information for all the resources in input.
:params:
:attr:`sites`: list of resource names (when not given, take every resource)
:returns:
{'ResourceName': {'SEVERITY': 'OUTAGE'|'AT_RISK',
'StartDate': 'aDate', ...} ... }
"""
if self.client is None:
from DIRAC.Core.LCG.GOCDBClient import GOCDBClient
self.client = GOCDBClient()
if resources is None:
# from DIRAC.Core.DISET.RPCClient import RPCClient
RPC = RPCClient("ResourceStatus/ResourceStatus")
resources = RPC.getResourcesList()
if not resources['OK']:
raise RSSException, where(
self, self.doCommand) + " " + resources['Message']
else:
resources = resources['Value']
try:
res = self.client.getStatus('Resource', resources, None, 120)
except:
gLogger.exception("Exception when calling GOCDBClient.")
return {}
if not res['OK']:
raise RSSException, where(self,
self.doCommand) + " " + res['Message']
else:
res = res['Value']
if res == None:
return {}
resToReturn = {}
for dt_ID in res:
dt = {}
dt['ID'] = dt_ID
dt['StartDate'] = res[dt_ID]['FORMATED_START_DATE']
dt['EndDate'] = res[dt_ID]['FORMATED_END_DATE']
dt['Severity'] = res[dt_ID]['SEVERITY']
dt['Description'] = res[dt_ID]['DESCRIPTION'].replace('\'', '')
dt['Link'] = res[dt_ID]['GOCDB_PORTAL_URL']
resToReturn[dt_ID] = dt
return resToReturn
def __init__(self, rsClient=None, rmClient=None):
self.GOCDBClient = GOCDBClient()
self.rsClient = ResourceStatusClient(
) if rsClient == None else rsClient
self.rmClient = ResourceManagementClient(
) if rmClient == None else rmClient
self.synclist = [
'Sites', 'Resources', 'StorageElements', 'Services',
'RegistryUsers'
]
def initialize(self):
""" Define the commands to be executed, and instantiate the clients that will be used.
"""
self.am_setOption('shifterProxy', 'DataManager')
self.rmClient = ResourceManagementClient()
self.commands['Downtime'] = [{'Downtime': {}}]
self.commands['SpaceTokenOccupancy'] = [{'SpaceTokenOccupancy': {}}]
self.commands['GOCDBSync'] = [{'GOCDBSync': {}}]
self.commands['FreeDiskSpace'] = [{'FreeDiskSpace': {}}]
# PilotsCommand
# self.commands[ 'Pilots' ] = [
# { 'PilotsWMS' : { 'element' : 'Site', 'siteName' : None } },
# { 'PilotsWMS' : { 'element' : 'Resource', 'siteName' : None } }
# ]
# FIXME: do not forget about hourly vs Always ...etc
# AccountingCacheCommand
# self.commands[ 'AccountingCache' ] = [
# {'SuccessfullJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'FailedJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'SuccessfullPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'SuccessfullPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'RunningJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :168, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :720, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :8760, 'plotType' :'Job' }},
# ]
# VOBOXAvailability
# self.commands[ 'VOBOXAvailability' ] = [
# { 'VOBOXAvailability' : {} }
#
# Reuse clients for the commands
self.clients['GOCDBClient'] = GOCDBClient()
self.clients['ReportGenerator'] = RPCClient(
'Accounting/ReportGenerator')
self.clients['ReportsClient'] = ReportsClient()
self.clients['ResourceStatusClient'] = ResourceStatusClient()
self.clients['ResourceManagementClient'] = ResourceManagementClient()
self.clients['WMSAdministrator'] = RPCClient(
'WorkloadManagement/WMSAdministrator')
self.cCaller = CommandCaller
return S_OK()
def __init__(self, args=None, clients=None):
super(DowntimeCommand, self).__init__(args, clients)
if 'GOCDBClient' in self.apis:
self.gClient = self.apis['GOCDBClient']
else:
self.gClient = GOCDBClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(GOCDBSyncCommand, self).__init__(args, clients)
if 'GOCDBClient' in self.apis:
self.gClient = self.apis['GOCDBClient']
else:
self.gClient = GOCDBClient()
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
self.seenHostnames = set()
def initialize(self):
self.am_setOption('shifterProxy', 'DataManager')
self.rmClient = ResourceManagementIHEPClient()
self.commands['JobIHEP'] = [{'JobIHEP': {}}]
self.commands['StorageIHEP'] = [{'StorageIHEP': {}}]
self.commands['WorkNodeIHEP'] = [{'WorkNodeIHEP': {}}]
# Reuse clients for the commands
self.clients['GOCDBClient'] = GOCDBClient()
self.clients['ReportGenerator'] = RPCClient(
'Accounting/ReportGenerator')
self.clients['ReportsClient'] = ReportsClient()
self.clients['ResourceStatusClient'] = ResourceStatusClient()
self.clients[
'ResourceManagementIHEPClient'] = ResourceManagementIHEPClient()
self.clients['WMSAdministrator'] = RPCClient(
'WorkloadManagement/WMSAdministrator')
self.cCaller = CommandCaller
return S_OK()
""" Few unit tests for LCG clients """ __RCSID__ = "$Id$" import mock from datetime import datetime, timedelta from DIRAC.Core.LCG.GOCDBClient import GOCDBClient mockRSS = mock.MagicMock() GOCCli = GOCDBClient() # data now = datetime.utcnow().replace(microsecond=0, second=0) tomorrow = datetime.utcnow().replace(microsecond=0, second=0) + timedelta(hours=24) inAWeek = datetime.utcnow().replace(microsecond=0, second=0) + timedelta(days=7) nowLess12h = str(now - timedelta(hours=12))[:-3] nowPlus8h = str(now + timedelta(hours=8))[:-3] nowPlus24h = str(now + timedelta(hours=24))[:-3] nowPlus40h = str(now + timedelta(hours=40))[:-3] nowPlus50h = str(now + timedelta(hours=50))[:-3] nowPlus60h = str(now + timedelta(hours=60))[:-3] XML_site_ongoing = '<?xml version="1.0"?>\n<ROOT><DOWNTIME ID="78505456" PRIMARY_KEY="28490G0"' XML_site_ongoing += ' CLASSIFICATION="SCHEDULED"><SITENAME>GRISU-ENEA-GRID</SITENAME>' XML_site_ongoing += '<SEVERITY>OUTAGE</SEVERITY>' XML_site_ongoing += '<DESCRIPTION>Software problems SITE</DESCRIPTION>' XML_site_ongoing += '<INSERT_DATE>1276273965</INSERT_DATE>'
def doCommand(self, sites=None):
"""
Returns downtimes information for all the sites in input.
:params:
:attr:`sites`: list of site names (when not given, take every site)
:returns:
{'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK',
'StartDate': 'aDate', ...} ... }
"""
if self.client is None:
from DIRAC.Core.LCG.GOCDBClient import GOCDBClient
self.client = GOCDBClient()
if sites is None:
# from DIRAC.Core.DISET.RPCClient import RPCClient
RPC = RPCClient("ResourceStatus/ResourceStatus")
GOC_sites = RPC.getGridSitesList()
if not GOC_sites['OK']:
raise RSSException, where(
self, self.doCommand) + " " + sites['Message']
else:
GOC_sites = GOC_sites['Value']
else:
GOC_sites = [getGOCSiteName(x)['Value'] for x in sites]
try:
res = self.client.getStatus('Site', GOC_sites, None, 120)
except:
gLogger.exception("Exception when calling GOCDBClient.")
return {}
if not res['OK']:
raise RSSException, where(self,
self.doCommand) + " " + res['Message']
else:
res = res['Value']
if res == None:
return {}
resToReturn = {}
for dt_ID in res:
try:
dt = {}
dt['ID'] = dt_ID
dt['StartDate'] = res[dt_ID]['FORMATED_START_DATE']
dt['EndDate'] = res[dt_ID]['FORMATED_END_DATE']
dt['Severity'] = res[dt_ID]['SEVERITY']
dt['Description'] = res[dt_ID]['DESCRIPTION'].replace('\'', '')
dt['Link'] = res[dt_ID]['GOCDB_PORTAL_URL']
DIRACnames = getDIRACSiteName(res[dt_ID]['SITENAME'])
if not DIRACnames['OK']:
raise RSSException, DIRACnames['Message']
DIRACnames = DIRACnames['Value']
for DIRACname in DIRACnames:
resToReturn[dt_ID.split()[0] + ' ' + DIRACname] = dt
except KeyError:
continue
return resToReturn
def setUp(self):
self.mockRSS = mock.MagicMock()
self.GOCCli = GOCDBClient()
def doCommand(self):
"""
Return getStatus from GOC DB Client
:attr:`args`:
- args[0]: string: should be a ValidRes
- args[1]: string: should be the name of the ValidRes
- args[2]: string: optional, number of hours in which
the down time is starting
"""
super(GOCDBStatus_Command, self).doCommand()
if self.client is None:
# use standard GOC DB Client
from DIRAC.Core.LCG.GOCDBClient import GOCDBClient
self.client = GOCDBClient()
granularity = self.args[0]
name = self.args[1]
try:
hours = self.args[2]
except IndexError:
hours = None
if granularity in ('Site', 'Sites'):
name = getGOCSiteName(name)
if not name['OK']:
raise RSSException, name['Message']
name = name['Value']
try:
res = self.client.getStatus(granularity, name, None, hours,
self.timeout)
if not res['OK']:
return {'Result': 'Unknown'}
res = res['Value']
if res is None or res == {}:
return {'Result': {'DT': None}}
DT_dict_result = {}
now = datetime.datetime.utcnow().replace(microsecond=0, second=0)
if len(res) > 1:
#there's more than one DT
for dt_ID in res:
#looking for an ongoing one
startSTR = res[dt_ID]['FORMATED_START_DATE']
start_datetime = datetime.datetime(
*time.strptime(startSTR, "%Y-%m-%d %H:%M")[0:5])
if start_datetime < now:
resDT = res[dt_ID]
break
try:
resDT
except:
#if I'm here, there's no OnGoing DT
resDT = res[res.keys()[0]]
res = resDT
else:
res = res[res.keys()[0]]
DT_dict_result['DT'] = res['SEVERITY']
DT_dict_result['EndDate'] = res['FORMATED_END_DATE']
startSTR = res['FORMATED_START_DATE']
start_datetime = datetime.datetime(
*time.strptime(startSTR, "%Y-%m-%d %H:%M")[0:5])
if start_datetime > now:
diff = convertTime(start_datetime - now, 'hours')
DT_dict_result['DT'] = DT_dict_result['DT'] + " in " + str(
diff) + ' hours'
return {'Result': DT_dict_result}
except urllib2.URLError:
gLogger.error("GOCDB timed out for " + granularity + " " + name)
return {'Result': 'Unknown'}
except:
gLogger.exception("Exception when calling GOCDBClient for " +
granularity + " " + name)
return {'Result': 'Unknown'}
def initialize(self):
"""Define the commands to be executed, and instantiate the clients that will be used."""
res = ObjectLoader().loadObject(
"DIRAC.ResourceStatusSystem.Client.ResourceStatusClient")
if not res["OK"]:
self.log.error("Failed to load ResourceStatusClient class: %s" %
res["Message"])
return res
rsClass = res["Value"]
res = ObjectLoader().loadObject(
"DIRAC.ResourceStatusSystem.Client.ResourceManagementClient")
if not res["OK"]:
self.log.error(
"Failed to load ResourceManagementClient class: %s" %
res["Message"])
return res
rmClass = res["Value"]
self.commands["Downtime"] = [{"Downtime": {}}]
self.commands["GOCDBSync"] = [{"GOCDBSync": {}}]
self.commands["FreeDiskSpace"] = [{"FreeDiskSpace": {}}]
# PilotsCommand
self.commands["Pilot"] = [
{
"Pilot": {
"element": "Site",
"siteName": None
}
},
{
"Pilot": {
"element": "Resource",
"siteName": None
}
},
]
# FIXME: do not forget about hourly vs Always ...etc
# AccountingCacheCommand
# self.commands[ 'AccountingCache' ] = [
# {'SuccessfullJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'FailedJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'SuccessfullPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'SuccessfullPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'RunningJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :168, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :720, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :8760, 'plotType' :'Job' }},
# ]
# VOBOXAvailability
# self.commands[ 'VOBOXAvailability' ] = [
# { 'VOBOXAvailability' : {} }
#
# Reuse clients for the commands
self.clients["GOCDBClient"] = GOCDBClient()
self.clients["ReportsClient"] = ReportsClient()
self.clients["ResourceStatusClient"] = rsClass()
self.clients["ResourceManagementClient"] = rmClass()
self.clients["WMSAdministrator"] = WMSAdministratorClient()
self.clients["Pilots"] = PilotManagerClient()
self.cCaller = CommandCaller
return S_OK()
