def doCommand(self): """ Returns transfer quality as it is cached :attr:`args`: - args[0]: string: should be a ValidRes - args[1]: string should be the name of the ValidRes :returns: {'Result': None | a float between 0.0 and 100.0} """ super(TransferQualityCached_Command, self).doCommand() if self.client is None: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient self.client = ResourceManagementClient(timeout=self.timeout) name = self.args[1] try: res = self.client.getCachedResult(name, 'TransferQualityEverySEs', 'TQ', 'NULL') if res == []: return {'Result': None} except: gLogger.exception( "Exception when calling ResourceManagementClient for %s" % (name)) return {'Result': 'Unknown'} return {'Result': float(res[0])}
class PilotsEffSimpleCached_Command(Command): def doCommand(self): """ Returns simple pilots efficiency :attr:`args`: - args[0]: string: should be a ValidRes - args[1]: string should be the name of the ValidRes returns: { 'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad' } """ super(PilotsEffSimpleCached_Command, self).doCommand() client = self.client if client is None: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient self.client = ResourceStatusClient(timeout=self.timeout) if self.args[0] in ('Service', 'Services'): try: name = self.client.getGeneralName(self.args[0], self.args[1], 'Site')['Value'][0] except: gLogger.error( "PilotsEffSimpleCached_Command: can't get a general name for %s %s" % (self.args[0], self.args[1])) return {'Result': 'Unknown'} granularity = 'Site' elif self.args[0] in ('Site', 'Sites'): name = self.args[1] granularity = self.args[0] else: raise InvalidRes, where(self, self.doCommand) try: if client is None: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient self.client = ResourceManagementClient(timeout=self.timeout) res = self.client.getCachedResult(name, 'PilotsEffSimpleEverySites', 'PE_S', 'NULL')['Value'] if res == None: return {'Result': 'Idle'} if res == []: return {'Result': 'Idle'} except: gLogger.exception( "Exception when calling ResourceManagementClient for %s %s" % (granularity, name)) return {'Result': 'Unknown'} return {'Result': res[0]} doCommand.__doc__ = Command.doCommand.__doc__ + doCommand.__doc__
def __init__(self, args=None, clients=None): super(SpaceTokenOccupancyCommand, self).__init__(args, clients) if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None): super(CachedPlotCommand, self).__init__(args, clients) if "ResourceManagementClient" in self.apis: self.rmClient = self.apis["ResourceManagementClient"] else: self.rmClient = ResourceManagementClient()
def __init__( self, args = None, clients = None ): super( TransferQualityFromCachedPlotCommand, self ).__init__( args, clients ) if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient()
def __init__(self, am): super(SpaceTokenOccupancyTest, self).__init__(am) self.xmlPath = rootPath + "/" + self.getAgentOption( "webRoot") + self.getTestOption("dir") self.rmClient = ResourceManagementClient() mkDir(self.xmlPath) self.generate_xml_and_dashboard()
class PilotsEffSimpleCached_Command(Command): def doCommand(self): """ Returns simple pilots efficiency :attr:`args`: - args[0]: string: should be a ValidRes - args[1]: string should be the name of the ValidRes returns: { 'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad' } """ super(PilotsEffSimpleCached_Command, self).doCommand() client = self.client if client is None: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient self.client = ResourceStatusClient(timeout = self.timeout) if self.args[0] in ('Service', 'Services'): try: name = self.client.getGeneralName(self.args[0], self.args[1], 'Site')[0] except: gLogger.error("PilotsEffSimpleCached_Command: can't get a general name for %s %s" %(self.args[0], self.args[1])) return {'Result':'Unknown'} granularity = 'Site' elif self.args[0] in ('Site', 'Sites'): name = self.args[1] granularity = self.args[0] else: raise InvalidRes, where(self, self.doCommand) try: if client is None: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient self.client = ResourceManagementClient(timeout = self.timeout) res = self.client.getCachedResult(name, 'PilotsEffSimpleEverySites', 'PE_S', 'NULL') if res == None: return {'Result':'Idle'} if res == []: return {'Result':'Idle'} except: gLogger.exception("Exception when calling ResourceManagementClient for %s %s" %(granularity, name)) return {'Result':'Unknown'} return {'Result':res[0]} doCommand.__doc__ = Command.doCommand.__doc__ + doCommand.__doc__
def __init__(self, rsClient=None, rmClient=None): self.GOCDBClient = GOCDBClient() self.rsClient = ResourceStatusClient( ) if rsClient == None else rsClient self.rmClient = ResourceManagementClient( ) if rmClient == None else rmClient self.synclist = [ 'Sites', 'Resources', 'StorageElements', 'Services', 'RegistryUsers' ]
def __init__(self, granularity, name, status_type, pdp_decision, **kw): ActionBase.__init__(self, granularity, name, status_type, pdp_decision, **kw) try: self.rsClient = self.kw["Clients"]['ResourceStatusClient'] except KeyError: self.rsClient = ResourceStatusClient() try: self.rmClient = self.kw["Clients"]['ResourceManagementClient'] except KeyError: self.rmClient = ResourceManagementClient()
def initialize( self ): self.am_setOption( 'shifterProxy', 'DataManager' ) self.rmClient = ResourceManagementClient() self.commands[ 'Downtime' ] = [ { 'Downtime' : {} } ] self.commands[ 'SpaceTokenOccupancy' ] = [ { 'SpaceTokenOccupancy' : {} } ] self.commands[ 'Pilot' ] = [ { 'Pilot' : { 'timespan' : 1800 } },] # { 'Pilot' : { 'timespan' : 86400 } }, # { 'Pilot' : { 'timespan' : 604800 } }] #PilotsCommand # self.commands[ 'Pilots' ] = [ # { 'PilotsWMS' : { 'element' : 'Site', 'siteName' : None } }, # { 'PilotsWMS' : { 'element' : 'Resource', 'siteName' : None } } # ] #FIXME: do not forget about hourly vs Always ...etc #AccountingCacheCommand # self.commands[ 'AccountingCache' ] = [ # {'SuccessfullJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }}, # {'FailedJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }}, # {'SuccessfullPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }}, # {'FailedPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }}, # {'SuccessfullPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }}, # {'FailedPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }}, # {'RunningJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }}, ## {'RunningJobsBySiteSplitted' :{'hours' :168, 'plotType' :'Job' }}, ## {'RunningJobsBySiteSplitted' :{'hours' :720, 'plotType' :'Job' }}, ## {'RunningJobsBySiteSplitted' :{'hours' :8760, 'plotType' :'Job' }}, # ] #VOBOXAvailability # self.commands[ 'VOBOXAvailability' ] = [ # { 'VOBOXAvailability' : {} } # #Reuse clients for the commands self.clients[ 'GOCDBClient' ] = GOCDBClient() self.clients[ 'ReportGenerator' ] = RPCClient( 'Accounting/ReportGenerator' ) self.clients[ 'ReportsClient' ] = ReportsClient() self.clients[ 'ResourceStatusClient' ] = ResourceStatusClient() self.clients[ 'ResourceManagementClient' ] = ResourceManagementClient() self.clients[ 'PilotsDB' ] = PilotAgentsDB() self.clients[ 'WMSAdministrator' ] = RPCClient( 'WorkloadManagement/WMSAdministrator' ) self.cCaller = CommandCaller return S_OK()
def __init__(self, args=None, clients=None): super(JobsEffSimpleCachedCommand, self).__init__(args, clients) if 'ResourceStatusClient' in self.apis: self.rsClient = self.apis['ResourceStatusClient'] else: self.rsClient = ResourceStatusClient() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None): super(JobCommand, self).__init__(args, clients) if 'JobDB' in self.apis: self.jobDB = self.apis['JobDB'] else: self.jobDB = JobDB() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None): super(PilotCommand, self).__init__(args, clients) if "Pilots" in self.apis: self.pilots = self.apis["Pilots"] else: self.pilots = PilotManagerClient() if "ResourceManagementClient" in self.apis: self.rmClient = self.apis["ResourceManagementClient"] else: self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None): super(PilotCommand, self).__init__(args, clients) if 'Pilots' in self.apis: self.pilots = self.apis['Pilots'] else: self.pilots = PilotManagerClient() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None): super(DowntimeCommand, self).__init__(args, clients) if 'GOCDBClient' in self.apis: self.gClient = self.apis['GOCDBClient'] else: self.gClient = GOCDBClient() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None): super(TransferCommand, self).__init__(args, clients) if "ReportsClient" in self.apis: self.rClient = self.apis["ReportsClient"] else: self.rClient = ReportsClient() if "ResourceManagementClient" in self.apis: self.rmClient = self.apis["ResourceManagementClient"] else: self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None): super(GGUSTicketsCommand, self).__init__(args, clients) if 'GGUSTicketsClient' in self.apis: self.gClient = self.apis['GGUSTicketsClient'] else: self.gClient = GGUSTicketsClient() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None): super(JobCommand, self).__init__(args, clients) if 'WMSAdministrator' in self.apis: self.wmsAdmin = self.apis['WMSAdministrator'] else: self.wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator') if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None): super(JobCommand, self).__init__(args, clients) if "WMSAdministrator" in self.apis: self.wmsAdmin = self.apis["WMSAdministrator"] else: self.wmsAdmin = WMSAdministratorClient() if "ResourceManagementClient" in self.apis: self.rmClient = self.apis["ResourceManagementClient"] else: self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None): super(PilotCommand, self).__init__(args, clients) if 'WMSAdministrator' in self.apis: self.wmsAdmin = self.apis['WMSAdministrator'] else: self.wmsAdmin = WMSAdministratorClient() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient()
def doCommand(self): """ Returns transfer quality from the plot cached in the accounting cache. :attr:`args`: - args[0]: string: should be a ValidRes - args[1]: string should be the name of the ValidRes :returns: {'Result': None | a float between 0.0 and 100.0} """ super(TransferQualityFromCachedPlot_Command, self).doCommand() if self.client is None: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient self.client = ResourceManagementClient(timeout=self.timeout) granularity = self.args[0] name = self.args[1] plotType = self.args[2] plotName = self.args[3] try: res = self.client.getCachedAccountingResult( name, plotType, plotName) res = res['Value'] if res == []: return {'Result': None} res = eval(res[0]) s = 0 n = 0 try: SE = res['data'].keys()[0] except IndexError: return {'Result': None} n = n + len(res['data'][SE]) s = s + sum(res['data'][SE].values()) meanQuality = s / n except: gLogger.exception( "Exception when calling ResourcePolicyClient for %s" % (name)) return {'Result': 'Unknown'} return {'Result': meanQuality}
def __init__(self, args=None, clients=None): super(GOCDBSyncCommand, self).__init__(args, clients) if 'GOCDBClient' in self.apis: self.gClient = self.apis['GOCDBClient'] else: self.gClient = GOCDBClient() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient() self.seenHostnames = set()
def __init__(self, name, decissionParams, enforcementResult, singlePolicyResults, clients=None): super(LogPolicyResultAction, self).__init__(name, decissionParams, enforcementResult, singlePolicyResults, clients) if clients is not None and 'ResourceManagementClient' in clients: self.rmClient = clients['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient()
class TransferQualityFromCachedPlotCommand(Command): def __init__(self, args=None, clients=None): super(TransferQualityFromCachedPlotCommand, self).__init__(args, clients) if "ResourceManagementClient" in self.apis: self.rmClient = self.apis["ResourceManagementClient"] else: self.rmClient = ResourceManagementClient() def doCommand(self): """ Returns transfer quality from the plot cached in the accounting cache. :attr:`args`: - args[0]: string: should be a ValidElement - args[1]: string should be the name of the ValidElement :returns: {'Result': None | a float between 0.0 and 100.0} """ if "name" not in self.args: return S_ERROR("Name no specified") name = self.args["name"] if "plotType" not in self.args: return S_ERROR("plotType no specified") plotType = self.args["plotType"] if "plotName" not in self.args: return S_ERROR("plotName no specified") plotName = self.args["plotName"] meta = {"columns": "Result"} results = self.rmClient.selectAccountingCache(name=name, plotType=plotType, plotName=plotName, meta=meta) if not results["OK"]: return results results = results["Value"] if results == []: results = None else: # FIXME: remove the eval from here !! results = eval(results[0][0]) num, den = 0, 0 se = results["data"].keys()[0] num = num + len(results["data"][se]) den = den + sum(results["data"][se].values()) meanQuality = den / num results = meanQuality return S_OK(results)
def __init__(self, granularity, name, status_type, pdp_decision, **kw): ActionBase.__init__( self, granularity, name, status_type, pdp_decision, **kw ) try: self.rsClient = self.kw["Clients"][ 'ResourceStatusClient' ] except KeyError: self.rsClient = ResourceStatusClient() try: self.rmClient = self.kw["Clients"][ 'ResourceManagementClient' ] except KeyError: self.rmClient = ResourceManagementClient()
def __init__(self, clients=None): """ Constructor examples: >>> pep = PEP() >>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } ) >>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } ) :Parameters: **clients** - [ None, `dict` ] dictionary with clients to be used in the commands issued by the policies. If not defined, the commands will import them. It is a measure to avoid opening the same connection every time a policy is evaluated. """ if clients is None: clients = {} # PEP uses internally two of the clients: ResourceStatusClient and ResouceManagementClient if 'ResourceStatusClient' in clients: self.rsClient = clients['ResourceStatusClient'] else: self.rsClient = ResourceStatusClient() if 'ResourceManagementClient' in clients: self.rmClient = clients['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient() self.clients = clients # Pass to the PDP the clients that are going to be used on the Commands self.pdp = PDP(clients)
def __init__( self, rsClient = None, rmClient = None ): self.GOCDBClient = GOCDBClient() self.rsClient = ResourceStatusClient() if rsClient == None else rsClient self.rmClient = ResourceManagementClient() if rmClient == None else rmClient self.synclist = [ 'Sites', 'Resources', 'StorageElements', 'Services', 'RegistryUsers' ]
class TransferQualityFromCachedPlotCommand(Command): def __init__(self, args=None, clients=None): super(TransferQualityFromCachedPlotCommand, self).__init__(args, clients) if "ResourceManagementClient" in self.apis: self.rmClient = self.apis["ResourceManagementClient"] else: self.rmClient = ResourceManagementClient() def doCommand(self): """ Returns transfer quality from the plot cached in the accounting cache. :attr:`args`: - args[0]: string: should be a ValidElement - args[1]: string should be the name of the ValidElement :returns: {'Result': None | a float between 0.0 and 100.0} """ if "name" not in self.args: return S_ERROR("Name no specified") name = self.args["name"] if "plotType" not in self.args: return S_ERROR("plotType no specified") plotType = self.args["plotType"] if "plotName" not in self.args: return S_ERROR("plotName no specified") plotName = self.args["plotName"] meta = {"columns": "Result"} results = self.rmClient.selectAccountingCache(name=name, plotType=plotType, plotName=plotName, meta=meta) if not results["OK"]: return results results = results["Value"] if results == []: results = None else: # FIXME: remove the eval from here !! results = eval(results[0][0]) num, den = 0, 0 se = list(results["data"])[0] num = num + len(results["data"][se]) den = den + sum(results["data"][se].values()) meanQuality = den / num results = meanQuality return S_OK(results)
def doCommand(self): """ Returns transfer quality as it is cached :attr:`args`: - args[0]: string: should be a ValidRes - args[1]: string should be the name of the ValidRes :returns: {'Result': None | a float between 0.0 and 100.0} """ super(TransferQualityCached_Command, self).doCommand() if self.client is None: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient self.client = ResourceManagementClient(timeout = self.timeout) name = self.args[1] try: res = self.client.getCachedResult(name, 'TransferQualityEverySEs', 'TQ', 'NULL') if res == []: return {'Result':None} except: gLogger.exception("Exception when calling ResourceManagementClient for %s" %(name)) return {'Result':'Unknown'} return {'Result':float(res[0])}
def __init__(self, args=None, clients=None): super(CachedPlotCommand, self).__init__(args, clients) if "ResourceManagementClient" in self.apis: self.rmClient = self.apis["ResourceManagementClient"] else: self.rmClient = ResourceManagementClient()
def setUp( self ): self.mockRSS = Mock() self.RSCli = ResourceStatusClient( serviceIn = self.mockRSS ) self.RMCli = ResourceManagementClient( serviceIn = self.mockRSS ) self.PilotsCli = PilotsClient() self.JobsCli = JobsClient()
def __init__( self, args = None, clients = None ): super( SpaceTokenOccupancyCommand, self ).__init__( args, clients ) if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient()
def __init__( self, name, decissionParams, enforcementResult, singlePolicyResults, clients = None ): super( LogPolicyResultAction, self ).__init__( name, decissionParams, enforcementResult, singlePolicyResults, clients ) if clients is not None and 'ResourceManagementClient' in clients: self.rmClient = clients[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient()
def initialize(self): ''' TokenAgent initialization ''' # Attribute defined outside __init__ # pylint: disable-msg=W0201 self.notifyHours = self.am_getOption('notifyHours', 10) try: self.rsClient = ResourceStatusClient() self.rmClient = ResourceManagementClient() self.noClient = NotificationClient() return S_OK() except Exception: errorStr = "TokenAgent initialization" self.log.exception(errorStr) return S_ERROR(errorStr)
def initializePublisherHandler( _serviceInfo ): ''' Handler initialization in the usual horrible way. ''' global rsClient rsClient = ResourceStatusClient() global rmClient rmClient = ResourceManagementClient() return S_OK()
def __init__(self, args=None, clients=None): super(TransferCommand, self).__init__(args, clients) if 'ReportsClient' in self.apis: self.rClient = self.apis['ReportsClient'] else: self.rClient = ReportsClient() if 'ReportGenerator' in self.apis: self.rgClient = self.apis['ReportGenerator'] else: self.rgClient = RPCClient('Accounting/ReportGenerator') self.rClient.rpcClient = self.rgClient if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient()
def initialize(self): # Attribute defined outside __init__ # pylint: disable-msg=W0201 try: self.rsClient = ResourceStatusClient() self.rmClient = ResourceManagementClient() validElements = RssConfiguration.getValidElements() self.historyTables = ['%sHistory' % x for x in validElements] return S_OK() except Exception: errorStr = "CacheCleanerAgent initialization" self.log.exception(errorStr) return S_ERROR(errorStr)
def doCommand(self): """ Returns transfer quality plot as it is cached in the accounting cache. :attr:`args`: - args[0]: string - should be a ValidRes - args[1]: string - should be the name of the ValidRes - args[2]: string - should be the plot type - args[3]: string - should be the plot name :returns: a plot """ super(CachedPlot_Command, self).doCommand() if self.client is None: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient self.client = ResourceManagementClient(timeout=self.timeout) granularity = self.args[0] name = self.args[1] plotType = self.args[2] plotName = self.args[3] if granularity == 'Service': name = name.split('@')[1] try: res = self.client.getCachedAccountingResult( name, plotType, plotName) if res == []: return {'Result': {'data': {}, 'granularity': 900}} except: gLogger.exception( "Exception when calling ResourcePolicyClient for %s" % (name)) return {'Result': 'Unknown'} return {'Result': eval(res[0])}
class TransferQualityFromCachedPlot_Command(Command): def doCommand(self): """ Returns transfer quality from the plot cached in the accounting cache. :attr:`args`: - args[0]: string: should be a ValidRes - args[1]: string should be the name of the ValidRes :returns: {'Result': None | a float between 0.0 and 100.0} """ super(TransferQualityFromCachedPlot_Command, self).doCommand() if self.client is None: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient self.client = ResourceManagementClient(timeout = self.timeout) granularity = self.args[0] name = self.args[1] plotType = self.args[2] plotName = self.args[3] try: res = self.client.getCachedAccountingResult(name, plotType, plotName) res = res[ 'Value' ] if res == []: return {'Result':None} res = eval(res[0]) s = 0 n = 0 try: SE = res['data'].keys()[0] except IndexError: return {'Result':None} n = n + len(res['data'][SE]) s = s + sum(res['data'][SE].values()) meanQuality = s/n except: gLogger.exception("Exception when calling ResourcePolicyClient for %s" %(name)) return {'Result':'Unknown'} return {'Result':meanQuality} doCommand.__doc__ = Command.doCommand.__doc__ + doCommand.__doc__
def __init__( self, args = None, clients = None ): super( DowntimeCommand, self ).__init__( args, clients ) if 'GOCDBClient' in self.apis: self.gClient = self.apis[ 'GOCDBClient' ] else: self.gClient = GOCDBClient() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None): super(GGUSTicketsCommand, self).__init__(args, clients) if "GGUSTicketsClient" in self.apis: self.gClient = self.apis["GGUSTicketsClient"] else: self.gClient = GGUSTicketsClient() if "ResourceManagementClient" in self.apis: self.rmClient = self.apis["ResourceManagementClient"] else: self.rmClient = ResourceManagementClient()
def __init__( self, args = None, clients = None ): super( PilotCommand, self ).__init__( args, clients ) if 'WMSAdministrator' in self.apis: self.wmsAdmin = self.apis[ 'WMSAdministrator' ] else: self.wmsAdmin = RPCClient( 'WorkloadManagement/WMSAdministrator' ) if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient()
def __init__( self, args = None, clients = None ): super( JobCommand, self ).__init__( args, clients ) if 'JobDB' in self.apis: self.jobDB = self.apis[ 'JobDB' ] else: self.jobDB = JobDB() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient()
class CachedPlot_Command(Command): def doCommand(self): """ Returns transfer quality plot as it is cached in the accounting cache. :attr:`args`: - args[0]: string - should be a ValidRes - args[1]: string - should be the name of the ValidRes - args[2]: string - should be the plot type - args[3]: string - should be the plot name :returns: a plot """ super(CachedPlot_Command, self).doCommand() if self.client is None: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient self.client = ResourceManagementClient(timeout = self.timeout) granularity = self.args[0] name = self.args[1] plotType = self.args[2] plotName = self.args[3] if granularity == 'Service': name = name.split('@')[1] try: res = self.client.getCachedAccountingResult(name, plotType, plotName) if res == []: return {'Result':{'data':{}, 'granularity':900}} except: gLogger.exception("Exception when calling ResourcePolicyClient for %s" %(name)) return {'Result':'Unknown'} return {'Result':eval(res[0])} doCommand.__doc__ = Command.doCommand.__doc__ + doCommand.__doc__
def __init__( self, args = None, clients = None ): super( TransferCommand, self ).__init__( args, clients ) if 'ReportsClient' in self.apis: self.rClient = self.apis[ 'ReportsClient' ] else: self.rClient = ReportsClient() if 'ReportGenerator' in self.apis: self.rgClient = self.apis[ 'ReportGenerator' ] else: self.rgClient = RPCClient( 'Accounting/ReportGenerator' ) self.rClient.rpcClient = self.rgClient if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient()
def initialize( self ): # Attribute defined outside __init__ # pylint: disable-msg=W0201 try: self.rsClient = ResourceStatusClient() self.rmClient = ResourceManagementClient() validElements = RssConfiguration.getValidElements() self.historyTables = [ '%sHistory' % x for x in validElements ] return S_OK() except Exception: errorStr = "CacheCleanerAgent initialization" self.log.exception( errorStr ) return S_ERROR( errorStr )
def initialize( self ): ''' TokenAgent initialization ''' # Attribute defined outside __init__ # pylint: disable-msg=W0201 self.notifyHours = self.am_getOption( 'notifyHours', 10 ) try: self.rsClient = ResourceStatusClient() self.rmClient = ResourceManagementClient() self.noClient = NotificationClient() return S_OK() except Exception: errorStr = "TokenAgent initialization" self.log.exception( errorStr ) return S_ERROR( errorStr )
def __init__( self, args = None, clients = None ): """ Constructor. :Parameters: **args** - [, `dict` ] arguments to be passed to be used in the _prepareCommand method ( name and timespan are the expected ones ) **clients - [, `dict` ] clients from where information is fetched. Mainly used to avoid creating new connections on agents looping over clients. ResourceManagementClient and PilotsDB are most welcome. """ super( PilotCommand, self ).__init__( args, clients ) if 'PilotsDB' in self.apis: self.pilotsDB = self.apis[ 'PilotsDB' ] else: self.pilotsDB = PilotAgentsDB() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient()
class DowntimeCommand( Command ): ''' Downtime "master" Command. ''' def __init__( self, args = None, clients = None ): super( DowntimeCommand, self ).__init__( args, clients ) if 'GOCDBClient' in self.apis: self.gClient = self.apis[ 'GOCDBClient' ] else: self.gClient = GOCDBClient() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient() def _storeCommand( self, result ): ''' Stores the results of doNew method on the database. ''' for dt in result: resQuery = self.rmClient.addOrModifyDowntimeCache( downtimeID = dt[ 'DowntimeID' ], element = dt[ 'Element' ], name = dt[ 'Name' ], startDate = dt[ 'StartDate' ], endDate = dt[ 'EndDate' ], severity = dt[ 'Severity' ], description = dt[ 'Description' ], link = dt[ 'Link' ], gocdbServiceType = dt[ 'GOCDBServiceType' ] ) return resQuery def _cleanCommand( self, element, elementNames): ''' Clear Cache from expired DT. ''' resQuery = [] for elementName in elementNames: #reading all the cache entries result = self.rmClient.selectDowntimeCache( element = element, name = elementName ) if not result[ 'OK' ]: return result uniformResult = [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ] currentDate = datetime.utcnow() if len(uniformResult) == 0: return S_OK( None ) for dt in uniformResult: if dt[ 'EndDate' ] < currentDate: result = self.rmClient.deleteDowntimeCache ( downtimeID = dt[ 'DowntimeID' ] ) resQuery.append(result) return S_OK( resQuery ) def _prepareCommand( self ): ''' DowntimeCommand requires four arguments: - name : <str> - element : Site / Resource - elementType: <str> If the elements are Site(s), we need to get their GOCDB names. They may not have, so we ignore them if they do not have. ''' if 'name' not in self.args: return S_ERROR( '"name" not found in self.args' ) elementName = self.args[ 'name' ] if 'element' not in self.args: return S_ERROR( '"element" not found in self.args' ) element = self.args[ 'element' ] if 'elementType' not in self.args: return S_ERROR( '"elementType" not found in self.args' ) elementType = self.args[ 'elementType' ] if not element in [ 'Site', 'Resource' ]: return S_ERROR( 'element is neither Site nor Resource' ) hours = None if 'hours' in self.args: hours = self.args[ 'hours' ] gocdbServiceType = None # Transform DIRAC site names into GOCDB topics if element == 'Site': gocSite = getGOCSiteName( elementName ) if not gocSite[ 'OK' ]: return gocSite elementName = gocSite[ 'Value' ] # The DIRAC se names mean nothing on the grid, but their hosts do mean. elif elementType == 'StorageElement': # We need to distinguish if it's tape or disk if getStorageElementOptions( elementName )['Value']['TapeSE']: gocdbServiceType = "srm.nearline" elif getStorageElementOptions( elementName )['Value']['DiskSE']: gocdbServiceType = "srm" seHost = CSHelpers.getSEHost( elementName ) if not seHost: return S_ERROR( 'No seHost for %s' % elementName ) elementName = seHost elif elementType == 'FTS' or elementType == 'FTS3': gocdbServiceType = 'FTS' try: #WARNING: this method presupposes that the server is an FTS3 type elementName = getGOCFTSName(elementName) except: return S_ERROR( 'No FTS3 server specified in dirac.cfg (see Resources/FTSEndpoints)' ) return S_OK( ( element, elementName, hours, gocdbServiceType ) ) def doNew( self, masterParams = None ): ''' Gets the parameters to run, either from the master method or from its own arguments. For every elementName, unless it is given a list, in which case it contacts the gocdb client. The server is not very stable, so in case of failure tries a second time. If there are downtimes, are recorded and then returned. ''' if masterParams is not None: element, elementNames = masterParams #translate DIRAC CS elementNames into GOCDB elementNames translatedElementNames = [] for e in elementNames: translatedElementNames.append(CSHelpers.getSEHost( e )) elementNames = translatedElementNames hours = None elementName = None gocdbServiceType = None else: params = self._prepareCommand() if not params[ 'OK' ]: return params element, elementName, hours, gocdbServiceType = params[ 'Value' ] elementNames = [ elementName ] #WARNING: checking all the DT that are ongoing or starting in given <hours> from now startDate = None if hours is not None: startDate = datetime.utcnow() + timedelta( hours = hours ) try: results = self.gClient.getStatus( element, elementNames, startDate ) except urllib2.URLError: try: #Let's give it a second chance.. results = self.gClient.getStatus( element, elementNames, startDate ) except urllib2.URLError, e: return S_ERROR( e ) if not results[ 'OK' ]: return results results = results[ 'Value' ] if results is None: return S_OK( None ) #cleaning the Cache cleanRes = self._cleanCommand(element, elementNames) if not cleanRes[ 'OK' ]: return cleanRes uniformResult = [] # Humanize the results into a dictionary, not the most optimal, but readable for downtime, downDic in results.items(): dt = {} if 'HOSTNAME' in downDic.keys(): dt[ 'Name' ] = downDic[ 'HOSTNAME' ] elif 'SITENAME' in downDic.keys(): dt[ 'Name' ] = downDic[ 'SITENAME' ] else: return S_ERROR( "SITENAME or HOSTNAME are missing" ) if 'SERVICE_TYPE' in downDic.keys(): dt[ 'GOCDBServiceType' ] = downDic[ 'SERVICE_TYPE' ] if gocdbServiceType: gocdbST = gocdbServiceType.lower() csST = downDic[ 'SERVICE_TYPE' ].lower() if gocdbST != csST: return S_ERROR( "SERVICE_TYPE mismatch between GOCDB (%s) and CS (%s) for %s" % (gocdbST, csST, dt[ 'Name' ]) ) else: #WARNING: do we want None as default value? dt[ 'GOCDBServiceType' ] = None dt[ 'DowntimeID' ] = downtime dt[ 'Element' ] = element dt[ 'StartDate' ] = downDic[ 'FORMATED_START_DATE' ] dt[ 'EndDate' ] = downDic[ 'FORMATED_END_DATE' ] dt[ 'Severity' ] = downDic[ 'SEVERITY' ] dt[ 'Description' ] = downDic[ 'DESCRIPTION' ].replace( '\'', '' ) dt[ 'Link' ] = downDic[ 'GOCDB_PORTAL_URL' ] uniformResult.append( dt ) storeRes = self._storeCommand( uniformResult ) if not storeRes[ 'OK' ]: return storeRes return S_OK()
class AlarmAction( ActionBase ): def __init__(self, granularity, name, status_type, pdp_decision, **kw): ActionBase.__init__( self, granularity, name, status_type, pdp_decision, **kw ) try: self.rsClient = self.kw["Clients"][ 'ResourceStatusClient' ] except KeyError: self.rsClient = ResourceStatusClient() try: self.rmClient = self.kw["Clients"][ 'ResourceManagementClient' ] except KeyError: self.rmClient = ResourceManagementClient() def _getUsersToNotify(self): groups = CS.getTypedDictRootedAtOperations("AssigneeGroups/" + CS.getSetup()).values() concerned_groups = [g for g in groups if Utils.dictMatch(self.kw["Params"], g)] return [{'Users':g['Users'], 'Notifications':g['Notifications']} for g in concerned_groups] def run(self): """ Do actions required to notify users. Mandatory keyword arguments: - Granularity Optional keyword arguments: - SiteType - ServiceType - ResourceType """ # Initializing variables nc = NotificationClient() # raise alarms, right now makes a simple notification if 'Granularity' not in self.kw['Params'].keys(): raise ValueError, "You have to provide a argument Granularity = <desired_granularity>" if self.new_status['Action']: notif = "%s %s is perceived as" % (self.granularity, self.name) notif = notif + " %s. Reason: %s." % (self.new_status['Status'], self.new_status['Reason']) users_to_notify = self._getUsersToNotify() for notif in users_to_notify: for user in notif['Users']: if 'Web' in notif['Notifications']: gLogger.info("Sending web notification to user %s" % user) nc.addNotificationForUser(user, notif) if 'Mail' in notif['Notifications']: gLogger.info("Sending mail notification to user %s" % user) was = self.rsClient.getElementHistory( self.granularity, elementName = self.name, statusType = self.status_type, meta = {"order": "DESC", 'limit' : 1, "columns": ['Status', 'Reason', 'DateEffective']})#[0] if not was[ 'OK' ]: gLogger.error( was[ 'Message' ] ) continue was = was[ 'Value' ][ 0 ] mailMessage = """ ---TESTING--- -------------------------------------------------------------------------------- RSS changed the status of the following resource: Granularity:\t%s Name:\t\t%s New status:\t%s Reason:\t\t%s Was:\t\t%s (%s) since %s Setup:\t\t%s If you think RSS took the wrong decision, please set the status manually: Use: dirac-rss-set-status -g <granularity> -n <element_name> -s <desired_status> [-t status_type] (if you omit the optional last part of the command, all status types are matched.) This notification has been sent according to those parameters: %s """ % (self.granularity, self.name, self.new_status['Status'], self.new_status['Reason'], was[0], was[1], was[2], CS.getSetup(), str(users_to_notify)) # Actually send the mail! resUser = self.rmClient.getUserRegistryCache( user ) if not resUser[ 'OK' ]: gLogger.error( resUser[ 'Message' ] ) continue resUser = resUser[ 'Value' ][ 0 ][ 2 ] nc.sendMail(resUser, '[RSS][%s][%s] %s -> %s' % (self.granularity, self.name, self.new_status['Status'], was[0]), mailMessage) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class Synchronizer( object ): def __init__( self, rsClient = None, rmClient = None ): self.GOCDBClient = GOCDBClient() self.rsClient = ResourceStatusClient() if rsClient == None else rsClient self.rmClient = ResourceManagementClient() if rmClient == None else rmClient self.synclist = [ 'Sites', 'Resources', 'StorageElements', 'Services', 'RegistryUsers' ] ################################################################################ def sync( self, _a, _b ): """ :params: :attr:`thingsToSync`: list of things to sync """ gLogger.info( "!!! Sync DB content with CS content for %s !!!" % ( ", ".join(self.synclist) ) ) for thing in self.synclist: getattr( self, '_sync' + thing )() return S_OK() ################################################################################ def __purge_resource( self, resourceName ): # Maybe remove attached SEs #SEs = Utils.unpack(self.rsClient.getStorageElement(resourceName=resourceName)) SEs = self.rsClient.getStorageElement( resourceName = resourceName ) if not SEs[ 'OK' ]: gLogger.error( SEs[ 'Message' ] ) return SEs #Utils.unpack(self.rsClient.removeElement("StorageElement", [s[0] for s in SEs])) SEs = [ se[0] for se in SEs ] res = self.rsClient.removeElement( 'StorageElement', SEs ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res # Remove resource itself. #Utils.unpack(self.rsClient.removeElement("Resource", resourceName)) res = self.rsClient.removeElement( 'Resource', resourceName ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res def __purge_site( self, siteName ): # Remove associated resources and services #resources = Utils.unpack(self.rsClient.getResource(siteName=siteName)) resources = self.rsClient.getResource( siteName = siteName ) if not resources[ 'OK' ]: gLogger.error( resources[ 'Message' ] ) return resources #services = Utils.unpack(self.rsClient.getService(siteName=siteName)) services = self.rsClient.getService( siteName = siteName ) if not services[ 'OK' ]: gLogger.error( services[ 'Message' ] ) return services #_ = [self.__purge_resource(r[0]) for r in resources] for resource in resources: res = self.__purge_resource( resource[ 0 ] ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res #Utils.unpack(self.rsClient.removeElement("Service", [s[0] for s in services])) services = [ service[ 0 ] for service in services[ 'Value' ] ] res = self.rsClient.removeElement( 'Service', services ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res # Remove site itself #Utils.unpack(self.rsClient.removeElement("Site", siteName)) res = self.rsClient.removeElement( 'Site', siteName ) if not res[ 'OK' ]: gLogger.info( res[ 'Message' ] ) return res def _syncSites( self ): """ Sync DB content with sites that are in the CS """ def getGOCTier(sitesList): return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)])) # sites in the DB now #sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite()))) sites = self.rsClient.getSite() if not sites[ 'OK' ]: gLogger.error( sites[ 'Message' ] ) return sites sitesDB = set( [ site[0] for site in sites[ 'Value' ] ] ) # sites in CS now sitesCS = set( CS.getSites() ) gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" % (len(sitesCS), len(sitesDB))) # remove sites and associated resources, services, and storage # elements from the DB that are not in the CS: for s in sitesDB - sitesCS: gLogger.info("Purging Site %s (not in CS anymore)" % s) self.__purge_site(s) # add to DB what is missing gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB)) for site in sitesCS - sitesDB: siteType = site.split(".")[0] # DIRAC Tier tier = "T" + str(CS.getSiteTier( site )) if siteType == "LCG": # Grid Name of the site #gridSiteName = Utils.unpack(getGOCSiteName(site)) gridSiteName = getGOCSiteName( site ) if not gridSiteName[ 'OK' ]: gLogger.error( gridSiteName[ 'Message' ] ) return gridSiteName gridSiteName = gridSiteName[ 'Value' ] # Grid Tier (with a workaround!) #DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName)) DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName ) if not DIRACSitesOfGridSites[ 'OK' ]: gLogger.error( DIRACSitesOfGridSites[ 'Message' ] ) return DIRACSitesOfGridSites DIRACSitesOfGridSites = DIRACSitesOfGridSites[ 'Value' ] if len( DIRACSitesOfGridSites ) == 1: gt = tier else: gt = getGOCTier( DIRACSitesOfGridSites ) #Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt) res = self.rsClient.addOrModifyGridSite( gridSiteName, gt ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res #Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName ) res = self.rsClient.addOrModifySite( site, tier, gridSiteName ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res elif siteType == "DIRAC": #Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" ) res = self.rsClient.addOrModifySite( site, tier, "NULL" ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res ################################################################################ # _syncResources HELPER functions def __updateService(self, site, type_): service = type_ + '@' + site #Utils.protect2(self.rsClient.addOrModifyService, service, type_, site ) res = self.rsClient.addOrModifyService( service, type_, site ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res def __getServiceEndpointInfo(self, node): #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) ) res = self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) if res['OK']: res = res[ 'Value' ] else: gLogger.warn( 'Error getting hostname info for %s' % node ) return [] if res == []: #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo('hostname', Utils.canonicalURL(node)) ) url = Utils.canonicalURL(node) res = self.GOCDBClient.getServiceEndpointInfo('hostname', url ) if res['OK']: res = res[ 'Value' ] else: gLogger.warn( 'Error getting canonical hostname info for %s' % node ) res = [] return res def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site = "NULL"): nodesToUpdate = NodeInCS - resourcesInDB if len(nodesToUpdate) > 0: gLogger.debug(str(NodeInCS)) gLogger.debug(str(nodesToUpdate)) # Update Service table siteInGOCDB = [self.__getServiceEndpointInfo(node) for node in nodesToUpdate] siteInGOCDB = Utils.list_sanitize(siteInGOCDB) #sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB] sites = [] for sInGOCDB in siteInGOCDB: siteName = getDIRACSiteName( sInGOCDB[ 0 ][ 'SITENAME' ] ) if not siteName[ 'OK' ]: gLogger.error( siteName[ 'Message' ] ) return siteName sites.append( siteName[ 'Value' ] ) sites = Utils.list_sanitize( Utils.list_flatten( sites ) ) _ = [ self.__updateService(s, serviceType) for s in sites ] # Update Resource table for node in NodeInCS: if serviceType == "Computing": resourceType = CS.getCEType(site, node) if node not in resourcesInDB and node is not None: try: siteInGOCDB = self.__getServiceEndpointInfo(node)[0]['SITENAME'] except IndexError: # No INFO in GOCDB: Node does not exist gLogger.warn("Node %s is not in GOCDB!! Considering that it does not exists!" % node) continue assert(type(siteInGOCDB) == str) #Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB ) res = self.rsClient.addOrModifyResource( node, resourceType, serviceType, site, siteInGOCDB ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res resourcesInDB.add( node ) ################################################################################ def _syncResources( self ): gLogger.info("Starting sync of Resources") # resources in the DB now #resourcesInDB = set((r[0] for r in Utils.unpack(self.rsClient.getResource()))) resources = self.rsClient.getResource() if not resources[ 'OK' ]: gLogger.error( resources[ 'Message' ] ) return resources resourcesInDB = set( [ resource[ 0 ] for resource in resources[ 'Value' ] ] ) # Site-CE / Site-SE mapping in CS now #CEinCS = Utils.unpack(getSiteCEMapping( 'LCG' )) CEinCS = getSiteCEMapping( 'LCG' ) if not CEinCS[ 'OK' ]: gLogger.error( CEinCS[ 'Message' ] ) return CEinCS CEinCS = CEinCS[ 'Value' ] # All CEs in CS now CEInCS = Utils.set_sanitize([CE for celist in CEinCS.values() for CE in celist]) # All SE Nodes in CS now SENodeInCS = set(CS.getSENodes()) # LFC Nodes in CS now LFCNodeInCS_L = set(CS.getLFCNode(readable = "ReadOnly")) LFCNodeInCS_C = set(CS.getLFCNode(readable = "ReadWrite")) # FTS Nodes in CS now FTSNodeInCS = set([v.split("/")[2][0:-5] for v in CS.getTypedDictRootedAt(root="/Resources/FTSEndpoints").values()]) # VOMS Nodes in CS now VOMSNodeInCS = set(CS.getVOMSEndpoints()) # complete list of resources in CS now resourcesInCS = CEInCS | SENodeInCS | LFCNodeInCS_L | LFCNodeInCS_C | FTSNodeInCS | VOMSNodeInCS gLogger.info(" %d resources in CS, %s resources in DB, updating %d resources" % (len(resourcesInCS), len(resourcesInDB), len(resourcesInCS)-len(resourcesInDB))) # Remove resources that are not in the CS anymore for res in resourcesInDB - resourcesInCS: gLogger.info("Purging resource %s. Reason: not in CS anywore." % res) self.__purge_resource(res) # Add to DB what is in CS now and wasn't before # CEs for site in CEinCS: self.__syncNode(set(CEinCS[site]), resourcesInDB, "", "Computing", site) # SRMs self.__syncNode(SENodeInCS, resourcesInDB, "SE", "Storage") # LFC_C self.__syncNode(LFCNodeInCS_C, resourcesInDB, "LFC_C", "Storage") # LFC_L self.__syncNode(LFCNodeInCS_L, resourcesInDB, "LFC_L", "Storage") # FTSs self.__syncNode(FTSNodeInCS, resourcesInDB, "FTS", "Storage") # VOMSs self.__syncNode(VOMSNodeInCS, resourcesInDB, "VOMS", "VOMS") ################################################################################ def _syncStorageElements( self ): # Get StorageElements from the CS and the DB CSSEs = set(CS.getSEs()) #DBSEs = set((s[0] for s in Utils.unpack(self.rsClient.getStorageElement()))) ses = self.rsClient.getStorageElement() if not ses[ 'OK' ]: gLogger.error( ses[ 'Message' ] ) return ses DBSEs = set( [ se[0] for se in ses[ 'Value' ] ] ) # Remove storageElements that are in DB but not in CS for se in DBSEs - CSSEs: #Utils.protect2(self.rsClient.removeElement, 'StorageElement', se ) res = self.rsClient.removeElement( 'StorageElement', se ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res # Add new storage elements gLogger.info("Updating %d StorageElements in DB (%d on CS vs %d on DB)" % (len(CSSEs - DBSEs), len(CSSEs), len(DBSEs))) for SE in CSSEs - DBSEs: srm = CS.getSEHost( SE ) if not srm: gLogger.warn("%s has no srm URL in CS!!!" % SE) continue #siteInGOCDB = Utils.unpack(self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )) siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm ) if siteInGOCDB[ 'OK' ]: siteInGOCDB = siteInGOCDB[ 'Value' ] else: gLogger.error("Error getting hostname for %s from GOCDB!!!" % srm) continue if siteInGOCDB == []: gLogger.warn("%s is not in GOCDB!!!" % srm) continue siteInGOCDB = siteInGOCDB[ 0 ][ 'SITENAME' ] #Utils.protect2(self.rsClient.addOrModifyStorageElement, SE, srm, siteInGOCDB ) res = self.rsClient.addOrModifyStorageElement( SE, srm, siteInGOCDB ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res ################################################################################ def _syncServices(self): """This function is in charge of cleaning the Service table in DB in case of obsolescence.""" # services in the DB now #servicesInDB = Utils.unpack(self.rsClient.getService()) servicesInDB = self.rsClient.getService() if not servicesInDB[ 'OK' ]: gLogger.error( servicesInDB[ 'Message' ] ) return servicesInDB servicesInDB = servicesInDB[ 'Value' ] for service_name, service_type, site_name in servicesInDB: if not service_type in ["VO-BOX", "CondDB", "VOMS", "Storage"]: #if Utils.unpack(self.rsClient.getResource(siteName=site_name, serviceType=service_type)) == []: resource = self.rsClient.getResource( siteName = site_name, serviceType = service_type ) if not resource[ 'OK' ]: gLogger.error( resource[ 'Message' ] ) return resource if resource[ 'Value' ] == []: gLogger.info("Deleting Service %s since it has no corresponding resources." % service_name) #Utils.protect2(self.rsClient.removeElement, "Service", service_name) res = self.rsClient.removeElement( "Service", service_name ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res elif service_type == "Storage": res = self.rsClient.getSite( siteName = site_name, meta = { 'columns' : 'GridSiteName'} ) if res[ 'OK' ]: res = res[ 'Value' ] else: res = [] if res: if self.rsClient.getResource( gridSiteName = res[0], serviceType = service_type ) == []: gLogger.info("Deleting Service %s since it has no corresponding resources." % service_name) #Utils.protect2(self.rsClient.removeElement, "Service", service_name) res = self.rsClient.removeElement( "Service", service_name ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res def _syncRegistryUsers(self): users = CS.getTypedDictRootedAt("Users", root= "/Registry") usersInCS = set(users.keys()) #usersInDB = set((u[0] for u in Utils.unpack(self.rmClient.getUserRegistryCache()))) usersInCache = self.rmClient.getUserRegistryCache() if not usersInCache[ 'OK' ]: gLogger.error( usersInCache[ 'Message' ] ) return usersInCache usersInDB = set( [ userInCache[ 0 ] for userInCache in usersInCache[ 'Value' ] ] ) usersToAdd = usersInCS - usersInDB usersToDel = usersInDB - usersInCS gLogger.info("Updating Registry Users: + %d, - %d" % (len(usersToAdd), len(usersToDel))) if len(usersToAdd) > 0: gLogger.debug(str(usersToAdd)) if len(usersToDel) > 0: gLogger.debug(str(usersToDel)) for u in usersToAdd: if type(users[u]['DN']) == list: users[u]['DN'] = users[u]['DN'][0] if type(users[u]['Email']) == list: users[u]['Email'] = users[u]['Email'][0] users[u]['DN'] = users[u]['DN'].split('=')[-1] #Utils.unpack(self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower())) res = self.rmClient.addOrModifyUserRegistryCache( u, users[u]['DN'], users[u]['Email'].lower() ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res for u in usersToDel: #Utils.protect2(self.rmClient.deleteUserRegistryCache, u) res = self.rmClient.deleteUserRegistryCache( u ) if not res[ 'OK' ]: gLogger.error( res[ 'Message' ] ) return res ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class TransferCommand( Command ): ''' Transfer "master" Command ''' def __init__( self, args = None, clients = None ): super( TransferCommand, self ).__init__( args, clients ) if 'ReportsClient' in self.apis: self.rClient = self.apis[ 'ReportsClient' ] else: self.rClient = ReportsClient() if 'ReportGenerator' in self.apis: self.rgClient = self.apis[ 'ReportGenerator' ] else: self.rgClient = RPCClient( 'Accounting/ReportGenerator' ) self.rClient.rpcClient = self.rgClient if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient() def _storeCommand( self, results ): ''' Stores the results of doNew method on the database. ''' for result in results: resQuery = self.rmClient.addOrModifyTransferCache( result[ 'SourceName' ], result[ 'DestinationName' ], result[ 'Metric' ], result[ 'Value' ] ) if not resQuery[ 'OK' ]: return resQuery return S_OK() def _prepareCommand( self ): ''' TransferChannelCommand requires four arguments: - hours : <int> - direction : Source | Destination - elementName : <str> - metric : Quality | FailedTransfers GGUSTickets are associated with gocDB names, so we have to transform the diracSiteName into a gocSiteName. ''' if not 'hours' in self.args: return S_ERROR( 'Number of hours not specified' ) hours = self.args[ 'hours' ] if not 'direction' in self.args: return S_ERROR( 'direction is missing' ) direction = self.args[ 'direction' ] if direction not in [ 'Source', 'Destination' ]: return S_ERROR( 'direction is not Source nor Destination' ) if not 'name' in self.args: return S_ERROR( '"name" is missing' ) name = self.args[ 'name' ] if not 'metric' in self.args: return S_ERROR( 'metric is missing' ) metric = self.args[ 'metric' ] if metric not in [ 'Quality', 'FailedTransfers' ]: return S_ERROR( 'metric is not Quality nor FailedTransfers' ) return S_OK( ( hours, name, direction, metric ) ) def doNew( self, masterParams = None ): ''' Gets the parameters to run, either from the master method or from its own arguments. For every elementName ( cannot process bulk queries.. ) contacts the accounting client. It reurns dictionaries like { 'X -> Y' : { id: 100%.. } } If there are ggus tickets, are recorded and then returned. ''' if masterParams is not None: hours, name, direction, metric = masterParams else: params = self._prepareCommand() if not params[ 'OK' ]: return params hours, name, direction, metric = params[ 'Value' ] toD = datetime.utcnow() fromD = toD - timedelta( hours = hours ) # dictionary with conditions for the accounting transferDict = { 'OperationType' : 'putAndRegister', direction : name } if metric == 'FailedTransfers': transferDict[ 'FinalStatus' ] = [ 'Failed' ] transferResults = self.rClient.getReport( 'DataOperation', metric, fromD, toD, transferDict, 'Channel' ) if not transferResults[ 'OK' ]: return transferResults transferResults = transferResults[ 'Value' ] if not 'data' in transferResults: return S_ERROR( 'Missing data key' ) transferResults = transferResults[ 'data' ] uniformResult = [] for channel, elementDict in transferResults.items(): try: source, destination = channel.split( ' -> ' ) except ValueError: continue channelDict = {} channelDict[ 'SourceName' ] = source channelDict[ 'DestinationName' ] = destination channelDict[ 'Metric' ] = metric channelDict[ 'Value' ] = sum( elementDict.values() ) / len( elementDict.values() ) uniformResult.append( channelDict ) storeRes = self._storeCommand( uniformResult ) if not storeRes[ 'OK' ]: return storeRes # Compute mean of all transfer channels value = 0 for channelDict in uniformResult: value += channelDict[ 'Value' ] if uniformResult: value = float( value ) / len( uniformResult ) else: value = None return S_OK( { 'Mean' : value, 'Name' : name } ) def doCache( self ): ''' Method that reads the cache table and tries to read from it. It will return a list of dictionaries if there are results. ''' params = self._prepareCommand() if not params[ 'OK' ]: return params _hours, name, direction, metric = params[ 'Value' ] sourceName, destinationName = None, None if direction == 'Source': sourceName = name if direction == 'Destination': destinationName = name result = self.rmClient.selectTransferCache( sourceName, destinationName, metric ) if not result[ 'OK' ]: return result result = [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ] # Compute mean of all transfer channels value = 0 for channelDict in result: value += channelDict[ 'Value' ] if result: value = float( value ) / len( result ) else: value = None return S_OK( { 'Mean' : value, 'Name' : name } ) def doMaster( self ): ''' Master method, which looks little bit spaguetti code, sorry ! - It gets all Sites. - It gets all StorageElements As there is no bulk query, it compares with what we have on the database. It queries a portion of them. ''' sites = CSHelpers.getSites() if not sites[ 'OK' ]: return sites sites = sites[ 'Value' ] ses = CSHelpers.getStorageElements() if not ses[ 'OK' ]: return ses ses = ses[ 'Value' ] elementNames = sites + ses # sourceQuery = self.rmClient.selectTransferCache( meta = { 'columns' : [ 'SourceName' ] } ) # if not sourceQuery[ 'OK' ]: # return sourceQuery # sourceQuery = [ element[0] for element in sourceQuery[ 'Value' ] ] # # sourceElementsToQuery = list( set( elementNames ).difference( set( sourceQuery ) ) ) gLogger.info( 'Processing %s' % ', '.join( elementNames ) ) for metric in [ 'Quality', 'FailedTransfers' ]: for direction in [ 'Source', 'Destination' ]: # 2 hours of window result = self.doNew( ( 2, elementNames, direction, metric ) ) if not result[ 'OK' ]: self.metrics[ 'failed' ].append( result ) return S_OK( self.metrics ) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class FreeDiskSpaceCommand(Command): """ Uses diskSpace method to get the free space """ def __init__(self, args=None, clients=None): super(FreeDiskSpaceCommand, self).__init__(args, clients=clients) self.rpc = None self.rsClient = ResourceManagementClient() def _prepareCommand(self): """ FreeDiskSpaceCommand requires one argument: - name : <str> """ if "name" not in self.args: return S_ERROR('"name" not found in self.args') elementName = self.args["name"] return S_OK(elementName) def doNew(self, masterParams=None): """ Gets the total and the free disk space of a DIPS storage element that is found in the CS and inserts the results in the SpaceTokenOccupancyCache table of ResourceManagementDB database. """ if masterParams is not None: elementName = masterParams else: elementName = self._prepareCommand() if not elementName["OK"]: return elementName se = StorageElement(elementName) elementURL = se.getStorageParameters(protocol="dips") if elementURL["OK"]: elementURL = se.getStorageParameters(protocol="dips")["Value"]["URLBase"] else: gLogger.verbose("Not a DIPS storage element, skipping...") return S_OK() self.rpc = RPCClient(elementURL, timeout=120) free = self.rpc.getFreeDiskSpace("/") if not free["OK"]: return free free = free["Value"] total = self.rpc.getTotalDiskSpace("/") if not total["OK"]: return total total = total["Value"] if free and free < 1: free = 1 if total and total < 1: total = 1 result = self.rsClient.addOrModifySpaceTokenOccupancyCache( endpoint=elementURL, lastCheckTime=datetime.utcnow(), free=free, total=total, token=elementName ) if not result["OK"]: return result return S_OK() def doCache(self): """ This is a method that gets the element's details from the spaceTokenOccupancy cache. """ elementName = self._prepareCommand() if not elementName["OK"]: return elementName result = self.rsClient.selectSpaceTokenOccupancyCache(token=elementName) if not result["OK"]: return result return S_OK(result) def doMaster(self): """ This method calls the doNew method for each storage element that exists in the CS. """ elements = CSHelpers.getStorageElements() for name in elements["Value"]: diskSpace = self.doNew(name) if not diskSpace["OK"]: gLogger.error("Unable to calculate free disk space") continue return S_OK()
class GOCDBSyncCommand(Command): def __init__(self, args=None, clients=None): super(GOCDBSyncCommand, self).__init__(args, clients) if 'GOCDBClient' in self.apis: self.gClient = self.apis['GOCDBClient'] else: self.gClient = GOCDBClient() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient() self.seenHostnames = set() def doNew(self, masterParams=None): """ Gets the downtime IDs and dates of a given hostname from the local database and compares the results with the remote database of GOCDB. If the downtime dates have been changed it updates the local database. :param: `masterParams` - string :return: S_OK / S_ERROR """ if masterParams: hostname = masterParams else: return S_ERROR(errno.EINVAL, 'masterParams is not provided') result = self.rmClient.selectDowntimeCache(name=hostname) if not result['OK']: return result for downtimes in result['Value']: localDBdict = {'DowntimeID': downtimes[3], 'FORMATED_START_DATE': downtimes[6].strftime('%Y-%m-%d %H:%M'), 'FORMATED_END_DATE': downtimes[7].strftime('%Y-%m-%d %H:%M')} response = self.gClient.getHostnameDowntime(hostname, ongoing=True) if not response['OK']: return response doc = minidom.parseString(response['Value']) downtimeElements = doc.getElementsByTagName("DOWNTIME") for dtElement in downtimeElements: GOCDBdict = _parseSingleElement(dtElement, ['PRIMARY_KEY', 'ENDPOINT', 'FORMATED_START_DATE', 'FORMATED_END_DATE']) localDowntimeID = localDBdict['DowntimeID'] GOCDBDowntimeID = GOCDBdict['PRIMARY_KEY'] + ' ' + GOCDBdict['ENDPOINT'] if localDowntimeID == GOCDBDowntimeID: if localDBdict['FORMATED_START_DATE'] != GOCDBdict['FORMATED_START_DATE']: result = self.rmClient.addOrModifyDowntimeCache(downtimeID=localDBdict['DowntimeID'], startDate=GOCDBdict['FORMATED_START_DATE']) gLogger.verbose("The start date of %s has been changed!" % downtimes[3]) if not result['OK']: return result if localDBdict['FORMATED_END_DATE'] != GOCDBdict['FORMATED_END_DATE']: result = self.rmClient.addOrModifyDowntimeCache(downtimeID=localDBdict['DowntimeID'], endDate=GOCDBdict['FORMATED_END_DATE']) gLogger.verbose("The end date of %s has been changed!" % downtimes[3]) if not result['OK']: return result return S_OK() def doCache(self): return S_OK() def doMaster(self): """ This method calls the doNew method for each hostname that exists in the DowntimeCache table of the local database. :return: S_OK / S_ERROR """ # Query DB for all downtimes result = self.rmClient.selectDowntimeCache() if not result['OK']: return result for data in result['Value']: # If already processed don't do it again if data[0] in self.seenHostnames: continue # data[0] contains the hostname gLogger.verbose("Checking if the downtime of %s has been changed" % data[0]) result = self.doNew(data[0]) if not result['OK']: return result self.seenHostnames.add(data[0]) return S_OK()
def __init__(self, args=None, clients=None): super(FreeDiskSpaceCommand, self).__init__(args, clients=clients) self.rpc = None self.rsClient = ResourceManagementClient()
class DowntimeCommand( Command ): ''' Downtime "master" Command. ''' def __init__( self, args = None, clients = None ): super( DowntimeCommand, self ).__init__( args, clients ) if 'GOCDBClient' in self.apis: self.gClient = self.apis[ 'GOCDBClient' ] else: self.gClient = GOCDBClient() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient() def _storeCommand( self, result ): ''' Stores the results of doNew method on the database. ''' for dt in result: resQuery = self.rmClient.addOrModifyDowntimeCache( dt[ 'DowntimeID' ], dt[ 'Element' ], dt[ 'Name' ], dt[ 'StartDate' ], dt[ 'EndDate' ], dt[ 'Severity' ], dt[ 'Description' ], dt[ 'Link' ] ) if not resQuery[ 'OK' ]: return resQuery return S_OK() def _prepareCommand( self ): ''' DowntimeCommand requires three arguments: - name : <str> - element : Site / Resource - elementType: <str> If the elements are Site(s), we need to get their GOCDB names. They may not have, so we ignore them if they do not have. ''' if 'name' not in self.args: return S_ERROR( '"name" not found in self.args' ) elementName = self.args[ 'name' ] if 'element' not in self.args: return S_ERROR( '"element" not found in self.args' ) element = self.args[ 'element' ] if 'elementType' not in self.args: return S_ERROR( '"elementType" not found in self.args' ) elementType = self.args[ 'elementType' ] if not element in [ 'Site', 'Resource' ]: return S_ERROR( 'element is not Site nor Resource' ) hours = None if 'hours' in self.args: hours = self.args[ 'hours' ] # Transform DIRAC site names into GOCDB topics if element == 'Site': gocSite = getGOCSiteName( elementName ) if not gocSite[ 'OK' ]: return gocSite elementName = gocSite[ 'Value' ] # The DIRAC se names mean nothing on the grid, but their hosts do mean. elif elementType == 'StorageElement': seHost = CSHelpers.getSEHost( elementName ) if not seHost: return S_ERROR( 'No seHost for %s' % elementName ) elementName = seHost return S_OK( ( element, elementName, hours ) ) def doNew( self, masterParams = None ): ''' Gets the parameters to run, either from the master method or from its own arguments. For every elementName, unless it is given a list, in which case it contacts the gocdb client. The server is not very stable, so in case of failure tries a second time. If there are downtimes, are recorded and then returned. ''' if masterParams is not None: element, elementNames = masterParams hours = None elementName = None else: params = self._prepareCommand() if not params[ 'OK' ]: return params element, elementName, hours = params[ 'Value' ] elementNames = [ elementName ] startDate = datetime.utcnow() - timedelta( days = 2 ) try: results = self.gClient.getStatus( element, elementName, startDate, 120 ) except urllib2.URLError: try: #Let's give it a second chance.. results = self.gClient.getStatus( element, elementName, startDate, 120 ) except urllib2.URLError, e: return S_ERROR( e ) if not results[ 'OK' ]: return results results = results[ 'Value' ] if results is None: return S_OK( None ) uniformResult = [] # Humanize the results into a dictionary, not the most optimal, but readable for downtime, downDic in results.items(): dt = {} if element == 'Resource': dt[ 'Name' ] = downDic[ 'HOSTNAME' ] else: dt[ 'Name' ] = downDic[ 'SITENAME' ] if not dt[ 'Name' ] in elementNames: continue dt[ 'DowntimeID' ] = downtime dt[ 'Element' ] = element dt[ 'StartDate' ] = downDic[ 'FORMATED_START_DATE' ] dt[ 'EndDate' ] = downDic[ 'FORMATED_END_DATE' ] dt[ 'Severity' ] = downDic[ 'SEVERITY' ] dt[ 'Description' ] = downDic[ 'DESCRIPTION' ].replace( '\'', '' ) dt[ 'Link' ] = downDic[ 'GOCDB_PORTAL_URL' ] uniformResult.append( dt ) storeRes = self._storeCommand( uniformResult ) if not storeRes[ 'OK' ]: return storeRes # We return only one downtime, if its ongoind at dtDate dtDate = datetime.now() if hours: dtDate = dtDate + timedelta( hours = hours ) result = None for dt in uniformResult: if ( dt[ 'StartDate' ] < str( dtDate ) ) and ( dt[ 'EndDate' ] > str( dtDate ) ): result = dt break return S_OK( result )
class DowntimeCommand( Command ): ''' Downtime "master" Command. ''' def __init__( self, args = None, clients = None ): super( DowntimeCommand, self ).__init__( args, clients ) if 'GOCDBClient' in self.apis: self.gClient = self.apis[ 'GOCDBClient' ] else: self.gClient = GOCDBClient() if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient() def _storeCommand( self, result ): ''' Stores the results of doNew method on the database. ''' for dt in result: resQuery = self.rmClient.addOrModifyDowntimeCache( downtimeID = dt[ 'DowntimeID' ], element = dt[ 'Element' ], name = dt[ 'Name' ], startDate = dt[ 'StartDate' ], endDate = dt[ 'EndDate' ], severity = dt[ 'Severity' ], description = dt[ 'Description' ], link = dt[ 'Link' ], gocdbServiceType = dt[ 'GOCDBServiceType' ] ) if not resQuery[ 'OK' ]: return resQuery return S_OK() def _prepareCommand( self ): ''' DowntimeCommand requires four arguments: - name : <str> - element : Site / Resource - elementType: <str> If the elements are Site(s), we need to get their GOCDB names. They may not have, so we ignore them if they do not have. ''' if 'name' not in self.args: return S_ERROR( '"name" not found in self.args' ) elementName = self.args[ 'name' ] if 'element' not in self.args: return S_ERROR( '"element" not found in self.args' ) element = self.args[ 'element' ] if 'elementType' not in self.args: return S_ERROR( '"elementType" not found in self.args' ) elementType = self.args[ 'elementType' ] if not element in [ 'Site', 'Resource' ]: return S_ERROR( 'element is not Site nor Resource' ) hours = None if 'hours' in self.args: hours = self.args[ 'hours' ] gocdbServiceType = None # Transform DIRAC site names into GOCDB topics if element == 'Site': gocSite = getGOCSiteName( elementName ) if not gocSite[ 'OK' ]: return gocSite elementName = gocSite[ 'Value' ] # The DIRAC se names mean nothing on the grid, but their hosts do mean. elif elementType == 'StorageElement': # We need to distinguish if it's tape or disk if getStorageElementOptions( elementName )['Value']['TapeSE']: gocdbServiceType = "srm" elif getStorageElementOptions( elementName )['Value']['DiskSE']: gocdbServiceType = "srm.nearline" seHost = CSHelpers.getSEHost( elementName ) if not seHost: return S_ERROR( 'No seHost for %s' % elementName ) elementName = seHost return S_OK( ( element, elementName, hours, gocdbServiceType ) ) def doNew( self, masterParams = None ): ''' Gets the parameters to run, either from the master method or from its own arguments. For every elementName, unless it is given a list, in which case it contacts the gocdb client. The server is not very stable, so in case of failure tries a second time. If there are downtimes, are recorded and then returned. ''' if masterParams is not None: element, elementNames = masterParams hours = None elementName = None gocdbServiceType = None else: params = self._prepareCommand() if not params[ 'OK' ]: return params element, elementName, hours, gocdbServiceType = params[ 'Value' ] elementNames = [ elementName ] startDate = datetime.utcnow() - timedelta( days = 14 ) try: results = self.gClient.getStatus( element, elementName, startDate, 120 ) except urllib2.URLError: try: #Let's give it a second chance.. results = self.gClient.getStatus( element, elementName, startDate, 120 ) except urllib2.URLError, e: return S_ERROR( e ) if not results[ 'OK' ]: return results results = results[ 'Value' ] if results is None: return S_OK( None ) uniformResult = [] # Humanize the results into a dictionary, not the most optimal, but readable for downtime, downDic in results.items(): dt = {} if gocdbServiceType and downDic[ 'SERVICE_TYPE' ]: if gocdbServiceType.lower() != downDic[ 'SERVICE_TYPE' ].lower(): continue if element == 'Resource': dt[ 'Name' ] = downDic[ 'HOSTNAME' ] else: dt[ 'Name' ] = downDic[ 'SITENAME' ] if not dt[ 'Name' ] in elementNames: continue dt[ 'DowntimeID' ] = downtime dt[ 'Element' ] = element dt[ 'StartDate' ] = downDic[ 'FORMATED_START_DATE' ] dt[ 'EndDate' ] = downDic[ 'FORMATED_END_DATE' ] dt[ 'Severity' ] = downDic[ 'SEVERITY' ] dt[ 'Description' ] = downDic[ 'DESCRIPTION' ].replace( '\'', '' ) dt[ 'Link' ] = downDic[ 'GOCDB_PORTAL_URL' ] try: dt[ 'GOCDBServiceType' ] = downDic[ 'SERVICE_TYPE' ] except KeyError: # SERVICE_TYPE is not always defined pass uniformResult.append( dt ) storeRes = self._storeCommand( uniformResult ) if not storeRes[ 'OK' ]: return storeRes # We return only one downtime, if its ongoing at dtDate startDate = datetime.utcnow() if hours: startDate = startDate + timedelta( hours = hours ) endDate = startDate result = None dtOutages = [] dtWarnings = [] for dt in uniformResult: if ( dt[ 'StartDate' ] < str( startDate ) ) and ( dt[ 'EndDate' ] > str( endDate ) ): if dt[ 'Severity' ] == 'Outage': dtOutages.append( dt ) else: dtWarnings.append( dt ) #In case many overlapping downtimes have been declared, the first one in #severity and then time order will be selected. We want to get the latest one #( they are sorted by insertion time ) if len( dtOutages ) > 0: result = dtOutages[-1] elif len( dtWarnings ) > 0: result = dtWarnings[-1] return S_OK( result )
class CachedPlotCommand( Command ): def __init__( self, args = None, clients = None ): super( CachedPlotCommand, self ).__init__( args, clients ) if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient() def doCommand( self ): """ Returns transfer quality plot as it is cached in the accounting cache. :attr:`args`: - args[0]: string - should be a ValidElement - args[1]: string - should be the name of the ValidElement - args[2]: string - should be the plot type - args[3]: string - should be the plot name :returns: a plot """ if not 'element' in self.args: return S_ERROR( 'element no specified' ) element = self.args[ 'element' ] if not 'name' in self.args: return S_ERROR( 'Name no specified' ) name = self.args[ 'name' ] if not 'plotType' in self.args: return S_ERROR( 'plotType no specified' ) plotType = self.args[ 'plotType' ] if not 'plotName' in self.args: return S_ERROR( 'plotName no specified' ) plotName = self.args[ 'plotName' ] #FIXME: we have no any longer Service granularity ! if element == 'Service': name = name.split('@')[1] meta = { 'columns' : 'Result' } results = self.rmClient.selectAccountingCache( name = name, plotType = plotType, plotName = plotName, meta = meta ) if not results[ 'OK' ]: return results results = results[ 'Value' ] if results == []: results = { 'data' : {}, 'granularity' : 900 } else: #FIXME: WTH is an eval doing here !!!! results = eval( results[0] ) return results