class FreeDiskSpaceCommand(Command): """ Uses diskSpace method to get the free space """ def __init__(self, args=None, clients=None): super(FreeDiskSpaceCommand, self).__init__(args, clients=clients) self.rpc = None self.rsClient = ResourceManagementClient() def _prepareCommand(self): """ FreeDiskSpaceCommand requires one argument: - name : <str> """ if "name" not in self.args: return S_ERROR('"name" not found in self.args') elementName = self.args["name"] return S_OK(elementName) def doNew(self, masterParams=None): """ Gets the total and the free disk space of a DIPS storage element that is found in the CS and inserts the results in the SpaceTokenOccupancyCache table of ResourceManagementDB database. """ if masterParams is not None: elementName = masterParams else: elementName = self._prepareCommand() if not elementName["OK"]: return elementName se = StorageElement(elementName) elementURL = se.getStorageParameters(protocol="dips") if elementURL["OK"]: elementURL = se.getStorageParameters(protocol="dips")["Value"]["URLBase"] else: gLogger.verbose("Not a DIPS storage element, skipping...") return S_OK() self.rpc = RPCClient(elementURL, timeout=120) free = self.rpc.getFreeDiskSpace("/") if not free["OK"]: return free free = free["Value"] total = self.rpc.getTotalDiskSpace("/") if not total["OK"]: return total total = total["Value"] if free and free < 1: free = 1 if total and total < 1: total = 1 result = self.rsClient.addOrModifySpaceTokenOccupancyCache( endpoint=elementURL, lastCheckTime=datetime.utcnow(), free=free, total=total, token=elementName ) if not result["OK"]: return result return S_OK() def doCache(self): """ This is a method that gets the element's details from the spaceTokenOccupancy cache. """ elementName = self._prepareCommand() if not elementName["OK"]: return elementName result = self.rsClient.selectSpaceTokenOccupancyCache(token=elementName) if not result["OK"]: return result return S_OK(result) def doMaster(self): """ This method calls the doNew method for each storage element that exists in the CS. """ elements = CSHelpers.getStorageElements() for name in elements["Value"]: diskSpace = self.doNew(name) if not diskSpace["OK"]: gLogger.error("Unable to calculate free disk space") continue return S_OK()
class SpaceTokenOccupancyCommand(Command): ''' Uses lcg_util to query status of endpoint for a given token. ''' def __init__(self, args=None, clients=None): super(SpaceTokenOccupancyCommand, self).__init__(args, clients) if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient() def _storeCommand(self, results): ''' Stores the results of doNew method on the database. ''' for result in results: resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache( result['Endpoint'], result['Token'], result['Total'], result['Guaranteed'], result['Free']) if not resQuery['OK']: return resQuery return S_OK() def _prepareCommand(self): ''' SpaceTokenOccupancy requires one argument: - elementName : <str> Given a (storage)elementName, we calculate its endpoint and spaceToken, which are used to query the srm interface. ''' if 'name' not in self.args: return S_ERROR('"name" not found in self.args') elementName = self.args['name'] endpoint = CSHelpers.getStorageElementEndpoint(elementName) if not endpoint['OK']: return endpoint endpoint = endpoint['Value'] spaceToken = CSHelpers.getSEToken(elementName) if not spaceToken['OK']: return spaceToken spaceToken = spaceToken['Value'] return S_OK((endpoint, spaceToken)) def doNew(self, masterParams=None): ''' Gets the parameters to run, either from the master method or from its own arguments. It queries the srm interface, and hopefully it will not crash. Out of the results, we keep totalsize, guaranteedsuze, and unusedsize. Then, they are recorded and returned. ''' if masterParams is not None: spaceTokenEndpoint, spaceToken = masterParams else: params = self._prepareCommand() if not params['OK']: return params spaceTokenEndpoint, spaceToken = params['Value'] # 10 secs of timeout. If it works, the reply is immediate. occupancyResult = pythonCall(10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0) if not occupancyResult['OK']: self.log.error( "Could not get spaceToken occupancy", "from endPoint/spaceToken %s/%s : %s" % \ ( spaceTokenEndpoint, spaceToken, occupancyResult['Message'] ) ) return occupancyResult else: occupancy = occupancyResult['Value'] # Timeout does not work here... # occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 ) if occupancy[0] != 0: return S_ERROR(occupancy) output = occupancy[1][0] sTokenDict = {} sTokenDict['Endpoint'] = spaceTokenEndpoint sTokenDict['Token'] = spaceToken sTokenDict['Total'] = float(output.get( 'totalsize', '0')) / 1e12 # Bytes to Terabytes sTokenDict['Guaranteed'] = float(output.get('guaranteedsize', '0')) / 1e12 sTokenDict['Free'] = float(output.get('unusedsize', '0')) / 1e12 storeRes = self._storeCommand([sTokenDict]) if not storeRes['OK']: return storeRes return S_OK([sTokenDict]) def doCache(self): ''' Method that reads the cache table and tries to read from it. It will return a list of dictionaries if there are results. ''' params = self._prepareCommand() if not params['OK']: return params spaceTokenEndpoint, spaceToken = params['Value'] result = self.rmClient.selectSpaceTokenOccupancyCache( spaceTokenEndpoint, spaceToken) if result['OK']: result = S_OK( [dict(zip(result['Columns'], res)) for res in result['Value']]) return result def doMaster(self): ''' Master method. Gets all endpoints from the storage elements and all the spaceTokens. Could have taken from Shares/Disk as well. It queries for all their possible combinations, unless there are records in the database for those combinations, which then are not queried. ''' self.log.verbose("Getting all SEs defined in the CS") storageElementNames = CSHelpers.getStorageElements() if not storageElementNames['OK']: self.log.warn(storageElementNames['Message']) return storageElementNames storageElementNames = storageElementNames['Value'] endpointTokenSet = set() for storageElementName in storageElementNames: endpoint = CSHelpers.getStorageElementEndpoint(storageElementName) if not endpoint['OK']: self.log.warn(endpoint['Message']) continue endpoint = endpoint['Value'] spaceToken = CSHelpers.getSEToken(storageElementName) if not spaceToken['OK']: self.log.warn(spaceToken['Message']) continue spaceToken = spaceToken['Value'] endpointTokenSet.add((endpoint, spaceToken)) self.log.verbose('Processing %s' % endpointTokenSet) for elementToQuery in endpointTokenSet: result = self.doNew(elementToQuery) if not result['OK']: self.metrics['failed'].append(result) return S_OK(self.metrics)
class FreeDiskSpaceCommand(Command): ''' Uses diskSpace method to get the free space ''' def __init__(self, args=None, clients=None): super(FreeDiskSpaceCommand, self).__init__(args, clients=clients) self.rmClient = ResourceManagementClient() def _prepareCommand(self): ''' FreeDiskSpaceCommand requires one argument: - name : <str> ''' if 'name' not in self.args: return S_ERROR('"name" not found in self.args') elementName = self.args['name'] # We keep TB as default as this is what was used (and will still be used) # in the policy for "space tokens" ("real", "data" SEs) unit = self.args.get('unit', 'TB') return S_OK((elementName, unit)) def doNew(self, masterParams=None): """ Gets the parameters to run, either from the master method or from its own arguments. Gets the total and the free disk space of a storage element and inserts the results in the SpaceTokenOccupancyCache table of ResourceManagementDB database. The result is also returned to the caller, not only inserted. What is inserted in the DB will normally be in MB, what is returned will be in the specified unit. """ if masterParams is not None: elementName, unit = masterParams else: params = self._prepareCommand() if not params['OK']: return params elementName, unit = params['Value'] endpointResult = CSHelpers.getStorageElementEndpoint(elementName) if not endpointResult['OK']: return endpointResult se = StorageElement(elementName) occupancyResult = se.getOccupancy(unit=unit) if not occupancyResult['OK']: return occupancyResult occupancy = occupancyResult['Value'] free = occupancy['Free'] total = occupancy['Total'] results = { 'Endpoint': endpointResult['Value'], 'Free': free, 'Total': total, 'ElementName': elementName } result = self._storeCommand(results) if not result['OK']: return result return S_OK({'Free': free, 'Total': total}) def _storeCommand(self, results): """ Here purely for extensibility """ return self.rmClient.addOrModifySpaceTokenOccupancyCache( endpoint=results['Endpoint'], lastCheckTime=datetime.utcnow(), free=results['Free'], total=results['Total'], token=results['ElementName']) def doCache(self): """ This is a method that gets the element's details from the spaceTokenOccupancyCache DB table. It will return a dictionary with th results, converted to "correct" unit. """ params = self._prepareCommand() if not params['OK']: return params elementName, unit = params['Value'] result = self.rmClient.selectSpaceTokenOccupancyCache( token=elementName) if not result['OK']: return result # results are normally in 'MB' free = result['Value'][0][3] total = result['Value'][0][4] free = convertSizeUnits(free, 'MB', unit) total = convertSizeUnits(total, 'MB', unit) if free == -sys.maxsize or total == -sys.maxsize: return S_ERROR("No valid unit specified") return S_OK({'Free': free, 'Total': total}) def doMaster(self): """ This method calls the doNew method for each storage element that exists in the CS. """ elements = CSHelpers.getStorageElements() for name in elements['Value']: # keeping TB as default diskSpace = self.doNew((name, 'MB')) if not diskSpace['OK']: gLogger.warn("Unable to calculate free/total disk space", "name: %s" % name) gLogger.warn(diskSpace['Message']) continue return S_OK()
class FreeDiskSpaceCommand(Command): """ Uses diskSpace method to get the free space """ def __init__(self, args=None, clients=None): super(FreeDiskSpaceCommand, self).__init__(args, clients=clients) self.rmClient = ResourceManagementClient() def _prepareCommand(self): """ FreeDiskSpaceCommand requires one argument: - name : <str> """ if "name" not in self.args: return S_ERROR('"name" not found in self.args') elementName = self.args["name"] # We keep TB as default as this is what was used (and will still be used) # in the policy for "space tokens" ("real", "data" SEs) unit = self.args.get("unit", "TB") return S_OK((elementName, unit)) def doNew(self, masterParams=None): """ Gets the parameters to run, either from the master method or from its own arguments. Gets the total and the free disk space of a storage element and inserts the results in the SpaceTokenOccupancyCache table of ResourceManagementDB database. The result is also returned to the caller, not only inserted. What is inserted in the DB will normally be in MB, what is returned will be in the specified unit. """ if masterParams is not None: elementName, unit = masterParams else: params = self._prepareCommand() if not params["OK"]: return params elementName, unit = params["Value"] se = StorageElement(elementName) occupancyResult = se.getOccupancy(unit=unit) if not occupancyResult["OK"]: return occupancyResult occupancy = occupancyResult["Value"] free = occupancy["Free"] total = occupancy["Total"] endpointResult = CSHelpers.getStorageElementEndpoint(elementName) if not endpointResult["OK"]: return endpointResult # We only take the first endpoint, in case there are severals of them (which is normal). # Most probably not ideal, because it would be nice to stay consistent, but well... endpoint = endpointResult["Value"][0] results = { "Endpoint": endpoint, "Free": free, "Total": total, "ElementName": elementName } result = self._storeCommand(results) if not result["OK"]: return result return S_OK({"Free": free, "Total": total}) def _storeCommand(self, results): """ Stores the results in the cache (SpaceTokenOccupancyCache), and adds records to the StorageOccupancy accounting. :param dict results: something like {'ElementName': 'CERN-HIST-EOS', 'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server', 'Free': 3264963586.10073, 'Total': 8000000000.0} :returns: S_OK/S_ERROR dict """ # Stores in cache res = self.rmClient.addOrModifySpaceTokenOccupancyCache( endpoint=results["Endpoint"], lastCheckTime=datetime.utcnow(), free=results["Free"], total=results["Total"], token=results["ElementName"], ) if not res["OK"]: self.log.error("Error calling addOrModifySpaceTokenOccupancyCache", res["Message"]) return res # Now proceed with the accounting siteRes = DMSHelpers().getLocalSiteForSE(results["ElementName"]) if not siteRes["OK"]: return siteRes accountingDict = { "StorageElement": results["ElementName"], "Endpoint": results["Endpoint"], "Site": siteRes["Value"] if siteRes["Value"] else "unassigned", } results["Used"] = results["Total"] - results["Free"] for sType in ["Total", "Free", "Used"]: spaceTokenAccounting = StorageOccupancy() spaceTokenAccounting.setNowAsStartAndEndTime() spaceTokenAccounting.setValuesFromDict(accountingDict) spaceTokenAccounting.setValueByKey("SpaceType", sType) spaceTokenAccounting.setValueByKey( "Space", int(convertSizeUnits(results[sType], "MB", "B"))) res = gDataStoreClient.addRegister(spaceTokenAccounting) if not res["OK"]: self.log.warn("Could not commit register", res["Message"]) continue return gDataStoreClient.commit() def doCache(self): """ This is a method that gets the element's details from the spaceTokenOccupancyCache DB table. It will return a dictionary with th results, converted to "correct" unit. """ params = self._prepareCommand() if not params["OK"]: return params elementName, unit = params["Value"] result = self.rmClient.selectSpaceTokenOccupancyCache( token=elementName) if not result["OK"]: return result if not result["Value"]: return S_ERROR(errno.ENODATA, "No occupancy recorded") # results are normally in 'MB' free = result["Value"][0][3] total = result["Value"][0][4] free = convertSizeUnits(free, "MB", unit) total = convertSizeUnits(total, "MB", unit) if free == -sys.maxsize or total == -sys.maxsize: return S_ERROR("No valid unit specified") return S_OK({"Free": free, "Total": total}) def doMaster(self): """ This method calls the doNew method for each storage element that exists in the CS. """ for name in DMSHelpers().getStorageElements(): try: # keeping TB as default diskSpace = self.doNew((name, "MB")) if not diskSpace["OK"]: self.log.warn("Unable to calculate free/total disk space", "name: %s" % name) self.log.warn(diskSpace["Message"]) continue except Exception as excp: # pylint: disable=broad-except self.log.error( "Failed to get SE FreeDiskSpace information ==> SE skipped", name) self.log.exception("Operation finished with exception: ", lException=excp) # Clear the cache return self._cleanCommand() def _cleanCommand(self, toDelete=None): """Clean the spaceTokenOccupancy table from old endpoints :param tuple toDelete: endpoint to remove (endpoint, storage_element_name), e.g. ('httpg://srm-lhcb.cern.ch:8443/srm/managerv2', CERN-RAW) """ if not toDelete: toDelete = [] res = self.rmClient.selectSpaceTokenOccupancyCache() if not res["OK"]: return res storedSEsSet = set([(sse[0], sse[1]) for sse in res["Value"]]) currentSEsSet = set() currentSEs = DMSHelpers().getStorageElements() for cse in currentSEs: res = CSHelpers.getStorageElementEndpoint(cse) if not res["OK"]: self.log.warn("Could not get endpoint", res["Message"]) continue endpoint = res["Value"][0] currentSEsSet.add((endpoint, cse)) toDelete = list(storedSEsSet - currentSEsSet) else: toDelete = [toDelete] for ep in toDelete: res = self.rmClient.deleteSpaceTokenOccupancyCache(ep[0], ep[1]) if not res["OK"]: self.log.warn( "Could not delete entry from SpaceTokenOccupancyCache", res["Message"]) return S_OK()
class SpaceTokenOccupancyCommand( Command ): ''' Uses lcg_util to query status of endpoint for a given token. ''' def __init__( self, args = None, clients = None ): super( SpaceTokenOccupancyCommand, self ).__init__( args, clients ) if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient() def _storeCommand( self, results ): ''' Stores the results of doNew method on the database. ''' for result in results: resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache( result[ 'Endpoint' ], result[ 'Token' ], result[ 'Total' ], result[ 'Guaranteed' ], result[ 'Free' ] ) if not resQuery[ 'OK' ]: return resQuery return S_OK() def _prepareCommand( self ): ''' SpaceTokenOccupancy requires one argument: - elementName : <str> Given a (storage)elementName, we calculate its endpoint and spaceToken, which are used to query the srm interface. ''' if not 'name' in self.args: return S_ERROR( '"name" not found in self.args' ) elementName = self.args[ 'name' ] endpoint = CSHelpers.getStorageElementEndpoint( elementName ) if not endpoint[ 'OK' ]: return endpoint endpoint = endpoint[ 'Value' ] spaceToken = CSHelpers.getStorageElementSpaceToken( elementName ) if not spaceToken[ 'OK' ]: return spaceToken spaceToken = spaceToken[ 'Value'] return S_OK( ( endpoint, spaceToken ) ) def doNew( self, masterParams = None ): ''' Gets the parameters to run, either from the master method or from its own arguments. It queries the srm interface, and hopefully it will not crash. Out of the results, we keep totalsize, guaranteedsuze, and unusedsize. Then, they are recorded and returned. ''' if masterParams is not None: spaceTokenEndpoint, spaceToken = masterParams else: params = self._prepareCommand() if not params[ 'OK' ]: return params spaceTokenEndpoint, spaceToken = params[ 'Value' ] # 10 secs of timeout. If it works, the reply is immediate. occupancy = pythonCall( 10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0 ) if not occupancy[ 'OK' ]: return occupancy occupancy = occupancy[ 'Value' ] #Timeout does not work here... #occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 ) if occupancy[ 0 ] != 0: return S_ERROR( occupancy ) output = occupancy[ 1 ][ 0 ] sTokenDict = {} sTokenDict[ 'Endpoint' ] = spaceTokenEndpoint sTokenDict[ 'Token' ] = spaceToken sTokenDict[ 'Total' ] = float( output.get( 'totalsize', '0' ) ) / 1e12 # Bytes to Terabytes sTokenDict[ 'Guaranteed' ] = float( output.get( 'guaranteedsize', '0' ) ) / 1e12 sTokenDict[ 'Free' ] = float( output.get( 'unusedsize', '0' ) ) / 1e12 storeRes = self._storeCommand( [ sTokenDict ] ) if not storeRes[ 'OK' ]: return storeRes return S_OK( [ sTokenDict ] ) def doCache( self ): ''' Method that reads the cache table and tries to read from it. It will return a list of dictionaries if there are results. ''' params = self._prepareCommand() if not params[ 'OK' ]: return params spaceTokenEndpoint, spaceToken = params[ 'Value' ] result = self.rmClient.selectSpaceTokenOccupancyCache( spaceTokenEndpoint, spaceToken ) if result[ 'OK' ]: result = S_OK( [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ] ) return result def doMaster( self ): ''' Master method. Gets all endpoints from the storage elements and all the spaceTokens. Could have taken from Shares/Disk as well. It queries for all their possible combinations, unless there are records in the database for those combinations, which then are not queried. ''' spaceTokens = CSHelpers.getSpaceTokens() if not spaceTokens[ 'OK' ]: return spaceTokens spaceTokens = spaceTokens[ 'Value' ] elementsToCheck = [] seEndpoints = CSHelpers.getStorageElementEndpoints() if not seEndpoints[ 'OK' ]: return seEndpoints seEndpoints = seEndpoints[ 'Value' ] for seEndpoint in seEndpoints: for spaceToken in spaceTokens: elementsToCheck.append( ( seEndpoint, spaceToken ) ) # resQuery = self.rmClient.selectSpaceTokenOccupancyCache( meta = { 'columns' : [ 'Endpoint', 'Token' ] } ) # if not resQuery[ 'OK' ]: # return resQuery # resQuery = resQuery[ 'Value' ] # # elementsToQuery = list( set( elementsToCheck ).difference( set( resQuery ) ) ) gLogger.verbose( 'Processing %s' % elementsToCheck ) for elementToQuery in elementsToCheck: result = self.doNew( elementToQuery ) if not result[ 'OK' ]: self.metrics[ 'failed' ].append( result ) return S_OK( self.metrics ) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class SpaceTokenOccupancyCommand(Command): """ Uses lcg_util to query status of endpoint for a given token. """ def __init__(self, args=None, clients=None): super(SpaceTokenOccupancyCommand, self).__init__(args, clients) if "ResourceManagementClient" in self.apis: self.rmClient = self.apis["ResourceManagementClient"] else: self.rmClient = ResourceManagementClient() def _storeCommand(self, results): """ Stores the results of doNew method on the database. """ for result in results: resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache( result["Endpoint"], result["Token"], result["Total"], result["Guaranteed"], result["Free"] ) if not resQuery["OK"]: return resQuery return S_OK() def _prepareCommand(self): """ SpaceTokenOccupancy requires one argument: - elementName : <str> Given a (storage)elementName, we calculate its endpoint and spaceToken, which are used to query the srm interface. """ if "name" not in self.args: return S_ERROR('"name" not found in self.args') elementName = self.args["name"] endpoint = CSHelpers.getStorageElementEndpoint(elementName) if not endpoint["OK"]: return endpoint endpoint = endpoint["Value"] spaceToken = CSHelpers.getSEToken(elementName) if not spaceToken["OK"]: return spaceToken spaceToken = spaceToken["Value"] return S_OK((endpoint, spaceToken)) def doNew(self, masterParams=None): """ Gets the parameters to run, either from the master method or from its own arguments. It queries the srm interface, and hopefully it will not crash. Out of the results, we keep totalsize, guaranteedsuze, and unusedsize. Then, they are recorded and returned. """ if masterParams is not None: spaceTokenEndpoint, spaceToken = masterParams else: params = self._prepareCommand() if not params["OK"]: return params spaceTokenEndpoint, spaceToken = params["Value"] # 10 secs of timeout. If it works, the reply is immediate. occupancy = pythonCall(10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0) if not occupancy["OK"]: return occupancy occupancy = occupancy["Value"] # Timeout does not work here... # occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 ) if occupancy[0] != 0: return S_ERROR(occupancy) output = occupancy[1][0] sTokenDict = {} sTokenDict["Endpoint"] = spaceTokenEndpoint sTokenDict["Token"] = spaceToken sTokenDict["Total"] = float(output.get("totalsize", "0")) / 1e12 # Bytes to Terabytes sTokenDict["Guaranteed"] = float(output.get("guaranteedsize", "0")) / 1e12 sTokenDict["Free"] = float(output.get("unusedsize", "0")) / 1e12 storeRes = self._storeCommand([sTokenDict]) if not storeRes["OK"]: return storeRes return S_OK([sTokenDict]) def doCache(self): """ Method that reads the cache table and tries to read from it. It will return a list of dictionaries if there are results. """ params = self._prepareCommand() if not params["OK"]: return params spaceTokenEndpoint, spaceToken = params["Value"] result = self.rmClient.selectSpaceTokenOccupancyCache(spaceTokenEndpoint, spaceToken) if result["OK"]: result = S_OK([dict(zip(result["Columns"], res)) for res in result["Value"]]) return result def doMaster(self): """ Master method. Gets all endpoints from the storage elements and all the spaceTokens. Could have taken from Shares/Disk as well. It queries for all their possible combinations, unless there are records in the database for those combinations, which then are not queried. """ self.log.verbose("Getting all SEs defined in the CS") storageElementNames = CSHelpers.getStorageElements() if not storageElementNames["OK"]: self.log.warn(storageElementNames["Message"]) return storageElementNames storageElementNames = storageElementNames["Value"] endpointTokenSet = set() for storageElementName in storageElementNames: endpoint = CSHelpers.getStorageElementEndpoint(storageElementName) if not endpoint["OK"]: self.log.warn(endpoint["Message"]) continue endpoint = endpoint["Value"] spaceToken = CSHelpers.getSEToken(storageElementName) if not spaceToken["OK"]: self.log.warn(spaceToken["Message"]) continue spaceToken = spaceToken["Value"] endpointTokenSet.add((endpoint, spaceToken)) self.log.verbose("Processing %s" % endpointTokenSet) for elementToQuery in endpointTokenSet: result = self.doNew(elementToQuery) if not result["OK"]: self.metrics["failed"].append(result) return S_OK(self.metrics)
class CacheFeederAgent(AgentModule): ''' The CacheFeederAgent feeds the cache tables for the client and the accounting. It runs periodically a set of commands, and stores it's results on the tables. ''' # Too many public methods # pylint: disable-msg=R0904 def initialize(self): # Attribute defined outside __init__ # pylint: disable-msg=W0201 try: self.rmClient = ResourceManagementClient() self.clientsInvoker = ClientsInvoker() commandsListClientsCache = [ ('ClientsCache_Command', 'JobsEffSimpleEveryOne_Command'), ('ClientsCache_Command', 'PilotsEffSimpleEverySites_Command'), ('ClientsCache_Command', 'DTEverySites_Command'), ('ClientsCache_Command', 'DTEveryResources_Command') ] commandsListAccountingCache = [ ('AccountingCache_Command', 'TransferQualityByDestSplitted_Command', (2, ), 'Always'), ('AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', (2, ), 'Always'), ('AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'FailedPilotsByCESplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (24, ), 'Hourly'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (168, ), 'Hourly'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (720, ), 'Daily'), ('AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', (8760, ), 'Daily'), ] commandsVOBOXAvailability = ( 'VOBOXAvailabilityCommand', 'VOBOXAvailabilityCommand', ) commandsSpaceTokenOccupancy = ( 'SpaceTokenOccupancyCommand', 'SpaceTokenOccupancyCommand', ) self.commandObjectsListClientsCache = [] self.commandObjectsListAccountingCache = [] self.commandObjectsVOBOXAvailability = [] self.commandObjectsSpaceTokenOccupancy = [] cc = CommandCaller() # We know beforehand which APIs are we going to need, so we initialize them # first, making everything faster. knownAPIs = [ 'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator', 'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient' ] knownAPIs = initAPIs(knownAPIs, {}) for command in commandsListClientsCache: cObj = cc.setCommandObject(command) for apiName, apiInstance in knownAPIs.items(): cc.setAPI(cObj, apiName, apiInstance) self.commandObjectsListClientsCache.append((command, cObj)) for command in commandsListAccountingCache: cObj = cc.setCommandObject(command) for apiName, apiInstance in knownAPIs.items(): cc.setAPI(cObj, apiName, apiInstance) cArgs = command[2] self.commandObjectsListAccountingCache.append( (command, cObj, cArgs)) for cArgs in self.__getVOBOXAvailabilityCandidates(): cObj = cc.setCommandObject(commandsVOBOXAvailability) self.commandObjectsVOBOXAvailability.append( (commandsVOBOXAvailability, cObj, cArgs)) for cArgs in self.__getSpaceTokenOccupancyCandidates(): cObj = cc.setCommandObject(commandsSpaceTokenOccupancy) self.commandObjectsSpaceTokenOccupancy.append( (commandsSpaceTokenOccupancy, cObj, cArgs)) return S_OK() except Exception: errorStr = "CacheFeederAgent initialization" self.log.exception(errorStr) return S_ERROR(errorStr) ################################################################################ def __getVOBOXAvailabilityCandidates(self): ''' Gets the candidates to execute the command ''' # This is horrible, future me, change this. request_management_urls = gConfig.getValue( '/Systems/RequestManagement/Development/URLs/allURLS', []) configuration_urls = gConfig.getServersList() framework_urls = gConfig.getValue( '/DIRAC/Framework/SystemAdministrator', []) elementsToCheck = request_management_urls + configuration_urls + framework_urls # This may look stupid, but the Command is expecting a tuple return [(el, ) for el in elementsToCheck] def __getSpaceTokenOccupancyCandidates(self): ''' Gets the candidates to execute the command ''' elementsToCheck = [] spaceEndpoints = CS.getSpaceTokenEndpoints() spaceTokens = CS.getSpaceTokens() for site, siteDict in spaceEndpoints.items(): if not isinstance(siteDict, dict): continue if not siteDict.has_key('Endpoint'): continue for spaceToken in spaceTokens: elementsToCheck.append(( siteDict['Endpoint'], spaceToken, )) return elementsToCheck def execute(self): try: now = datetime.utcnow() #VOBOX for co in self.commandObjectsVOBOXAvailability: commandName = co[0][1].split('_')[0] self.log.info('Executed %s with %s' % (commandName, str(co[2]))) co[1].setArgs(co[2]) self.clientsInvoker.setCommand(co[1]) res = self.clientsInvoker.doCommand()['Result'] if not res['OK']: self.log.warn(str(res['Message'])) continue res = res['Value'] serviceUp = res['serviceUpTime'] machineUp = res['machineUpTime'] site = res['site'] system = res['system'] resQuery = self.rmClient.addOrModifyVOBOXCache( site, system, serviceUp, machineUp, now) if not resQuery['OK']: self.log.error(str(resQuery['Message'])) #SpaceTokenOccupancy for co in self.commandObjectsSpaceTokenOccupancy: commandName = co[0][1].split('_')[0] self.log.info('Executed %s with %s' % (commandName, str(co[2]))) co[1].setArgs(co[2]) self.clientsInvoker.setCommand(co[1]) res = self.clientsInvoker.doCommand()['Result'] if not res['OK']: self.log.warn(res['Message']) continue site, token = co[2] res = res['Value'] total = res['total'] guaranteed = res['guaranteed'] free = res['free'] resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache( site, token, total, guaranteed, free, now) if not resQuery['OK']: self.log.error(str(resQuery['Message'])) for co in self.commandObjectsListClientsCache: commandName = co[0][1].split('_')[0] self.log.info('Executed %s' % commandName) try: self.clientsInvoker.setCommand(co[1]) res = self.clientsInvoker.doCommand()['Result'] if not res['OK']: self.log.warn(res['Message']) continue res = res['Value'] if not res or res is None: self.log.info(' returned empty...') continue self.log.debug(res) for key in res.keys(): clientCache = () if 'ID' in res[key].keys(): for value in res[key].keys(): if value != 'ID': clientCache = (key.split()[1], commandName, res[key]['ID'], value, res[key][value], None, None) resQuery = self.rmClient.addOrModifyClientCache( *clientCache) if not resQuery['OK']: self.log.error(resQuery['Message']) else: for value in res[key].keys(): clientCache = (key, commandName, None, value, res[key][value], None, None) resQuery = self.rmClient.addOrModifyClientCache( *clientCache) if not resQuery['OK']: self.log.error(resQuery['Message']) except: self.log.exception("Exception when executing " + co[0][1]) continue now = datetime.utcnow().replace(microsecond=0) for co in self.commandObjectsListAccountingCache: if co[0][3] == 'Hourly': if now.minute >= 10: continue elif co[0][3] == 'Daily': if now.hour >= 1: continue commandName = co[0][1].split('_')[0] plotName = commandName + '_' + str(co[2][0]) self.log.info('Executed %s with args %s %s' % (commandName, co[0][2], co[0][3])) try: co[1].setArgs(co[2]) self.clientsInvoker.setCommand(co[1]) res = self.clientsInvoker.doCommand()['Result'] if not res['OK']: self.log.warn(res['Message']) continue res = res['Value'] if not res or res is None: self.log.info(' returned empty...') continue self.log.debug(res) plotType = res.keys()[0] if not res[plotType]: self.log.info(' returned empty...') self.log.debug(res) for name in res[plotType].keys(): #name, plotType, plotName, result, dateEffective, lastCheckTime accountingClient = (name, plotType, plotName, str(res[plotType][name]), None, None) resQuery = self.rmClient.addOrModifyAccountingCache( *accountingClient) if not resQuery['OK']: self.log.error(resQuery['Message']) except: self.log.exception("Exception when executing " + commandName) continue return S_OK() except Exception: errorStr = "CacheFeederAgent execution" self.log.exception(errorStr) return S_ERROR(errorStr) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class CacheFeederAgent( AgentModule ): ''' The CacheFeederAgent feeds the cache tables for the client and the accounting. It runs periodically a set of commands, and stores it's results on the tables. ''' # Too many public methods # pylint: disable-msg=R0904 def initialize( self ): # Attribute defined outside __init__ # pylint: disable-msg=W0201 try: self.rmClient = ResourceManagementClient() self.clientsInvoker = ClientsInvoker() commandsListClientsCache = [ ( 'ClientsCache_Command', 'JobsEffSimpleEveryOne_Command' ), ( 'ClientsCache_Command', 'PilotsEffSimpleEverySites_Command' ), ( 'ClientsCache_Command', 'DTEverySites_Command' ), ( 'ClientsCache_Command', 'DTEveryResources_Command' ) ] commandsListAccountingCache = [ ( 'AccountingCache_Command', 'TransferQualityByDestSplitted_Command', ( 2, ), 'Always' ), ( 'AccountingCache_Command', 'FailedTransfersBySourceSplitted_Command', ( 2, ), 'Always' ), ( 'AccountingCache_Command', 'TransferQualityByDestSplittedSite_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'SuccessfullJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'FailedJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'SuccessfullPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'FailedPilotsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'SuccessfullPilotsByCESplitted_Command' , ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'FailedPilotsByCESplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 24, ), 'Hourly' ), ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 168, ), 'Hourly' ), ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 720, ), 'Daily' ), ( 'AccountingCache_Command', 'RunningJobsBySiteSplitted_Command', ( 8760, ), 'Daily' ), ] commandsVOBOXAvailability = ( 'VOBOXAvailabilityCommand', 'VOBOXAvailabilityCommand', ) commandsSpaceTokenOccupancy = ( 'SpaceTokenOccupancyCommand', 'SpaceTokenOccupancyCommand', ) self.commandObjectsListClientsCache = [] self.commandObjectsListAccountingCache = [] self.commandObjectsVOBOXAvailability = [] self.commandObjectsSpaceTokenOccupancy = [] cc = CommandCaller() # We know beforehand which APIs are we going to need, so we initialize them # first, making everything faster. knownAPIs = [ 'ResourceStatusClient', 'WMSAdministrator', 'ReportGenerator', 'JobsClient', 'PilotsClient', 'GOCDBClient', 'ReportsClient' ] knownAPIs = initAPIs( knownAPIs, {} ) for command in commandsListClientsCache: cObj = cc.setCommandObject( command ) for apiName, apiInstance in knownAPIs.items(): cc.setAPI( cObj, apiName, apiInstance ) self.commandObjectsListClientsCache.append( ( command, cObj ) ) for command in commandsListAccountingCache: cObj = cc.setCommandObject( command ) for apiName, apiInstance in knownAPIs.items(): cc.setAPI( cObj, apiName, apiInstance ) cArgs = command[ 2 ] self.commandObjectsListAccountingCache.append( ( command, cObj, cArgs ) ) for cArgs in self.__getVOBOXAvailabilityCandidates(): cObj = cc.setCommandObject( commandsVOBOXAvailability ) self.commandObjectsVOBOXAvailability.append( ( commandsVOBOXAvailability, cObj, cArgs ) ) for cArgs in self.__getSpaceTokenOccupancyCandidates(): cObj = cc.setCommandObject( commandsSpaceTokenOccupancy ) self.commandObjectsSpaceTokenOccupancy.append( ( commandsSpaceTokenOccupancy, cObj, cArgs ) ) return S_OK() except Exception: errorStr = "CacheFeederAgent initialization" self.log.exception( errorStr ) return S_ERROR( errorStr ) ################################################################################ def __getVOBOXAvailabilityCandidates( self ): ''' Gets the candidates to execute the command ''' # This is horrible, future me, change this. request_management_urls = gConfig.getValue( '/Systems/RequestManagement/Development/URLs/allURLS', [] ) configuration_urls = gConfig.getServersList() framework_urls = gConfig.getValue( '/DIRAC/Framework/SystemAdministrator', [] ) elementsToCheck = request_management_urls + configuration_urls + framework_urls # This may look stupid, but the Command is expecting a tuple return [ ( el, ) for el in elementsToCheck ] def __getSpaceTokenOccupancyCandidates( self ): ''' Gets the candidates to execute the command ''' elementsToCheck = [] spaceEndpoints = CS.getSpaceTokenEndpoints() spaceTokens = CS.getSpaceTokens() for site,siteDict in spaceEndpoints.items(): if not isinstance( siteDict, dict ): continue if not siteDict.has_key( 'Endpoint' ): continue for spaceToken in spaceTokens: elementsToCheck.append( ( siteDict[ 'Endpoint' ], spaceToken, ) ) return elementsToCheck def execute( self ): try: now = datetime.utcnow() #VOBOX for co in self.commandObjectsVOBOXAvailability: commandName = co[0][1].split( '_' )[0] self.log.info( 'Executed %s with %s' % ( commandName, str( co[2] ) ) ) co[1].setArgs( co[2] ) self.clientsInvoker.setCommand( co[1] ) res = self.clientsInvoker.doCommand()[ 'Result' ] if not res[ 'OK' ]: self.log.warn( str( res[ 'Message' ] ) ) continue res = res[ 'Value' ] serviceUp = res[ 'serviceUpTime' ] machineUp = res[ 'machineUpTime' ] site = res[ 'site' ] system = res[ 'system' ] resQuery = self.rmClient.addOrModifyVOBOXCache( site, system, serviceUp, machineUp, now ) if not resQuery[ 'OK' ]: self.log.error( str( resQuery[ 'Message' ] ) ) #SpaceTokenOccupancy for co in self.commandObjectsSpaceTokenOccupancy: commandName = co[0][1].split( '_' )[0] self.log.info( 'Executed %s with %s' % ( commandName, str( co[2] ) ) ) co[1].setArgs( co[2] ) self.clientsInvoker.setCommand( co[1] ) res = self.clientsInvoker.doCommand()[ 'Result' ] if not res[ 'OK' ]: self.log.warn( res[ 'Message' ] ) continue site, token = co[ 2 ] res = res[ 'Value' ] total = res[ 'total' ] guaranteed = res[ 'guaranteed' ] free = res[ 'free' ] resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache( site, token, total, guaranteed, free, now ) if not resQuery[ 'OK' ]: self.log.error( str( resQuery[ 'Message' ] ) ) for co in self.commandObjectsListClientsCache: commandName = co[0][1].split( '_' )[0] self.log.info( 'Executed %s' % commandName ) try: self.clientsInvoker.setCommand( co[1] ) res = self.clientsInvoker.doCommand()['Result'] if not res['OK']: self.log.warn( res['Message'] ) continue res = res[ 'Value' ] if not res or res is None: self.log.info(' returned empty...') continue self.log.debug( res ) for key in res.keys(): clientCache = () if 'ID' in res[key].keys(): for value in res[key].keys(): if value != 'ID': clientCache = ( key.split()[1], commandName, res[key]['ID'], value, res[key][value], None, None ) resQuery = self.rmClient.addOrModifyClientCache( *clientCache ) if not resQuery[ 'OK' ]: self.log.error( resQuery[ 'Message' ] ) else: for value in res[key].keys(): clientCache = ( key, commandName, None, value, res[key][value], None, None ) resQuery = self.rmClient.addOrModifyClientCache( *clientCache ) if not resQuery[ 'OK' ]: self.log.error( resQuery[ 'Message' ] ) except: self.log.exception( "Exception when executing " + co[0][1] ) continue now = datetime.utcnow().replace( microsecond = 0 ) for co in self.commandObjectsListAccountingCache: if co[0][3] == 'Hourly': if now.minute >= 10: continue elif co[0][3] == 'Daily': if now.hour >= 1: continue commandName = co[0][1].split( '_' )[0] plotName = commandName + '_' + str( co[2][0] ) self.log.info( 'Executed %s with args %s %s' % ( commandName, co[0][2], co[0][3] ) ) try: co[1].setArgs( co[2] ) self.clientsInvoker.setCommand( co[1] ) res = self.clientsInvoker.doCommand()['Result'] if not res['OK']: self.log.warn( res['Message'] ) continue res = res[ 'Value' ] if not res or res is None: self.log.info(' returned empty...') continue self.log.debug( res ) plotType = res.keys()[ 0 ] if not res[ plotType ]: self.log.info(' returned empty...') self.log.debug( res ) for name in res[ plotType ].keys(): #name, plotType, plotName, result, dateEffective, lastCheckTime accountingClient = ( name, plotType, plotName, str(res[plotType][name]), None, None ) resQuery = self.rmClient.addOrModifyAccountingCache( *accountingClient ) if not resQuery[ 'OK' ]: self.log.error( resQuery[ 'Message' ] ) except: self.log.exception( "Exception when executing " + commandName ) continue return S_OK() except Exception: errorStr = "CacheFeederAgent execution" self.log.exception( errorStr ) return S_ERROR( errorStr ) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class FreeDiskSpaceCommand(Command): ''' Uses diskSpace method to get the free space ''' def __init__(self, args=None, clients=None): super(FreeDiskSpaceCommand, self).__init__(args, clients=clients) self.rmClient = ResourceManagementClient() def _prepareCommand(self): ''' FreeDiskSpaceCommand requires one argument: - name : <str> ''' if 'name' not in self.args: return S_ERROR('"name" not found in self.args') elementName = self.args['name'] # We keep TB as default as this is what was used (and will still be used) # in the policy for "space tokens" ("real", "data" SEs) unit = self.args.get('unit', 'TB') return S_OK((elementName, unit)) def doNew(self, masterParams=None): """ Gets the parameters to run, either from the master method or from its own arguments. Gets the total and the free disk space of a storage element and inserts the results in the SpaceTokenOccupancyCache table of ResourceManagementDB database. The result is also returned to the caller, not only inserted. What is inserted in the DB will normally be in MB, what is returned will be in the specified unit. """ if masterParams is not None: elementName, unit = masterParams else: params = self._prepareCommand() if not params['OK']: return params elementName, unit = params['Value'] endpointResult = CSHelpers.getStorageElementEndpoint(elementName) if not endpointResult['OK']: return endpointResult se = StorageElement(elementName) occupancyResult = se.getOccupancy(unit=unit) if not occupancyResult['OK']: return occupancyResult occupancy = occupancyResult['Value'] free = occupancy['Free'] total = occupancy['Total'] results = {'Endpoint': endpointResult['Value'], 'Free': free, 'Total': total, 'ElementName': elementName} result = self._storeCommand(results) if not result['OK']: return result return S_OK({'Free': free, 'Total': total}) def _storeCommand(self, results): """ Here purely for extensibility """ return self.rmClient.addOrModifySpaceTokenOccupancyCache(endpoint=results['Endpoint'], lastCheckTime=datetime.utcnow(), free=results['Free'], total=results['Total'], token=results['ElementName']) def doCache(self): """ This is a method that gets the element's details from the spaceTokenOccupancyCache DB table. It will return a dictionary with th results, converted to "correct" unit. """ params = self._prepareCommand() if not params['OK']: return params elementName, unit = params['Value'] result = self.rmClient.selectSpaceTokenOccupancyCache(token=elementName) if not result['OK']: return result if not result['Value']: return S_ERROR(errno.ENODATA, "No occupancy recorded") # results are normally in 'MB' free = result['Value'][0][3] total = result['Value'][0][4] free = convertSizeUnits(free, 'MB', unit) total = convertSizeUnits(total, 'MB', unit) if free == -sys.maxsize or total == -sys.maxsize: return S_ERROR("No valid unit specified") return S_OK({'Free': free, 'Total': total}) def doMaster(self): """ This method calls the doNew method for each storage element that exists in the CS. """ elements = CSHelpers.getStorageElements() for name in elements['Value']: # keeping TB as default diskSpace = self.doNew((name, 'MB')) if not diskSpace['OK']: gLogger.warn("Unable to calculate free/total disk space", "name: %s" % name) gLogger.warn(diskSpace['Message']) continue return S_OK()
class FreeDiskSpaceCommand( Command ): ''' Uses diskSpace method to get the free space ''' def __init__( self, args = None, clients = None ): super( FreeDiskSpaceCommand, self ).__init__( args, clients = clients ) self.rpc = None self.rsClient = ResourceManagementClient() def _prepareCommand( self ): ''' FreeDiskSpaceCommand requires one argument: - name : <str> ''' if 'name' not in self.args: return S_ERROR( '"name" not found in self.args' ) elementName = self.args[ 'name' ] return S_OK( elementName ) def doNew( self, masterParams = None ): """ Gets the total and the free disk space of a DIPS storage element that is found in the CS and inserts the results in the SpaceTokenOccupancyCache table of ResourceManagementDB database. """ if masterParams is not None: elementName = masterParams else: elementName = self._prepareCommand() if not elementName[ 'OK' ]: return elementName se = StorageElement(elementName) elementURL = se.getStorageParameters(protocol = "dips") if elementURL['OK']: elementURL = se.getStorageParameters(protocol = "dips")['Value']['URLBase'] else: gLogger.verbose( "Not a DIPS storage element, skipping..." ) return S_OK() self.rpc = RPCClient( elementURL, timeout=120 ) free = self.rpc.getFreeDiskSpace("/") if not free[ 'OK' ]: return free free = free['Value'] total = self.rpc.getTotalDiskSpace("/") if not total[ 'OK' ]: return total total = total['Value'] if free and free < 1: free = 1 if total and total < 1: total = 1 result = self.rsClient.addOrModifySpaceTokenOccupancyCache( endpoint = elementURL, lastCheckTime = datetime.utcnow(), free = free, total = total, token = elementName ) if not result[ 'OK' ]: return result return S_OK() def doCache( self ): """ This is a method that gets the element's details from the spaceTokenOccupancy cache. """ elementName = self._prepareCommand() if not elementName[ 'OK' ]: return elementName result = self.rsClient.selectSpaceTokenOccupancyCache(token = elementName) if not result[ 'OK' ]: return result return S_OK( result ) def doMaster( self ): """ This method calls the doNew method for each storage element that exists in the CS. """ elements = CSHelpers.getStorageElements() for name in elements['Value']: diskSpace = self.doNew( name ) if not diskSpace[ 'OK' ]: gLogger.error( "Unable to calculate free disk space" ) continue return S_OK()