class FileCatalogDBTestCase(unittest.TestCase): """ Base class for the FileCatalogDB test cases """ def setUp(self): self.db = FileCatalogDB() # for table in self.db._query( "Show tables;" )["Value"]: # self.db.deleteEntries( table[0] ) self.db.setConfig(DATABASE_CONFIG) def tearDown(self): pass
class FileCatalogDBTestCase( unittest.TestCase ): """ Base class for the FileCatalogDB test cases """ def setUp( self ): self.db = FileCatalogDB() # for table in self.db._query( "Show tables;" )["Value"]: # self.db.deleteEntries( table[0] ) self.db.setConfig( DATABASE_CONFIG ) def tearDown(self): pass
def __init__(self, args=None, clients=None): super(StorageIHEPCommand, self).__init__(args, clients) if 'ResourceManagementIHEPClient' in self.apis: self.rmIHEPClient = self.apis['ResourceManagementIHEPClient'] else: self.rmIHEPClient = ResourceManagementIHEPClient() if 'FileCatalogDB' in self.apis: self.fcDB = self.apis['FileCatalogDB'] else: self.fcDB = FileCatalogDB()
def initializeFileCatalogHandler(serviceInfo): global fcDB serviceCS = serviceInfo['serviceSectionPath'] # Instantiate the requested database dbLocation = gConfig.getValue('%s/Database' % serviceCS, 'DataManagement/FileCatalogDB') fcDB = FileCatalogDB(dbLocation) databaseConfig = {} # Obtain the plugins to be used for DB interaction gLogger.info("Initializing with FileCatalog with following managers:") defaultManagers = { 'UserGroupManager': 'UserAndGroupManagerDB', 'SEManager': 'SEManagerDB', 'SecurityManager': 'NoSecurityManager', 'DirectoryManager': 'DirectoryLevelTree', 'FileManager': 'FileManager', 'DirectoryMetadata': 'DirectoryMetadata', 'FileMetadata': 'FileMetadata' } for configKey in sortList(defaultManagers.keys()): defaultValue = defaultManagers[configKey] configValue = gConfig.getValue('%s/%s' % (serviceCS, configKey), defaultValue) gLogger.info("%s : %s" % (str(configKey).ljust(20), str(configValue).ljust(20))) databaseConfig[configKey] = configValue # Obtain some general configuration of the database gLogger.info( "Initializing the FileCatalog with the following configuration:") defaultConfig = { 'UniqueGUID': False, 'GlobalReadAccess': True, 'LFNPFNConvention': True, 'ResolvePFN': True, 'DefaultUmask': 0775, 'VisibleStatus': ['AprioriGood'] } for configKey in sortList(defaultConfig.keys()): defaultValue = defaultConfig[configKey] configValue = gConfig.getValue('%s/%s' % (serviceCS, configKey), defaultValue) gLogger.info("%s : %s" % (str(configKey).ljust(20), str(configValue).ljust(20))) databaseConfig[configKey] = configValue res = fcDB.setConfig(databaseConfig) return res
def initializeFileCatalogHandler(serviceInfo): """ handler initialisation """ global gFileCatalogDB dbLocation = getServiceOption(serviceInfo, 'Database', 'DataManagement/FileCatalogDB') gFileCatalogDB = FileCatalogDB(dbLocation) databaseConfig = {} # Obtain the plugins to be used for DB interaction gLogger.info("Initializing with FileCatalog with following managers:") defaultManagers = { 'UserGroupManager': 'UserAndGroupManagerDB', 'SEManager': 'SEManagerDB', 'SecurityManager': 'NoSecurityManager', 'DirectoryManager': 'DirectoryLevelTree', 'FileManager': 'FileManager', 'DirectoryMetadata': 'DirectoryMetadata', 'FileMetadata': 'FileMetadata', 'DatasetManager': 'DatasetManager' } for configKey in sortList(defaultManagers.keys()): defaultValue = defaultManagers[configKey] configValue = getServiceOption(serviceInfo, configKey, defaultValue) gLogger.info("%-20s : %-20s" % (str(configKey), str(configValue))) databaseConfig[configKey] = configValue # Obtain some general configuration of the database gLogger.info( "Initializing the FileCatalog with the following configuration:") defaultConfig = { 'UniqueGUID': False, 'GlobalReadAccess': True, 'LFNPFNConvention': 'Strong', 'ResolvePFN': True, 'DefaultUmask': 0775, 'ValidFileStatus': ['AprioriGood', 'Trash', 'Removing', 'Probing'], 'ValidReplicaStatus': ['AprioriGood', 'Trash', 'Removing', 'Probing'], 'VisibleFileStatus': ['AprioriGood'], 'VisibleReplicaStatus': ['AprioriGood'] } for configKey in sortList(defaultConfig.keys()): defaultValue = defaultConfig[configKey] configValue = getServiceOption(serviceInfo, configKey, defaultValue) gLogger.info("%-20s : %-20s" % (str(configKey), str(configValue))) databaseConfig[configKey] = configValue res = gFileCatalogDB.setConfig(databaseConfig) return res
def initializeFileCatalogHandler( serviceInfo ): """ handler initialisation """ global gFileCatalogDB dbLocation = getServiceOption( serviceInfo, 'Database', 'DataManagement/FileCatalogDB' ) gFileCatalogDB = FileCatalogDB( dbLocation ) databaseConfig = {} # Obtain the plugins to be used for DB interaction gLogger.info( "Initializing with FileCatalog with following managers:" ) defaultManagers = { 'UserGroupManager' : 'UserAndGroupManagerDB', 'SEManager' : 'SEManagerDB', 'SecurityManager' : 'NoSecurityManager', 'DirectoryManager' : 'DirectoryLevelTree', 'FileManager' : 'FileManager', 'DirectoryMetadata' : 'DirectoryMetadata', 'FileMetadata' : 'FileMetadata', 'DatasetManager' : 'DatasetManager' } for configKey in sortList( defaultManagers.keys() ): defaultValue = defaultManagers[configKey] configValue = getServiceOption( serviceInfo, configKey, defaultValue ) gLogger.info( "%-20s : %-20s" % ( str( configKey ), str( configValue ) ) ) databaseConfig[configKey] = configValue # Obtain some general configuration of the database gLogger.info( "Initializing the FileCatalog with the following configuration:" ) defaultConfig = { 'UniqueGUID' : False, 'GlobalReadAccess' : True, 'LFNPFNConvention' : 'Strong', 'ResolvePFN' : True, 'DefaultUmask' : 0775, 'ValidFileStatus' : ['AprioriGood','Trash','Removing','Probing'], 'ValidReplicaStatus' : ['AprioriGood','Removing','Probing'], 'VisibleFileStatus' : ['AprioriGood'], 'VisibleReplicaStatus': ['AprioriGood']} for configKey in sortList( defaultConfig.keys() ): defaultValue = defaultConfig[configKey] configValue = getServiceOption( serviceInfo, configKey, defaultValue ) gLogger.info( "%-20s : %-20s" % ( str( configKey ), str( configValue ) ) ) databaseConfig[configKey] = configValue res = gFileCatalogDB.setConfig( databaseConfig ) return res
def initializeFileCatalogHandler(serviceInfo): global fcDB serviceCS = serviceInfo ['serviceSectionPath'] # Instantiate the requested database dbLocation = gConfig.getValue('%s/Database' % serviceCS,'DataManagement/FileCatalogDB') fcDB = FileCatalogDB(dbLocation) databaseConfig = {} # Obtain the plugins to be used for DB interaction gLogger.info("Initializing with FileCatalog with following managers:") defaultManagers = { 'UserGroupManager' : 'UserAndGroupManagerDB', 'SEManager' : 'SEManagerDB', 'SecurityManager' : 'NoSecurityManager', 'DirectoryManager' : 'DirectoryLevelTree', 'FileManager' : 'FileManager', 'DirectoryMetadata' : 'DirectoryMetadata', 'FileMetadata' : 'FileMetadata'} for configKey in sortList(defaultManagers.keys()): defaultValue = defaultManagers[configKey] configValue = gConfig.getValue('%s/%s' % (serviceCS,configKey),defaultValue) gLogger.info("%s : %s" % (str(configKey).ljust(20),str(configValue).ljust(20))) databaseConfig[configKey] = configValue # Obtain some general configuration of the database gLogger.info("Initializing the FileCatalog with the following configuration:") defaultConfig = { 'UniqueGUID' : False, 'GlobalReadAccess' : True, 'LFNPFNConvention' : True, 'ResolvePFN' : True, 'DefaultUmask' : 0775, 'VisibleStatus' : ['AprioriGood']} for configKey in sortList(defaultConfig.keys()): defaultValue = defaultConfig[configKey] configValue = gConfig.getValue('%s/%s' % (serviceCS,configKey),defaultValue) gLogger.info("%s : %s" % (str(configKey).ljust(20),str(configValue).ljust(20))) databaseConfig[configKey] = configValue res = fcDB.setConfig(databaseConfig) return res
def __init__( self, args = None, clients = None ): super( StorageIHEPCommand, self ).__init__( args, clients ) if 'ResourceManagementIHEPClient' in self.apis: self.rmIHEPClient = self.apis[ 'ResourceManagementIHEPClient' ] else: self.rmIHEPClient = ResourceManagementIHEPClient() if 'FileCatalogDB' in self.apis: self.fcDB = self.apis[ 'FileCatalogDB' ] else: self.fcDB = FileCatalogDB()
def initializeHandler(cls, serviceInfo): """Handler initialization""" dbLocation = getServiceOption(serviceInfo, "Database", "DataManagement/FileCatalogDB") cls.fileCatalogDB = FileCatalogDB(dbLocation, parentLogger=cls.log) databaseConfig = {} # Obtain the plugins to be used for DB interaction cls.log.info("Initializing with FileCatalog with following managers:") defaultManagers = { "UserGroupManager": "UserAndGroupManagerDB", "SEManager": "SEManagerDB", "SecurityManager": "NoSecurityManager", "DirectoryManager": "DirectoryLevelTree", "FileManager": "FileManager", "DirectoryMetadata": "DirectoryMetadata", "FileMetadata": "FileMetadata", "DatasetManager": "DatasetManager", } for configKey in sorted(defaultManagers.keys()): defaultValue = defaultManagers[configKey] configValue = getServiceOption(serviceInfo, configKey, defaultValue) cls.log.info("%-20s : %-20s" % (str(configKey), str(configValue))) databaseConfig[configKey] = configValue # Obtain some general configuration of the database cls.log.info("Initializing the FileCatalog with the following configuration:") defaultConfig = { "UniqueGUID": False, "GlobalReadAccess": True, "LFNPFNConvention": "Strong", "ResolvePFN": True, "DefaultUmask": 0o775, "ValidFileStatus": ["AprioriGood", "Trash", "Removing", "Probing"], "ValidReplicaStatus": ["AprioriGood", "Trash", "Removing", "Probing"], "VisibleFileStatus": ["AprioriGood"], "VisibleReplicaStatus": ["AprioriGood"], } for configKey in sorted(defaultConfig.keys()): defaultValue = defaultConfig[configKey] configValue = getServiceOption(serviceInfo, configKey, defaultValue) cls.log.info("%-20s : %-20s" % (str(configKey), str(configValue))) databaseConfig[configKey] = configValue res = cls.fileCatalogDB.setConfig(databaseConfig) return res
def initializeHandler(cls, serviceInfo): """ Handler initialization """ dbLocation = getServiceOption(serviceInfo, 'Database', 'DataManagement/FileCatalogDB') cls.fileCatalogDB = FileCatalogDB(dbLocation) databaseConfig = {} # Obtain the plugins to be used for DB interaction cls.log.info("Initializing with FileCatalog with following managers:") defaultManagers = {'UserGroupManager': 'UserAndGroupManagerDB', 'SEManager': 'SEManagerDB', 'SecurityManager': 'NoSecurityManager', 'DirectoryManager': 'DirectoryLevelTree', 'FileManager': 'FileManager', 'DirectoryMetadata': 'DirectoryMetadata', 'FileMetadata': 'FileMetadata', 'DatasetManager': 'DatasetManager'} for configKey in sorted(defaultManagers.keys()): defaultValue = defaultManagers[configKey] configValue = getServiceOption(serviceInfo, configKey, defaultValue) cls.log.info("%-20s : %-20s" % (str(configKey), str(configValue))) databaseConfig[configKey] = configValue # Obtain some general configuration of the database cls.log.info("Initializing the FileCatalog with the following configuration:") defaultConfig = {'UniqueGUID': False, 'GlobalReadAccess': True, 'LFNPFNConvention': 'Strong', 'ResolvePFN': True, 'DefaultUmask': 0o775, 'ValidFileStatus': ['AprioriGood', 'Trash', 'Removing', 'Probing'], 'ValidReplicaStatus': ['AprioriGood', 'Trash', 'Removing', 'Probing'], 'VisibleFileStatus': ['AprioriGood'], 'VisibleReplicaStatus': ['AprioriGood']} for configKey in sorted(defaultConfig.keys()): defaultValue = defaultConfig[configKey] configValue = getServiceOption(serviceInfo, configKey, defaultValue) cls.log.info("%-20s : %-20s" % (str(configKey), str(configValue))) databaseConfig[configKey] = configValue res = cls.fileCatalogDB.setConfig(databaseConfig) gMonitor.registerActivity("AddFile", "Amount of addFile calls", "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) gMonitor.registerActivity("AddFileSuccessful", "Files successfully added", "FileCatalogHandler", "files/min", gMonitor.OP_SUM) gMonitor.registerActivity("AddFileFailed", "Files failed to add", "FileCatalogHandler", "files/min", gMonitor.OP_SUM) gMonitor.registerActivity("RemoveFile", "Amount of removeFile calls", "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) gMonitor.registerActivity("RemoveFileSuccessful", "Files successfully removed", "FileCatalogHandler", "files/min", gMonitor.OP_SUM) gMonitor.registerActivity("RemoveFileFailed", "Files failed to remove", "FileCatalogHandler", "files/min", gMonitor.OP_SUM) gMonitor.registerActivity("AddReplica", "Amount of addReplica calls", "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) gMonitor.registerActivity("AddReplicaSuccessful", "Replicas successfully added", "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM) gMonitor.registerActivity("AddReplicaFailed", "Replicas failed to add", "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM) gMonitor.registerActivity("RemoveReplica", "Amount of removeReplica calls", "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) gMonitor.registerActivity("RemoveReplicaSuccessful", "Replicas successfully removed", "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM) gMonitor.registerActivity("RemoveReplicaFailed", "Replicas failed to remove", "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM) gMonitor.registerActivity("ListDirectory", "Amount of listDirectory calls", "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) return res
class StorageIHEPCommand( Command ): """ StorageIHEPCommand """ def __init__( self, args = None, clients = None ): super( StorageIHEPCommand, self ).__init__( args, clients ) if 'ResourceManagementIHEPClient' in self.apis: self.rmIHEPClient = self.apis[ 'ResourceManagementIHEPClient' ] else: self.rmIHEPClient = ResourceManagementIHEPClient() if 'FileCatalogDB' in self.apis: self.fcDB = self.apis[ 'FileCatalogDB' ] else: self.fcDB = FileCatalogDB() def _storeCommand( self, result): """ Stores the results of doNew method on the database. """ for storageDict in result: print storageDict resQuery = self.rmIHEPClient.addOrModifyStorageCache( sE = storageDict[ 'SE' ], occupied = storageDict[ 'Occupied' ], free = storageDict[ 'Free' ], usage = storageDict[ 'Usage' ] ) if not resQuery[ 'OK' ]: return resQuery return S_OK() def doNew( self, masterParams = None ): """ It searches FileCatalogDB to find out occupied storage. """ ses = masterParams seMaxStorage = {} for se in ses: maxStorage = gConfig.getValue('/Resources/StorageElements/%s/Capacity' % se, 0) * 1 << 40 seMaxStorage[ se ] = maxStorage sqlStr = """select SE.SEName, sum(F.Size) from FC_Replicas R, FC_Files F, FC_StorageElements SE where R.FileID=F.FileID and R.SEID=SE.SEID group by R.SEID;""" result = self.fcDB._query(sqlStr) if not result[ 'OK' ]: return result result = result[ 'Value' ] seOccupied = {} for se, occupied in result: seOccupied[ se ] = int(occupied) uniformResult = [] for se in ses: max = seMaxStorage.get(se, 0) occupied = seOccupied.get(se, 0) if max == 0: usage = 0.0 free = 0 else: usage = math.floor(float(occupied) / max * 1000) / 10 free = max - occupied uniformResult.append( { 'SE' : se, 'Occupied' : occupied, 'Free' : free, 'Usage' : usage } ) storeRes = self._storeCommand( uniformResult ) if not storeRes[ 'OK' ]: return storeRes return S_OK( result ) def doMaster(self): """ Master method Gets all ses and call doNew method """ ses = CSHelpers.getStorageElements() if not ses[ 'OK' ]: return ses storageResults = self.doNew( ses[ 'Value' ] ) if not storageResults[ 'OK' ]: self.metrics[ 'failed' ].append( storageResults[ 'Message' ] ) return S_OK( self.metrics )
def setUp(self): self.db = FileCatalogDB() # for table in self.db._query( "Show tables;" )["Value"]: # self.db.deleteEntries( table[0] ) self.db.setConfig(DATABASE_CONFIG)
def setUp( self ): self.db = FileCatalogDB() # for table in self.db._query( "Show tables;" )["Value"]: # self.db.deleteEntries( table[0] ) self.db.setConfig( DATABASE_CONFIG )
def initializeFileCatalogHandler( serviceInfo ): """ handler initialisation """ global gFileCatalogDB dbLocation = getServiceOption( serviceInfo, 'Database', 'DataManagement/FileCatalogDB' ) gFileCatalogDB = FileCatalogDB( dbLocation ) databaseConfig = {} # Obtain the plugins to be used for DB interaction gLogger.info( "Initializing with FileCatalog with following managers:" ) defaultManagers = { 'UserGroupManager' : 'UserAndGroupManagerDB', 'SEManager' : 'SEManagerDB', 'SecurityManager' : 'NoSecurityManager', 'DirectoryManager' : 'DirectoryLevelTree', 'FileManager' : 'FileManager', 'DirectoryMetadata' : 'DirectoryMetadata', 'FileMetadata' : 'FileMetadata', 'DatasetManager' : 'DatasetManager' } for configKey in sorted( defaultManagers.keys() ): defaultValue = defaultManagers[configKey] configValue = getServiceOption( serviceInfo, configKey, defaultValue ) gLogger.info( "%-20s : %-20s" % ( str( configKey ), str( configValue ) ) ) databaseConfig[configKey] = configValue # Obtain some general configuration of the database gLogger.info( "Initializing the FileCatalog with the following configuration:" ) defaultConfig = { 'UniqueGUID' : False, 'GlobalReadAccess' : True, 'LFNPFNConvention' : 'Strong', 'ResolvePFN' : True, 'DefaultUmask' : 0775, 'ValidFileStatus' : ['AprioriGood','Trash','Removing','Probing'], 'ValidReplicaStatus' : ['AprioriGood','Trash','Removing','Probing'], 'VisibleFileStatus' : ['AprioriGood'], 'VisibleReplicaStatus': ['AprioriGood']} for configKey in sorted( defaultConfig.keys() ): defaultValue = defaultConfig[configKey] configValue = getServiceOption( serviceInfo, configKey, defaultValue ) gLogger.info( "%-20s : %-20s" % ( str( configKey ), str( configValue ) ) ) databaseConfig[configKey] = configValue res = gFileCatalogDB.setConfig( databaseConfig ) gMonitor.registerActivity( "AddFile", "Amount of addFile calls", "FileCatalogHandler", "calls/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "AddFileSuccessful", "Files successfully added", "FileCatalogHandler", "files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "AddFileFailed", "Files failed to add", "FileCatalogHandler", "files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RemoveFile", "Amount of removeFile calls", "FileCatalogHandler", "calls/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RemoveFileSuccessful", "Files successfully removed", "FileCatalogHandler", "files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RemoveFileFailed", "Files failed to remove", "FileCatalogHandler", "files/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "AddReplica", "Amount of addReplica calls", "FileCatalogHandler", "calls/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "AddReplicaSuccessful", "Replicas successfully added", "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "AddReplicaFailed", "Replicas failed to add", "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RemoveReplica", "Amount of removeReplica calls", "FileCatalogHandler", "calls/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RemoveReplicaSuccessful", "Replicas successfully removed", "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "RemoveReplicaFailed", "Replicas failed to remove", "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM ) gMonitor.registerActivity( "ListDirectory", "Amount of listDirectory calls", "FileCatalogHandler", "calls/min", gMonitor.OP_SUM ) return res
class StorageIHEPCommand(Command): """ StorageIHEPCommand """ def __init__(self, args=None, clients=None): super(StorageIHEPCommand, self).__init__(args, clients) if 'ResourceManagementIHEPClient' in self.apis: self.rmIHEPClient = self.apis['ResourceManagementIHEPClient'] else: self.rmIHEPClient = ResourceManagementIHEPClient() if 'FileCatalogDB' in self.apis: self.fcDB = self.apis['FileCatalogDB'] else: self.fcDB = FileCatalogDB() def _storeCommand(self, result): """ Stores the results of doNew method on the database. """ for storageDict in result: resQuery = self.rmIHEPClient.addOrModifyStorageCache( sE=storageDict['SE'], occupied=storageDict['Occupied'], free=storageDict['Free'], usage=storageDict['Usage']) if not resQuery['OK']: return resQuery return S_OK() def doNew(self, masterParams=None): """ It searches FileCatalogDB to find out occupied storage. """ ses = masterParams seMaxStorage = {} for se in ses: maxStorage = gConfig.getValue( '/Resources/StorageElements/%s/Capacity' % se, 0) * 1 << 40 seMaxStorage[se] = maxStorage sqlStr = """select SE.SEName, sum(F.Size) from FC_Replicas R, FC_Files F, FC_StorageElements SE where R.FileID=F.FileID and R.SEID=SE.SEID group by R.SEID;""" result = self.fcDB._query(sqlStr) if not result['OK']: return result result = result['Value'] seOccupied = {} for se, occupied in result: seOccupied[se] = int(occupied) uniformResult = [] for se in ses: max = seMaxStorage.get(se, 0) occupied = seOccupied.get(se, 0) if max == 0: usage = 0.0 free = 0 else: usage = math.floor(float(occupied) / max * 1000) / 10 free = max - occupied uniformResult.append({ 'SE': se, 'Occupied': occupied, 'Free': free, 'Usage': usage }) storeRes = self._storeCommand(uniformResult) if not storeRes['OK']: return storeRes return S_OK(result) def doMaster(self): """ Master method Gets all ses and call doNew method """ ses = CSHelpers.getStorageElements() if not ses['OK']: return ses storageResults = self.doNew(ses['Value']) if not storageResults['OK']: self.metrics['failed'].append(storageResults['Message']) return S_OK(self.metrics)