def __generateStorageObject( self, storageName, protocolName, protocol, path = None, host = None, port = None, spaceToken = None, wsUrl = None ): moduleRootPaths = getInstalledExtensions() moduleLoaded = False for moduleRootPath in moduleRootPaths: if moduleLoaded: break gLogger.verbose( "Trying to load from root path %s" % moduleRootPath ) moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Storage", "%sStorage.py" % protocolName ) gLogger.verbose( "Looking for file %s" % moduleFile ) if not os.path.isfile( moduleFile ): continue try: # This inforces the convention that the plug in must be named after the protocol moduleName = "%sStorage" % ( protocolName ) storageModule = __import__( '%s.Resources.Storage.%s' % ( moduleRootPath, moduleName ), globals(), locals(), [moduleName] ) except Exception, x: errStr = "StorageFactory._generateStorageObject: Failed to import %s: %s" % ( storageName, x ) gLogger.exception( errStr ) return S_ERROR( errStr ) try: evalString = "storageModule.%s(storageName,protocol,path,host,port,spaceToken,wsUrl)" % moduleName storage = eval( evalString ) if not storage.isOK(): errStr = "StorageFactory._generateStorageObject: Failed to instatiate storage plug in." gLogger.error( errStr, "%s" % ( moduleName ) ) return S_ERROR( errStr ) except Exception, x: errStr = "StorageFactory._generateStorageObject: Failed to instatiate %s(): %s" % ( moduleName, x ) gLogger.exception( errStr ) return S_ERROR( errStr )
def _generateCatalogObject( self, catalogName ): moduleRootPaths = getInstalledExtensions() moduleLoaded = False for moduleRootPath in moduleRootPaths: if moduleLoaded: break gLogger.verbose( "Trying to load from root path %s" % moduleRootPath ) moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Catalog", "%sClient.py" % catalogName ) gLogger.verbose( "Looking for file %s" % moduleFile ) if not os.path.isfile( moduleFile ): continue try: # This inforces the convention that the plug in must be named after the file catalog moduleName = "%sClient" % ( catalogName ) catalogModule = __import__( '%s.Resources.Catalog.%s' % ( moduleRootPath, moduleName ), globals(), locals(), [moduleName] ) except Exception, x: errStr = "FileCatalog._generateCatalogObject: Failed to import %s: %s" % ( catalogName, x ) gLogger.exception( errStr ) return S_ERROR( errStr ) try: evalString = "catalogModule.%s()" % moduleName catalog = eval( evalString ) if not catalog.isOK(): errStr = "FileCatalog._generateCatalogObject: Failed to instantiate catalog plug in." gLogger.error( errStr, moduleName ) return S_ERROR( errStr ) return S_OK( catalog ) except Exception, x: errStr = "FileCatalog._generateCatalogObject: Failed to instantiate %s()" % ( moduleName ) gLogger.exception( errStr, lException = x ) return S_ERROR( errStr )
def loadModules( self, modulesList, hideExceptions = False ): """ Load all modules required in moduleList """ for modName in modulesList: gLogger.verbose( "Checking %s" % modName ) #if it's a executor modName name just load it and be done with it if modName.find( "/" ) > -1: gLogger.verbose( "Module %s seems to be a valid name. Try to load it!" % modName ) result = self.loadModule( modName, hideExceptions = hideExceptions ) if not result[ 'OK' ]: return result continue #Check if it's a system name #Look in the CS system = modName #Can this be generated with sectionFinder? csPath = "%s/Executors" % PathFinder.getSystemSection ( system, ( system, ) ) gLogger.verbose( "Exploring %s to discover modules" % csPath ) result = gConfig.getSections( csPath ) if result[ 'OK' ]: #Add all modules in the CS :P for modName in result[ 'Value' ]: result = self.loadModule( "%s/%s" % ( system, modName ), hideExceptions = hideExceptions ) if not result[ 'OK' ]: return result #Look what is installed parentModule = None for rootModule in getInstalledExtensions(): if system.find( "System" ) != len( system ) - 6: parentImport = "%s.%sSystem.%s" % ( rootModule, system, self.__csSuffix ) else: parentImport = "%s.%s.%s" % ( rootModule, system, self.__csSuffix ) #HERE! result = self.__recurseImport( parentImport ) if not result[ 'OK' ]: return result parentModule = result[ 'Value' ] if parentModule: break if not parentModule: continue parentPath = parentModule.__path__[0] gLogger.notice( "Found modules path at %s" % parentImport ) for entry in os.listdir( parentPath ): if entry[-3:] != ".py" or entry == "__init__.py": continue if not os.path.isfile( os.path.join( parentPath, entry ) ): continue modName = "%s/%s" % ( system, entry[:-3] ) gLogger.verbose( "Trying to import %s" % modName ) result = self.loadModule( modName, hideExceptions = hideExceptions, parentModule = parentModule ) return S_OK()
def __generateStorageObject(self, storageName, protocolName, protocol, path=None, host=None, port=None, spaceToken=None, wsUrl=None): storageType = protocolName if self.proxy: storageType = 'Proxy' moduleRootPaths = getInstalledExtensions() moduleLoaded = False path = path.rstrip('/') if not path: path = '/' for moduleRootPath in moduleRootPaths: if moduleLoaded: break gLogger.verbose("Trying to load from root path %s" % moduleRootPath) moduleFile = os.path.join(rootPath, moduleRootPath, "Resources", "Storage", "%sStorage.py" % storageType) gLogger.verbose("Looking for file %s" % moduleFile) if not os.path.isfile(moduleFile): continue try: # This inforces the convention that the plug in must be named after the protocol moduleName = "%sStorage" % (storageType) storageModule = __import__( '%s.Resources.Storage.%s' % (moduleRootPath, moduleName), globals(), locals(), [moduleName]) except Exception, x: errStr = "StorageFactory._generateStorageObject: Failed to import %s: %s" % ( storageName, x) gLogger.exception(errStr) return S_ERROR(errStr) try: evalString = "storageModule.%s(storageName,protocol,path,host,port,spaceToken,wsUrl)" % moduleName storage = eval(evalString) if not storage.isOK(): errStr = "StorageFactory._generateStorageObject: Failed to instantiate storage plug in." gLogger.error(errStr, "%s" % (moduleName)) return S_ERROR(errStr) except Exception, x: errStr = "StorageFactory._generateStorageObject: Failed to instantiate %s(): %s" % ( moduleName, x) gLogger.exception(errStr) return S_ERROR(errStr)
def loadModule( self, modName, hideExceptions = False, parentModule = False ): """ Load module name. name must take the form [DIRAC System Name]/[DIRAC module] """ while modName and modName[0] == "/": modName = modName[1:] if modName in self.__modules: return S_OK() modList = modName.split( "/" ) if len( modList ) != 2: return S_ERROR( "Can't load %s: Invalid module name" % ( modName ) ) csSection = self.__sectionFinder( modName ) loadGroup = gConfig.getValue( "%s/Load" % csSection, [] ) #Check if it's a load group if loadGroup: gLogger.info( "Found load group %s. Will load %s" % ( modName, ", ".join( loadGroup ) ) ) for loadModName in loadGroup: if loadModName.find( "/" ) == -1: loadModName = "%s/%s" % ( modList[0], loadModName ) result = self.loadModule( loadModName, hideExceptions = hideExceptions, parentModule = False ) if not result[ 'OK' ]: return result return S_OK() #Normal load loadName = gConfig.getValue( "%s/Module" % csSection, "" ) if not loadName: loadName = modName gLogger.info( "Loading %s" % ( modName ) ) else: if loadName.find( "/" ) == -1: loadName = "%s/%s" % ( modList[0], loadName ) gLogger.info( "Loading %s (%s)" % ( modName, loadName ) ) #If already loaded, skip loadList = loadName.split( "/" ) if len( loadList ) != 2: return S_ERROR( "Can't load %s: Invalid module name" % ( loadName ) ) system, module = loadList #Load className = module if self.__modSuffix: className = "%s%s" % ( className, self.__modSuffix ) if loadName not in self.__loadedModules: #Check if handler is defined loadCSSection = self.__sectionFinder( loadName ) handlerPath = gConfig.getValue( "%s/HandlerPath" % loadCSSection, "" ) if handlerPath: gLogger.info( "Trying to %s from CS defined path %s" % ( loadName, handlerPath ) ) gLogger.verbose( "Found handler for %s: %s" % ( loadName, handlerPath ) ) handlerPath = handlerPath.replace( "/", "." ) if handlerPath.find( ".py", len( handlerPath ) -3 ) > -1: handlerPath = handlerPath[ :-3 ] className = List.fromChar( handlerPath, "." )[-1] result = self.__recurseImport( handlerPath ) if not result[ 'OK' ]: return S_ERROR( "Cannot load user defined handler %s: %s" % ( handlerPath, result[ 'Message' ] ) ) gLogger.verbose( "Loaded %s" % handlerPath ) elif parentModule: gLogger.info( "Trying to autodiscover %s from parent" % loadName ) #If we've got a parent module, load from there. modImport = module if self.__modSuffix: modImport = "%s%s" % ( modImport, self.__modSuffix ) result = self.__recurseImport( modImport, parentModule, hideExceptions = hideExceptions ) else: #Check to see if the module exists in any of the root modules gLogger.info( "Trying to autodiscover %s" % loadName ) rootModulesToLook = getInstalledExtensions() for rootModule in rootModulesToLook: importString = '%s.%sSystem.%s.%s' % ( rootModule, system, self.__importLocation, module ) if self.__modSuffix: importString = "%s%s" % ( importString, self.__modSuffix ) gLogger.verbose( "Trying to load %s" % importString ) result = self.__recurseImport( importString, hideExceptions = hideExceptions ) #Error while loading if not result[ 'OK' ]: return result #Something has been found! break :) if result[ 'Value' ]: gLogger.verbose( "Found %s" % importString ) break #Nothing found if not result[ 'Value' ]: return S_ERROR( "Could not find %s" % loadName ) modObj = result[ 'Value' ] try: #Try to get the class from the module modClass = getattr( modObj, className ) except AttributeError: location = "" if '__file__' in dir( modObj ): location = modObj.__file__ else: location = modObj.__path__ gLogger.exception( "%s module does not have a %s class!" % ( location, module ) ) return S_ERROR( "Cannot load %s" % module ) #Check if it's subclass if not issubclass( modClass, self.__superClass ): return S_ERROR( "%s has to inherit from %s" % ( loadName, self.__superClass.__name__ ) ) self.__loadedModules[ loadName ] = { 'classObj' : modClass, 'moduleObj' : modObj } #End of loading of 'loadName' module #A-OK :) self.__modules[ modName ] = self.__loadedModules[ loadName ].copy() #keep the name of the real code module self.__modules[ modName ][ 'modName' ] = modName self.__modules[ modName ][ 'loadName' ] = loadName gLogger.notice( "Loaded module %s" % modName ) return S_OK()
def loadAgentModule( self, fullName, hideExceptions = False ): """ Load module fullName. fullName must take the form [DIRAC System Name]/[DIRAC Agent Name] then: - calls the am_initialize method of the imported Agent - determines the pooling interval: am_getPollingTime, - determines the number of executions: am_getMaxCycles - creates a periodic Task in the ThreadScheduler: am_go """ modList = fullName.split( "/" ) if len( modList ) != 2: return S_ERROR( "Can't load %s: Invalid agent name" % ( fullName ) ) if fullName in self.__agentModules: gLogger.notice( "Agent already loaded:", fullName ) return S_OK() gLogger.info( "Loading %s" % fullName ) system, agentName = modList rootModulesToLook = getInstalledExtensions() moduleLoaded = False for rootModule in rootModulesToLook: if moduleLoaded: break gLogger.verbose( "Trying to load from root module %s" % rootModule ) moduleFile = os.path.join( rootPath, rootModule, "%sSystem" % system, "Agent", "%s.py" % agentName ) gLogger.verbose( "Looking for file %s" % moduleFile ) if not os.path.isfile( moduleFile ): continue try: importString = '%s.%sSystem.Agent.%s' % ( rootModule, system, agentName ) gLogger.debug( "Trying to load %s" % importString ) agentModule = __import__( importString, globals(), locals(), agentName ) agentClass = getattr( agentModule, agentName ) if not self.__checkHandler( agentClass ): gLogger.error( "Invalid Agent module", "%s has to inherit from AgentModule" % fullName ) continue agent = agentClass( fullName, self.__baseAgentName ) if not isinstance( agent, AgentModule ): gLogger.error( "%s is not a subclass of AgentModule" % moduleFile ) continue result = agent.am_initialize() if not result[ 'OK' ]: return S_ERROR( "Error while calling initialize method of %s: %s" % ( fullName, result[ 'Message' ] ) ) moduleLoaded = True except Exception: if not hideExceptions: gLogger.exception( "Can't load agent %s" % fullName ) if not moduleLoaded: return S_ERROR( "Can't load agent %s in root modules %s" % ( fullName, ", ".join ( rootModulesToLook ) ) ) self.__agentModules[ fullName ] = { 'instance' : agent, 'class' : agentClass, 'module' : agentModule, 'running' : False } agentPeriod = agent.am_getPollingTime() result = self.__scheduler.addPeriodicTask( agentPeriod, agent.am_go, executions = agent.am_getMaxCycles(), elapsedTime = agentPeriod ) if not result[ 'OK' ]: return result taskId = result[ 'Value' ] self.__tasks[ result[ 'Value' ] ] = fullName self.__agentModules[ fullName ][ 'taskId' ] = taskId self.__agentModules[ fullName ][ 'running' ] = True return S_OK()