def __generateStorageObject( self, storageName, protocolName, protocol, path = None,
host = None, port = None, spaceToken = None, wsUrl = None ):
moduleRootPaths = getInstalledExtensions()
moduleLoaded = False
for moduleRootPath in moduleRootPaths:
if moduleLoaded:
break
gLogger.verbose( "Trying to load from root path %s" % moduleRootPath )
moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Storage", "%sStorage.py" % protocolName )
gLogger.verbose( "Looking for file %s" % moduleFile )
if not os.path.isfile( moduleFile ):
continue
try:
# This inforces the convention that the plug in must be named after the protocol
moduleName = "%sStorage" % ( protocolName )
storageModule = __import__( '%s.Resources.Storage.%s' % ( moduleRootPath, moduleName ),
globals(), locals(), [moduleName] )
except Exception, x:
errStr = "StorageFactory._generateStorageObject: Failed to import %s: %s" % ( storageName, x )
gLogger.exception( errStr )
return S_ERROR( errStr )
try:
evalString = "storageModule.%s(storageName,protocol,path,host,port,spaceToken,wsUrl)" % moduleName
storage = eval( evalString )
if not storage.isOK():
errStr = "StorageFactory._generateStorageObject: Failed to instatiate storage plug in."
gLogger.error( errStr, "%s" % ( moduleName ) )
return S_ERROR( errStr )
except Exception, x:
errStr = "StorageFactory._generateStorageObject: Failed to instatiate %s(): %s" % ( moduleName, x )
gLogger.exception( errStr )
return S_ERROR( errStr )
def _generateCatalogObject( self, catalogName ):
moduleRootPaths = getInstalledExtensions()
moduleLoaded = False
for moduleRootPath in moduleRootPaths:
if moduleLoaded:
break
gLogger.verbose( "Trying to load from root path %s" % moduleRootPath )
moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Catalog", "%sClient.py" % catalogName )
gLogger.verbose( "Looking for file %s" % moduleFile )
if not os.path.isfile( moduleFile ):
continue
try:
# This inforces the convention that the plug in must be named after the file catalog
moduleName = "%sClient" % ( catalogName )
catalogModule = __import__( '%s.Resources.Catalog.%s' % ( moduleRootPath, moduleName ),
globals(), locals(), [moduleName] )
except Exception, x:
errStr = "FileCatalog._generateCatalogObject: Failed to import %s: %s" % ( catalogName, x )
gLogger.exception( errStr )
return S_ERROR( errStr )
try:
evalString = "catalogModule.%s()" % moduleName
catalog = eval( evalString )
if not catalog.isOK():
errStr = "FileCatalog._generateCatalogObject: Failed to instantiate catalog plug in."
gLogger.error( errStr, moduleName )
return S_ERROR( errStr )
return S_OK( catalog )
except Exception, x:
errStr = "FileCatalog._generateCatalogObject: Failed to instantiate %s()" % ( moduleName )
gLogger.exception( errStr, lException = x )
return S_ERROR( errStr )
def loadModules( self, modulesList, hideExceptions = False ):
"""
Load all modules required in moduleList
"""
for modName in modulesList:
gLogger.verbose( "Checking %s" % modName )
#if it's a executor modName name just load it and be done with it
if modName.find( "/" ) > -1:
gLogger.verbose( "Module %s seems to be a valid name. Try to load it!" % modName )
result = self.loadModule( modName, hideExceptions = hideExceptions )
if not result[ 'OK' ]:
return result
continue
#Check if it's a system name
#Look in the CS
system = modName
#Can this be generated with sectionFinder?
csPath = "%s/Executors" % PathFinder.getSystemSection ( system, ( system, ) )
gLogger.verbose( "Exploring %s to discover modules" % csPath )
result = gConfig.getSections( csPath )
if result[ 'OK' ]:
#Add all modules in the CS :P
for modName in result[ 'Value' ]:
result = self.loadModule( "%s/%s" % ( system, modName ), hideExceptions = hideExceptions )
if not result[ 'OK' ]:
return result
#Look what is installed
parentModule = None
for rootModule in getInstalledExtensions():
if system.find( "System" ) != len( system ) - 6:
parentImport = "%s.%sSystem.%s" % ( rootModule, system, self.__csSuffix )
else:
parentImport = "%s.%s.%s" % ( rootModule, system, self.__csSuffix )
#HERE!
result = self.__recurseImport( parentImport )
if not result[ 'OK' ]:
return result
parentModule = result[ 'Value' ]
if parentModule:
break
if not parentModule:
continue
parentPath = parentModule.__path__[0]
gLogger.notice( "Found modules path at %s" % parentImport )
for entry in os.listdir( parentPath ):
if entry[-3:] != ".py" or entry == "__init__.py":
continue
if not os.path.isfile( os.path.join( parentPath, entry ) ):
continue
modName = "%s/%s" % ( system, entry[:-3] )
gLogger.verbose( "Trying to import %s" % modName )
result = self.loadModule( modName,
hideExceptions = hideExceptions,
parentModule = parentModule )
return S_OK()
def __generateStorageObject(self,
storageName,
protocolName,
protocol,
path=None,
host=None,
port=None,
spaceToken=None,
wsUrl=None):
storageType = protocolName
if self.proxy:
storageType = 'Proxy'
moduleRootPaths = getInstalledExtensions()
moduleLoaded = False
path = path.rstrip('/')
if not path:
path = '/'
for moduleRootPath in moduleRootPaths:
if moduleLoaded:
break
gLogger.verbose("Trying to load from root path %s" %
moduleRootPath)
moduleFile = os.path.join(rootPath, moduleRootPath, "Resources",
"Storage", "%sStorage.py" % storageType)
gLogger.verbose("Looking for file %s" % moduleFile)
if not os.path.isfile(moduleFile):
continue
try:
# This inforces the convention that the plug in must be named after the protocol
moduleName = "%sStorage" % (storageType)
storageModule = __import__(
'%s.Resources.Storage.%s' % (moduleRootPath, moduleName),
globals(), locals(), [moduleName])
except Exception, x:
errStr = "StorageFactory._generateStorageObject: Failed to import %s: %s" % (
storageName, x)
gLogger.exception(errStr)
return S_ERROR(errStr)
try:
evalString = "storageModule.%s(storageName,protocol,path,host,port,spaceToken,wsUrl)" % moduleName
storage = eval(evalString)
if not storage.isOK():
errStr = "StorageFactory._generateStorageObject: Failed to instantiate storage plug in."
gLogger.error(errStr, "%s" % (moduleName))
return S_ERROR(errStr)
except Exception, x:
errStr = "StorageFactory._generateStorageObject: Failed to instantiate %s(): %s" % (
moduleName, x)
gLogger.exception(errStr)
return S_ERROR(errStr)
def loadModule( self, modName, hideExceptions = False, parentModule = False ):
"""
Load module name.
name must take the form [DIRAC System Name]/[DIRAC module]
"""
while modName and modName[0] == "/":
modName = modName[1:]
if modName in self.__modules:
return S_OK()
modList = modName.split( "/" )
if len( modList ) != 2:
return S_ERROR( "Can't load %s: Invalid module name" % ( modName ) )
csSection = self.__sectionFinder( modName )
loadGroup = gConfig.getValue( "%s/Load" % csSection, [] )
#Check if it's a load group
if loadGroup:
gLogger.info( "Found load group %s. Will load %s" % ( modName, ", ".join( loadGroup ) ) )
for loadModName in loadGroup:
if loadModName.find( "/" ) == -1:
loadModName = "%s/%s" % ( modList[0], loadModName )
result = self.loadModule( loadModName, hideExceptions = hideExceptions, parentModule = False )
if not result[ 'OK' ]:
return result
return S_OK()
#Normal load
loadName = gConfig.getValue( "%s/Module" % csSection, "" )
if not loadName:
loadName = modName
gLogger.info( "Loading %s" % ( modName ) )
else:
if loadName.find( "/" ) == -1:
loadName = "%s/%s" % ( modList[0], loadName )
gLogger.info( "Loading %s (%s)" % ( modName, loadName ) )
#If already loaded, skip
loadList = loadName.split( "/" )
if len( loadList ) != 2:
return S_ERROR( "Can't load %s: Invalid module name" % ( loadName ) )
system, module = loadList
#Load
className = module
if self.__modSuffix:
className = "%s%s" % ( className, self.__modSuffix )
if loadName not in self.__loadedModules:
#Check if handler is defined
loadCSSection = self.__sectionFinder( loadName )
handlerPath = gConfig.getValue( "%s/HandlerPath" % loadCSSection, "" )
if handlerPath:
gLogger.info( "Trying to %s from CS defined path %s" % ( loadName, handlerPath ) )
gLogger.verbose( "Found handler for %s: %s" % ( loadName, handlerPath ) )
handlerPath = handlerPath.replace( "/", "." )
if handlerPath.find( ".py", len( handlerPath ) -3 ) > -1:
handlerPath = handlerPath[ :-3 ]
className = List.fromChar( handlerPath, "." )[-1]
result = self.__recurseImport( handlerPath )
if not result[ 'OK' ]:
return S_ERROR( "Cannot load user defined handler %s: %s" % ( handlerPath, result[ 'Message' ] ) )
gLogger.verbose( "Loaded %s" % handlerPath )
elif parentModule:
gLogger.info( "Trying to autodiscover %s from parent" % loadName )
#If we've got a parent module, load from there.
modImport = module
if self.__modSuffix:
modImport = "%s%s" % ( modImport, self.__modSuffix )
result = self.__recurseImport( modImport, parentModule, hideExceptions = hideExceptions )
else:
#Check to see if the module exists in any of the root modules
gLogger.info( "Trying to autodiscover %s" % loadName )
rootModulesToLook = getInstalledExtensions()
for rootModule in rootModulesToLook:
importString = '%s.%sSystem.%s.%s' % ( rootModule, system, self.__importLocation, module )
if self.__modSuffix:
importString = "%s%s" % ( importString, self.__modSuffix )
gLogger.verbose( "Trying to load %s" % importString )
result = self.__recurseImport( importString, hideExceptions = hideExceptions )
#Error while loading
if not result[ 'OK' ]:
return result
#Something has been found! break :)
if result[ 'Value' ]:
gLogger.verbose( "Found %s" % importString )
break
#Nothing found
if not result[ 'Value' ]:
return S_ERROR( "Could not find %s" % loadName )
modObj = result[ 'Value' ]
try:
#Try to get the class from the module
modClass = getattr( modObj, className )
except AttributeError:
location = ""
if '__file__' in dir( modObj ):
location = modObj.__file__
else:
location = modObj.__path__
gLogger.exception( "%s module does not have a %s class!" % ( location, module ) )
return S_ERROR( "Cannot load %s" % module )
#Check if it's subclass
if not issubclass( modClass, self.__superClass ):
return S_ERROR( "%s has to inherit from %s" % ( loadName, self.__superClass.__name__ ) )
self.__loadedModules[ loadName ] = { 'classObj' : modClass, 'moduleObj' : modObj }
#End of loading of 'loadName' module
#A-OK :)
self.__modules[ modName ] = self.__loadedModules[ loadName ].copy()
#keep the name of the real code module
self.__modules[ modName ][ 'modName' ] = modName
self.__modules[ modName ][ 'loadName' ] = loadName
gLogger.notice( "Loaded module %s" % modName )
return S_OK()
def loadAgentModule( self, fullName, hideExceptions = False ):
"""
Load module fullName.
fullName must take the form [DIRAC System Name]/[DIRAC Agent Name]
then:
- calls the am_initialize method of the imported Agent
- determines the pooling interval: am_getPollingTime,
- determines the number of executions: am_getMaxCycles
- creates a periodic Task in the ThreadScheduler: am_go
"""
modList = fullName.split( "/" )
if len( modList ) != 2:
return S_ERROR( "Can't load %s: Invalid agent name" % ( fullName ) )
if fullName in self.__agentModules:
gLogger.notice( "Agent already loaded:", fullName )
return S_OK()
gLogger.info( "Loading %s" % fullName )
system, agentName = modList
rootModulesToLook = getInstalledExtensions()
moduleLoaded = False
for rootModule in rootModulesToLook:
if moduleLoaded:
break
gLogger.verbose( "Trying to load from root module %s" % rootModule )
moduleFile = os.path.join( rootPath, rootModule, "%sSystem" % system, "Agent", "%s.py" % agentName )
gLogger.verbose( "Looking for file %s" % moduleFile )
if not os.path.isfile( moduleFile ):
continue
try:
importString = '%s.%sSystem.Agent.%s' % ( rootModule, system, agentName )
gLogger.debug( "Trying to load %s" % importString )
agentModule = __import__( importString,
globals(),
locals(), agentName )
agentClass = getattr( agentModule, agentName )
if not self.__checkHandler( agentClass ):
gLogger.error( "Invalid Agent module", "%s has to inherit from AgentModule" % fullName )
continue
agent = agentClass( fullName, self.__baseAgentName )
if not isinstance( agent, AgentModule ):
gLogger.error( "%s is not a subclass of AgentModule" % moduleFile )
continue
result = agent.am_initialize()
if not result[ 'OK' ]:
return S_ERROR( "Error while calling initialize method of %s: %s" % ( fullName, result[ 'Message' ] ) )
moduleLoaded = True
except Exception:
if not hideExceptions:
gLogger.exception( "Can't load agent %s" % fullName )
if not moduleLoaded:
return S_ERROR( "Can't load agent %s in root modules %s" % ( fullName, ", ".join ( rootModulesToLook ) ) )
self.__agentModules[ fullName ] = { 'instance' : agent,
'class' : agentClass,
'module' : agentModule,
'running' : False }
agentPeriod = agent.am_getPollingTime()
result = self.__scheduler.addPeriodicTask( agentPeriod,
agent.am_go,
executions = agent.am_getMaxCycles(),
elapsedTime = agentPeriod )
if not result[ 'OK' ]:
return result
taskId = result[ 'Value' ]
self.__tasks[ result[ 'Value' ] ] = fullName
self.__agentModules[ fullName ][ 'taskId' ] = taskId
self.__agentModules[ fullName ][ 'running' ] = True
return S_OK()
