def createCatalog(self, catalogName, useProxy=False): """ Create a file catalog object from its name and CS description """ catalogPath = getCatalogPath(catalogName) catalogType = gConfig.getValue(catalogPath + '/CatalogType', catalogName) catalogURL = gConfig.getValue(catalogPath + '/CatalogURL', "DataManagement/" + catalogType) optionsDict = {} result = gConfig.getOptionsDict(catalogPath) if result['OK']: optionsDict = result['Value'] if useProxy: result = self.__getCatalogClass(catalogType) if not result['OK']: return result catalogClass = result['Value'] methods = catalogClass.getInterfaceMethods() catalog = FileCatalogProxyClient(catalogName) catalog.setInterfaceMethods(methods) return S_OK(catalog) return self.__createCatalog(catalogName, catalogType, catalogURL, optionsDict)
def createCatalog(self, catalogName): """ Create a file catalog object from its name and CS description """ # get the CS description first catalogPath = getCatalogPath(catalogName) catalogType = gConfig.getValue(catalogPath + "/CatalogType", catalogName) catalogURL = gConfig.getValue(catalogPath + "/CatalogURL", "") self.log.verbose("Creating %s client" % catalogName) moduleRootPaths = getInstalledExtensions() for moduleRootPath in moduleRootPaths: gLogger.verbose("Trying to load from root path %s" % moduleRootPath) # moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Catalog", "%sClient.py" % catalogType ) # gLogger.verbose( "Looking for file %s" % moduleFile ) # if not os.path.isfile( moduleFile ): # continue try: # This enforces the convention that the plug in must be named after the file catalog moduleName = "%sClient" % (catalogType) catalogModule = __import__( "%s.Resources.Catalog.%s" % (moduleRootPath, moduleName), globals(), locals(), [moduleName] ) except ImportError, x: if "No module" in str(x): gLogger.debug("Catalog module %s not found in %s" % (catalogType, moduleRootPath)) else: errStr = "Failed attempt to import %s from the path %s: %s" % (catalogType, moduleRootPath, x) gLogger.error(errStr) continue except Exception, x: errStr = "Failed attempt to import %s from the path %s: %s" % (catalogType, moduleRootPath, x) gLogger.error(errStr) continue
def createCatalog(self, catalogName, useProxy=False): """ Create a file catalog object from its name and CS description """ if useProxy: catalog = FileCatalogProxyClient(catalogName) return S_OK(catalog) # get the CS description first catalogPath = getCatalogPath(catalogName) catalogType = gConfig.getValue(catalogPath + '/CatalogType', catalogName) catalogURL = gConfig.getValue(catalogPath + '/CatalogURL', "DataManagement/" + catalogType) self.log.debug('Creating %s client' % catalogName) objectLoader = ObjectLoader.ObjectLoader() result = objectLoader.loadObject( 'Resources.Catalog.%sClient' % catalogType, catalogType + 'Client') if not result['OK']: gLogger.error('Failed to load catalog object: %s' % result['Message']) return result catalogClass = result['Value'] try: # FIXME: is it really needed? This is the factory, can't this be moved out? if catalogType in ['LcgFileCatalogCombined', 'LcgFileCatalog']: # The LFC special case infoSys = gConfig.getValue(catalogPath + '/LcgGfalInfosys', '') host = gConfig.getValue(catalogPath + '/MasterHost', '') catalog = catalogClass(infoSys, host) else: if catalogURL: catalog = catalogClass(url=catalogURL) else: catalog = catalogClass() self.log.debug('Loaded module %sClient' % catalogType) return S_OK(catalog) except Exception, x: errStr = "Failed to instantiate %s()" % (catalogType) gLogger.exception(errStr, lException=x) return S_ERROR(errStr)
def createCatalog(self, catalogName, useProxy=False): """ Create a file catalog object from its name and CS description """ if useProxy: catalog = FileCatalogProxyClient(catalogName) return S_OK(catalog) # get the CS description first catalogPath = getCatalogPath(catalogName) catalogType = gConfig.getValue(catalogPath + '/CatalogType', catalogName) catalogURL = gConfig.getValue(catalogPath + '/CatalogURL', '') self.log.verbose('Creating %s client' % catalogName) moduleRootPaths = getInstalledExtensions() for moduleRootPath in moduleRootPaths: gLogger.verbose("Trying to load from root path %s" % moduleRootPath) #moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Catalog", "%sClient.py" % catalogType ) #gLogger.verbose( "Looking for file %s" % moduleFile ) #if not os.path.isfile( moduleFile ): # continue try: # This enforces the convention that the plug in must be named after the file catalog moduleName = "%sClient" % (catalogType) catalogModule = __import__( '%s.Resources.Catalog.%s' % (moduleRootPath, moduleName), globals(), locals(), [moduleName]) except ImportError, x: if "No module" in str(x): gLogger.debug('Catalog module %s not found in %s' % (catalogType, moduleRootPath)) else: errStr = "Failed attempt to import %s from the path %s: %s" % ( catalogType, moduleRootPath, x) gLogger.error(errStr) continue except Exception, x: errStr = "Failed attempt to import %s from the path %s: %s" % ( catalogType, moduleRootPath, x) gLogger.error(errStr) continue
def createCatalog( self, catalogName, useProxy = False ): """ Create a file catalog object from its name and CS description """ if useProxy: catalog = FileCatalogProxyClient( catalogName ) return S_OK( catalog ) # get the CS description first catalogPath = getCatalogPath( catalogName ) catalogType = gConfig.getValue( catalogPath + '/CatalogType', catalogName ) catalogURL = gConfig.getValue( catalogPath + '/CatalogURL', "DataManagement/" + catalogType ) self.log.debug( 'Creating %s client' % catalogName ) objectLoader = ObjectLoader.ObjectLoader() result = objectLoader.loadObject( 'Resources.Catalog.%sClient' % catalogType, catalogType + 'Client' ) if not result['OK']: gLogger.error( 'Failed to load catalog object', '%s' % result['Message'] ) return result catalogClass = result['Value'] try: # FIXME: is it really needed? This is the factory, can't this be moved out? if catalogType in [ 'LcgFileCatalogCombined', 'LcgFileCatalog' ]: # The LFC special case infoSys = gConfig.getValue( catalogPath + '/LcgGfalInfosys', '' ) host = gConfig.getValue( catalogPath + '/MasterHost', '' ) catalog = catalogClass( infoSys, host ) else: if catalogURL: catalog = catalogClass( url = catalogURL ) else: catalog = catalogClass() self.log.debug( 'Loaded module %sClient' % catalogType ) return S_OK( catalog ) except Exception, x: errStr = "Failed to instantiate %s()" % ( catalogType ) gLogger.exception( errStr, lException = x ) return S_ERROR( errStr )
def createCatalog( self, catalogName, useProxy = False ): """ Create a file catalog object from its name and CS description """ catalogPath = getCatalogPath( catalogName ) catalogType = gConfig.getValue( catalogPath + '/CatalogType', catalogName ) catalogURL = gConfig.getValue( catalogPath + '/CatalogURL', "DataManagement/" + catalogType ) optionsDict = {} result = gConfig.getOptionsDict( catalogPath ) if result['OK']: optionsDict = result['Value'] if useProxy: result = self.__getCatalogClass( catalogType ) if not result['OK']: return result catalogClass = result['Value'] methods = catalogClass.getInterfaceMethods() catalog = FileCatalogProxyClient( catalogName ) catalog.setInterfaceMethods( methods ) return S_OK( catalog ) return self.__createCatalog( catalogName, catalogType, catalogURL, optionsDict )
def createCatalog(self, catalogName, useProxy=False): """ Create a file catalog object from its name and CS description """ if useProxy: catalog = FileCatalogProxyClient(catalogName) return S_OK(catalog) # get the CS description first catalogPath = getCatalogPath(catalogName) catalogType = gConfig.getValue(catalogPath + "/CatalogType", catalogName) catalogURL = gConfig.getValue(catalogPath + "/CatalogURL", "DataManagement/" + catalogName) self.log.verbose("Creating %s client" % catalogName) objectLoader = ObjectLoader.ObjectLoader() result = objectLoader.loadObject("Resources.Catalog.%sClient" % catalogType, catalogType + "Client") if not result["OK"]: gLogger.error("Failed to load catalog object: %s" % result["Message"]) return result catalogClass = result["Value"] try: if catalogType in ["LcgFileCatalogCombined", "LcgFileCatalog"]: # The LFC special case infoSys = gConfig.getValue(catalogPath + "/LcgGfalInfosys", "") host = gConfig.getValue(catalogPath + "/MasterHost", "") catalog = catalogClass(infoSys, host) else: if catalogURL: catalog = catalogClass(url=catalogURL) else: catalog = catalogClass() self.log.debug("Loaded module %sClient" % catalogType) return S_OK(catalog) except Exception, x: errStr = "Failed to instantiate %s()" % (catalogType) gLogger.exception(errStr, lException=x) return S_ERROR(errStr)
def configHelper(voList): """ A helper function to gather necessary Rucio client options from the CS. :param volist: list of VO names, or a VO name (str) :return: a dictionary of a form {vo: params, vo: params,} :rtype: dict """ log = gLogger.getLocalSubLogger("RucioSynchronizerHelper") if isinstance(voList, str): voList = [voList] clientConfig = {} log.debug("VO list to consider for synchronization: ", voList) # locate RucioFileCatalog type in resources first result = gConfig.getSections("/Resources/FileCatalogs") catNames = [] if result["OK"]: catalogs = result["Value"] log.debug("File catalogs defined in Resources", catalogs) for catalog in catalogs: result = gConfig.getOptionsDict(getCatalogPath(catalog)) if result["OK"]: options = result["Value"] log.debug("Rucio Catalog candidate options", options) if options.get("Status", None) == "Active" and options.get( "CatalogType", None) == "RucioFileCatalog": catNames.append(catalog) else: log.error("No catalogs defined in Resources.") return S_ERROR("No catalogs defined in Resources.") log.info( "Active FileCatalogs candidates of type RucioFileCatalog found in Resources:", catNames) # we found (possibly more that one) candidate, now we look for it in Operations # to find out which one is used by which VO. There can be only one # Rucio catalog per VO. for vo in voList: opHelper = Operations(vo=vo) result = opHelper.getSections("/Services/Catalogs") if result["OK"]: catSections = set(result["Value"]) else: log.warn("No Services/Catalogs section in Operations, for ", "VO=%s (skipped)" % vo) continue selectedCatalog = list(catSections.intersection(catNames)) if len(selectedCatalog) > 1: log.error( "VO %s: Services/Catalogs section mis-configured." " More that one Rucio file catalog", "[VO: %s, Catalogs: %s]" % (vo, selectedCatalog), ) continue if not selectedCatalog: log.warn("VO is not using RucioFileCatalog (VO skipped)", "[VO: %s]" % vo) continue # check if the section name is in the catalog list to use. # if the list is not empty it has to contain the selected catalog. fileCatalogs = opHelper.getValue("/Services/Catalogs/CatalogList", []) if fileCatalogs and selectedCatalog[0] not in fileCatalogs: log.warn( "VO is not using RucioFileCatalog - it is not in the catalog list", "[VO: %s]" % vo) continue # now collect Rucio specific parameters for the VO params = {} result = gConfig.getOptionsDict(getCatalogPath(selectedCatalog[0])) if result["OK"]: optDict = result["Value"] params["rucioHost"] = optDict.get("RucioHost", None) params["authHost"] = optDict.get("AuthHost", None) params["privilegedAccount"] = optDict.get("PrivilegedAccount", "root") clientConfig[vo] = params log.info("RSEs and users will be configured in Rucio for the VO:", vo) else: log.error(result["Message"]) return clientConfig