def createCatalog(self, catalogName, useProxy=False):
""" Create a file catalog object from its name and CS description
"""
catalogPath = getCatalogPath(catalogName)
catalogType = gConfig.getValue(catalogPath + '/CatalogType',
catalogName)
catalogURL = gConfig.getValue(catalogPath + '/CatalogURL',
"DataManagement/" + catalogType)
optionsDict = {}
result = gConfig.getOptionsDict(catalogPath)
if result['OK']:
optionsDict = result['Value']
if useProxy:
result = self.__getCatalogClass(catalogType)
if not result['OK']:
return result
catalogClass = result['Value']
methods = catalogClass.getInterfaceMethods()
catalog = FileCatalogProxyClient(catalogName)
catalog.setInterfaceMethods(methods)
return S_OK(catalog)
return self.__createCatalog(catalogName, catalogType, catalogURL,
optionsDict)
def createCatalog(self, catalogName):
""" Create a file catalog object from its name and CS description
"""
# get the CS description first
catalogPath = getCatalogPath(catalogName)
catalogType = gConfig.getValue(catalogPath + "/CatalogType", catalogName)
catalogURL = gConfig.getValue(catalogPath + "/CatalogURL", "")
self.log.verbose("Creating %s client" % catalogName)
moduleRootPaths = getInstalledExtensions()
for moduleRootPath in moduleRootPaths:
gLogger.verbose("Trying to load from root path %s" % moduleRootPath)
# moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Catalog", "%sClient.py" % catalogType )
# gLogger.verbose( "Looking for file %s" % moduleFile )
# if not os.path.isfile( moduleFile ):
# continue
try:
# This enforces the convention that the plug in must be named after the file catalog
moduleName = "%sClient" % (catalogType)
catalogModule = __import__(
"%s.Resources.Catalog.%s" % (moduleRootPath, moduleName), globals(), locals(), [moduleName]
)
except ImportError, x:
if "No module" in str(x):
gLogger.debug("Catalog module %s not found in %s" % (catalogType, moduleRootPath))
else:
errStr = "Failed attempt to import %s from the path %s: %s" % (catalogType, moduleRootPath, x)
gLogger.error(errStr)
continue
except Exception, x:
errStr = "Failed attempt to import %s from the path %s: %s" % (catalogType, moduleRootPath, x)
gLogger.error(errStr)
continue
def createCatalog(self, catalogName, useProxy=False):
""" Create a file catalog object from its name and CS description
"""
if useProxy:
catalog = FileCatalogProxyClient(catalogName)
return S_OK(catalog)
# get the CS description first
catalogPath = getCatalogPath(catalogName)
catalogType = gConfig.getValue(catalogPath + '/CatalogType',
catalogName)
catalogURL = gConfig.getValue(catalogPath + '/CatalogURL',
"DataManagement/" + catalogType)
self.log.debug('Creating %s client' % catalogName)
objectLoader = ObjectLoader.ObjectLoader()
result = objectLoader.loadObject(
'Resources.Catalog.%sClient' % catalogType, catalogType + 'Client')
if not result['OK']:
gLogger.error('Failed to load catalog object: %s' %
result['Message'])
return result
catalogClass = result['Value']
try:
# FIXME: is it really needed? This is the factory, can't this be moved out?
if catalogType in ['LcgFileCatalogCombined', 'LcgFileCatalog']:
# The LFC special case
infoSys = gConfig.getValue(catalogPath + '/LcgGfalInfosys', '')
host = gConfig.getValue(catalogPath + '/MasterHost', '')
catalog = catalogClass(infoSys, host)
else:
if catalogURL:
catalog = catalogClass(url=catalogURL)
else:
catalog = catalogClass()
self.log.debug('Loaded module %sClient' % catalogType)
return S_OK(catalog)
except Exception, x:
errStr = "Failed to instantiate %s()" % (catalogType)
gLogger.exception(errStr, lException=x)
return S_ERROR(errStr)
def createCatalog(self, catalogName, useProxy=False):
""" Create a file catalog object from its name and CS description
"""
if useProxy:
catalog = FileCatalogProxyClient(catalogName)
return S_OK(catalog)
# get the CS description first
catalogPath = getCatalogPath(catalogName)
catalogType = gConfig.getValue(catalogPath + '/CatalogType',
catalogName)
catalogURL = gConfig.getValue(catalogPath + '/CatalogURL', '')
self.log.verbose('Creating %s client' % catalogName)
moduleRootPaths = getInstalledExtensions()
for moduleRootPath in moduleRootPaths:
gLogger.verbose("Trying to load from root path %s" %
moduleRootPath)
#moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Catalog", "%sClient.py" % catalogType )
#gLogger.verbose( "Looking for file %s" % moduleFile )
#if not os.path.isfile( moduleFile ):
# continue
try:
# This enforces the convention that the plug in must be named after the file catalog
moduleName = "%sClient" % (catalogType)
catalogModule = __import__(
'%s.Resources.Catalog.%s' % (moduleRootPath, moduleName),
globals(), locals(), [moduleName])
except ImportError, x:
if "No module" in str(x):
gLogger.debug('Catalog module %s not found in %s' %
(catalogType, moduleRootPath))
else:
errStr = "Failed attempt to import %s from the path %s: %s" % (
catalogType, moduleRootPath, x)
gLogger.error(errStr)
continue
except Exception, x:
errStr = "Failed attempt to import %s from the path %s: %s" % (
catalogType, moduleRootPath, x)
gLogger.error(errStr)
continue
def createCatalog( self, catalogName, useProxy = False ):
""" Create a file catalog object from its name and CS description
"""
if useProxy:
catalog = FileCatalogProxyClient( catalogName )
return S_OK( catalog )
# get the CS description first
catalogPath = getCatalogPath( catalogName )
catalogType = gConfig.getValue( catalogPath + '/CatalogType', catalogName )
catalogURL = gConfig.getValue( catalogPath + '/CatalogURL', "DataManagement/" + catalogType )
self.log.debug( 'Creating %s client' % catalogName )
objectLoader = ObjectLoader.ObjectLoader()
result = objectLoader.loadObject( 'Resources.Catalog.%sClient' % catalogType, catalogType + 'Client' )
if not result['OK']:
gLogger.error( 'Failed to load catalog object', '%s' % result['Message'] )
return result
catalogClass = result['Value']
try:
# FIXME: is it really needed? This is the factory, can't this be moved out?
if catalogType in [ 'LcgFileCatalogCombined', 'LcgFileCatalog' ]:
# The LFC special case
infoSys = gConfig.getValue( catalogPath + '/LcgGfalInfosys', '' )
host = gConfig.getValue( catalogPath + '/MasterHost', '' )
catalog = catalogClass( infoSys, host )
else:
if catalogURL:
catalog = catalogClass( url = catalogURL )
else:
catalog = catalogClass()
self.log.debug( 'Loaded module %sClient' % catalogType )
return S_OK( catalog )
except Exception, x:
errStr = "Failed to instantiate %s()" % ( catalogType )
gLogger.exception( errStr, lException = x )
return S_ERROR( errStr )
def createCatalog( self, catalogName, useProxy = False ):
""" Create a file catalog object from its name and CS description
"""
catalogPath = getCatalogPath( catalogName )
catalogType = gConfig.getValue( catalogPath + '/CatalogType', catalogName )
catalogURL = gConfig.getValue( catalogPath + '/CatalogURL', "DataManagement/" + catalogType )
optionsDict = {}
result = gConfig.getOptionsDict( catalogPath )
if result['OK']:
optionsDict = result['Value']
if useProxy:
result = self.__getCatalogClass( catalogType )
if not result['OK']:
return result
catalogClass = result['Value']
methods = catalogClass.getInterfaceMethods()
catalog = FileCatalogProxyClient( catalogName )
catalog.setInterfaceMethods( methods )
return S_OK( catalog )
return self.__createCatalog( catalogName, catalogType, catalogURL, optionsDict )
def createCatalog(self, catalogName, useProxy=False):
""" Create a file catalog object from its name and CS description
"""
if useProxy:
catalog = FileCatalogProxyClient(catalogName)
return S_OK(catalog)
# get the CS description first
catalogPath = getCatalogPath(catalogName)
catalogType = gConfig.getValue(catalogPath + "/CatalogType", catalogName)
catalogURL = gConfig.getValue(catalogPath + "/CatalogURL", "DataManagement/" + catalogName)
self.log.verbose("Creating %s client" % catalogName)
objectLoader = ObjectLoader.ObjectLoader()
result = objectLoader.loadObject("Resources.Catalog.%sClient" % catalogType, catalogType + "Client")
if not result["OK"]:
gLogger.error("Failed to load catalog object: %s" % result["Message"])
return result
catalogClass = result["Value"]
try:
if catalogType in ["LcgFileCatalogCombined", "LcgFileCatalog"]:
# The LFC special case
infoSys = gConfig.getValue(catalogPath + "/LcgGfalInfosys", "")
host = gConfig.getValue(catalogPath + "/MasterHost", "")
catalog = catalogClass(infoSys, host)
else:
if catalogURL:
catalog = catalogClass(url=catalogURL)
else:
catalog = catalogClass()
self.log.debug("Loaded module %sClient" % catalogType)
return S_OK(catalog)
except Exception, x:
errStr = "Failed to instantiate %s()" % (catalogType)
gLogger.exception(errStr, lException=x)
return S_ERROR(errStr)
def configHelper(voList):
"""
A helper function to gather necessary Rucio client options from the CS.
:param volist: list of VO names, or a VO name (str)
:return: a dictionary of a form {vo: params, vo: params,}
:rtype: dict
"""
log = gLogger.getLocalSubLogger("RucioSynchronizerHelper")
if isinstance(voList, str):
voList = [voList]
clientConfig = {}
log.debug("VO list to consider for synchronization: ", voList)
# locate RucioFileCatalog type in resources first
result = gConfig.getSections("/Resources/FileCatalogs")
catNames = []
if result["OK"]:
catalogs = result["Value"]
log.debug("File catalogs defined in Resources", catalogs)
for catalog in catalogs:
result = gConfig.getOptionsDict(getCatalogPath(catalog))
if result["OK"]:
options = result["Value"]
log.debug("Rucio Catalog candidate options", options)
if options.get("Status", None) == "Active" and options.get(
"CatalogType", None) == "RucioFileCatalog":
catNames.append(catalog)
else:
log.error("No catalogs defined in Resources.")
return S_ERROR("No catalogs defined in Resources.")
log.info(
"Active FileCatalogs candidates of type RucioFileCatalog found in Resources:",
catNames)
# we found (possibly more that one) candidate, now we look for it in Operations
# to find out which one is used by which VO. There can be only one
# Rucio catalog per VO.
for vo in voList:
opHelper = Operations(vo=vo)
result = opHelper.getSections("/Services/Catalogs")
if result["OK"]:
catSections = set(result["Value"])
else:
log.warn("No Services/Catalogs section in Operations, for ",
"VO=%s (skipped)" % vo)
continue
selectedCatalog = list(catSections.intersection(catNames))
if len(selectedCatalog) > 1:
log.error(
"VO %s: Services/Catalogs section mis-configured."
" More that one Rucio file catalog",
"[VO: %s, Catalogs: %s]" % (vo, selectedCatalog),
)
continue
if not selectedCatalog:
log.warn("VO is not using RucioFileCatalog (VO skipped)",
"[VO: %s]" % vo)
continue
# check if the section name is in the catalog list to use.
# if the list is not empty it has to contain the selected catalog.
fileCatalogs = opHelper.getValue("/Services/Catalogs/CatalogList", [])
if fileCatalogs and selectedCatalog[0] not in fileCatalogs:
log.warn(
"VO is not using RucioFileCatalog - it is not in the catalog list",
"[VO: %s]" % vo)
continue
# now collect Rucio specific parameters for the VO
params = {}
result = gConfig.getOptionsDict(getCatalogPath(selectedCatalog[0]))
if result["OK"]:
optDict = result["Value"]
params["rucioHost"] = optDict.get("RucioHost", None)
params["authHost"] = optDict.get("AuthHost", None)
params["privilegedAccount"] = optDict.get("PrivilegedAccount",
"root")
clientConfig[vo] = params
log.info("RSEs and users will be configured in Rucio for the VO:",
vo)
else:
log.error(result["Message"])
return clientConfig