def createCatalog(self,
catalogName,
useProxy=False,
vo=None,
catalogConfig={}):
""" Create a file catalog object from its name and CS description
"""
if useProxy:
catalog = FileCatalogProxyClient(catalogName)
return S_OK(catalog)
# get the CS description first
catConfig = catalogConfig
if not catConfig:
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
vo = result['Value']
reHelper = Resources(vo=vo)
result = reHelper.getCatalogOptionsDict(catalogName)
if not result['OK']:
return result
catConfig = result['Value']
catalogType = catConfig.get('CatalogType', catalogName)
catalogURL = catConfig.get('CatalogURL', '')
self.log.verbose('Creating %s client' % catalogName)
objectLoader = ObjectLoader()
result = objectLoader.loadObject(
'Resources.Catalog.%sClient' % catalogType, catalogType + 'Client')
if not result['OK']:
gLogger.error('Failed to load catalog object: %s' %
result['Message'])
return result
catalogClass = result['Value']
try:
if catalogType in ['LcgFileCatalogCombined', 'LcgFileCatalog']:
# The LFC special case
infoSys = catConfig.get('LcgGfalInfosys', '')
host = catConfig.get('MasterHost', '')
catalog = catalogClass(infoSys, host)
else:
if catalogURL:
catalog = catalogClass(url=catalogURL)
else:
catalog = catalogClass()
self.log.debug('Loaded module %sClient' % catalogType)
return S_OK(catalog)
except Exception, x:
errStr = "Failed to instantiate %s()" % (catalogType)
gLogger.exception(errStr, lException=x)
return S_ERROR(errStr)
def createCatalog(self,
catalogName,
useProxy=False,
vo=None,
catalogConfig={}):
""" Create a file catalog object from its name and CS description
"""
if useProxy:
catalog = FileCatalogProxyClient(catalogName)
return S_OK(catalog)
# get the CS description first
catConfig = catalogConfig
if not catConfig:
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
vo = result['Value']
reHelper = Resources(vo=vo)
result = reHelper.getCatalogOptionsDict(catalogName)
if not result['OK']:
return result
catConfig = result['Value']
catalogType = catConfig.get('CatalogType', catalogName)
catalogURL = catConfig.get('CatalogURL', '')
self.log.verbose('Creating %s client' % catalogName)
moduleRootPaths = getInstalledExtensions()
for moduleRootPath in moduleRootPaths:
gLogger.verbose("Trying to load from root path %s" %
moduleRootPath)
#moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Catalog", "%sClient.py" % catalogType )
#gLogger.verbose( "Looking for file %s" % moduleFile )
#if not os.path.isfile( moduleFile ):
# continue
try:
# This enforces the convention that the plug in must be named after the file catalog
moduleName = "%sClient" % (catalogType)
catalogModule = __import__(
'%s.Resources.Catalog.%s' % (moduleRootPath, moduleName),
globals(), locals(), [moduleName])
except ImportError, x:
if "No module" in str(x):
gLogger.debug('Catalog module %s not found in %s' %
(catalogType, moduleRootPath))
else:
errStr = "Failed attempt to import %s from the path %s: %s" % (
catalogType, moduleRootPath, x)
gLogger.error(errStr)
continue
except Exception, x:
errStr = "Failed attempt to import %s from the path %s: %s" % (
catalogType, moduleRootPath, x)
gLogger.error(errStr)
continue
def createCatalog( self, catalogName, useProxy = False, vo = None, catalogConfig = {} ):
""" Create a file catalog object from its name and CS description
"""
if useProxy:
catalog = FileCatalogProxyClient( catalogName )
return S_OK( catalog )
# get the CS description first
catConfig = catalogConfig
if not catConfig:
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
vo = result['Value']
reHelper = Resources( vo = vo )
result = reHelper.getCatalogOptionsDict( catalogName )
if not result['OK']:
return result
catConfig = result['Value']
catalogType = catConfig.get('CatalogType',catalogName)
catalogURL = catConfig.get('CatalogURL','')
self.log.verbose( 'Creating %s client' % catalogName )
objectLoader = ObjectLoader()
result = objectLoader.loadObject( 'Resources.Catalog.%sClient' % catalogType, catalogType+'Client' )
if not result['OK']:
gLogger.error( 'Failed to load catalog object: %s' % result['Message'] )
return result
catalogClass = result['Value']
try:
if catalogType in ['LcgFileCatalogCombined','LcgFileCatalog']:
# The LFC special case
infoSys = catConfig.get('LcgGfalInfosys','')
host = catConfig.get('MasterHost','')
catalog = catalogClass( infoSys, host )
else:
if catalogURL:
catalog = catalogClass( url = catalogURL )
else:
catalog = catalogClass()
self.log.debug('Loaded module %sClient' % catalogType )
return S_OK( catalog )
except Exception, x:
errStr = "Failed to instantiate %s()" % ( catalogType )
gLogger.exception( errStr, lException = x )
return S_ERROR( errStr )
class FileCatalog:
ro_methods = [
'exists', 'isLink', 'readLink', 'isFile', 'getFileMetadata',
'getReplicas', 'getReplicaStatus', 'getFileSize', 'isDirectory',
'getDirectoryReplicas', 'listDirectory', 'getDirectoryMetadata',
'getDirectorySize', 'getDirectoryContents', 'resolveDataset',
'getPathPermissions', 'getLFNForPFN', 'getUsers', 'getGroups',
'getFileUserMetadata'
]
write_methods = [
'createLink', 'removeLink', 'addFile', 'setFileStatus', 'addReplica',
'removeReplica', 'removeFile', 'setReplicaStatus', 'setReplicaHost',
'createDirectory', 'setDirectoryStatus', 'removeDirectory',
'removeDataset', 'removeFileFromDataset', 'createDataset'
]
def __init__(self, catalogs=[], vo=None):
""" Default constructor
"""
self.valid = True
self.timeout = 180
self.readCatalogs = []
self.writeCatalogs = []
self.vo = vo
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
self.vo = result['Value']
self.opHelper = Operations(vo=self.vo)
self.reHelper = Resources(vo=self.vo)
if type(catalogs) in types.StringTypes:
catalogs = [catalogs]
if catalogs:
res = self._getSelectedCatalogs(catalogs)
else:
res = self._getCatalogs()
if not res['OK']:
self.valid = False
elif (len(self.readCatalogs) == 0) and (len(self.writeCatalogs) == 0):
self.valid = False
def isOK(self):
return self.valid
def getReadCatalogs(self):
return self.readCatalogs
def getWriteCatalogs(self):
return self.writeCatalogs
def __getattr__(self, name):
self.call = name
if name in FileCatalog.write_methods:
return self.w_execute
elif name in FileCatalog.ro_methods:
return self.r_execute
else:
raise AttributeError
def __checkArgumentFormat(self, path):
if type(path) in types.StringTypes:
urls = {path: False}
elif type(path) == types.ListType:
urls = {}
for url in path:
urls[url] = False
elif type(path) == types.DictType:
urls = path
else:
return S_ERROR(
"FileCatalog.__checkArgumentFormat: Supplied path is not of the correct format."
)
return S_OK(urls)
def w_execute(self, *parms, **kws):
""" Write method executor.
"""
successful = {}
failed = {}
failedCatalogs = []
fileInfo = parms[0]
res = self.__checkArgumentFormat(fileInfo)
if not res['OK']:
return res
fileInfo = res['Value']
allLfns = fileInfo.keys()
for catalogName, oCatalog, master in self.writeCatalogs:
method = getattr(oCatalog, self.call)
res = method(fileInfo, **kws)
if not res['OK']:
if master:
# If this is the master catalog and it fails we dont want to continue with the other catalogs
gLogger.error(
"FileCatalog.w_execute: Failed to execute %s on master catalog %s."
% (self.call, catalogName), res['Message'])
return res
else:
# Otherwise we keep the failed catalogs so we can update their state later
failedCatalogs.append((catalogName, res['Message']))
else:
for lfn, message in res['Value']['Failed'].items():
# Save the error message for the failed operations
if not failed.has_key(lfn):
failed[lfn] = {}
failed[lfn][catalogName] = message
if master:
# If this is the master catalog then we should not attempt the operation on other catalogs
fileInfo.pop(lfn)
for lfn, result in res['Value']['Successful'].items():
# Save the result return for each file for the successful operations
if not successful.has_key(lfn):
successful[lfn] = {}
successful[lfn][catalogName] = result
# This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog
for catalogName, errorMessage in failedCatalogs:
for lfn in allLfns:
if not failed.has_key(lfn):
failed[lfn] = {}
failed[lfn][catalogName] = errorMessage
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def r_execute(self, *parms, **kws):
""" Read method executor.
"""
successful = {}
failed = {}
for catalogTuple in self.readCatalogs:
oCatalog = catalogTuple[1]
method = getattr(oCatalog, self.call)
res = method(*parms, **kws)
if res['OK']:
if 'Successful' in res['Value']:
for key, item in res['Value']['Successful'].items():
if not successful.has_key(key):
successful[key] = item
if failed.has_key(key):
failed.pop(key)
for key, item in res['Value']['Failed'].items():
if not successful.has_key(key):
failed[key] = item
if len(failed) == 0:
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
else:
return res
if (len(successful) == 0) and (len(failed) == 0):
return S_ERROR('Failed to perform %s from any catalog' % self.call)
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
###########################################################################################
#
# Below is the method for obtaining the objects instantiated for a provided catalogue configuration
#
def addCatalog(self, catalogName, mode="Write", master=False):
""" Add a new catalog with catalogName to the pool of catalogs in mode:
"Read","Write" or "ReadWrite"
"""
result = self._generateCatalogObject(catalogName)
if not result['OK']:
return result
oCatalog = result['Value']
if mode.lower().find("read") != -1:
self.readCatalogs.append((catalogName, oCatalog, master))
if mode.lower().find("write") != -1:
self.writeCatalogs.append((catalogName, oCatalog, master))
return S_OK()
def removeCatalog(self, catalogName):
""" Remove the specified catalog from the internal pool
"""
catalog_removed = False
for i in range(len(self.readCatalogs)):
catalog = self.readCatalogs[i][0]
if catalog == catalogName:
del self.readCatalogs[i]
catalog_removed = True
break
for i in range(len(self.writeCatalogs)):
catalog = self.writeCatalogs[i][0]
if catalog == catalogName:
del self.writeCatalogs[i]
catalog_removed = True
break
if catalog_removed:
return S_OK()
else:
return S_OK('Catalog does not exist')
def _getSelectedCatalogs(self, desiredCatalogs):
for catalogName in desiredCatalogs:
res = self._generateCatalogObject(catalogName)
if not res['OK']:
return res
oCatalog = res['Value']
self.readCatalogs.append((catalogName, oCatalog, True))
self.writeCatalogs.append((catalogName, oCatalog, True))
return S_OK()
def _getCatalogs(self):
# Get the eligible catalogs first
# First, look in the Operations, if nothing defined look in /Resources
result = self.opHelper.getSections('/Services/Catalogs')
fileCatalogs = []
operationsFlag = False
if result['OK']:
fileCatalogs = result['Value']
operationsFlag = True
else:
res = self.reHelper.getEligibleResources('Catalog')
if not res['OK']:
errStr = "FileCatalog._getCatalogs: Failed to get file catalog configuration."
gLogger.error(errStr, res['Message'])
return S_ERROR(errStr)
fileCatalogs = res['Value']
# Get the catalogs now
for catalogName in fileCatalogs:
res = self._getCatalogConfigDetails(catalogName)
if not res['OK']:
return res
catalogConfig = res['Value']
if operationsFlag:
result = self.opHelper.getOptionsDict('/Services/Catalogs/%s' %
catalogName)
if not result['OK']:
return result
catalogConfig.update(result['Value'])
if catalogConfig['Status'] == 'Active':
res = self._generateCatalogObject(catalogName)
if not res['OK']:
return res
oCatalog = res['Value']
master = catalogConfig['Master']
# If the catalog is read type
if re.search('Read', catalogConfig['AccessType']):
if master:
self.readCatalogs.insert(
0, (catalogName, oCatalog, master))
else:
self.readCatalogs.append(
(catalogName, oCatalog, master))
# If the catalog is write type
if re.search('Write', catalogConfig['AccessType']):
if master:
self.writeCatalogs.insert(
0, (catalogName, oCatalog, master))
else:
self.writeCatalogs.append(
(catalogName, oCatalog, master))
return S_OK()
def _getCatalogConfigDetails(self, catalogName):
# First obtain the options that are available
result = self.reHelper.getCatalogOptionsDict(catalogName)
if not result['OK']:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options"
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
catalogConfig = result['Value']
# The 'Status' option should be defined (default = 'Active')
if not catalogConfig.has_key('Status'):
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined"
gLogger.warn(warnStr, catalogName)
catalogConfig['Status'] = 'Active'
# The 'AccessType' option must be defined
if not catalogConfig.has_key('AccessType'):
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined"
gLogger.error(errStr, catalogName)
return S_ERROR(errStr)
# Anything other than 'True' in the 'Master' option means it is not
if not catalogConfig.has_key('Master'):
catalogConfig['Master'] = False
elif catalogConfig['Master'] == 'True':
catalogConfig['Master'] = True
else:
catalogConfig['Master'] = False
return S_OK(catalogConfig)
def _generateCatalogObject(self, catalogName):
""" Create a file catalog object from its name and CS description
"""
useProxy = gConfig.getValue(
'/LocalSite/Catalogs/%s/UseProxy' % catalogName, False)
if not useProxy:
useProxy = self.opHelper.getValue(
'/Services/Catalogs/%s/UseProxy' % catalogName, False)
return FileCatalogFactory().createCatalog(catalogName, useProxy)