def setUp(self):
self.lfc = LcgFileCatalogClient(host='lfc-lhcb.cern.ch',infosys='lcg-bdii.cern.ch:2170')
######################################################
#
# Clean the test directory before starting
#
dir = '/lhcb/test/unit-test'
res = self.lfc.listDirectory(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
lfnsToDelete = res['Value']['Successful'][dir]
res = self.lfc.getReplicas(lfnsToDelete)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
replicas = res['Value']['Successful']
replicaTupleList = []
for lfn in replicas.keys():
for se in replicas[lfn].keys():
replicaTuple = (lfn,replicas[lfn][se],se)
replicaTupleList.append(replicaTuple)
res = self.lfc.removeReplica(replicaTupleList)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assertFalse(res['Value']['Failed'])
res = self.lfc.removeFile(lfnsToDelete)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assertFalse(res['Value']['Failed'])
def setUp(self):
self.lfc = LcgFileCatalogClient(host='lfc-lhcb.cern.ch',infosys='lcg-bdii.cern.ch:2170')
######################################################
#
# Clean the test directory before starting
#
dir = '/lhcb/test/unit-test'
res = self.lfc.listDirectory(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
lfnsToDelete = res['Value']['Successful'][dir]
res = self.lfc.getReplicas(lfnsToDelete)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
replicas = res['Value']['Successful']
replicaTupleList = []
for lfn in replicas.keys():
for se in replicas[lfn].keys():
replicaTuple = (lfn,replicas[lfn][se],se)
replicaTupleList.append(replicaTuple)
res = self.lfc.removeReplica(replicaTupleList)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assertFalse(res['Value']['Failed'])
res = self.lfc.removeFile(lfnsToDelete)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assertFalse(res['Value']['Failed'])
def setUp(self):
self.lfc = LcgFileCatalogClient(host="lfc-lhcb.cern.ch", infosys="lcg-bdii.cern.ch:2170")
######################################################
#
# Clean the test directory before starting
#
dir = "/lhcb/test/unit-test"
res = self.lfc.listDirectory(dir)
self.assert_(res["OK"])
self.assert_(res["Value"].has_key("Successful"))
self.assert_(res["Value"].has_key("Failed"))
self.assert_(res["Value"]["Successful"].has_key(dir))
lfnsToDelete = res["Value"]["Successful"][dir]
res = self.lfc.getReplicas(lfnsToDelete)
self.assert_(res["OK"])
self.assert_(res["Value"].has_key("Successful"))
self.assert_(res["Value"].has_key("Failed"))
replicas = res["Value"]["Successful"]
replicaTupleList = []
for lfn in replicas.keys():
for se in replicas[lfn].keys():
replicaTuple = (lfn, replicas[lfn][se], se)
replicaTupleList.append(replicaTuple)
res = self.lfc.removeReplica(replicaTupleList)
self.assert_(res["OK"])
self.assert_(res["Value"].has_key("Successful"))
self.assert_(res["Value"].has_key("Failed"))
self.assertFalse(res["Value"]["Failed"])
res = self.lfc.removeFile(lfnsToDelete)
self.assert_(res["OK"])
self.assert_(res["Value"].has_key("Successful"))
self.assert_(res["Value"].has_key("Failed"))
self.assertFalse(res["Value"]["Failed"])
def __init__( self, infosys = None, master_host = None, mirrors = [] ):
""" Default constructor
"""
if not infosys:
configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/LcgGfalInfosys'
infosys = gConfig.getValue( configPath )
self.valid = False
if not master_host:
configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/MasterHost'
master_host = gConfig.getValue( configPath )
if master_host:
# Create the master LFC client first
self.lfc = LcgFileCatalogClient( infosys, master_host )
if self.lfc.isOK():
self.valid = True
if not mirrors:
siteName = DIRAC.siteName()
res = getLocationOrderedCatalogs( siteName = siteName )
if not res['OK']:
mirrors = []
else:
mirrors = res['Value']
# Create the mirror LFC instances
self.mirrors = []
for mirror in mirrors:
lfc = LcgFileCatalogClient( infosys, mirror )
self.mirrors.append( lfc )
self.nmirrors = len( self.mirrors )
# Keep the environment for the master instance
self.master_host = self.lfc.host
os.environ['LFC_HOST'] = self.master_host
os.environ['LCG_GFAL_INFOSYS'] = infosys
self.name = 'LFC'
self.timeout = 3000
def __init__( self, infosys = None, master_host = None, mirrors = [] ):
""" Default constructor
"""
if not infosys:
configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/LcgGfalInfosys'
infosys = gConfig.getValue( configPath )
self.valid = False
if not master_host:
configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/MasterHost'
master_host = gConfig.getValue( configPath )
if master_host:
# Create the master LFC client first
self.lfc = LcgFileCatalogClient( infosys, master_host )
if self.lfc.isOK():
self.valid = True
if not mirrors:
siteName = DIRAC.siteName()
res = getLocationOrderedCatalogs( siteName = siteName )
if not res['OK']:
mirrors = []
else:
mirrors = res['Value']
# Create the mirror LFC instances
self.mirrors = []
for mirror in mirrors:
lfc = LcgFileCatalogClient( infosys, mirror )
self.mirrors.append( lfc )
self.nmirrors = len( self.mirrors )
# Keep the environment for the master instance
self.master_host = self.lfc.host
os.environ['LFC_HOST'] = self.master_host
os.environ['LCG_GFAL_INFOSYS'] = infosys
self.name = 'LFC'
self.timeout = 3000
fcType = res['Value']
if not fcType:
res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True)
if res['OK']:
fcType = res['Value'][0]
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "f" or switch[0].lower() == "file-catalog":
fcType = switch[1]
from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI
if fcType == "LcgFileCatalog" or fcType == "LFC":
from DIRAC.Resources.Catalog.LcgFileCatalogClient import LcgFileCatalogClient
cli = FileCatalogClientCLI(LcgFileCatalogClient())
try:
host = os.environ['LFC_HOST']
except Exception, x:
print "LFC_HOST environment variable not defined"
sys.exit(1)
print "Starting LFC FileCatalog client"
cli.cmdloop()
elif fcType == "LcgFileCatalogProxy" or fcType == "LFCproxy":
from DIRAC.Resources.Catalog.LcgFileCatalogProxyClient import LcgFileCatalogProxyClient
cli = FileCatalogClientCLI(LcgFileCatalogProxyClient())
print "Starting LFC Proxy FileCatalog client"
cli.cmdloop()
elif fcType == "LcgFileCatalogCombined" or fcType == "LFCCombined":
from DIRAC.Resources.Catalog.LcgFileCatalogCombinedClient import LcgFileCatalogCombinedClient
cli = FileCatalogClientCLI(LcgFileCatalogCombinedClient())
class LcgFileCatalogCombinedClient( object ):
ro_methods = ['exists', 'isLink', 'readLink', 'isFile', 'getFileMetadata', 'getReplicas',
'getReplicaStatus', 'getFileSize', 'isDirectory', 'getDirectoryReplicas',
'listDirectory', 'getDirectoryMetadata', 'getDirectorySize', 'getDirectoryContents',
'resolveDataset', 'getPathPermissions', 'getLFNForPFN']
write_methods = ['createLink', 'removeLink', 'addFile', 'addReplica', 'removeReplica',
'removeFile', 'setReplicaStatus', 'setReplicaHost', 'createDirectory',
'removeDirectory', 'removeDataset', 'removeFileFromDataset', 'createDataset']
def __init__( self, infosys = None, master_host = None, mirrors = [] ):
""" Default constructor
"""
if not infosys:
configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/LcgGfalInfosys'
infosys = gConfig.getValue( configPath )
self.valid = False
if not master_host:
configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/MasterHost'
master_host = gConfig.getValue( configPath )
if master_host:
# Create the master LFC client first
self.lfc = LcgFileCatalogClient( infosys, master_host )
if self.lfc.isOK():
self.valid = True
if not mirrors:
siteName = DIRAC.siteName()
res = getLocationOrderedCatalogs( siteName = siteName )
if not res['OK']:
mirrors = []
else:
mirrors = res['Value']
# Create the mirror LFC instances
self.mirrors = []
for mirror in mirrors:
lfc = LcgFileCatalogClient( infosys, mirror )
self.mirrors.append( lfc )
self.nmirrors = len( self.mirrors )
# Keep the environment for the master instance
self.master_host = self.lfc.host
os.environ['LFC_HOST'] = self.master_host
os.environ['LCG_GFAL_INFOSYS'] = infosys
self.name = 'LFC'
self.timeout = 3000
def isOK( self ):
return self.valid
def getName( self, DN = '' ):
""" Get the file catalog type name
"""
return self.name
def __getattr__( self, name ):
self.call = name
if name in LcgFileCatalogCombinedClient.write_methods:
return self.w_execute
elif name in LcgFileCatalogCombinedClient.ro_methods:
return self.r_execute
else:
raise AttributeError
def w_execute( self, *parms, **kws ):
""" Write method executor.
Dispatches execution of the methods which need Read/Write
access to the master LFC instance
"""
# If the DN argument is given, this is an operation on behalf
# of the user with this DN, prepare setAuthorizationId call
userDN = ''
if kws.has_key( 'DN' ):
userDN = kws['DN']
del kws['DN']
# Try the method 3 times just in case of intermittent errors
max_retry = 2
count = 0
result = S_ERROR()
while ( not result['OK'] ) and ( count <= max_retry ):
if count > 0:
# If retrying, wait a bit
time.sleep( 1 )
try:
result = S_OK()
if userDN:
resAuth = pythonCall( self.timeout, self.lfc.setAuthorizationId, userDN )
if not resAuth['OK']:
result = S_ERROR( 'Failed to set user authorization' )
if result['OK']:
method = getattr( self.lfc, self.call )
resMeth = method( *parms, **kws )
if not resMeth['OK']:
return resMeth
else:
result = resMeth
except Exception, x:
gLogger.exception( 'Exception while calling LFC Master service', '', x )
result = S_ERROR( 'Exception while calling LFC Master service ' + str( x ) )
count += 1
return result
class LcgFileCatalogCombinedClient:
ro_methods = ['exists', 'isLink', 'readLink', 'isFile', 'getFileMetadata', 'getReplicas',
'getReplicaStatus', 'getFileSize', 'isDirectory', 'getDirectoryReplicas',
'listDirectory', 'getDirectoryMetadata', 'getDirectorySize', 'getDirectoryContents',
'resolveDataset', 'getPathPermissions', 'getLFNForPFN']
write_methods = ['createLink', 'removeLink', 'addFile', 'addReplica', 'removeReplica',
'removeFile', 'setReplicaStatus', 'setReplicaHost', 'createDirectory',
'removeDirectory', 'removeDataset', 'removeFileFromDataset', 'createDataset']
def __init__( self, infosys = None, master_host = None, mirrors = [] ):
""" Default constructor
"""
if not infosys:
configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/LcgGfalInfosys'
infosys = gConfig.getValue( configPath )
self.valid = False
if not master_host:
configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/MasterHost'
master_host = gConfig.getValue( configPath )
if master_host:
# Create the master LFC client first
self.lfc = LcgFileCatalogClient( infosys, master_host )
if self.lfc.isOK():
self.valid = True
if not mirrors:
siteName = DIRAC.siteName()
res = getLocationOrderedCatalogs( siteName = siteName )
if not res['OK']:
mirrors = []
else:
mirrors = res['Value']
# Create the mirror LFC instances
self.mirrors = []
for mirror in mirrors:
lfc = LcgFileCatalogClient( infosys, mirror )
self.mirrors.append( lfc )
self.nmirrors = len( self.mirrors )
# Keep the environment for the master instance
self.master_host = self.lfc.host
os.environ['LFC_HOST'] = self.master_host
os.environ['LCG_GFAL_INFOSYS'] = infosys
self.name = 'LFC'
self.timeout = 3000
def isOK( self ):
return self.valid
def getName( self, DN = '' ):
""" Get the file catalog type name
"""
return self.name
def __getattr__( self, name ):
self.call = name
if name in LcgFileCatalogCombinedClient.write_methods:
return self.w_execute
elif name in LcgFileCatalogCombinedClient.ro_methods:
return self.r_execute
else:
raise AttributeError
def w_execute( self, *parms, **kws ):
""" Write method executor.
Dispatches execution of the methods which need Read/Write
access to the master LFC instance
"""
# If the DN argument is given, this is an operation on behalf
# of the user with this DN, prepare setAuthorizationId call
userDN = ''
if kws.has_key( 'DN' ):
userDN = kws['DN']
del kws['DN']
# Try the method 3 times just in case of intermittent errors
max_retry = 2
count = 0
result = S_ERROR()
while ( not result['OK'] ) and ( count <= max_retry ):
if count > 0:
# If retrying, wait a bit
time.sleep( 1 )
try:
result = S_OK()
if userDN:
resAuth = pythonCall( self.timeout, self.lfc.setAuthorizationId, userDN )
if not resAuth['OK']:
result = S_ERROR( 'Failed to set user authorization' )
if result['OK']:
method = getattr( self.lfc, self.call )
resMeth = method( *parms, **kws )
if not resMeth['OK']:
return resMeth
else:
result = resMeth
except Exception, x:
gLogger.exception( 'Exception while calling LFC Master service', '', x )
result = S_ERROR( 'Exception while calling LFC Master service ' + str( x ) )
count += 1
return result