def setUp(self): self.lfc = LcgFileCatalogClient(host='lfc-lhcb.cern.ch',infosys='lcg-bdii.cern.ch:2170') ###################################################### # # Clean the test directory before starting # dir = '/lhcb/test/unit-test' res = self.lfc.listDirectory(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) lfnsToDelete = res['Value']['Successful'][dir] res = self.lfc.getReplicas(lfnsToDelete) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) replicas = res['Value']['Successful'] replicaTupleList = [] for lfn in replicas.keys(): for se in replicas[lfn].keys(): replicaTuple = (lfn,replicas[lfn][se],se) replicaTupleList.append(replicaTuple) res = self.lfc.removeReplica(replicaTupleList) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assertFalse(res['Value']['Failed']) res = self.lfc.removeFile(lfnsToDelete) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assertFalse(res['Value']['Failed'])
def setUp(self): self.lfc = LcgFileCatalogClient(host="lfc-lhcb.cern.ch", infosys="lcg-bdii.cern.ch:2170") ###################################################### # # Clean the test directory before starting # dir = "/lhcb/test/unit-test" res = self.lfc.listDirectory(dir) self.assert_(res["OK"]) self.assert_(res["Value"].has_key("Successful")) self.assert_(res["Value"].has_key("Failed")) self.assert_(res["Value"]["Successful"].has_key(dir)) lfnsToDelete = res["Value"]["Successful"][dir] res = self.lfc.getReplicas(lfnsToDelete) self.assert_(res["OK"]) self.assert_(res["Value"].has_key("Successful")) self.assert_(res["Value"].has_key("Failed")) replicas = res["Value"]["Successful"] replicaTupleList = [] for lfn in replicas.keys(): for se in replicas[lfn].keys(): replicaTuple = (lfn, replicas[lfn][se], se) replicaTupleList.append(replicaTuple) res = self.lfc.removeReplica(replicaTupleList) self.assert_(res["OK"]) self.assert_(res["Value"].has_key("Successful")) self.assert_(res["Value"].has_key("Failed")) self.assertFalse(res["Value"]["Failed"]) res = self.lfc.removeFile(lfnsToDelete) self.assert_(res["OK"]) self.assert_(res["Value"].has_key("Successful")) self.assert_(res["Value"].has_key("Failed")) self.assertFalse(res["Value"]["Failed"])
def __init__( self, infosys = None, master_host = None, mirrors = [] ): """ Default constructor """ if not infosys: configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/LcgGfalInfosys' infosys = gConfig.getValue( configPath ) self.valid = False if not master_host: configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/MasterHost' master_host = gConfig.getValue( configPath ) if master_host: # Create the master LFC client first self.lfc = LcgFileCatalogClient( infosys, master_host ) if self.lfc.isOK(): self.valid = True if not mirrors: siteName = DIRAC.siteName() res = getLocationOrderedCatalogs( siteName = siteName ) if not res['OK']: mirrors = [] else: mirrors = res['Value'] # Create the mirror LFC instances self.mirrors = [] for mirror in mirrors: lfc = LcgFileCatalogClient( infosys, mirror ) self.mirrors.append( lfc ) self.nmirrors = len( self.mirrors ) # Keep the environment for the master instance self.master_host = self.lfc.host os.environ['LFC_HOST'] = self.master_host os.environ['LCG_GFAL_INFOSYS'] = infosys self.name = 'LFC' self.timeout = 3000
fcType = res['Value'] if not fcType: res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True) if res['OK']: fcType = res['Value'][0] for switch in Script.getUnprocessedSwitches(): if switch[0].lower() == "f" or switch[0].lower() == "file-catalog": fcType = switch[1] from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI if fcType == "LcgFileCatalog" or fcType == "LFC": from DIRAC.Resources.Catalog.LcgFileCatalogClient import LcgFileCatalogClient cli = FileCatalogClientCLI(LcgFileCatalogClient()) try: host = os.environ['LFC_HOST'] except Exception, x: print "LFC_HOST environment variable not defined" sys.exit(1) print "Starting LFC FileCatalog client" cli.cmdloop() elif fcType == "LcgFileCatalogProxy" or fcType == "LFCproxy": from DIRAC.Resources.Catalog.LcgFileCatalogProxyClient import LcgFileCatalogProxyClient cli = FileCatalogClientCLI(LcgFileCatalogProxyClient()) print "Starting LFC Proxy FileCatalog client" cli.cmdloop() elif fcType == "LcgFileCatalogCombined" or fcType == "LFCCombined": from DIRAC.Resources.Catalog.LcgFileCatalogCombinedClient import LcgFileCatalogCombinedClient cli = FileCatalogClientCLI(LcgFileCatalogCombinedClient())
class LcgFileCatalogCombinedClient( object ): ro_methods = ['exists', 'isLink', 'readLink', 'isFile', 'getFileMetadata', 'getReplicas', 'getReplicaStatus', 'getFileSize', 'isDirectory', 'getDirectoryReplicas', 'listDirectory', 'getDirectoryMetadata', 'getDirectorySize', 'getDirectoryContents', 'resolveDataset', 'getPathPermissions', 'getLFNForPFN'] write_methods = ['createLink', 'removeLink', 'addFile', 'addReplica', 'removeReplica', 'removeFile', 'setReplicaStatus', 'setReplicaHost', 'createDirectory', 'removeDirectory', 'removeDataset', 'removeFileFromDataset', 'createDataset'] def __init__( self, infosys = None, master_host = None, mirrors = [] ): """ Default constructor """ if not infosys: configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/LcgGfalInfosys' infosys = gConfig.getValue( configPath ) self.valid = False if not master_host: configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/MasterHost' master_host = gConfig.getValue( configPath ) if master_host: # Create the master LFC client first self.lfc = LcgFileCatalogClient( infosys, master_host ) if self.lfc.isOK(): self.valid = True if not mirrors: siteName = DIRAC.siteName() res = getLocationOrderedCatalogs( siteName = siteName ) if not res['OK']: mirrors = [] else: mirrors = res['Value'] # Create the mirror LFC instances self.mirrors = [] for mirror in mirrors: lfc = LcgFileCatalogClient( infosys, mirror ) self.mirrors.append( lfc ) self.nmirrors = len( self.mirrors ) # Keep the environment for the master instance self.master_host = self.lfc.host os.environ['LFC_HOST'] = self.master_host os.environ['LCG_GFAL_INFOSYS'] = infosys self.name = 'LFC' self.timeout = 3000 def isOK( self ): return self.valid def getName( self, DN = '' ): """ Get the file catalog type name """ return self.name def __getattr__( self, name ): self.call = name if name in LcgFileCatalogCombinedClient.write_methods: return self.w_execute elif name in LcgFileCatalogCombinedClient.ro_methods: return self.r_execute else: raise AttributeError def w_execute( self, *parms, **kws ): """ Write method executor. Dispatches execution of the methods which need Read/Write access to the master LFC instance """ # If the DN argument is given, this is an operation on behalf # of the user with this DN, prepare setAuthorizationId call userDN = '' if kws.has_key( 'DN' ): userDN = kws['DN'] del kws['DN'] # Try the method 3 times just in case of intermittent errors max_retry = 2 count = 0 result = S_ERROR() while ( not result['OK'] ) and ( count <= max_retry ): if count > 0: # If retrying, wait a bit time.sleep( 1 ) try: result = S_OK() if userDN: resAuth = pythonCall( self.timeout, self.lfc.setAuthorizationId, userDN ) if not resAuth['OK']: result = S_ERROR( 'Failed to set user authorization' ) if result['OK']: method = getattr( self.lfc, self.call ) resMeth = method( *parms, **kws ) if not resMeth['OK']: return resMeth else: result = resMeth except Exception, x: gLogger.exception( 'Exception while calling LFC Master service', '', x ) result = S_ERROR( 'Exception while calling LFC Master service ' + str( x ) ) count += 1 return result
class LcgFileCatalogCombinedClient: ro_methods = ['exists', 'isLink', 'readLink', 'isFile', 'getFileMetadata', 'getReplicas', 'getReplicaStatus', 'getFileSize', 'isDirectory', 'getDirectoryReplicas', 'listDirectory', 'getDirectoryMetadata', 'getDirectorySize', 'getDirectoryContents', 'resolveDataset', 'getPathPermissions', 'getLFNForPFN'] write_methods = ['createLink', 'removeLink', 'addFile', 'addReplica', 'removeReplica', 'removeFile', 'setReplicaStatus', 'setReplicaHost', 'createDirectory', 'removeDirectory', 'removeDataset', 'removeFileFromDataset', 'createDataset'] def __init__( self, infosys = None, master_host = None, mirrors = [] ): """ Default constructor """ if not infosys: configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/LcgGfalInfosys' infosys = gConfig.getValue( configPath ) self.valid = False if not master_host: configPath = '/Resources/FileCatalogs/LcgFileCatalogCombined/MasterHost' master_host = gConfig.getValue( configPath ) if master_host: # Create the master LFC client first self.lfc = LcgFileCatalogClient( infosys, master_host ) if self.lfc.isOK(): self.valid = True if not mirrors: siteName = DIRAC.siteName() res = getLocationOrderedCatalogs( siteName = siteName ) if not res['OK']: mirrors = [] else: mirrors = res['Value'] # Create the mirror LFC instances self.mirrors = [] for mirror in mirrors: lfc = LcgFileCatalogClient( infosys, mirror ) self.mirrors.append( lfc ) self.nmirrors = len( self.mirrors ) # Keep the environment for the master instance self.master_host = self.lfc.host os.environ['LFC_HOST'] = self.master_host os.environ['LCG_GFAL_INFOSYS'] = infosys self.name = 'LFC' self.timeout = 3000 def isOK( self ): return self.valid def getName( self, DN = '' ): """ Get the file catalog type name """ return self.name def __getattr__( self, name ): self.call = name if name in LcgFileCatalogCombinedClient.write_methods: return self.w_execute elif name in LcgFileCatalogCombinedClient.ro_methods: return self.r_execute else: raise AttributeError def w_execute( self, *parms, **kws ): """ Write method executor. Dispatches execution of the methods which need Read/Write access to the master LFC instance """ # If the DN argument is given, this is an operation on behalf # of the user with this DN, prepare setAuthorizationId call userDN = '' if kws.has_key( 'DN' ): userDN = kws['DN'] del kws['DN'] # Try the method 3 times just in case of intermittent errors max_retry = 2 count = 0 result = S_ERROR() while ( not result['OK'] ) and ( count <= max_retry ): if count > 0: # If retrying, wait a bit time.sleep( 1 ) try: result = S_OK() if userDN: resAuth = pythonCall( self.timeout, self.lfc.setAuthorizationId, userDN ) if not resAuth['OK']: result = S_ERROR( 'Failed to set user authorization' ) if result['OK']: method = getattr( self.lfc, self.call ) resMeth = method( *parms, **kws ) if not resMeth['OK']: return resMeth else: result = resMeth except Exception, x: gLogger.exception( 'Exception while calling LFC Master service', '', x ) result = S_ERROR( 'Exception while calling LFC Master service ' + str( x ) ) count += 1 return result