class CatalogPlugInTestCase(unittest.TestCase): """Base class for the CatalogPlugin test case""" def setUp(self): self.fullMetadata = [ "Status", "ChecksumType", "OwnerRole", "CreationDate", "Checksum", "ModificationDate", "OwnerDN", "Mode", "GUID", "Size", ] self.dirMetadata = self.fullMetadata + ["NumberOfSubPaths"] self.fileMetadata = self.fullMetadata + ["NumberOfLinks"] self.catalog = FileCatalog(catalogs=[catalogClientToTest]) valid = self.catalog.isOK() self.assertTrue(valid) self.destDir = "/lhcb/test/unit-test/TestCatalogPlugin" self.link = "%s/link" % self.destDir # Clean the existing directory self.cleanDirectory() res = self.catalog.createDirectory(self.destDir) returnValue = self.parseResult(res, self.destDir) # Register some files to work with self.numberOfFiles = 2 self.files = [] for i in range(self.numberOfFiles): lfn = "%s/testFile_%d" % (self.destDir, i) res = self.registerFile(lfn) self.assertTrue(res) self.files.append(lfn) def registerFile(self, lfn): pfn = "protocol://host:port/storage/path%s" % lfn size = 10000000 se = "DIRAC-storage" guid = makeGuid() adler = stringAdler(guid) fileDict = {} fileDict[lfn] = { "PFN": pfn, "Size": size, "SE": se, "GUID": guid, "Checksum": adler } res = self.catalog.addFile(fileDict) return self.parseResult(res, lfn) def parseResult(self, res, path): self.assertTrue(res["OK"]) self.assertTrue(res["Value"]) self.assertTrue(res["Value"]["Successful"]) self.assertTrue(path in res["Value"]["Successful"]) return res["Value"]["Successful"][path] def parseError(self, res, path): self.assertTrue(res["OK"]) self.assertTrue(res["Value"]) self.assertTrue(res["Value"]["Failed"]) self.assertTrue(path in res["Value"]["Failed"]) return res["Value"]["Failed"][path] def cleanDirectory(self): res = self.catalog.exists(self.destDir) returnValue = self.parseResult(res, self.destDir) if not returnValue: return res = self.catalog.listDirectory(self.destDir) returnValue = self.parseResult(res, self.destDir) toRemove = list(returnValue["Files"]) if toRemove: self.purgeFiles(toRemove) res = self.catalog.removeDirectory(self.destDir) returnValue = self.parseResult(res, self.destDir) self.assertTrue(returnValue) def purgeFiles(self, lfns): for lfn in lfns: res = self.catalog.getReplicas(lfn, True) replicas = self.parseResult(res, lfn) for se, pfn in replicas.items(): repDict = {} repDict[lfn] = {"PFN": pfn, "SE": se} res = self.catalog.removeReplica(repDict) self.parseResult(res, lfn) res = self.catalog.removeFile(lfn) self.parseResult(res, lfn) def tearDown(self): self.cleanDirectory()
class CatalogPlugInTestCase(unittest.TestCase): """ Base class for the CatalogPlugin test case """ def setUp(self): self.fullMetadata = [ 'Status', 'ChecksumType', 'OwnerRole', 'CreationDate', 'Checksum', 'ModificationDate', 'OwnerDN', 'Mode', 'GUID', 'Size' ] self.dirMetadata = self.fullMetadata + ['NumberOfSubPaths'] self.fileMetadata = self.fullMetadata + ['NumberOfLinks'] self.catalog = FileCatalog(catalogs=[catalogClientToTest]) valid = self.catalog.isOK() self.assertTrue(valid) self.destDir = '/lhcb/test/unit-test/TestCatalogPlugin' self.link = "%s/link" % self.destDir # Clean the existing directory self.cleanDirectory() res = self.catalog.createDirectory(self.destDir) returnValue = self.parseResult(res, self.destDir) # Register some files to work with self.numberOfFiles = 2 self.files = [] for i in xrange(self.numberOfFiles): lfn = "%s/testFile_%d" % (self.destDir, i) res = self.registerFile(lfn) self.assertTrue(res) self.files.append(lfn) def registerFile(self, lfn): pfn = 'protocol://host:port/storage/path%s' % lfn size = 10000000 se = 'DIRAC-storage' guid = makeGuid() adler = stringAdler(guid) fileDict = {} fileDict[lfn] = { 'PFN': pfn, 'Size': size, 'SE': se, 'GUID': guid, 'Checksum': adler } res = self.catalog.addFile(fileDict) return self.parseResult(res, lfn) def parseResult(self, res, path): self.assertTrue(res['OK']) self.assertTrue(res['Value']) self.assertTrue(res['Value']['Successful']) self.assertTrue(path in res['Value']['Successful']) return res['Value']['Successful'][path] def parseError(self, res, path): self.assertTrue(res['OK']) self.assertTrue(res['Value']) self.assertTrue(res['Value']['Failed']) self.assertTrue(path in res['Value']['Failed']) return res['Value']['Failed'][path] def cleanDirectory(self): res = self.catalog.exists(self.destDir) returnValue = self.parseResult(res, self.destDir) if not returnValue: return res = self.catalog.listDirectory(self.destDir) returnValue = self.parseResult(res, self.destDir) toRemove = returnValue['Files'].keys() if toRemove: self.purgeFiles(toRemove) res = self.catalog.removeDirectory(self.destDir) returnValue = self.parseResult(res, self.destDir) self.assertTrue(returnValue) def purgeFiles(self, lfns): for lfn in lfns: res = self.catalog.getReplicas(lfn, True) replicas = self.parseResult(res, lfn) for se, pfn in replicas.items(): repDict = {} repDict[lfn] = {'PFN': pfn, 'SE': se} res = self.catalog.removeReplica(repDict) self.parseResult(res, lfn) res = self.catalog.removeFile(lfn) self.parseResult(res, lfn) def tearDown(self): self.cleanDirectory()
class CatalogPlugInTestCase(unittest.TestCase): """ Base class for the CatalogPlugin test case """ def setUp(self): self.fullMetadata = [ "Status", "ChecksumType", "OwnerRole", "CreationDate", "Checksum", "ModificationDate", "OwnerDN", "Mode", "GUID", "Size", ] self.dirMetadata = self.fullMetadata + ["NumberOfSubPaths"] self.fileMetadata = self.fullMetadata + ["NumberOfLinks"] self.catalog = FileCatalog(catalogs=[catalogClientToTest]) valid = self.catalog.isOK() self.assert_(valid) self.destDir = "/lhcb/test/unit-test/TestCatalogPlugin" self.link = "%s/link" % self.destDir # Clean the existing directory self.cleanDirectory() res = self.catalog.createDirectory(self.destDir) returnValue = self.parseResult(res, self.destDir) # Register some files to work with self.numberOfFiles = 2 self.files = [] for i in range(self.numberOfFiles): lfn = "%s/testFile_%d" % (self.destDir, i) res = self.registerFile(lfn) self.assert_(res) self.files.append(lfn) def registerFile(self, lfn): pfn = "protocol://host:port/storage/path%s" % lfn size = 10000000 se = "DIRAC-storage" guid = makeGuid() adler = stringAdler(guid) fileDict = {} fileDict[lfn] = {"PFN": pfn, "Size": size, "SE": se, "GUID": guid, "Checksum": adler} res = self.catalog.addFile(fileDict) return self.parseResult(res, lfn) def parseResult(self, res, path): self.assert_(res["OK"]) self.assert_(res["Value"]) self.assert_(res["Value"]["Successful"]) self.assert_(res["Value"]["Successful"].has_key(path)) return res["Value"]["Successful"][path] def parseError(self, res, path): self.assert_(res["OK"]) self.assert_(res["Value"]) self.assert_(res["Value"]["Failed"]) self.assert_(res["Value"]["Failed"].has_key(path)) return res["Value"]["Failed"][path] def cleanDirectory(self): res = self.catalog.exists(self.destDir) returnValue = self.parseResult(res, self.destDir) if not returnValue: return res = self.catalog.listDirectory(self.destDir) returnValue = self.parseResult(res, self.destDir) toRemove = returnValue["Files"].keys() if toRemove: self.purgeFiles(toRemove) res = self.catalog.removeDirectory(self.destDir) returnValue = self.parseResult(res, self.destDir) self.assert_(returnValue) def purgeFiles(self, lfns): for lfn in lfns: res = self.catalog.getReplicas(lfn, True) replicas = self.parseResult(res, lfn) for se, pfn in replicas.items(): repDict = {} repDict[lfn] = {"PFN": pfn, "SE": se} res = self.catalog.removeReplica(repDict) self.parseResult(res, lfn) res = self.catalog.removeFile(lfn) self.parseResult(res, lfn) def tearDown(self): self.cleanDirectory()
class CatalogPlugInTestCase(unittest.TestCase): """ Base class for the CatalogPlugin test case """ def setUp(self): self.fullMetadata = ['Status', 'CheckSumType', 'OwnerRole', 'CreationDate', 'Checksum', 'ModificationDate', 'OwnerDN', 'Mode', 'GUID', 'Size'] self.dirMetadata = self.fullMetadata + ['NumberOfSubPaths'] self.fileMetadata = self.fullMetadata + ['NumberOfLinks'] self.catalog = FileCatalog(catalogs=[catalogClientToTest]) valid = self.catalog.isOK() self.assert_(valid) self.destDir = '/lhcb/test/unit-test/TestCatalogPlugin' self.link = "%s/link" % self.destDir # Clean the existing directory self.cleanDirectory() res = self.catalog.createDirectory(self.destDir) returnValue = self.parseResult(res,self.destDir) # Register some files to work with self.numberOfFiles = 2 self.files = [] for i in range(self.numberOfFiles): lfn = "%s/testFile_%d" % (self.destDir,i) res = self.registerFile(lfn) self.assert_(res) self.files.append(lfn) def registerFile(self,lfn): pfn = 'protocol://host:port/storage/path%s' % lfn size = 10000000 se = 'DIRAC-storage' guid = makeGuid() adler = stringAdler(guid) fileDict = {} fileDict[lfn] = {'PFN':pfn,'Size':size,'SE':se,'GUID':guid,'Checksum':adler} res = self.catalog.addFile(fileDict) return self.parseResult(res,lfn) def parseResult(self,res,path): self.assert_(res['OK']) self.assert_(res['Value']) self.assert_(res['Value']['Successful']) self.assert_(res['Value']['Successful'].has_key(path)) return res['Value']['Successful'][path] def parseError(self,res,path): self.assert_(res['OK']) self.assert_(res['Value']) self.assert_(res['Value']['Failed']) self.assert_(res['Value']['Failed'].has_key(path)) return res['Value']['Failed'][path] def cleanDirectory(self): res = self.catalog.exists(self.destDir) returnValue = self.parseResult(res,self.destDir) if not returnValue: return res = self.catalog.listDirectory(self.destDir) returnValue = self.parseResult(res,self.destDir) toRemove = returnValue['Files'].keys() if toRemove: self.purgeFiles(toRemove) res = self.catalog.removeDirectory(self.destDir) returnValue = self.parseResult(res,self.destDir) self.assert_(returnValue) def purgeFiles(self,lfns): for lfn in lfns: res = self.catalog.getReplicas(lfn,True) replicas = self.parseResult(res,lfn) for se,pfn in replicas.items(): repDict = {} repDict[lfn] = {'PFN':pfn,'SE':se} res = self.catalog.removeReplica(repDict) self.parseResult(res,lfn) res = self.catalog.removeFile(lfn) self.parseResult(res,lfn) def tearDown(self): self.cleanDirectory()
class DIRACBackend(GridBackend): """Grid backend using the GFAL command line tools `gfal-*`.""" def __init__(self, **kwargs): GridBackend.__init__(self, catalogue_prefix='', **kwargs) from DIRAC.Core.Base import Script Script.initialize() from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient self.pm = ProxyManagerClient() proxy = self.pm.getUserProxiesInfo() if not proxy['OK']: raise BackendException("Proxy error.") from DIRAC.Interfaces.API.Dirac import Dirac self.dirac = Dirac() from DIRAC.Resources.Catalog.FileCatalog import FileCatalog self.fc = FileCatalog() from DIRAC.DataManagementSystem.Client.DataManager import DataManager self.dm = DataManager() self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False) self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False) self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False) self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False) self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False) self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False) self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False) self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False) self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False) @staticmethod def _check_return_value(ret): if not ret['OK']: raise BackendException("Failed: %s", ret['Message']) for path, error in ret['Value']['Failed'].items(): if ('No such' in error) or ('Directory does not' in error): raise DoesNotExistException("No such file or directory.") else: raise BackendException(error) def _is_dir(self, lurl): isdir = self.fc.isDirectory(lurl) self._check_return_value(isdir) return isdir['Value']['Successful'][lurl] def _is_file(self, lurl): isfile = self.fc.isFile(lurl) self._check_return_value(isfile) return isfile['Value']['Successful'][lurl] def _get_dir_entry(self, lurl, infodict=None): """Take a lurl and return a DirEntry.""" # If no dctionary with the information is specified, get it from the catalogue try: md = infodict['MetaData'] except TypeError: md = self.fc.getFileMetadata(lurl) if not md['OK']: raise BackendException("Failed to list path '%s': %s", lurl, md['Message']) for path, error in md['Value']['Failed'].items(): if 'No such file' in error: # File does not exist, maybe a directory? md = self.fc.getDirectoryMetadata(lurl) for path, error in md['Value']['Failed'].items(): raise DoesNotExistException("No such file or directory.") else: raise BackendException(md['Value']['Failed'][lurl]) md = md['Value']['Successful'][lurl] return DirEntry(posixpath.basename(lurl), mode=oct(md.get('Mode', -1)), links=md.get('links', -1), gid=md['OwnerGroup'], uid=md['Owner'], size=md.get('Size', -1), modified=str(md.get('ModificationDate', '?'))) def _iter_directory(self, lurl): """Iterate over entries in a directory.""" ret = self.fc.listDirectory(lurl) if not ret['OK']: raise BackendException("Failed to list path '%s': %s", lurl, ret['Message']) for path, error in ret['Value']['Failed'].items(): if 'Directory does not' in error: # Dir does not exist, maybe a File? if self.fc.isFile(lurl): lst = [(lurl, None)] break else: raise DoesNotExistException("No such file or Directory.") else: raise BackendException(ret['Value']['Failed'][lurl]) else: # Sort items by keys, i.e. paths lst = sorted(ret['Value']['Successful'][lurl]['Files'].items() + ret['Value']['Successful'][lurl]['SubDirs'].items()) for item in lst: yield item # = path, dict def _ls(self, lurl, **kwargs): # Translate keyword arguments d = kwargs.pop('directory', False) if d: # Just the requested entry itself yield self._get_dir_entry(lurl) return for path, info in self._iter_directory(lurl): yield self._get_dir_entry(path, info) def _ls_se(self, surl, **kwargs): # Translate keyword arguments d = kwargs.pop('directory', False) args = [] if -d: args.append('-d') args.append('-l') args.append(surl) try: output = self._ls_se_cmd(*args, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or Directory.") else: raise BackendException(e.stderr) for line in output: fields = line.split() mode, links, gid, uid, size = fields[:5] name = fields[-1] modified = ' '.join(fields[5:-1]) yield DirEntry(name, mode=mode, links=int(links), gid=gid, uid=uid, size=int(size), modified=modified) def _replicas(self, lurl, **kwargs): # Check the lurl actually exists self._ls(lurl, directory=True) rep = self.dirac.getReplicas(lurl) self._check_return_value(rep) rep = rep['Value']['Successful'][lurl] return rep.values() def _exists(self, surl, **kwargs): try: ret = self._ls_se_cmd(surl, '-d', '-l', **kwargs).strip() except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: return False else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) else: return ret[0] != 'd' # Return `False` for directories def _register(self, surl, lurl, verbose=False, **kwargs): # Register an existing physical copy in the file catalogue se = storage.get_SE(surl).name # See if file already exists in DFC ret = self.fc.getFileMetadata(lurl) try: self._check_return_value(ret) except DoesNotExistException: # Add new file size = next(self._ls_se(surl, directory=True)).size checksum = self.checksum(surl) guid = str(uuid.uuid4()) # The guid does not seem to be important. Make it unique if possible. ret = self.dm.registerFile((lurl, surl, size, se, guid, checksum)) else: # Add new replica ret = self.dm.registerReplica((lurl, surl, se)) self._check_return_value(ret) if verbose: print_("Successfully registered replica %s of %s from %s."%(surl, lurl, se)) return True def _deregister(self, surl, lurl, verbose=False, **kwargs): # DIRAC only needs to know the SE name to deregister a replica se = storage.get_SE(surl).name ret = self.dm.removeReplicaFromCatalog(se, [lurl]) self._check_return_value(ret) if verbose: print_("Successfully deregistered replica of %s from %s."%(lurl, se)) return True def _state(self, surl, **kwargs): try: state = self._xattr_cmd(surl, 'user.status', **kwargs).strip() except sh.ErrorReturnCode as e: if "No such file" in e.stderr: raise DoesNotExistException("No such file or Directory.") state = '?' except sh.SignalException_SIGSEGV: state = '?' return state def _checksum(self, surl, **kwargs): try: checksum = self._replica_checksum_cmd(surl, 'ADLER32', **kwargs).split()[1] except sh.ErrorReturnCode: checksum = '?' except sh.SignalException_SIGSEGV: checksum = '?' except IndexError: checksum = '?' return checksum def _bringonline(self, surl, timeout, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None # gfal does not notice when files come online, it seems # Just send a single short request, then check regularly if verbose: out = sys.stdout else: out = None end = time.time() + timeout try: self._bringonline_cmd('-t', 10, surl, _out=out, **kwargs) except sh.ErrorReturnCode as e: # The command fails if the file is not online # To be expected after 10 seconds if "No such file" in e.stderr: # Except when the file does not actually exist on the tape storage raise DoesNotExistException("No such file or Directory.") wait = 5 while(True): if verbose: print_("Checking replica state...") if self.is_online(surl): if verbose: print_("Replica brought online.") return True time_left = end - time.time() if time_left <= 0: if verbose: print_("Could not bring replica online.") return False wait *= 2 if time_left < wait: wait = time_left if verbose: print_("Timeout remaining: %d s"%(time_left)) print_("Checking again in: %d s"%(wait)) time.sleep(wait) def _replicate(self, source_surl, destination_surl, lurl, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None source = storage.get_SE(source_surl).name destination = storage.get_SE(destination_surl).name try: self._replicate_cmd(lurl, destination, source, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return True def _get(self, surl, localpath, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None try: self._cp_cmd('-f', '--checksum', 'ADLER32', surl, localpath, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return os.path.isfile(localpath) def _put(self, localpath, surl, lurl, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None se = storage.get_SE(surl).name try: self._add_cmd(lurl, localpath, se, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return True def _remove(self, surl, lurl, last=False, verbose=False, **kwargs): se = storage.get_SE(surl).name if last: # Delete lfn if verbose: print_("Removing all replicas of %s."%(lurl,)) ret = self.dm.removeFile([lurl]) else: if verbose: print_("Removing replica of %s from %s."%(lurl, se)) ret = self.dm.removeReplica(se, [lurl]) if not ret['OK']: raise BackendException('Failed: %s'%(ret['Message'])) for lurl, error in ret['Value']['Failed'].items(): if 'No such file' in error: raise DoesNotExistException("No such file or directory.") else: raise BackendException(error) return True def _rmdir(self, lurl, verbose=False): """Remove the an empty directory from the catalogue.""" rep = self.fc.removeDirectory(lurl) self._check_return_value(rep) return True def _move_replica(self, surl, new_surl, verbose=False, **kwargs): if verbose: out = sys.stdout else: out = None try: folder = posixpath.dirname(new_surl) self._mkdir_cmd(folder, '-p', _out=out, **kwargs) self._move_cmd(surl, new_surl, _out=out, **kwargs) except sh.ErrorReturnCode as e: if 'No such file' in e.stderr: raise DoesNotExistException("No such file or directory.") else: if len(e.stderr) == 0: raise BackendException(e.stdout) else: raise BackendException(e.stderr) return True