class CatalogPlugInTestCase(unittest.TestCase):
"""Base class for the CatalogPlugin test case"""
def setUp(self):
self.fullMetadata = [
"Status",
"ChecksumType",
"OwnerRole",
"CreationDate",
"Checksum",
"ModificationDate",
"OwnerDN",
"Mode",
"GUID",
"Size",
]
self.dirMetadata = self.fullMetadata + ["NumberOfSubPaths"]
self.fileMetadata = self.fullMetadata + ["NumberOfLinks"]
self.catalog = FileCatalog(catalogs=[catalogClientToTest])
valid = self.catalog.isOK()
self.assertTrue(valid)
self.destDir = "/lhcb/test/unit-test/TestCatalogPlugin"
self.link = "%s/link" % self.destDir
# Clean the existing directory
self.cleanDirectory()
res = self.catalog.createDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
# Register some files to work with
self.numberOfFiles = 2
self.files = []
for i in range(self.numberOfFiles):
lfn = "%s/testFile_%d" % (self.destDir, i)
res = self.registerFile(lfn)
self.assertTrue(res)
self.files.append(lfn)
def registerFile(self, lfn):
pfn = "protocol://host:port/storage/path%s" % lfn
size = 10000000
se = "DIRAC-storage"
guid = makeGuid()
adler = stringAdler(guid)
fileDict = {}
fileDict[lfn] = {
"PFN": pfn,
"Size": size,
"SE": se,
"GUID": guid,
"Checksum": adler
}
res = self.catalog.addFile(fileDict)
return self.parseResult(res, lfn)
def parseResult(self, res, path):
self.assertTrue(res["OK"])
self.assertTrue(res["Value"])
self.assertTrue(res["Value"]["Successful"])
self.assertTrue(path in res["Value"]["Successful"])
return res["Value"]["Successful"][path]
def parseError(self, res, path):
self.assertTrue(res["OK"])
self.assertTrue(res["Value"])
self.assertTrue(res["Value"]["Failed"])
self.assertTrue(path in res["Value"]["Failed"])
return res["Value"]["Failed"][path]
def cleanDirectory(self):
res = self.catalog.exists(self.destDir)
returnValue = self.parseResult(res, self.destDir)
if not returnValue:
return
res = self.catalog.listDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
toRemove = list(returnValue["Files"])
if toRemove:
self.purgeFiles(toRemove)
res = self.catalog.removeDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
self.assertTrue(returnValue)
def purgeFiles(self, lfns):
for lfn in lfns:
res = self.catalog.getReplicas(lfn, True)
replicas = self.parseResult(res, lfn)
for se, pfn in replicas.items():
repDict = {}
repDict[lfn] = {"PFN": pfn, "SE": se}
res = self.catalog.removeReplica(repDict)
self.parseResult(res, lfn)
res = self.catalog.removeFile(lfn)
self.parseResult(res, lfn)
def tearDown(self):
self.cleanDirectory()
class CatalogPlugInTestCase(unittest.TestCase):
""" Base class for the CatalogPlugin test case """
def setUp(self):
self.fullMetadata = [
'Status', 'ChecksumType', 'OwnerRole', 'CreationDate', 'Checksum',
'ModificationDate', 'OwnerDN', 'Mode', 'GUID', 'Size'
]
self.dirMetadata = self.fullMetadata + ['NumberOfSubPaths']
self.fileMetadata = self.fullMetadata + ['NumberOfLinks']
self.catalog = FileCatalog(catalogs=[catalogClientToTest])
valid = self.catalog.isOK()
self.assertTrue(valid)
self.destDir = '/lhcb/test/unit-test/TestCatalogPlugin'
self.link = "%s/link" % self.destDir
# Clean the existing directory
self.cleanDirectory()
res = self.catalog.createDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
# Register some files to work with
self.numberOfFiles = 2
self.files = []
for i in xrange(self.numberOfFiles):
lfn = "%s/testFile_%d" % (self.destDir, i)
res = self.registerFile(lfn)
self.assertTrue(res)
self.files.append(lfn)
def registerFile(self, lfn):
pfn = 'protocol://host:port/storage/path%s' % lfn
size = 10000000
se = 'DIRAC-storage'
guid = makeGuid()
adler = stringAdler(guid)
fileDict = {}
fileDict[lfn] = {
'PFN': pfn,
'Size': size,
'SE': se,
'GUID': guid,
'Checksum': adler
}
res = self.catalog.addFile(fileDict)
return self.parseResult(res, lfn)
def parseResult(self, res, path):
self.assertTrue(res['OK'])
self.assertTrue(res['Value'])
self.assertTrue(res['Value']['Successful'])
self.assertTrue(path in res['Value']['Successful'])
return res['Value']['Successful'][path]
def parseError(self, res, path):
self.assertTrue(res['OK'])
self.assertTrue(res['Value'])
self.assertTrue(res['Value']['Failed'])
self.assertTrue(path in res['Value']['Failed'])
return res['Value']['Failed'][path]
def cleanDirectory(self):
res = self.catalog.exists(self.destDir)
returnValue = self.parseResult(res, self.destDir)
if not returnValue:
return
res = self.catalog.listDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
toRemove = returnValue['Files'].keys()
if toRemove:
self.purgeFiles(toRemove)
res = self.catalog.removeDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
self.assertTrue(returnValue)
def purgeFiles(self, lfns):
for lfn in lfns:
res = self.catalog.getReplicas(lfn, True)
replicas = self.parseResult(res, lfn)
for se, pfn in replicas.items():
repDict = {}
repDict[lfn] = {'PFN': pfn, 'SE': se}
res = self.catalog.removeReplica(repDict)
self.parseResult(res, lfn)
res = self.catalog.removeFile(lfn)
self.parseResult(res, lfn)
def tearDown(self):
self.cleanDirectory()
class CatalogPlugInTestCase(unittest.TestCase):
""" Base class for the CatalogPlugin test case """
def setUp(self):
self.fullMetadata = [
"Status",
"ChecksumType",
"OwnerRole",
"CreationDate",
"Checksum",
"ModificationDate",
"OwnerDN",
"Mode",
"GUID",
"Size",
]
self.dirMetadata = self.fullMetadata + ["NumberOfSubPaths"]
self.fileMetadata = self.fullMetadata + ["NumberOfLinks"]
self.catalog = FileCatalog(catalogs=[catalogClientToTest])
valid = self.catalog.isOK()
self.assert_(valid)
self.destDir = "/lhcb/test/unit-test/TestCatalogPlugin"
self.link = "%s/link" % self.destDir
# Clean the existing directory
self.cleanDirectory()
res = self.catalog.createDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
# Register some files to work with
self.numberOfFiles = 2
self.files = []
for i in range(self.numberOfFiles):
lfn = "%s/testFile_%d" % (self.destDir, i)
res = self.registerFile(lfn)
self.assert_(res)
self.files.append(lfn)
def registerFile(self, lfn):
pfn = "protocol://host:port/storage/path%s" % lfn
size = 10000000
se = "DIRAC-storage"
guid = makeGuid()
adler = stringAdler(guid)
fileDict = {}
fileDict[lfn] = {"PFN": pfn, "Size": size, "SE": se, "GUID": guid, "Checksum": adler}
res = self.catalog.addFile(fileDict)
return self.parseResult(res, lfn)
def parseResult(self, res, path):
self.assert_(res["OK"])
self.assert_(res["Value"])
self.assert_(res["Value"]["Successful"])
self.assert_(res["Value"]["Successful"].has_key(path))
return res["Value"]["Successful"][path]
def parseError(self, res, path):
self.assert_(res["OK"])
self.assert_(res["Value"])
self.assert_(res["Value"]["Failed"])
self.assert_(res["Value"]["Failed"].has_key(path))
return res["Value"]["Failed"][path]
def cleanDirectory(self):
res = self.catalog.exists(self.destDir)
returnValue = self.parseResult(res, self.destDir)
if not returnValue:
return
res = self.catalog.listDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
toRemove = returnValue["Files"].keys()
if toRemove:
self.purgeFiles(toRemove)
res = self.catalog.removeDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
self.assert_(returnValue)
def purgeFiles(self, lfns):
for lfn in lfns:
res = self.catalog.getReplicas(lfn, True)
replicas = self.parseResult(res, lfn)
for se, pfn in replicas.items():
repDict = {}
repDict[lfn] = {"PFN": pfn, "SE": se}
res = self.catalog.removeReplica(repDict)
self.parseResult(res, lfn)
res = self.catalog.removeFile(lfn)
self.parseResult(res, lfn)
def tearDown(self):
self.cleanDirectory()
class CatalogPlugInTestCase(unittest.TestCase):
""" Base class for the CatalogPlugin test case """
def setUp(self):
self.fullMetadata = ['Status', 'CheckSumType', 'OwnerRole', 'CreationDate', 'Checksum', 'ModificationDate', 'OwnerDN', 'Mode', 'GUID', 'Size']
self.dirMetadata = self.fullMetadata + ['NumberOfSubPaths']
self.fileMetadata = self.fullMetadata + ['NumberOfLinks']
self.catalog = FileCatalog(catalogs=[catalogClientToTest])
valid = self.catalog.isOK()
self.assert_(valid)
self.destDir = '/lhcb/test/unit-test/TestCatalogPlugin'
self.link = "%s/link" % self.destDir
# Clean the existing directory
self.cleanDirectory()
res = self.catalog.createDirectory(self.destDir)
returnValue = self.parseResult(res,self.destDir)
# Register some files to work with
self.numberOfFiles = 2
self.files = []
for i in range(self.numberOfFiles):
lfn = "%s/testFile_%d" % (self.destDir,i)
res = self.registerFile(lfn)
self.assert_(res)
self.files.append(lfn)
def registerFile(self,lfn):
pfn = 'protocol://host:port/storage/path%s' % lfn
size = 10000000
se = 'DIRAC-storage'
guid = makeGuid()
adler = stringAdler(guid)
fileDict = {}
fileDict[lfn] = {'PFN':pfn,'Size':size,'SE':se,'GUID':guid,'Checksum':adler}
res = self.catalog.addFile(fileDict)
return self.parseResult(res,lfn)
def parseResult(self,res,path):
self.assert_(res['OK'])
self.assert_(res['Value'])
self.assert_(res['Value']['Successful'])
self.assert_(res['Value']['Successful'].has_key(path))
return res['Value']['Successful'][path]
def parseError(self,res,path):
self.assert_(res['OK'])
self.assert_(res['Value'])
self.assert_(res['Value']['Failed'])
self.assert_(res['Value']['Failed'].has_key(path))
return res['Value']['Failed'][path]
def cleanDirectory(self):
res = self.catalog.exists(self.destDir)
returnValue = self.parseResult(res,self.destDir)
if not returnValue:
return
res = self.catalog.listDirectory(self.destDir)
returnValue = self.parseResult(res,self.destDir)
toRemove = returnValue['Files'].keys()
if toRemove:
self.purgeFiles(toRemove)
res = self.catalog.removeDirectory(self.destDir)
returnValue = self.parseResult(res,self.destDir)
self.assert_(returnValue)
def purgeFiles(self,lfns):
for lfn in lfns:
res = self.catalog.getReplicas(lfn,True)
replicas = self.parseResult(res,lfn)
for se,pfn in replicas.items():
repDict = {}
repDict[lfn] = {'PFN':pfn,'SE':se}
res = self.catalog.removeReplica(repDict)
self.parseResult(res,lfn)
res = self.catalog.removeFile(lfn)
self.parseResult(res,lfn)
def tearDown(self):
self.cleanDirectory()
class DIRACBackend(GridBackend):
"""Grid backend using the GFAL command line tools `gfal-*`."""
def __init__(self, **kwargs):
GridBackend.__init__(self, catalogue_prefix='', **kwargs)
from DIRAC.Core.Base import Script
Script.initialize()
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient
self.pm = ProxyManagerClient()
proxy = self.pm.getUserProxiesInfo()
if not proxy['OK']:
raise BackendException("Proxy error.")
from DIRAC.Interfaces.API.Dirac import Dirac
self.dirac = Dirac()
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
self.fc = FileCatalog()
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
self.dm = DataManager()
self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False)
self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False)
self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False)
self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False)
self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False)
self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False)
self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False)
self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False)
self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False)
@staticmethod
def _check_return_value(ret):
if not ret['OK']:
raise BackendException("Failed: %s", ret['Message'])
for path, error in ret['Value']['Failed'].items():
if ('No such' in error) or ('Directory does not' in error):
raise DoesNotExistException("No such file or directory.")
else:
raise BackendException(error)
def _is_dir(self, lurl):
isdir = self.fc.isDirectory(lurl)
self._check_return_value(isdir)
return isdir['Value']['Successful'][lurl]
def _is_file(self, lurl):
isfile = self.fc.isFile(lurl)
self._check_return_value(isfile)
return isfile['Value']['Successful'][lurl]
def _get_dir_entry(self, lurl, infodict=None):
"""Take a lurl and return a DirEntry."""
# If no dctionary with the information is specified, get it from the catalogue
try:
md = infodict['MetaData']
except TypeError:
md = self.fc.getFileMetadata(lurl)
if not md['OK']:
raise BackendException("Failed to list path '%s': %s", lurl, md['Message'])
for path, error in md['Value']['Failed'].items():
if 'No such file' in error:
# File does not exist, maybe a directory?
md = self.fc.getDirectoryMetadata(lurl)
for path, error in md['Value']['Failed'].items():
raise DoesNotExistException("No such file or directory.")
else:
raise BackendException(md['Value']['Failed'][lurl])
md = md['Value']['Successful'][lurl]
return DirEntry(posixpath.basename(lurl), mode=oct(md.get('Mode', -1)), links=md.get('links', -1), gid=md['OwnerGroup'], uid=md['Owner'], size=md.get('Size', -1), modified=str(md.get('ModificationDate', '?')))
def _iter_directory(self, lurl):
"""Iterate over entries in a directory."""
ret = self.fc.listDirectory(lurl)
if not ret['OK']:
raise BackendException("Failed to list path '%s': %s", lurl, ret['Message'])
for path, error in ret['Value']['Failed'].items():
if 'Directory does not' in error:
# Dir does not exist, maybe a File?
if self.fc.isFile(lurl):
lst = [(lurl, None)]
break
else:
raise DoesNotExistException("No such file or Directory.")
else:
raise BackendException(ret['Value']['Failed'][lurl])
else:
# Sort items by keys, i.e. paths
lst = sorted(ret['Value']['Successful'][lurl]['Files'].items() + ret['Value']['Successful'][lurl]['SubDirs'].items())
for item in lst:
yield item # = path, dict
def _ls(self, lurl, **kwargs):
# Translate keyword arguments
d = kwargs.pop('directory', False)
if d:
# Just the requested entry itself
yield self._get_dir_entry(lurl)
return
for path, info in self._iter_directory(lurl):
yield self._get_dir_entry(path, info)
def _ls_se(self, surl, **kwargs):
# Translate keyword arguments
d = kwargs.pop('directory', False)
args = []
if -d:
args.append('-d')
args.append('-l')
args.append(surl)
try:
output = self._ls_se_cmd(*args, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or Directory.")
else:
raise BackendException(e.stderr)
for line in output:
fields = line.split()
mode, links, gid, uid, size = fields[:5]
name = fields[-1]
modified = ' '.join(fields[5:-1])
yield DirEntry(name, mode=mode, links=int(links), gid=gid, uid=uid, size=int(size), modified=modified)
def _replicas(self, lurl, **kwargs):
# Check the lurl actually exists
self._ls(lurl, directory=True)
rep = self.dirac.getReplicas(lurl)
self._check_return_value(rep)
rep = rep['Value']['Successful'][lurl]
return rep.values()
def _exists(self, surl, **kwargs):
try:
ret = self._ls_se_cmd(surl, '-d', '-l', **kwargs).strip()
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
return False
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
else:
return ret[0] != 'd' # Return `False` for directories
def _register(self, surl, lurl, verbose=False, **kwargs):
# Register an existing physical copy in the file catalogue
se = storage.get_SE(surl).name
# See if file already exists in DFC
ret = self.fc.getFileMetadata(lurl)
try:
self._check_return_value(ret)
except DoesNotExistException:
# Add new file
size = next(self._ls_se(surl, directory=True)).size
checksum = self.checksum(surl)
guid = str(uuid.uuid4()) # The guid does not seem to be important. Make it unique if possible.
ret = self.dm.registerFile((lurl, surl, size, se, guid, checksum))
else:
# Add new replica
ret = self.dm.registerReplica((lurl, surl, se))
self._check_return_value(ret)
if verbose:
print_("Successfully registered replica %s of %s from %s."%(surl, lurl, se))
return True
def _deregister(self, surl, lurl, verbose=False, **kwargs):
# DIRAC only needs to know the SE name to deregister a replica
se = storage.get_SE(surl).name
ret = self.dm.removeReplicaFromCatalog(se, [lurl])
self._check_return_value(ret)
if verbose:
print_("Successfully deregistered replica of %s from %s."%(lurl, se))
return True
def _state(self, surl, **kwargs):
try:
state = self._xattr_cmd(surl, 'user.status', **kwargs).strip()
except sh.ErrorReturnCode as e:
if "No such file" in e.stderr:
raise DoesNotExistException("No such file or Directory.")
state = '?'
except sh.SignalException_SIGSEGV:
state = '?'
return state
def _checksum(self, surl, **kwargs):
try:
checksum = self._replica_checksum_cmd(surl, 'ADLER32', **kwargs).split()[1]
except sh.ErrorReturnCode:
checksum = '?'
except sh.SignalException_SIGSEGV:
checksum = '?'
except IndexError:
checksum = '?'
return checksum
def _bringonline(self, surl, timeout, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
# gfal does not notice when files come online, it seems
# Just send a single short request, then check regularly
if verbose:
out = sys.stdout
else:
out = None
end = time.time() + timeout
try:
self._bringonline_cmd('-t', 10, surl, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
# The command fails if the file is not online
# To be expected after 10 seconds
if "No such file" in e.stderr:
# Except when the file does not actually exist on the tape storage
raise DoesNotExistException("No such file or Directory.")
wait = 5
while(True):
if verbose:
print_("Checking replica state...")
if self.is_online(surl):
if verbose:
print_("Replica brought online.")
return True
time_left = end - time.time()
if time_left <= 0:
if verbose:
print_("Could not bring replica online.")
return False
wait *= 2
if time_left < wait:
wait = time_left
if verbose:
print_("Timeout remaining: %d s"%(time_left))
print_("Checking again in: %d s"%(wait))
time.sleep(wait)
def _replicate(self, source_surl, destination_surl, lurl, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
source = storage.get_SE(source_surl).name
destination = storage.get_SE(destination_surl).name
try:
self._replicate_cmd(lurl, destination, source, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return True
def _get(self, surl, localpath, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
try:
self._cp_cmd('-f', '--checksum', 'ADLER32', surl, localpath, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return os.path.isfile(localpath)
def _put(self, localpath, surl, lurl, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
se = storage.get_SE(surl).name
try:
self._add_cmd(lurl, localpath, se, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return True
def _remove(self, surl, lurl, last=False, verbose=False, **kwargs):
se = storage.get_SE(surl).name
if last:
# Delete lfn
if verbose:
print_("Removing all replicas of %s."%(lurl,))
ret = self.dm.removeFile([lurl])
else:
if verbose:
print_("Removing replica of %s from %s."%(lurl, se))
ret = self.dm.removeReplica(se, [lurl])
if not ret['OK']:
raise BackendException('Failed: %s'%(ret['Message']))
for lurl, error in ret['Value']['Failed'].items():
if 'No such file' in error:
raise DoesNotExistException("No such file or directory.")
else:
raise BackendException(error)
return True
def _rmdir(self, lurl, verbose=False):
"""Remove the an empty directory from the catalogue."""
rep = self.fc.removeDirectory(lurl)
self._check_return_value(rep)
return True
def _move_replica(self, surl, new_surl, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
try:
folder = posixpath.dirname(new_surl)
self._mkdir_cmd(folder, '-p', _out=out, **kwargs)
self._move_cmd(surl, new_surl, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return True