def setup(self):
"""SRM (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseSRM.tmpdir
self.rse_id = 'FZK-LCG2_SCRATCHDISK'
self.mtc = MgrTestCases(self.tmpdir, 'FZK-LCG2_SCRATCHDISK', TestRseSRM.user, TestRseSRM.static_file)
# self.mtc = MgrTestCases(self.tmpdir, 'FZK-LCG2_SCRATCHDISK', TestRseSRM.user, "srm://atlassrm-fzk.gridka.de/pnfs/gridka.de/atlas/disk-only/atlasscratchdisk/user/wguan/rucio.test.2")
self.mtc.setup_scheme('srm')
def setup(self):
"""SFTP (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseSFTP.tmpdir
self.static_file = TestRseSFTP.static_file
self.mtc = MgrTestCases(self.tmpdir, 'LXPLUS', TestRseSFTP.user, self.static_file)
class TestRseSFTP():
tmpdir = None
user = None
@classmethod
def setupClass(cls):
"""SFTP (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write('\0')
for f in MgrTestCases.files_local:
os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
# Load local credentials from file
with open('etc/rse-accounts.cfg') as f:
data = json.load(f)
credentials = data['LXPLUS']
lxplus = pysftp.Connection(**credentials)
with open('etc/rse_repository.json') as f:
prefix = json.load(f)['LXPLUS']['protocols']['supported']['sftp']['prefix']
lxplus.execute('mkdir %s' % prefix)
lxplus.execute('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % prefix)
cls.static_file = 'sftp://lxplus.cern.ch:22%sdata.raw' % prefix
protocol = mgr.create_protocol(mgr.get_rse_info('LXPLUS'), 'write')
for f in MgrTestCases.files_remote:
tmp = protocol.parse_pfns(protocol.lfns2pfns({'name': f, 'scope': 'user.%s' % cls.user}).values()[0]).values()[0]
for cmd in ['mkdir -p %s' % ''.join([tmp['prefix'], tmp['path']]), 'ln -s %sdata.raw %s' % (prefix, ''.join([tmp['prefix'], tmp['path'], tmp['name']]))]:
lxplus.execute(cmd)
lxplus.close()
@classmethod
def tearDownClass(cls):
"""SFTP (RSE/PROTOCOLS): Removing created directorie s and files """
# Load local creditentials from file
credentials = {}
with open('etc/rse-accounts.cfg') as f:
data = json.load(f)
credentials = data['LXPLUS']
lxplus = pysftp.Connection(**credentials)
with open('etc/rse_repository.json') as f:
prefix = json.load(f)['LXPLUS']['protocols']['supported']['sftp']['prefix']
lxplus.execute('rm -rf %s' % prefix)
lxplus.close()
shutil.rmtree(cls.tmpdir)
def setup(self):
"""SFTP (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseSFTP.tmpdir
self.static_file = TestRseSFTP.static_file
self.mtc = MgrTestCases(self.tmpdir, 'LXPLUS', TestRseSFTP.user, self.static_file)
# Mgr-Tests: GET
def test_multi_get_mgr_ok(self):
"""SFTP (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)"""
self.mtc.test_multi_get_mgr_ok()
def test_get_mgr_ok_single_lfn(self):
"""SFTP (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)"""
self.mtc.test_get_mgr_ok_single_lfn()
def test_get_mgr_ok_single_pfn(self):
"""SFTP (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)"""
self.mtc.test_get_mgr_ok_single_pfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_multi(self):
"""SFTP (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_lfn(self):
"""SFTP (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_pfn(self):
"""SFTP (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_pfn()
# Mgr-Tests: PUT
def test_put_mgr_ok_multi(self):
"""SFTP (RSE/PROTOCOLS): Put multiple files to storage (Success)"""
self.mtc.test_put_mgr_ok_multi()
def test_put_mgr_ok_single(self):
"""SFTP (RSE/PROTOCOLS): Put a single file to storage (Success)"""
self.mtc.test_put_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_multi(self):
"""SFTP (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_single(self):
"""SFTP (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_single()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_multi(self):
"""SFTP (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_single(self):
"""SFTP (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_single()
# MGR-Tests: DELETE
def test_delete_mgr_ok_multi(self):
"""SFTP (RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
self.mtc.test_delete_mgr_ok_multi()
def test_delete_mgr_ok_single(self):
"""SFTP (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
self.mtc.test_delete_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_multi(self):
"""SFTP (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_single(self):
"""SFTP (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_single()
# MGR-Tests: EXISTS
def test_exists_mgr_ok_multi(self):
"""SFTP (RSE/PROTOCOLS): Check multiple files on storage (Success)"""
self.mtc.test_exists_mgr_ok_multi()
def test_exists_mgr_ok_single_lfn(self):
"""SFTP (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)"""
self.mtc.test_exists_mgr_ok_single_lfn()
def test_exists_mgr_ok_single_pfn(self):
"""SFTP (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)"""
self.mtc.test_exists_mgr_ok_single_pfn()
def test_exists_mgr_false_multi(self):
"""SFTP (RSE/PROTOCOLS): Check multiple files on storage (Fail)"""
self.mtc.test_exists_mgr_false_multi()
def test_exists_mgr_false_single_lfn(self):
"""SFTP (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)"""
self.mtc.test_exists_mgr_false_single_lfn()
def test_exists_mgr_false_single_pfn(self):
"""SFTP (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)"""
self.mtc.test_exists_mgr_false_single_pfn()
# MGR-Tests: RENAME
def test_rename_mgr_ok_multi(self):
"""SFTP (RSE/PROTOCOLS): Rename multiple files on storage (Success)"""
self.mtc.test_rename_mgr_ok_multi()
def test_rename_mgr_ok_single_lfn(self):
"""SFTP (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)"""
self.mtc.test_rename_mgr_ok_single_lfn()
def test_rename_mgr_ok_single_pfn(self):
"""SFTP (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)"""
self.mtc.test_rename_mgr_ok_single_pfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_multi(self):
"""SFTP (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self):
"""SFTP (RSE/PROTOCOLS): Rename a single file on storage using LFN (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self):
"""SFTP (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_multi(self):
"""SFTP (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_lfn(self):
"""SFTP (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_pfn(self):
"""SFTP (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_pfn()
def test_change_scope_mgr_ok_single_lfn(self):
"""SFTP (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_lfn()
def test_change_scope_mgr_ok_single_pfn(self):
"""SFTP (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_pfn()
def setup(self):
"""S3 (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseS3.tmpdir
self.mtc = MgrTestCases(self.tmpdir, 'BNL-BOTO', TestRseS3.user, TestRseS3.static_file)
class TestRseS3():
tmpdir = None
user = None
@classmethod
def setupClass(cls):
"""S3 (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
# cls.user = '******' # use again when latency issue with S3 storage is resolved
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write('\0')
for f in MgrTestCases.files_local:
os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
rse_tag = 'BNL-BOTO'
rse_settings = mgr.get_rse_info(rse_tag)
try:
with open('etc/rse-accounts.cfg') as f:
data = json.load(f)
rse_settings['credentials'] = data[rse_tag]
except KeyError:
print 'No credentials found for this RSE.'
pass
for protocol in rse_settings['protocols']:
if protocol['scheme'] == 's3':
break
conn = boto.connect_s3(host=protocol['hostname'],
port=int(protocol.get('port', 80)),
aws_access_key_id=rse_settings['credentials']['access_key'],
aws_secret_access_key=rse_settings['credentials']['secret_key'],
is_secure=rse_settings['credentials'].get('is_secure', False),
calling_format=boto.s3.connection.OrdinaryCallingFormat())
cls.static_file = mgr.lfns2pfns(rse_settings, {'name': 'data.raw', 'scope': 'user.%s' % cls.user}).values()[0]
key = get_bucket_key(cls.static_file, conn, create=True)
key.set_contents_from_filename("%s/data.raw" % cls.tmpdir)
for f in MgrTestCases.files_remote:
pfn = mgr.lfns2pfns(rse_settings, {'name': f, 'scope': 'user.%s' % cls.user}).values()[0]
bucket_name, key_name = get_bucket_key_name(pfn)
key.copy(bucket_name, key_name)
def setup(self):
"""S3 (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseS3.tmpdir
self.mtc = MgrTestCases(self.tmpdir, 'BNL-BOTO', TestRseS3.user, TestRseS3.static_file)
@classmethod
def tearDownClass(cls):
"""S3 (RSE/PROTOCOLS): Removing created directories and files """
# Remove test files from storage
# rse_tag = 'AMAZON-BOTO'
rse_tag = 'BNL-BOTO'
rse_settings = mgr.get_rse_info(rse_tag)
try:
with open('etc/rse-accounts.cfg') as f:
data = json.load(f)
rse_settings['credentials'] = data[rse_tag]
except KeyError:
print 'No credentials found for this RSE.'
pass
for protocol in rse_settings['protocols']:
if protocol['scheme'] == 's3':
break
conn = boto.connect_s3(host=protocol['hostname'],
port=int(protocol.get('port', 80)),
aws_access_key_id=rse_settings['credentials']['access_key'],
aws_secret_access_key=rse_settings['credentials']['secret_key'],
is_secure=rse_settings['credentials'].get('is_secure', False),
calling_format=boto.s3.connection.OrdinaryCallingFormat())
for protocol in rse_settings['protocols']:
if protocol['scheme'] == 's3':
break
bucket_name = protocol['prefix']
bucket = conn.get_bucket(bucket_name)
keys = bucket.list()
for key in keys:
key.delete()
# Mgr-Tests: GET
def test_multi_get_mgr_ok(self):
"""S3 (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)"""
self.mtc.test_multi_get_mgr_ok()
def test_get_mgr_ok_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)"""
self.mtc.test_get_mgr_ok_single_lfn()
def test_get_mgr_ok_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)"""
self.mtc.test_get_mgr_ok_single_pfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_multi(self):
"""S3 (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_pfn()
# Mgr-Tests: PUT
def test_put_mgr_ok_multi(self):
"""S3 (RSE/PROTOCOLS): Put multiple files to storage (Success)"""
self.mtc.test_put_mgr_ok_multi()
def test_put_mgr_ok_single(self):
"""S3 (RSE/PROTOCOLS): Put a single file to storage (Success)"""
self.mtc.test_put_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_multi(self):
"""S3 (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_single(self):
"""S3 (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_single()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_multi(self):
"""S3 (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_single(self):
"""S3 (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_single()
# MGR-Tests: DELETE
def test_delete_mgr_ok_multi(self):
"""S3 (RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
self.mtc.test_delete_mgr_ok_multi()
def test_delete_mgr_ok_single(self):
"""S3 (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
self.mtc.test_delete_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_multi(self):
"""S3 (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_single(self):
"""S3 (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_single()
# MGR-Tests: EXISTS
def test_exists_mgr_ok_multi(self):
"""S3 (RSE/PROTOCOLS): Check multiple files on storage (Success)"""
self.mtc.test_exists_mgr_ok_multi()
def test_exists_mgr_ok_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)"""
self.mtc.test_exists_mgr_ok_single_lfn()
def test_exists_mgr_ok_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)"""
self.mtc.test_exists_mgr_ok_single_pfn()
def test_exists_mgr_false_multi(self):
"""S3 (RSE/PROTOCOLS): Check multiple files on storage (Fail)"""
self.mtc.test_exists_mgr_false_multi()
def test_exists_mgr_false_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)"""
self.mtc.test_exists_mgr_false_single_lfn()
def test_exists_mgr_false_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)"""
self.mtc.test_exists_mgr_false_single_pfn()
# MGR-Tests: RENAME
def test_rename_mgr_ok_multi(self):
"""S3 (RSE/PROTOCOLS): Rename multiple files on storage (Success)"""
self.mtc.test_rename_mgr_ok_multi()
def test_rename_mgr_ok_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)"""
self.mtc.test_rename_mgr_ok_single_lfn()
def test_rename_mgr_ok_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)"""
self.mtc.test_rename_mgr_ok_single_pfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_multi(self):
"""S3 (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_multi(self):
"""S3 (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_pfn()
def test_change_scope_mgr_ok_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_lfn()
def test_change_scope_mgr_ok_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_pfn()
class TestRseS3():
tmpdir = None
user = None
@classmethod
def setupClass(cls):
"""S3 (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
# cls.user = '******' # use again when latency issue with S3 storage is resolved
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write('\0')
for f in MgrTestCases.files_local:
os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
rse_tag = 'BNL-BOTO'
rse_settings = mgr.get_rse_info(rse_tag)
try:
with open('etc/rse-accounts.cfg') as f:
data = json.load(f)
rse_settings['credentials'] = data[rse_tag]
except KeyError:
print('No credentials found for this RSE.')
pass
for protocol in rse_settings['protocols']:
if protocol['scheme'] == 's3':
break
conn = boto.connect_s3(host=protocol['hostname'],
port=int(protocol.get('port', 80)),
aws_access_key_id=rse_settings['credentials']['access_key'],
aws_secret_access_key=rse_settings['credentials']['secret_key'],
is_secure=rse_settings['credentials'].get('is_secure', False),
calling_format=boto.s3.connection.OrdinaryCallingFormat())
cls.static_file = mgr.lfns2pfns(rse_settings, {'name': 'data.raw', 'scope': 'user.%s' % cls.user}).values()[0]
key = get_bucket_key(cls.static_file, conn, create=True)
key.set_contents_from_filename("%s/data.raw" % cls.tmpdir)
for f in MgrTestCases.files_remote:
pfn = mgr.lfns2pfns(rse_settings, {'name': f, 'scope': 'user.%s' % cls.user}).values()[0]
bucket_name, key_name = get_bucket_key_name(pfn)
key.copy(bucket_name, key_name)
def setup(self):
"""S3 (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseS3.tmpdir
self.mtc = MgrTestCases(self.tmpdir, 'BNL-BOTO', TestRseS3.user, TestRseS3.static_file)
@classmethod
def tearDownClass(cls):
"""S3 (RSE/PROTOCOLS): Removing created directories and files """
# Remove test files from storage
# rse_tag = 'AMAZON-BOTO'
rse_tag = 'BNL-BOTO'
rse_settings = mgr.get_rse_info(rse_tag)
try:
with open('etc/rse-accounts.cfg') as f:
data = json.load(f)
rse_settings['credentials'] = data[rse_tag]
except KeyError:
print('No credentials found for this RSE.')
pass
for protocol in rse_settings['protocols']:
if protocol['scheme'] == 's3':
break
conn = boto.connect_s3(host=protocol['hostname'],
port=int(protocol.get('port', 80)),
aws_access_key_id=rse_settings['credentials']['access_key'],
aws_secret_access_key=rse_settings['credentials']['secret_key'],
is_secure=rse_settings['credentials'].get('is_secure', False),
calling_format=boto.s3.connection.OrdinaryCallingFormat())
for protocol in rse_settings['protocols']:
if protocol['scheme'] == 's3':
break
bucket_name = protocol['prefix']
bucket = conn.get_bucket(bucket_name)
keys = bucket.list()
for key in keys:
key.delete()
# Mgr-Tests: GET
def test_multi_get_mgr_ok(self):
"""S3 (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)"""
self.mtc.test_multi_get_mgr_ok()
def test_get_mgr_ok_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)"""
self.mtc.test_get_mgr_ok_single_lfn()
def test_get_mgr_ok_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)"""
self.mtc.test_get_mgr_ok_single_pfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_multi(self):
"""S3 (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_pfn()
# Mgr-Tests: PUT
def test_put_mgr_ok_multi(self):
"""S3 (RSE/PROTOCOLS): Put multiple files to storage (Success)"""
self.mtc.test_put_mgr_ok_multi()
def test_put_mgr_ok_single(self):
"""S3 (RSE/PROTOCOLS): Put a single file to storage (Success)"""
self.mtc.test_put_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_multi(self):
"""S3 (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_single(self):
"""S3 (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_single()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_multi(self):
"""S3 (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_single(self):
"""S3 (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_single()
# MGR-Tests: DELETE
def test_delete_mgr_ok_multi(self):
"""S3 (RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
self.mtc.test_delete_mgr_ok_multi()
def test_delete_mgr_ok_single(self):
"""S3 (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
self.mtc.test_delete_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_multi(self):
"""S3 (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_single(self):
"""S3 (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_single()
# MGR-Tests: EXISTS
def test_exists_mgr_ok_multi(self):
"""S3 (RSE/PROTOCOLS): Check multiple files on storage (Success)"""
self.mtc.test_exists_mgr_ok_multi()
def test_exists_mgr_ok_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)"""
self.mtc.test_exists_mgr_ok_single_lfn()
def test_exists_mgr_ok_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)"""
self.mtc.test_exists_mgr_ok_single_pfn()
def test_exists_mgr_false_multi(self):
"""S3 (RSE/PROTOCOLS): Check multiple files on storage (Fail)"""
self.mtc.test_exists_mgr_false_multi()
def test_exists_mgr_false_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)"""
self.mtc.test_exists_mgr_false_single_lfn()
def test_exists_mgr_false_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)"""
self.mtc.test_exists_mgr_false_single_pfn()
# MGR-Tests: RENAME
def test_rename_mgr_ok_multi(self):
"""S3 (RSE/PROTOCOLS): Rename multiple files on storage (Success)"""
self.mtc.test_rename_mgr_ok_multi()
def test_rename_mgr_ok_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)"""
self.mtc.test_rename_mgr_ok_single_lfn()
def test_rename_mgr_ok_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)"""
self.mtc.test_rename_mgr_ok_single_pfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_multi(self):
"""S3 (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_multi(self):
"""S3 (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_pfn()
def test_change_scope_mgr_ok_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_lfn()
def test_change_scope_mgr_ok_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_pfn()
def setup(self):
"""S3 (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseS3.tmpdir
self.mtc = MgrTestCases(self.tmpdir, 'BNL-BOTO', TestRseS3.user, TestRseS3.static_file)
class TestRseMOCK():
tmpdir = '/tmp/'
user = None
static_file = 'mock:///tmp/rucio_rse/file1'
def setup(self):
"""MOCK (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseMOCK.tmpdir
self.rse_id = 'MOCK'
self.mtc = MgrTestCases(tmpdir=self.tmpdir, rse_tag='MOCK', user=TestRseMOCK.user, static_file=TestRseMOCK.static_file)
# Mgr-Tests: GET
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_multi(self):
"""MOCK (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)"""
for f in MgrTestCases.files_remote:
mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ])
self.mtc.test_get_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_lfn(self):
"""MOCK (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)"""
for f in MgrTestCases.files_remote:
mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ])
self.mtc.test_get_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_pfn(self):
"""MOCK (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)"""
for f in MgrTestCases.files_remote:
mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ])
self.mtc.test_get_mgr_SourceNotFound_single_pfn()
# Mgr-Tests: PUT
def test_put_mgr_ok_single(self):
"""MOCK (RSE/PROTOCOLS): Put a single file to storage (Success)"""
self.mtc.test_put_mgr_ok_single()
# MGR-Tests: DELETE
def test_delete_mgr_ok_multi(self):
"""MOCK (RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
for f in MgrTestCases.files_remote:
mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ])
self.mtc.test_delete_mgr_ok_multi()
def test_delete_mgr_ok_single(self):
"""MOCK (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
for f in MgrTestCases.files_remote:
mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ])
self.mtc.test_delete_mgr_ok_single()
# MGR-Tests: EXISTS
# ATTENTION: this tests won't work no more with the new RSEMgr as the protocol object is no longer cached and therefore the list of files is also not 'persisted'
# def test_exists_mgr_ok_multi(self):
# """MOCK (RSE/PROTOCOLS): Check multiple files on storage (Success)"""
# for f in MgrTestCases.files_remote:
# mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ])
# self.mtc.test_exists_mgr_ok_multi()
def test_exists_mgr_ok_single_lfn(self):
"""MOCK (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)"""
self.mtc.test_exists_mgr_ok_single_lfn()
def test_exists_mgr_ok_single_pfn(self):
"""MOCK (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)"""
self.mtc.test_exists_mgr_ok_single_pfn()
# ATTENTION: this tests won't work no more with the new RSEMgr as the protocol object is no longer cached and therefore the list of files is also not 'persisted'
# def test_exists_mgr_false_multi(self):
# """MOCK (RSE/PROTOCOLS): Check multiple files on storage (Fail)"""
# for f in MgrTestCases.files_remote:
# mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ])
# self.mtc.test_exists_mgr_false_multi()
def test_exists_mgr_false_single(self):
"""MOCK (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)"""
self.mtc.test_exists_mgr_false_single_lfn()
def test_exists_mgr_false_single_pfn(self):
"""MOCK (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)"""
self.mtc.test_exists_mgr_false_single_pfn()
# MGR-Tests: RENAME
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_lfn(self):
"""MOCK (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_pfn(self):
"""MOCK (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_pfn()
def setup(self):
"""XROOTD (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseXROOTD.tmpdir
self.rse_id = 'WJ-XROOTD'
self.mtc = MgrTestCases(self.tmpdir, 'WJ-XROOTD', TestRseXROOTD.user,
TestRseXROOTD.static_file)
def setup(self):
"""MOCK (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseMOCK.tmpdir
self.rse_id = 'MOCK'
self.mtc = MgrTestCases(tmpdir=self.tmpdir, rse_tag='MOCK', user=TestRseMOCK.user, static_file=TestRseMOCK.static_file)
class TestRseMOCK(object):
'''
class TestRseMOCK
'''
tmpdir = '/tmp/'
user = None
static_file = 'mock:///tmp/rucio_rse/file1'
def setup(self):
"""MOCK (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseMOCK.tmpdir
self.rse_id = 'MOCK'
self.mtc = MgrTestCases(tmpdir=self.tmpdir, rse_tag='MOCK', user=TestRseMOCK.user, static_file=TestRseMOCK.static_file)
# Mgr-Tests: GET
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_multi(self):
"""MOCK (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)"""
for fichier in MgrTestCases.files_remote:
mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': fichier, 'scope': 'user.%s' % self.user}, ])
self.mtc.test_get_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_lfn(self):
"""MOCK (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)"""
for fichier in MgrTestCases.files_remote:
mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': fichier, 'scope': 'user.%s' % self.user}, ])
self.mtc.test_get_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_pfn(self):
"""MOCK (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)"""
for fichier in MgrTestCases.files_remote:
mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': fichier, 'scope': 'user.%s' % self.user}, ])
self.mtc.test_get_mgr_SourceNotFound_single_pfn()
# Mgr-Tests: PUT
def test_put_mgr_ok_single(self):
"""MOCK (RSE/PROTOCOLS): Put a single file to storage (Success)"""
self.mtc.test_put_mgr_ok_single()
# MGR-Tests: DELETE
def test_delete_mgr_ok_multi(self):
"""MOCK (RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
for fichier in MgrTestCases.files_remote:
mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': fichier, 'scope': 'user.%s' % self.user}, ])
self.mtc.test_delete_mgr_ok_multi()
def test_delete_mgr_ok_single(self):
"""MOCK (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
for fichier in MgrTestCases.files_remote:
mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': fichier, 'scope': 'user.%s' % self.user}, ])
self.mtc.test_delete_mgr_ok_single()
# MGR-Tests: EXISTS
# ATTENTION: this tests won't work no more with the new RSEMgr as the protocol object is no longer cached and therefore the list of files is also not 'persisted'
# def test_exists_mgr_ok_multi(self):
# """MOCK (RSE/PROTOCOLS): Check multiple files on storage (Success)"""
# for f in MgrTestCases.files_remote:
# mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ])
# self.mtc.test_exists_mgr_ok_multi()
def test_exists_mgr_ok_single_lfn(self):
"""MOCK (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)"""
self.mtc.test_exists_mgr_ok_single_lfn()
def test_exists_mgr_ok_single_pfn(self):
"""MOCK (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)"""
self.mtc.test_exists_mgr_ok_single_pfn()
# ATTENTION: this tests won't work no more with the new RSEMgr as the protocol object is no longer cached and therefore the list of files is also not 'persisted'
# def test_exists_mgr_false_multi(self):
# """MOCK (RSE/PROTOCOLS): Check multiple files on storage (Fail)"""
# for f in MgrTestCases.files_remote:
# mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ])
# self.mtc.test_exists_mgr_false_multi()
def test_exists_mgr_false_single(self):
"""MOCK (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)"""
self.mtc.test_exists_mgr_false_single_lfn()
def test_exists_mgr_false_single_pfn(self):
"""MOCK (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)"""
self.mtc.test_exists_mgr_false_single_pfn()
# MGR-Tests: RENAME
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_lfn(self):
"""MOCK (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_pfn(self):
"""MOCK (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_pfn()
class TestRseS3ES():
tmpdir = None
user = None
# The setupClass and tearDownClass need some fixing, but can be ignored for this patch
@classmethod
def setupClass(cls):
"""S3ES (RSE/PROTOCOLS): Creating necessary directories and files """
cls.site = 'BNL-OSG2_ES'
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = '******'
with open('etc/rse_repository.json') as f:
data = json.load(f)
scheme = data[cls.site]['protocols']['supported']['s3+https']['scheme']
prefix = data[cls.site]['protocols']['supported']['s3+https']['prefix']
hostname = data[cls.site]['protocols']['supported']['s3+https']['hostname']
port = data[cls.site]['protocols']['supported']['s3+https']['port']
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024) - 1) # 1 kB
out.write('\0')
for f in MgrTestCases.files_local:
os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
cls.static_file = '%s://%s:%s/%s/user.%s/data.raw' % (scheme, hostname, port, prefix, cls.user)
rse_settings = rsemanager.get_rse_info(cls.site)
storage = rsemanager.create_protocol(rse_settings, operation='write', scheme='s3+https')
storage.connect()
for f in MgrTestCases.files_remote:
os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
destfile = rsemanager.lfns2pfns(rse_settings, [{'name': f, 'scope': 'user.%s' % (cls.user)}, ], operation='write', scheme='s3+https').values()[0]
try:
storage.put('%s/%s' % (cls.tmpdir, f), destfile)
except FileReplicaAlreadyExists as e:
print(e)
f = 'data.raw'
destfile = rsemanager.lfns2pfns(rse_settings, [{'name': f, 'scope': 'user.%s' % (cls.user)}, ], operation='write', scheme='s3+https').values()[0]
try:
storage.put('%s/%s' % (cls.tmpdir, f), destfile)
except FileReplicaAlreadyExists as e:
print(e)
@classmethod
def tearDownClass(cls):
"""S3ES (RSE/PROTOCOLS): Removing created directories and files """
rse_settings = rsemanager.get_rse_info(cls.site)
with open('etc/rse_repository.json') as f:
data = json.load(f)
scheme = data[cls.site]['protocols']['supported']['s3+https']['scheme']
prefix = data[cls.site]['protocols']['supported']['s3+https']['prefix']
hostname = data[cls.site]['protocols']['supported']['s3+https']['hostname']
port = data[cls.site]['protocols']['supported']['s3+https']['port']
for protocol in rse_settings['protocols']:
if protocol['impl'] == 'rucio.rse.protocols.signeds3.Default':
protocol['impl'] = 'rucio.rse.protocols.s3es.Default'
storage = rsemanager.create_protocol(rse_settings, operation='write', scheme='s3+https')
storage.connect()
try:
storage.delete('%s://%s:%s/%s/%s' % (scheme, hostname, port, prefix, 'user'))
except Exception as e:
print(e)
try:
storage.delete('%s://%s:%s/%s/%s' % (scheme, hostname, port, prefix, 'group'))
except Exception as e:
print(e)
def setup(self):
"""S3ES (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseS3ES.tmpdir
self.rse_id = 'BNL-OSG2_ES'
self.mtc = MgrTestCases(self.tmpdir, 'BNL-OSG2_ES', TestRseS3ES.user, TestRseS3ES.static_file)
self.mtc.setup_scheme('s3+https', 's3es')
# MGR-Tests: DELETE
def test_delete_mgr_ok_multi(self):
"""S3ES (RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
try:
self.mtc.test_delete_mgr_ok_multi()
except NotImplementedError:
pass
def test_delete_mgr_ok_single(self):
"""S3ES (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
try:
self.mtc.test_delete_mgr_ok_single()
except NotImplementedError:
pass
def test_delete_mgr_ok_dir(self):
"""S3ES (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
try:
self.mtc.test_delete_mgr_ok_dir()
except NotImplementedError:
pass
def setup(self):
"""S3ES (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseS3ES.tmpdir
self.rse_id = 'BNL-OSG2_ES'
self.mtc = MgrTestCases(self.tmpdir, 'BNL-OSG2_ES', TestRseS3ES.user, TestRseS3ES.static_file)
self.mtc.setup_scheme('s3+https', 's3es')
def setup(self):
"""MOCK (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseMOCK.tmpdir
self.rse_id = 'MOCK'
self.mtc = MgrTestCases(tmpdir=self.tmpdir, rse_tag='MOCK', user=TestRseMOCK.user, static_file=TestRseMOCK.static_file)
class TestRseXROOTD():
tmpdir = None
user = None
@classmethod
def setupClass(cls):
"""XROOTD (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write('\0')
for f in MgrTestCases.files_local:
shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
protocol = rsemanager.create_protocol(
rsemanager.get_rse_info('WJ-XROOTD'), 'write')
protocol.connect()
with open('etc/rse_repository.json') as f:
data = json.load(f)
prefix = data['WJ-XROOTD']['protocols']['supported']['xroot']['prefix']
hostname = data['WJ-XROOTD']['protocols']['supported']['xroot'][
'hostname']
port = data['WJ-XROOTD']['protocols']['supported']['xroot']['port']
try:
os.mkdir(prefix)
except Exception as e:
print(e)
os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' %
prefix)
cls.static_file = 'xroot://%s:%d/%s/data.raw' % (hostname, port,
prefix)
cmd = 'xrdcp %s/data.raw %s' % (prefix, cls.static_file)
execute(cmd)
for f in MgrTestCases.files_remote:
path = protocol.path2pfn(
prefix + protocol._get_path('user.%s' % cls.user, f))
cmd = 'xrdcp %s/data.raw %s' % (prefix, path)
execute(cmd)
@classmethod
def teardownClass(cls):
"""XROOTD (RSE/PROTOCOLS): Removing created directorie s and files"""
with open('etc/rse_repository.json') as f:
data = json.load(f)
prefix = data['WJ-XROOTD']['protocols']['supported']['xroot']['prefix']
hostname = data['WJ-XROOTD']['protocols']['supported']['xroot'][
'hostname']
shutil.rmtree(prefix)
shutil.rmtree(cls.tmpdir)
clean_raw = '%s/data.raw' % prefix
list_files_cmd_user = '******' % (hostname, prefix,
cls.user)
clean_files = str(execute(list_files_cmd_user)[1]).split('\n')
list_files_cmd_group = 'xrdfs %s ls %s/group.%s' % (hostname, prefix,
cls.user)
clean_files += str(execute(list_files_cmd_group)[1]).split('\n')
clean_files.append(clean_raw)
for files in clean_files:
clean_cmd = 'xrdfs %s rm %s' % (hostname, files)
execute(clean_cmd)
clean_prefix = '%s' % prefix
list_directory = 'xrdfs %s ls %s' % (hostname, prefix)
clean_directory = str(execute(list_directory)[1]).split('\n')
clean_directory.append(clean_prefix)
for directory in clean_directory:
clean_cmd = 'xrdfs %s rmdir %s' % (hostname, directory)
execute(clean_cmd)
def setup(self):
"""XROOTD (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseXROOTD.tmpdir
self.rse_id = 'WJ-XROOTD'
self.mtc = MgrTestCases(self.tmpdir, 'WJ-XROOTD', TestRseXROOTD.user,
TestRseXROOTD.static_file)
# Mgr-Tests: GET
def test_multi_get_mgr_ok(self):
"""XROOTD (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)"""
self.mtc.test_multi_get_mgr_ok()
def test_get_mgr_ok_single_lfn(self):
"""XROOTD (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)"""
self.mtc.test_get_mgr_ok_single_lfn()
def test_get_mgr_ok_single_pfn(self):
"""XROOTD (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)"""
self.mtc.test_get_mgr_ok_single_pfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_multi(self):
"""XROOTD (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_lfn(self):
"""XROOTD (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_pfn(self):
"""XROOTD (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_pfn()
# Mgr-Tests: PUT
def test_put_mgr_ok_multi(self):
"""XROOTD (RSE/PROTOCOLS): Put multiple files to storage providing LFNs and PFNs (Success)"""
self.mtc.test_put_mgr_ok_multi()
def test_put_mgr_ok_single(self):
"""XROOTD (RSE/PROTOCOLS): Put a single file to storage (Success)"""
self.mtc.test_put_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_multi(self):
"""XROOTD (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_single(self):
"""XROOTD (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_single()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_multi(self):
"""XROOTD (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_single(self):
"""XROOTD (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_single()
# MGR-Tests: DELETE
def test_delete_mgr_ok_multi(self):
"""XROOTD (RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
self.mtc.test_delete_mgr_ok_multi()
def test_delete_mgr_ok_single(self):
"""XROOTD (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
self.mtc.test_delete_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_multi(self):
"""XROOTD (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_single(self):
"""XROOTD (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_single()
# MGR-Tests: EXISTS
def test_exists_mgr_ok_multi(self):
"""XROOTD (RSE/PROTOCOLS): Check multiple files on storage (Success)"""
self.mtc.test_exists_mgr_ok_multi()
def test_exists_mgr_ok_single_lfn(self):
"""XROOTD (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)"""
self.mtc.test_exists_mgr_ok_single_lfn()
def test_exists_mgr_ok_single_pfn(self):
"""XROOTD (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)"""
self.mtc.test_exists_mgr_ok_single_pfn()
def test_exists_mgr_false_multi(self):
"""XROOTD (RSE/PROTOCOLS): Check multiple files on storage (Fail)"""
self.mtc.test_exists_mgr_false_multi()
def test_exists_mgr_false_single(self):
"""XROOTD (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)"""
self.mtc.test_exists_mgr_false_single_lfn()
def test_exists_mgr_false_single_pfn(self):
"""XROOTD (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)"""
self.mtc.test_exists_mgr_false_single_pfn()
# MGR-Tests: RENAME
def test_rename_mgr_ok_multi(self):
"""XROOTD (RSE/PROTOCOLS): Rename multiple files on storage (Success)"""
self.mtc.test_rename_mgr_ok_multi()
def test_rename_mgr_ok_single_lfn(self):
"""XROOTD (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)"""
self.mtc.test_rename_mgr_ok_single_lfn()
def test_rename_mgr_ok_single_pfn(self):
"""XROOTD (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)"""
self.mtc.test_rename_mgr_ok_single_pfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_multi(self):
"""XROOTD (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self):
"""XROOTD (RSE/PROTOCOLS): Rename a single file on storage using LFN(FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self):
"""XROOTD (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_multi(self):
"""XROOTD (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_lfn(self):
"""XROOTD (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_pfn(self):
"""XROOTD (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_pfn()
def test_change_scope_mgr_ok_single_lfn(self):
"""XROOTD (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_lfn()
def test_change_scope_mgr_ok_single_pfn(self):
"""XROOTD (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_pfn()
class TestRseSRM():
tmpdir = None
user = None
@classmethod
def setupClass(cls):
"""SRM (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write('\0')
for f in MgrTestCases.files_local:
shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
with open('etc/rse_repository.json') as f:
data = json.load(f)
prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix']
hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname']
if hostname.count("://"):
hostname = hostname.split("://")[1]
if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys():
port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['port'])
else:
port = 0
if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys() and 'web_service_path' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes'].keys():
web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes']['web_service_path']
else:
web_service_path = ''
os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % cls.tmpdir)
if port > 0:
cls.static_file = 'srm://%s:%s%s%s/data.raw' % (hostname, port, web_service_path, prefix)
else:
cls.static_file = 'srm://%s%s%s/data.raw' % (hostname, web_service_path, prefix)
cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % (cls.tmpdir, cls.static_file)
execute(cmd)
for f in MgrTestCases.files_remote:
tmp = mgr.lfns2pfns(mgr.get_rse_info('FZK-LCG2_SCRATCHDISK'), {'name': f, 'scope': 'user.%s' % cls.user}, scheme='srm').values()[0]
cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % (cls.tmpdir, tmp)
execute(cmd)
@classmethod
def teardownClass(cls):
"""SRM (RSE/PROTOCOLS): Removing created directorie s and files"""
with open('etc/rse_repository.json') as f:
data = json.load(f)
prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix']
hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname']
if hostname.count("://"):
hostname = hostname.split("://")[1]
if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys():
port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['port'])
else:
port = 0
if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys() and 'web_service_path' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes'].keys():
web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes']['web_service_path']
else:
web_service_path = ''
shutil.rmtree(cls.tmpdir)
clean_raw = '%s/data.raw' % prefix
if int(port) > 0:
srm_path = ''.join(["srm://", hostname, ":", port, web_service_path])
else:
srm_path = ''.join(["srm://", hostname, web_service_path])
list_files_cmd_user = '******' % (srm_path, prefix, cls.user)
clean_files = str(execute(list_files_cmd_user)[1]).split('\n')
list_files_cmd_user = '******' % (srm_path, prefix, cls.user)
clean_files += str(execute(list_files_cmd_user)[1]).split('\n')
clean_files.append("1024 " + clean_raw)
for files in clean_files:
if len(files.strip()) > 0:
file = files.split()[1]
if not file.endswith("/"):
clean_cmd = 'srmrm --debug=false -retry_num=0 %s/%s' % (srm_path, file)
execute(clean_cmd)
clean_directory = ['user', 'group']
for directory in clean_directory:
clean_cmd = 'srmrmdir --debug=false -retry_num=0 -recursive %s%s/%s/%s' % (srm_path, prefix, directory, cls.user)
execute(clean_cmd)
def setup(self):
"""SRM (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseSRM.tmpdir
self.rse_id = 'FZK-LCG2_SCRATCHDISK'
self.mtc = MgrTestCases(self.tmpdir, 'FZK-LCG2_SCRATCHDISK', TestRseSRM.user, TestRseSRM.static_file)
# self.mtc = MgrTestCases(self.tmpdir, 'FZK-LCG2_SCRATCHDISK', TestRseSRM.user, "srm://atlassrm-fzk.gridka.de/pnfs/gridka.de/atlas/disk-only/atlasscratchdisk/user/wguan/rucio.test.2")
self.mtc.setup_scheme('srm')
# Mgr-Tests: GET
def test_multi_get_mgr_ok(self):
"""SRM (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)"""
self.mtc.test_multi_get_mgr_ok()
def test_get_mgr_ok_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)"""
self.mtc.test_get_mgr_ok_single_lfn()
def test_get_mgr_ok_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)"""
self.mtc.test_get_mgr_ok_single_pfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_multi(self):
"""SRM (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_pfn()
# Mgr-Tests: PUT
def test_put_mgr_ok_multi(self):
"""SRM (RSE/PROTOCOLS): Put multiple files to storage providing LFNs and PFNs (Success)"""
self.mtc.test_put_mgr_ok_multi()
def test_put_mgr_ok_single(self):
"""SRM (RSE/PROTOCOLS): Put a single file to storage (Success)"""
self.mtc.test_put_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_multi(self):
"""SRM (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_single(self):
"""SRM (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_single()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_multi(self):
"""SRM (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_single(self):
"""SRM (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_single()
# MGR-Tests: DELETE
def test_delete_mgr_ok_multi(self):
"""SRM (RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
self.mtc.test_delete_mgr_ok_multi()
def test_delete_mgr_ok_single(self):
"""SRM (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
self.mtc.test_delete_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_multi(self):
"""SRM (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_single(self):
"""SRM (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_single()
# MGR-Tests: EXISTS
def test_exists_mgr_ok_multi(self):
"""SRM (RSE/PROTOCOLS): Check multiple files on storage (Success)"""
self.mtc.test_exists_mgr_ok_multi()
def test_exists_mgr_ok_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)"""
self.mtc.test_exists_mgr_ok_single_lfn()
def test_exists_mgr_ok_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)"""
self.mtc.test_exists_mgr_ok_single_pfn()
def test_exists_mgr_false_multi(self):
"""SRM (RSE/PROTOCOLS): Check multiple files on storage (Fail)"""
self.mtc.test_exists_mgr_false_multi()
def test_exists_mgr_false_single(self):
"""SRM (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)"""
self.mtc.test_exists_mgr_false_single_lfn()
def test_exists_mgr_false_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)"""
self.mtc.test_exists_mgr_false_single_pfn()
# MGR-Tests: RENAME
def test_rename_mgr_ok_multi(self):
"""SRM (RSE/PROTOCOLS): Rename multiple files on storage (Success)"""
self.mtc.test_rename_mgr_ok_multi()
def test_rename_mgr_ok_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)"""
self.mtc.test_rename_mgr_ok_single_lfn()
def test_rename_mgr_ok_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)"""
self.mtc.test_rename_mgr_ok_single_pfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_multi(self):
"""SRM (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Rename a single file on storage using LFN(FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_multi(self):
"""SRM (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_pfn()
def test_change_scope_mgr_ok_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_lfn()
def test_change_scope_mgr_ok_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_pfn()
class TestRseS3():
tmpdir = None
user = None
@classmethod
def setupClass(cls):
"""S3 (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
# cls.user = '******' # use again when latency issue with S3 storage is resolved
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write('\0')
for f in MgrTestCases.files_local:
os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
fnull = open(os.devnull, 'w')
# Create test files on storage
try:
subprocess.call(["s3cmd", "mb", "s3://USER"],
stdout=fnull,
stderr=fnull,
shell=False)
subprocess.call(["s3cmd", "mb", "s3://GROUP"],
stdout=fnull,
stderr=fnull,
shell=False)
subprocess.call(["s3cmd", "mb", "s3://NONDETERMINISTIC"],
stdout=fnull,
stderr=fnull,
shell=False)
except S3Error:
pass
cls.static_file = 's3://NONDETERMINISTIC/data.raw'
subprocess.call([
"s3cmd", "put",
"%s/data.raw" % cls.tmpdir, cls.static_file, "--no-progress"
],
stdout=fnull,
stderr=fnull)
for f in MgrTestCases.files_remote:
subprocess.call([
"s3cmd", "cp", cls.static_file,
mgr.lfns2pfns(mgr.get_rse_info('SWIFT'), {
'name': f,
'scope': 'user.%s' % cls.user
}).values()[0]
],
stdout=fnull,
stderr=fnull)
fnull.close()
def setup(self):
"""S3 (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseS3.tmpdir
self.mtc = MgrTestCases(self.tmpdir, 'SWIFT', TestRseS3.user,
TestRseS3.static_file)
@classmethod
def tearDownClass(cls):
"""S3 (RSE/PROTOCOLS): Removing created directories and files """
# Remove test files from storage
fnull = open(os.devnull, 'w')
subprocess.call(["s3cmd", "rb", "s3://USER", "--recursive"],
stdout=fnull,
stderr=fnull)
subprocess.call(["s3cmd", "rb", "s3://GROUP", "--recursive"],
stdout=fnull,
stderr=fnull)
shutil.rmtree(cls.tmpdir)
fnull.close()
# Mgr-Tests: GET
def test_multi_get_mgr_ok(self):
"""S3 (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)"""
self.mtc.test_multi_get_mgr_ok()
def test_get_mgr_ok_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)"""
self.mtc.test_get_mgr_ok_single_lfn()
def test_get_mgr_ok_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)"""
self.mtc.test_get_mgr_ok_single_pfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_multi(self):
"""S3 (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_pfn()
# Mgr-Tests: PUT
def test_put_mgr_ok_multi(self):
"""S3 (RSE/PROTOCOLS): Put multiple files to storage (Success)"""
self.mtc.test_put_mgr_ok_multi()
def test_put_mgr_ok_single(self):
"""S3 (RSE/PROTOCOLS): Put a single file to storage (Success)"""
self.mtc.test_put_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_multi(self):
"""S3 (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_single(self):
"""S3 (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_single()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_multi(self):
"""S3 (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_single(self):
"""S3 (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_single()
# MGR-Tests: DELETE
def test_delete_mgr_ok_multi(self):
"""S3 (RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
self.mtc.test_delete_mgr_ok_multi()
def test_delete_mgr_ok_single(self):
"""S3 (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
self.mtc.test_delete_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_multi(self):
"""S3 (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_single(self):
"""S3 (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_single()
# MGR-Tests: EXISTS
def test_exists_mgr_ok_multi(self):
"""S3 (RSE/PROTOCOLS): Check multiple files on storage (Success)"""
self.mtc.test_exists_mgr_ok_multi()
def test_exists_mgr_ok_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)"""
self.mtc.test_exists_mgr_ok_single_lfn()
def test_exists_mgr_ok_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)"""
self.mtc.test_exists_mgr_ok_single_pfn()
def test_exists_mgr_false_multi(self):
"""S3 (RSE/PROTOCOLS): Check multiple files on storage (Fail)"""
self.mtc.test_exists_mgr_false_multi()
def test_exists_mgr_false_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)"""
self.mtc.test_exists_mgr_false_single_lfn()
def test_exists_mgr_false_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)"""
self.mtc.test_exists_mgr_false_single_pfn()
# MGR-Tests: RENAME
def test_rename_mgr_ok_multi(self):
"""S3 (RSE/PROTOCOLS): Rename multiple files on storage (Success)"""
self.mtc.test_rename_mgr_ok_multi()
def test_rename_mgr_ok_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)"""
self.mtc.test_rename_mgr_ok_single_lfn()
def test_rename_mgr_ok_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)"""
self.mtc.test_rename_mgr_ok_single_pfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_multi(self):
"""S3 (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_multi(self):
"""S3 (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_pfn()
def test_change_scope_mgr_ok_single_lfn(self):
"""S3 (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_lfn()
def test_change_scope_mgr_ok_single_pfn(self):
"""S3 (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_pfn()