class TestRseSRM():
tmpdir = None
user = None
@classmethod
def setupClass(cls):
"""SRM (RSE/PROTOCOLS): Creating necessary directories and files """
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = uuid()
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024 * 1024) - 1) # 1 MB
out.write('\0')
for f in MgrTestCases.files_local:
shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
with open('etc/rse_repository.json') as f:
data = json.load(f)
prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix']
hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname']
if hostname.count("://"):
hostname = hostname.split("://")[1]
if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys():
port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['port'])
else:
port = 0
if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys() and 'web_service_path' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes'].keys():
web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes']['web_service_path']
else:
web_service_path = ''
os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % cls.tmpdir)
if port > 0:
cls.static_file = 'srm://%s:%s%s%s/data.raw' % (hostname, port, web_service_path, prefix)
else:
cls.static_file = 'srm://%s%s%s/data.raw' % (hostname, web_service_path, prefix)
cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % (cls.tmpdir, cls.static_file)
execute(cmd)
for f in MgrTestCases.files_remote:
tmp = mgr.lfns2pfns(mgr.get_rse_info('FZK-LCG2_SCRATCHDISK'), {'name': f, 'scope': 'user.%s' % cls.user}, scheme='srm').values()[0]
cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % (cls.tmpdir, tmp)
execute(cmd)
@classmethod
def teardownClass(cls):
"""SRM (RSE/PROTOCOLS): Removing created directorie s and files"""
with open('etc/rse_repository.json') as f:
data = json.load(f)
prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix']
hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname']
if hostname.count("://"):
hostname = hostname.split("://")[1]
if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys():
port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['port'])
else:
port = 0
if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys() and 'web_service_path' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes'].keys():
web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes']['web_service_path']
else:
web_service_path = ''
shutil.rmtree(cls.tmpdir)
clean_raw = '%s/data.raw' % prefix
if int(port) > 0:
srm_path = ''.join(["srm://", hostname, ":", port, web_service_path])
else:
srm_path = ''.join(["srm://", hostname, web_service_path])
list_files_cmd_user = '******' % (srm_path, prefix, cls.user)
clean_files = str(execute(list_files_cmd_user)[1]).split('\n')
list_files_cmd_user = '******' % (srm_path, prefix, cls.user)
clean_files += str(execute(list_files_cmd_user)[1]).split('\n')
clean_files.append("1024 " + clean_raw)
for files in clean_files:
if len(files.strip()) > 0:
file = files.split()[1]
if not file.endswith("/"):
clean_cmd = 'srmrm --debug=false -retry_num=0 %s/%s' % (srm_path, file)
execute(clean_cmd)
clean_directory = ['user', 'group']
for directory in clean_directory:
clean_cmd = 'srmrmdir --debug=false -retry_num=0 -recursive %s%s/%s/%s' % (srm_path, prefix, directory, cls.user)
execute(clean_cmd)
def setup(self):
"""SRM (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseSRM.tmpdir
self.rse_id = 'FZK-LCG2_SCRATCHDISK'
self.mtc = MgrTestCases(self.tmpdir, 'FZK-LCG2_SCRATCHDISK', TestRseSRM.user, TestRseSRM.static_file)
# self.mtc = MgrTestCases(self.tmpdir, 'FZK-LCG2_SCRATCHDISK', TestRseSRM.user, "srm://atlassrm-fzk.gridka.de/pnfs/gridka.de/atlas/disk-only/atlasscratchdisk/user/wguan/rucio.test.2")
self.mtc.setup_scheme('srm')
# Mgr-Tests: GET
def test_multi_get_mgr_ok(self):
"""SRM (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)"""
self.mtc.test_multi_get_mgr_ok()
def test_get_mgr_ok_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)"""
self.mtc.test_get_mgr_ok_single_lfn()
def test_get_mgr_ok_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)"""
self.mtc.test_get_mgr_ok_single_pfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_multi(self):
"""SRM (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_get_mgr_SourceNotFound_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)"""
self.mtc.test_get_mgr_SourceNotFound_single_pfn()
# Mgr-Tests: PUT
def test_put_mgr_ok_multi(self):
"""SRM (RSE/PROTOCOLS): Put multiple files to storage providing LFNs and PFNs (Success)"""
self.mtc.test_put_mgr_ok_multi()
def test_put_mgr_ok_single(self):
"""SRM (RSE/PROTOCOLS): Put a single file to storage (Success)"""
self.mtc.test_put_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_multi(self):
"""SRM (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_put_mgr_SourceNotFound_single(self):
"""SRM (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)"""
self.mtc.test_put_mgr_SourceNotFound_single()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_multi(self):
"""SRM (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_put_mgr_FileReplicaAlreadyExists_single(self):
"""SRM (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)"""
self.mtc.test_put_mgr_FileReplicaAlreadyExists_single()
# MGR-Tests: DELETE
def test_delete_mgr_ok_multi(self):
"""SRM (RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
self.mtc.test_delete_mgr_ok_multi()
def test_delete_mgr_ok_single(self):
"""SRM (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
self.mtc.test_delete_mgr_ok_single()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_multi(self):
"""SRM (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_delete_mgr_SourceNotFound_single(self):
"""SRM (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)"""
self.mtc.test_delete_mgr_SourceNotFound_single()
# MGR-Tests: EXISTS
def test_exists_mgr_ok_multi(self):
"""SRM (RSE/PROTOCOLS): Check multiple files on storage (Success)"""
self.mtc.test_exists_mgr_ok_multi()
def test_exists_mgr_ok_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)"""
self.mtc.test_exists_mgr_ok_single_lfn()
def test_exists_mgr_ok_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)"""
self.mtc.test_exists_mgr_ok_single_pfn()
def test_exists_mgr_false_multi(self):
"""SRM (RSE/PROTOCOLS): Check multiple files on storage (Fail)"""
self.mtc.test_exists_mgr_false_multi()
def test_exists_mgr_false_single(self):
"""SRM (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)"""
self.mtc.test_exists_mgr_false_single_lfn()
def test_exists_mgr_false_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)"""
self.mtc.test_exists_mgr_false_single_pfn()
# MGR-Tests: RENAME
def test_rename_mgr_ok_multi(self):
"""SRM (RSE/PROTOCOLS): Rename multiple files on storage (Success)"""
self.mtc.test_rename_mgr_ok_multi()
def test_rename_mgr_ok_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)"""
self.mtc.test_rename_mgr_ok_single_lfn()
def test_rename_mgr_ok_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)"""
self.mtc.test_rename_mgr_ok_single_pfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_multi(self):
"""SRM (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Rename a single file on storage using LFN(FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn()
@raises(exception.FileReplicaAlreadyExists)
def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)"""
self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_multi(self):
"""SRM (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_multi()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_lfn()
@raises(exception.SourceNotFound)
def test_rename_mgr_SourceNotFound_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)"""
self.mtc.test_rename_mgr_SourceNotFound_single_pfn()
def test_change_scope_mgr_ok_single_lfn(self):
"""SRM (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_lfn()
def test_change_scope_mgr_ok_single_pfn(self):
"""SRM (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)"""
self.mtc.test_change_scope_mgr_ok_single_pfn()
class TestRseS3ES():
tmpdir = None
user = None
# The setupClass and tearDownClass need some fixing, but can be ignored for this patch
@classmethod
def setupClass(cls):
"""S3ES (RSE/PROTOCOLS): Creating necessary directories and files """
cls.site = 'BNL-OSG2_ES'
# Creating local files
cls.tmpdir = tempfile.mkdtemp()
cls.user = '******'
with open('etc/rse_repository.json') as f:
data = json.load(f)
scheme = data[cls.site]['protocols']['supported']['s3+https']['scheme']
prefix = data[cls.site]['protocols']['supported']['s3+https']['prefix']
hostname = data[cls.site]['protocols']['supported']['s3+https']['hostname']
port = data[cls.site]['protocols']['supported']['s3+https']['port']
with open("%s/data.raw" % cls.tmpdir, "wb") as out:
out.seek((1024) - 1) # 1 kB
out.write('\0')
for f in MgrTestCases.files_local:
os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
cls.static_file = '%s://%s:%s/%s/user.%s/data.raw' % (scheme, hostname, port, prefix, cls.user)
rse_settings = rsemanager.get_rse_info(cls.site)
storage = rsemanager.create_protocol(rse_settings, operation='write', scheme='s3+https')
storage.connect()
for f in MgrTestCases.files_remote:
os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f))
destfile = rsemanager.lfns2pfns(rse_settings, [{'name': f, 'scope': 'user.%s' % (cls.user)}, ], operation='write', scheme='s3+https').values()[0]
try:
storage.put('%s/%s' % (cls.tmpdir, f), destfile)
except FileReplicaAlreadyExists as e:
print(e)
f = 'data.raw'
destfile = rsemanager.lfns2pfns(rse_settings, [{'name': f, 'scope': 'user.%s' % (cls.user)}, ], operation='write', scheme='s3+https').values()[0]
try:
storage.put('%s/%s' % (cls.tmpdir, f), destfile)
except FileReplicaAlreadyExists as e:
print(e)
@classmethod
def tearDownClass(cls):
"""S3ES (RSE/PROTOCOLS): Removing created directories and files """
rse_settings = rsemanager.get_rse_info(cls.site)
with open('etc/rse_repository.json') as f:
data = json.load(f)
scheme = data[cls.site]['protocols']['supported']['s3+https']['scheme']
prefix = data[cls.site]['protocols']['supported']['s3+https']['prefix']
hostname = data[cls.site]['protocols']['supported']['s3+https']['hostname']
port = data[cls.site]['protocols']['supported']['s3+https']['port']
for protocol in rse_settings['protocols']:
if protocol['impl'] == 'rucio.rse.protocols.signeds3.Default':
protocol['impl'] = 'rucio.rse.protocols.s3es.Default'
storage = rsemanager.create_protocol(rse_settings, operation='write', scheme='s3+https')
storage.connect()
try:
storage.delete('%s://%s:%s/%s/%s' % (scheme, hostname, port, prefix, 'user'))
except Exception as e:
print(e)
try:
storage.delete('%s://%s:%s/%s/%s' % (scheme, hostname, port, prefix, 'group'))
except Exception as e:
print(e)
def setup(self):
"""S3ES (RSE/PROTOCOLS): Creating Mgr-instance """
self.tmpdir = TestRseS3ES.tmpdir
self.rse_id = 'BNL-OSG2_ES'
self.mtc = MgrTestCases(self.tmpdir, 'BNL-OSG2_ES', TestRseS3ES.user, TestRseS3ES.static_file)
self.mtc.setup_scheme('s3+https', 's3es')
# MGR-Tests: DELETE
def test_delete_mgr_ok_multi(self):
"""S3ES (RSE/PROTOCOLS): Delete multiple files from storage (Success)"""
try:
self.mtc.test_delete_mgr_ok_multi()
except NotImplementedError:
pass
def test_delete_mgr_ok_single(self):
"""S3ES (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
try:
self.mtc.test_delete_mgr_ok_single()
except NotImplementedError:
pass
def test_delete_mgr_ok_dir(self):
"""S3ES (RSE/PROTOCOLS): Delete a single file from storage (Success)"""
try:
self.mtc.test_delete_mgr_ok_dir()
except NotImplementedError:
pass
