def setup(self): """SRM (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseSRM.tmpdir self.rse_id = 'FZK-LCG2_SCRATCHDISK' self.mtc = MgrTestCases(self.tmpdir, 'FZK-LCG2_SCRATCHDISK', TestRseSRM.user, TestRseSRM.static_file) # self.mtc = MgrTestCases(self.tmpdir, 'FZK-LCG2_SCRATCHDISK', TestRseSRM.user, "srm://atlassrm-fzk.gridka.de/pnfs/gridka.de/atlas/disk-only/atlasscratchdisk/user/wguan/rucio.test.2") self.mtc.setup_scheme('srm')
def setup(self): """SFTP (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseSFTP.tmpdir self.static_file = TestRseSFTP.static_file self.mtc = MgrTestCases(self.tmpdir, 'LXPLUS', TestRseSFTP.user, self.static_file)
class TestRseSFTP(): tmpdir = None user = None @classmethod def setupClass(cls): """SFTP (RSE/PROTOCOLS): Creating necessary directories and files """ # Creating local files cls.tmpdir = tempfile.mkdtemp() cls.user = uuid() with open("%s/data.raw" % cls.tmpdir, "wb") as out: out.seek((1024 * 1024) - 1) # 1 MB out.write('\0') for f in MgrTestCases.files_local: os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) # Load local credentials from file with open('etc/rse-accounts.cfg') as f: data = json.load(f) credentials = data['LXPLUS'] lxplus = pysftp.Connection(**credentials) with open('etc/rse_repository.json') as f: prefix = json.load(f)['LXPLUS']['protocols']['supported']['sftp']['prefix'] lxplus.execute('mkdir %s' % prefix) lxplus.execute('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % prefix) cls.static_file = 'sftp://lxplus.cern.ch:22%sdata.raw' % prefix protocol = mgr.create_protocol(mgr.get_rse_info('LXPLUS'), 'write') for f in MgrTestCases.files_remote: tmp = protocol.parse_pfns(protocol.lfns2pfns({'name': f, 'scope': 'user.%s' % cls.user}).values()[0]).values()[0] for cmd in ['mkdir -p %s' % ''.join([tmp['prefix'], tmp['path']]), 'ln -s %sdata.raw %s' % (prefix, ''.join([tmp['prefix'], tmp['path'], tmp['name']]))]: lxplus.execute(cmd) lxplus.close() @classmethod def tearDownClass(cls): """SFTP (RSE/PROTOCOLS): Removing created directorie s and files """ # Load local creditentials from file credentials = {} with open('etc/rse-accounts.cfg') as f: data = json.load(f) credentials = data['LXPLUS'] lxplus = pysftp.Connection(**credentials) with open('etc/rse_repository.json') as f: prefix = json.load(f)['LXPLUS']['protocols']['supported']['sftp']['prefix'] lxplus.execute('rm -rf %s' % prefix) lxplus.close() shutil.rmtree(cls.tmpdir) def setup(self): """SFTP (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseSFTP.tmpdir self.static_file = TestRseSFTP.static_file self.mtc = MgrTestCases(self.tmpdir, 'LXPLUS', TestRseSFTP.user, self.static_file) # Mgr-Tests: GET def test_multi_get_mgr_ok(self): """SFTP (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)""" self.mtc.test_multi_get_mgr_ok() def test_get_mgr_ok_single_lfn(self): """SFTP (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)""" self.mtc.test_get_mgr_ok_single_lfn() def test_get_mgr_ok_single_pfn(self): """SFTP (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)""" self.mtc.test_get_mgr_ok_single_pfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_multi(self): """SFTP (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_lfn(self): """SFTP (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_pfn(self): """SFTP (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_single_pfn() # Mgr-Tests: PUT def test_put_mgr_ok_multi(self): """SFTP (RSE/PROTOCOLS): Put multiple files to storage (Success)""" self.mtc.test_put_mgr_ok_multi() def test_put_mgr_ok_single(self): """SFTP (RSE/PROTOCOLS): Put a single file to storage (Success)""" self.mtc.test_put_mgr_ok_single() @raises(exception.SourceNotFound) def test_put_mgr_SourceNotFound_multi(self): """SFTP (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)""" self.mtc.test_put_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_put_mgr_SourceNotFound_single(self): """SFTP (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)""" self.mtc.test_put_mgr_SourceNotFound_single() @raises(exception.FileReplicaAlreadyExists) def test_put_mgr_FileReplicaAlreadyExists_multi(self): """SFTP (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)""" self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi() @raises(exception.FileReplicaAlreadyExists) def test_put_mgr_FileReplicaAlreadyExists_single(self): """SFTP (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)""" self.mtc.test_put_mgr_FileReplicaAlreadyExists_single() # MGR-Tests: DELETE def test_delete_mgr_ok_multi(self): """SFTP (RSE/PROTOCOLS): Delete multiple files from storage (Success)""" self.mtc.test_delete_mgr_ok_multi() def test_delete_mgr_ok_single(self): """SFTP (RSE/PROTOCOLS): Delete a single file from storage (Success)""" self.mtc.test_delete_mgr_ok_single() @raises(exception.SourceNotFound) def test_delete_mgr_SourceNotFound_multi(self): """SFTP (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)""" self.mtc.test_delete_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_delete_mgr_SourceNotFound_single(self): """SFTP (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)""" self.mtc.test_delete_mgr_SourceNotFound_single() # MGR-Tests: EXISTS def test_exists_mgr_ok_multi(self): """SFTP (RSE/PROTOCOLS): Check multiple files on storage (Success)""" self.mtc.test_exists_mgr_ok_multi() def test_exists_mgr_ok_single_lfn(self): """SFTP (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)""" self.mtc.test_exists_mgr_ok_single_lfn() def test_exists_mgr_ok_single_pfn(self): """SFTP (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)""" self.mtc.test_exists_mgr_ok_single_pfn() def test_exists_mgr_false_multi(self): """SFTP (RSE/PROTOCOLS): Check multiple files on storage (Fail)""" self.mtc.test_exists_mgr_false_multi() def test_exists_mgr_false_single_lfn(self): """SFTP (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)""" self.mtc.test_exists_mgr_false_single_lfn() def test_exists_mgr_false_single_pfn(self): """SFTP (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)""" self.mtc.test_exists_mgr_false_single_pfn() # MGR-Tests: RENAME def test_rename_mgr_ok_multi(self): """SFTP (RSE/PROTOCOLS): Rename multiple files on storage (Success)""" self.mtc.test_rename_mgr_ok_multi() def test_rename_mgr_ok_single_lfn(self): """SFTP (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)""" self.mtc.test_rename_mgr_ok_single_lfn() def test_rename_mgr_ok_single_pfn(self): """SFTP (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)""" self.mtc.test_rename_mgr_ok_single_pfn() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_multi(self): """SFTP (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self): """SFTP (RSE/PROTOCOLS): Rename a single file on storage using LFN (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self): """SFTP (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_multi(self): """SFTP (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_lfn(self): """SFTP (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_pfn(self): """SFTP (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_pfn() def test_change_scope_mgr_ok_single_lfn(self): """SFTP (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)""" self.mtc.test_change_scope_mgr_ok_single_lfn() def test_change_scope_mgr_ok_single_pfn(self): """SFTP (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)""" self.mtc.test_change_scope_mgr_ok_single_pfn()
def setup(self): """S3 (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseS3.tmpdir self.mtc = MgrTestCases(self.tmpdir, 'BNL-BOTO', TestRseS3.user, TestRseS3.static_file)
class TestRseS3(): tmpdir = None user = None @classmethod def setupClass(cls): """S3 (RSE/PROTOCOLS): Creating necessary directories and files """ # Creating local files cls.tmpdir = tempfile.mkdtemp() cls.user = uuid() # cls.user = '******' # use again when latency issue with S3 storage is resolved with open("%s/data.raw" % cls.tmpdir, "wb") as out: out.seek((1024 * 1024) - 1) # 1 MB out.write('\0') for f in MgrTestCases.files_local: os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) rse_tag = 'BNL-BOTO' rse_settings = mgr.get_rse_info(rse_tag) try: with open('etc/rse-accounts.cfg') as f: data = json.load(f) rse_settings['credentials'] = data[rse_tag] except KeyError: print 'No credentials found for this RSE.' pass for protocol in rse_settings['protocols']: if protocol['scheme'] == 's3': break conn = boto.connect_s3(host=protocol['hostname'], port=int(protocol.get('port', 80)), aws_access_key_id=rse_settings['credentials']['access_key'], aws_secret_access_key=rse_settings['credentials']['secret_key'], is_secure=rse_settings['credentials'].get('is_secure', False), calling_format=boto.s3.connection.OrdinaryCallingFormat()) cls.static_file = mgr.lfns2pfns(rse_settings, {'name': 'data.raw', 'scope': 'user.%s' % cls.user}).values()[0] key = get_bucket_key(cls.static_file, conn, create=True) key.set_contents_from_filename("%s/data.raw" % cls.tmpdir) for f in MgrTestCases.files_remote: pfn = mgr.lfns2pfns(rse_settings, {'name': f, 'scope': 'user.%s' % cls.user}).values()[0] bucket_name, key_name = get_bucket_key_name(pfn) key.copy(bucket_name, key_name) def setup(self): """S3 (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseS3.tmpdir self.mtc = MgrTestCases(self.tmpdir, 'BNL-BOTO', TestRseS3.user, TestRseS3.static_file) @classmethod def tearDownClass(cls): """S3 (RSE/PROTOCOLS): Removing created directories and files """ # Remove test files from storage # rse_tag = 'AMAZON-BOTO' rse_tag = 'BNL-BOTO' rse_settings = mgr.get_rse_info(rse_tag) try: with open('etc/rse-accounts.cfg') as f: data = json.load(f) rse_settings['credentials'] = data[rse_tag] except KeyError: print 'No credentials found for this RSE.' pass for protocol in rse_settings['protocols']: if protocol['scheme'] == 's3': break conn = boto.connect_s3(host=protocol['hostname'], port=int(protocol.get('port', 80)), aws_access_key_id=rse_settings['credentials']['access_key'], aws_secret_access_key=rse_settings['credentials']['secret_key'], is_secure=rse_settings['credentials'].get('is_secure', False), calling_format=boto.s3.connection.OrdinaryCallingFormat()) for protocol in rse_settings['protocols']: if protocol['scheme'] == 's3': break bucket_name = protocol['prefix'] bucket = conn.get_bucket(bucket_name) keys = bucket.list() for key in keys: key.delete() # Mgr-Tests: GET def test_multi_get_mgr_ok(self): """S3 (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)""" self.mtc.test_multi_get_mgr_ok() def test_get_mgr_ok_single_lfn(self): """S3 (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)""" self.mtc.test_get_mgr_ok_single_lfn() def test_get_mgr_ok_single_pfn(self): """S3 (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)""" self.mtc.test_get_mgr_ok_single_pfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_multi(self): """S3 (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_lfn(self): """S3 (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_pfn(self): """S3 (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_single_pfn() # Mgr-Tests: PUT def test_put_mgr_ok_multi(self): """S3 (RSE/PROTOCOLS): Put multiple files to storage (Success)""" self.mtc.test_put_mgr_ok_multi() def test_put_mgr_ok_single(self): """S3 (RSE/PROTOCOLS): Put a single file to storage (Success)""" self.mtc.test_put_mgr_ok_single() @raises(exception.SourceNotFound) def test_put_mgr_SourceNotFound_multi(self): """S3 (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)""" self.mtc.test_put_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_put_mgr_SourceNotFound_single(self): """S3 (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)""" self.mtc.test_put_mgr_SourceNotFound_single() @raises(exception.FileReplicaAlreadyExists) def test_put_mgr_FileReplicaAlreadyExists_multi(self): """S3 (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)""" self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi() @raises(exception.FileReplicaAlreadyExists) def test_put_mgr_FileReplicaAlreadyExists_single(self): """S3 (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)""" self.mtc.test_put_mgr_FileReplicaAlreadyExists_single() # MGR-Tests: DELETE def test_delete_mgr_ok_multi(self): """S3 (RSE/PROTOCOLS): Delete multiple files from storage (Success)""" self.mtc.test_delete_mgr_ok_multi() def test_delete_mgr_ok_single(self): """S3 (RSE/PROTOCOLS): Delete a single file from storage (Success)""" self.mtc.test_delete_mgr_ok_single() @raises(exception.SourceNotFound) def test_delete_mgr_SourceNotFound_multi(self): """S3 (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)""" self.mtc.test_delete_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_delete_mgr_SourceNotFound_single(self): """S3 (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)""" self.mtc.test_delete_mgr_SourceNotFound_single() # MGR-Tests: EXISTS def test_exists_mgr_ok_multi(self): """S3 (RSE/PROTOCOLS): Check multiple files on storage (Success)""" self.mtc.test_exists_mgr_ok_multi() def test_exists_mgr_ok_single_lfn(self): """S3 (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)""" self.mtc.test_exists_mgr_ok_single_lfn() def test_exists_mgr_ok_single_pfn(self): """S3 (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)""" self.mtc.test_exists_mgr_ok_single_pfn() def test_exists_mgr_false_multi(self): """S3 (RSE/PROTOCOLS): Check multiple files on storage (Fail)""" self.mtc.test_exists_mgr_false_multi() def test_exists_mgr_false_single_lfn(self): """S3 (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)""" self.mtc.test_exists_mgr_false_single_lfn() def test_exists_mgr_false_single_pfn(self): """S3 (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)""" self.mtc.test_exists_mgr_false_single_pfn() # MGR-Tests: RENAME def test_rename_mgr_ok_multi(self): """S3 (RSE/PROTOCOLS): Rename multiple files on storage (Success)""" self.mtc.test_rename_mgr_ok_multi() def test_rename_mgr_ok_single_lfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)""" self.mtc.test_rename_mgr_ok_single_lfn() def test_rename_mgr_ok_single_pfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)""" self.mtc.test_rename_mgr_ok_single_pfn() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_multi(self): """S3 (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_multi(self): """S3 (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_lfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_pfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_pfn() def test_change_scope_mgr_ok_single_lfn(self): """S3 (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)""" self.mtc.test_change_scope_mgr_ok_single_lfn() def test_change_scope_mgr_ok_single_pfn(self): """S3 (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)""" self.mtc.test_change_scope_mgr_ok_single_pfn()
class TestRseS3(): tmpdir = None user = None @classmethod def setupClass(cls): """S3 (RSE/PROTOCOLS): Creating necessary directories and files """ # Creating local files cls.tmpdir = tempfile.mkdtemp() cls.user = uuid() # cls.user = '******' # use again when latency issue with S3 storage is resolved with open("%s/data.raw" % cls.tmpdir, "wb") as out: out.seek((1024 * 1024) - 1) # 1 MB out.write('\0') for f in MgrTestCases.files_local: os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) rse_tag = 'BNL-BOTO' rse_settings = mgr.get_rse_info(rse_tag) try: with open('etc/rse-accounts.cfg') as f: data = json.load(f) rse_settings['credentials'] = data[rse_tag] except KeyError: print('No credentials found for this RSE.') pass for protocol in rse_settings['protocols']: if protocol['scheme'] == 's3': break conn = boto.connect_s3(host=protocol['hostname'], port=int(protocol.get('port', 80)), aws_access_key_id=rse_settings['credentials']['access_key'], aws_secret_access_key=rse_settings['credentials']['secret_key'], is_secure=rse_settings['credentials'].get('is_secure', False), calling_format=boto.s3.connection.OrdinaryCallingFormat()) cls.static_file = mgr.lfns2pfns(rse_settings, {'name': 'data.raw', 'scope': 'user.%s' % cls.user}).values()[0] key = get_bucket_key(cls.static_file, conn, create=True) key.set_contents_from_filename("%s/data.raw" % cls.tmpdir) for f in MgrTestCases.files_remote: pfn = mgr.lfns2pfns(rse_settings, {'name': f, 'scope': 'user.%s' % cls.user}).values()[0] bucket_name, key_name = get_bucket_key_name(pfn) key.copy(bucket_name, key_name) def setup(self): """S3 (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseS3.tmpdir self.mtc = MgrTestCases(self.tmpdir, 'BNL-BOTO', TestRseS3.user, TestRseS3.static_file) @classmethod def tearDownClass(cls): """S3 (RSE/PROTOCOLS): Removing created directories and files """ # Remove test files from storage # rse_tag = 'AMAZON-BOTO' rse_tag = 'BNL-BOTO' rse_settings = mgr.get_rse_info(rse_tag) try: with open('etc/rse-accounts.cfg') as f: data = json.load(f) rse_settings['credentials'] = data[rse_tag] except KeyError: print('No credentials found for this RSE.') pass for protocol in rse_settings['protocols']: if protocol['scheme'] == 's3': break conn = boto.connect_s3(host=protocol['hostname'], port=int(protocol.get('port', 80)), aws_access_key_id=rse_settings['credentials']['access_key'], aws_secret_access_key=rse_settings['credentials']['secret_key'], is_secure=rse_settings['credentials'].get('is_secure', False), calling_format=boto.s3.connection.OrdinaryCallingFormat()) for protocol in rse_settings['protocols']: if protocol['scheme'] == 's3': break bucket_name = protocol['prefix'] bucket = conn.get_bucket(bucket_name) keys = bucket.list() for key in keys: key.delete() # Mgr-Tests: GET def test_multi_get_mgr_ok(self): """S3 (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)""" self.mtc.test_multi_get_mgr_ok() def test_get_mgr_ok_single_lfn(self): """S3 (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)""" self.mtc.test_get_mgr_ok_single_lfn() def test_get_mgr_ok_single_pfn(self): """S3 (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)""" self.mtc.test_get_mgr_ok_single_pfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_multi(self): """S3 (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_lfn(self): """S3 (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_pfn(self): """S3 (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_single_pfn() # Mgr-Tests: PUT def test_put_mgr_ok_multi(self): """S3 (RSE/PROTOCOLS): Put multiple files to storage (Success)""" self.mtc.test_put_mgr_ok_multi() def test_put_mgr_ok_single(self): """S3 (RSE/PROTOCOLS): Put a single file to storage (Success)""" self.mtc.test_put_mgr_ok_single() @raises(exception.SourceNotFound) def test_put_mgr_SourceNotFound_multi(self): """S3 (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)""" self.mtc.test_put_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_put_mgr_SourceNotFound_single(self): """S3 (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)""" self.mtc.test_put_mgr_SourceNotFound_single() @raises(exception.FileReplicaAlreadyExists) def test_put_mgr_FileReplicaAlreadyExists_multi(self): """S3 (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)""" self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi() @raises(exception.FileReplicaAlreadyExists) def test_put_mgr_FileReplicaAlreadyExists_single(self): """S3 (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)""" self.mtc.test_put_mgr_FileReplicaAlreadyExists_single() # MGR-Tests: DELETE def test_delete_mgr_ok_multi(self): """S3 (RSE/PROTOCOLS): Delete multiple files from storage (Success)""" self.mtc.test_delete_mgr_ok_multi() def test_delete_mgr_ok_single(self): """S3 (RSE/PROTOCOLS): Delete a single file from storage (Success)""" self.mtc.test_delete_mgr_ok_single() @raises(exception.SourceNotFound) def test_delete_mgr_SourceNotFound_multi(self): """S3 (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)""" self.mtc.test_delete_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_delete_mgr_SourceNotFound_single(self): """S3 (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)""" self.mtc.test_delete_mgr_SourceNotFound_single() # MGR-Tests: EXISTS def test_exists_mgr_ok_multi(self): """S3 (RSE/PROTOCOLS): Check multiple files on storage (Success)""" self.mtc.test_exists_mgr_ok_multi() def test_exists_mgr_ok_single_lfn(self): """S3 (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)""" self.mtc.test_exists_mgr_ok_single_lfn() def test_exists_mgr_ok_single_pfn(self): """S3 (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)""" self.mtc.test_exists_mgr_ok_single_pfn() def test_exists_mgr_false_multi(self): """S3 (RSE/PROTOCOLS): Check multiple files on storage (Fail)""" self.mtc.test_exists_mgr_false_multi() def test_exists_mgr_false_single_lfn(self): """S3 (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)""" self.mtc.test_exists_mgr_false_single_lfn() def test_exists_mgr_false_single_pfn(self): """S3 (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)""" self.mtc.test_exists_mgr_false_single_pfn() # MGR-Tests: RENAME def test_rename_mgr_ok_multi(self): """S3 (RSE/PROTOCOLS): Rename multiple files on storage (Success)""" self.mtc.test_rename_mgr_ok_multi() def test_rename_mgr_ok_single_lfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)""" self.mtc.test_rename_mgr_ok_single_lfn() def test_rename_mgr_ok_single_pfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)""" self.mtc.test_rename_mgr_ok_single_pfn() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_multi(self): """S3 (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_multi(self): """S3 (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_lfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_pfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_pfn() def test_change_scope_mgr_ok_single_lfn(self): """S3 (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)""" self.mtc.test_change_scope_mgr_ok_single_lfn() def test_change_scope_mgr_ok_single_pfn(self): """S3 (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)""" self.mtc.test_change_scope_mgr_ok_single_pfn()
class TestRseMOCK(): tmpdir = '/tmp/' user = None static_file = 'mock:///tmp/rucio_rse/file1' def setup(self): """MOCK (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseMOCK.tmpdir self.rse_id = 'MOCK' self.mtc = MgrTestCases(tmpdir=self.tmpdir, rse_tag='MOCK', user=TestRseMOCK.user, static_file=TestRseMOCK.static_file) # Mgr-Tests: GET @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_multi(self): """MOCK (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)""" for f in MgrTestCases.files_remote: mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ]) self.mtc.test_get_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_lfn(self): """MOCK (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)""" for f in MgrTestCases.files_remote: mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ]) self.mtc.test_get_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_pfn(self): """MOCK (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)""" for f in MgrTestCases.files_remote: mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ]) self.mtc.test_get_mgr_SourceNotFound_single_pfn() # Mgr-Tests: PUT def test_put_mgr_ok_single(self): """MOCK (RSE/PROTOCOLS): Put a single file to storage (Success)""" self.mtc.test_put_mgr_ok_single() # MGR-Tests: DELETE def test_delete_mgr_ok_multi(self): """MOCK (RSE/PROTOCOLS): Delete multiple files from storage (Success)""" for f in MgrTestCases.files_remote: mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ]) self.mtc.test_delete_mgr_ok_multi() def test_delete_mgr_ok_single(self): """MOCK (RSE/PROTOCOLS): Delete a single file from storage (Success)""" for f in MgrTestCases.files_remote: mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ]) self.mtc.test_delete_mgr_ok_single() # MGR-Tests: EXISTS # ATTENTION: this tests won't work no more with the new RSEMgr as the protocol object is no longer cached and therefore the list of files is also not 'persisted' # def test_exists_mgr_ok_multi(self): # """MOCK (RSE/PROTOCOLS): Check multiple files on storage (Success)""" # for f in MgrTestCases.files_remote: # mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ]) # self.mtc.test_exists_mgr_ok_multi() def test_exists_mgr_ok_single_lfn(self): """MOCK (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)""" self.mtc.test_exists_mgr_ok_single_lfn() def test_exists_mgr_ok_single_pfn(self): """MOCK (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)""" self.mtc.test_exists_mgr_ok_single_pfn() # ATTENTION: this tests won't work no more with the new RSEMgr as the protocol object is no longer cached and therefore the list of files is also not 'persisted' # def test_exists_mgr_false_multi(self): # """MOCK (RSE/PROTOCOLS): Check multiple files on storage (Fail)""" # for f in MgrTestCases.files_remote: # mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ]) # self.mtc.test_exists_mgr_false_multi() def test_exists_mgr_false_single(self): """MOCK (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)""" self.mtc.test_exists_mgr_false_single_lfn() def test_exists_mgr_false_single_pfn(self): """MOCK (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)""" self.mtc.test_exists_mgr_false_single_pfn() # MGR-Tests: RENAME @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_lfn(self): """MOCK (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_pfn(self): """MOCK (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_pfn()
def setup(self): """XROOTD (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseXROOTD.tmpdir self.rse_id = 'WJ-XROOTD' self.mtc = MgrTestCases(self.tmpdir, 'WJ-XROOTD', TestRseXROOTD.user, TestRseXROOTD.static_file)
def setup(self): """MOCK (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseMOCK.tmpdir self.rse_id = 'MOCK' self.mtc = MgrTestCases(tmpdir=self.tmpdir, rse_tag='MOCK', user=TestRseMOCK.user, static_file=TestRseMOCK.static_file)
class TestRseMOCK(object): ''' class TestRseMOCK ''' tmpdir = '/tmp/' user = None static_file = 'mock:///tmp/rucio_rse/file1' def setup(self): """MOCK (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseMOCK.tmpdir self.rse_id = 'MOCK' self.mtc = MgrTestCases(tmpdir=self.tmpdir, rse_tag='MOCK', user=TestRseMOCK.user, static_file=TestRseMOCK.static_file) # Mgr-Tests: GET @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_multi(self): """MOCK (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)""" for fichier in MgrTestCases.files_remote: mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': fichier, 'scope': 'user.%s' % self.user}, ]) self.mtc.test_get_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_lfn(self): """MOCK (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)""" for fichier in MgrTestCases.files_remote: mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': fichier, 'scope': 'user.%s' % self.user}, ]) self.mtc.test_get_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_pfn(self): """MOCK (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)""" for fichier in MgrTestCases.files_remote: mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': fichier, 'scope': 'user.%s' % self.user}, ]) self.mtc.test_get_mgr_SourceNotFound_single_pfn() # Mgr-Tests: PUT def test_put_mgr_ok_single(self): """MOCK (RSE/PROTOCOLS): Put a single file to storage (Success)""" self.mtc.test_put_mgr_ok_single() # MGR-Tests: DELETE def test_delete_mgr_ok_multi(self): """MOCK (RSE/PROTOCOLS): Delete multiple files from storage (Success)""" for fichier in MgrTestCases.files_remote: mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': fichier, 'scope': 'user.%s' % self.user}, ]) self.mtc.test_delete_mgr_ok_multi() def test_delete_mgr_ok_single(self): """MOCK (RSE/PROTOCOLS): Delete a single file from storage (Success)""" for fichier in MgrTestCases.files_remote: mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': fichier, 'scope': 'user.%s' % self.user}, ]) self.mtc.test_delete_mgr_ok_single() # MGR-Tests: EXISTS # ATTENTION: this tests won't work no more with the new RSEMgr as the protocol object is no longer cached and therefore the list of files is also not 'persisted' # def test_exists_mgr_ok_multi(self): # """MOCK (RSE/PROTOCOLS): Check multiple files on storage (Success)""" # for f in MgrTestCases.files_remote: # mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ]) # self.mtc.test_exists_mgr_ok_multi() def test_exists_mgr_ok_single_lfn(self): """MOCK (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)""" self.mtc.test_exists_mgr_ok_single_lfn() def test_exists_mgr_ok_single_pfn(self): """MOCK (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)""" self.mtc.test_exists_mgr_ok_single_pfn() # ATTENTION: this tests won't work no more with the new RSEMgr as the protocol object is no longer cached and therefore the list of files is also not 'persisted' # def test_exists_mgr_false_multi(self): # """MOCK (RSE/PROTOCOLS): Check multiple files on storage (Fail)""" # for f in MgrTestCases.files_remote: # mgr.upload(mgr.get_rse_info(self.rse_id), [{'name': f, 'scope': 'user.%s' % self.user}, ]) # self.mtc.test_exists_mgr_false_multi() def test_exists_mgr_false_single(self): """MOCK (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)""" self.mtc.test_exists_mgr_false_single_lfn() def test_exists_mgr_false_single_pfn(self): """MOCK (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)""" self.mtc.test_exists_mgr_false_single_pfn() # MGR-Tests: RENAME @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_lfn(self): """MOCK (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_pfn(self): """MOCK (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_pfn()
class TestRseS3ES(): tmpdir = None user = None # The setupClass and tearDownClass need some fixing, but can be ignored for this patch @classmethod def setupClass(cls): """S3ES (RSE/PROTOCOLS): Creating necessary directories and files """ cls.site = 'BNL-OSG2_ES' # Creating local files cls.tmpdir = tempfile.mkdtemp() cls.user = '******' with open('etc/rse_repository.json') as f: data = json.load(f) scheme = data[cls.site]['protocols']['supported']['s3+https']['scheme'] prefix = data[cls.site]['protocols']['supported']['s3+https']['prefix'] hostname = data[cls.site]['protocols']['supported']['s3+https']['hostname'] port = data[cls.site]['protocols']['supported']['s3+https']['port'] with open("%s/data.raw" % cls.tmpdir, "wb") as out: out.seek((1024) - 1) # 1 kB out.write('\0') for f in MgrTestCases.files_local: os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) cls.static_file = '%s://%s:%s/%s/user.%s/data.raw' % (scheme, hostname, port, prefix, cls.user) rse_settings = rsemanager.get_rse_info(cls.site) storage = rsemanager.create_protocol(rse_settings, operation='write', scheme='s3+https') storage.connect() for f in MgrTestCases.files_remote: os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) destfile = rsemanager.lfns2pfns(rse_settings, [{'name': f, 'scope': 'user.%s' % (cls.user)}, ], operation='write', scheme='s3+https').values()[0] try: storage.put('%s/%s' % (cls.tmpdir, f), destfile) except FileReplicaAlreadyExists as e: print(e) f = 'data.raw' destfile = rsemanager.lfns2pfns(rse_settings, [{'name': f, 'scope': 'user.%s' % (cls.user)}, ], operation='write', scheme='s3+https').values()[0] try: storage.put('%s/%s' % (cls.tmpdir, f), destfile) except FileReplicaAlreadyExists as e: print(e) @classmethod def tearDownClass(cls): """S3ES (RSE/PROTOCOLS): Removing created directories and files """ rse_settings = rsemanager.get_rse_info(cls.site) with open('etc/rse_repository.json') as f: data = json.load(f) scheme = data[cls.site]['protocols']['supported']['s3+https']['scheme'] prefix = data[cls.site]['protocols']['supported']['s3+https']['prefix'] hostname = data[cls.site]['protocols']['supported']['s3+https']['hostname'] port = data[cls.site]['protocols']['supported']['s3+https']['port'] for protocol in rse_settings['protocols']: if protocol['impl'] == 'rucio.rse.protocols.signeds3.Default': protocol['impl'] = 'rucio.rse.protocols.s3es.Default' storage = rsemanager.create_protocol(rse_settings, operation='write', scheme='s3+https') storage.connect() try: storage.delete('%s://%s:%s/%s/%s' % (scheme, hostname, port, prefix, 'user')) except Exception as e: print(e) try: storage.delete('%s://%s:%s/%s/%s' % (scheme, hostname, port, prefix, 'group')) except Exception as e: print(e) def setup(self): """S3ES (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseS3ES.tmpdir self.rse_id = 'BNL-OSG2_ES' self.mtc = MgrTestCases(self.tmpdir, 'BNL-OSG2_ES', TestRseS3ES.user, TestRseS3ES.static_file) self.mtc.setup_scheme('s3+https', 's3es') # MGR-Tests: DELETE def test_delete_mgr_ok_multi(self): """S3ES (RSE/PROTOCOLS): Delete multiple files from storage (Success)""" try: self.mtc.test_delete_mgr_ok_multi() except NotImplementedError: pass def test_delete_mgr_ok_single(self): """S3ES (RSE/PROTOCOLS): Delete a single file from storage (Success)""" try: self.mtc.test_delete_mgr_ok_single() except NotImplementedError: pass def test_delete_mgr_ok_dir(self): """S3ES (RSE/PROTOCOLS): Delete a single file from storage (Success)""" try: self.mtc.test_delete_mgr_ok_dir() except NotImplementedError: pass
def setup(self): """S3ES (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseS3ES.tmpdir self.rse_id = 'BNL-OSG2_ES' self.mtc = MgrTestCases(self.tmpdir, 'BNL-OSG2_ES', TestRseS3ES.user, TestRseS3ES.static_file) self.mtc.setup_scheme('s3+https', 's3es')
class TestRseXROOTD(): tmpdir = None user = None @classmethod def setupClass(cls): """XROOTD (RSE/PROTOCOLS): Creating necessary directories and files """ # Creating local files cls.tmpdir = tempfile.mkdtemp() cls.user = uuid() with open("%s/data.raw" % cls.tmpdir, "wb") as out: out.seek((1024 * 1024) - 1) # 1 MB out.write('\0') for f in MgrTestCases.files_local: shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) protocol = rsemanager.create_protocol( rsemanager.get_rse_info('WJ-XROOTD'), 'write') protocol.connect() with open('etc/rse_repository.json') as f: data = json.load(f) prefix = data['WJ-XROOTD']['protocols']['supported']['xroot']['prefix'] hostname = data['WJ-XROOTD']['protocols']['supported']['xroot'][ 'hostname'] port = data['WJ-XROOTD']['protocols']['supported']['xroot']['port'] try: os.mkdir(prefix) except Exception as e: print(e) os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % prefix) cls.static_file = 'xroot://%s:%d/%s/data.raw' % (hostname, port, prefix) cmd = 'xrdcp %s/data.raw %s' % (prefix, cls.static_file) execute(cmd) for f in MgrTestCases.files_remote: path = protocol.path2pfn( prefix + protocol._get_path('user.%s' % cls.user, f)) cmd = 'xrdcp %s/data.raw %s' % (prefix, path) execute(cmd) @classmethod def teardownClass(cls): """XROOTD (RSE/PROTOCOLS): Removing created directorie s and files""" with open('etc/rse_repository.json') as f: data = json.load(f) prefix = data['WJ-XROOTD']['protocols']['supported']['xroot']['prefix'] hostname = data['WJ-XROOTD']['protocols']['supported']['xroot'][ 'hostname'] shutil.rmtree(prefix) shutil.rmtree(cls.tmpdir) clean_raw = '%s/data.raw' % prefix list_files_cmd_user = '******' % (hostname, prefix, cls.user) clean_files = str(execute(list_files_cmd_user)[1]).split('\n') list_files_cmd_group = 'xrdfs %s ls %s/group.%s' % (hostname, prefix, cls.user) clean_files += str(execute(list_files_cmd_group)[1]).split('\n') clean_files.append(clean_raw) for files in clean_files: clean_cmd = 'xrdfs %s rm %s' % (hostname, files) execute(clean_cmd) clean_prefix = '%s' % prefix list_directory = 'xrdfs %s ls %s' % (hostname, prefix) clean_directory = str(execute(list_directory)[1]).split('\n') clean_directory.append(clean_prefix) for directory in clean_directory: clean_cmd = 'xrdfs %s rmdir %s' % (hostname, directory) execute(clean_cmd) def setup(self): """XROOTD (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseXROOTD.tmpdir self.rse_id = 'WJ-XROOTD' self.mtc = MgrTestCases(self.tmpdir, 'WJ-XROOTD', TestRseXROOTD.user, TestRseXROOTD.static_file) # Mgr-Tests: GET def test_multi_get_mgr_ok(self): """XROOTD (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)""" self.mtc.test_multi_get_mgr_ok() def test_get_mgr_ok_single_lfn(self): """XROOTD (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)""" self.mtc.test_get_mgr_ok_single_lfn() def test_get_mgr_ok_single_pfn(self): """XROOTD (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)""" self.mtc.test_get_mgr_ok_single_pfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_multi(self): """XROOTD (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_lfn(self): """XROOTD (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_pfn(self): """XROOTD (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_single_pfn() # Mgr-Tests: PUT def test_put_mgr_ok_multi(self): """XROOTD (RSE/PROTOCOLS): Put multiple files to storage providing LFNs and PFNs (Success)""" self.mtc.test_put_mgr_ok_multi() def test_put_mgr_ok_single(self): """XROOTD (RSE/PROTOCOLS): Put a single file to storage (Success)""" self.mtc.test_put_mgr_ok_single() @raises(exception.SourceNotFound) def test_put_mgr_SourceNotFound_multi(self): """XROOTD (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)""" self.mtc.test_put_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_put_mgr_SourceNotFound_single(self): """XROOTD (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)""" self.mtc.test_put_mgr_SourceNotFound_single() @raises(exception.FileReplicaAlreadyExists) def test_put_mgr_FileReplicaAlreadyExists_multi(self): """XROOTD (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)""" self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi() @raises(exception.FileReplicaAlreadyExists) def test_put_mgr_FileReplicaAlreadyExists_single(self): """XROOTD (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)""" self.mtc.test_put_mgr_FileReplicaAlreadyExists_single() # MGR-Tests: DELETE def test_delete_mgr_ok_multi(self): """XROOTD (RSE/PROTOCOLS): Delete multiple files from storage (Success)""" self.mtc.test_delete_mgr_ok_multi() def test_delete_mgr_ok_single(self): """XROOTD (RSE/PROTOCOLS): Delete a single file from storage (Success)""" self.mtc.test_delete_mgr_ok_single() @raises(exception.SourceNotFound) def test_delete_mgr_SourceNotFound_multi(self): """XROOTD (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)""" self.mtc.test_delete_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_delete_mgr_SourceNotFound_single(self): """XROOTD (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)""" self.mtc.test_delete_mgr_SourceNotFound_single() # MGR-Tests: EXISTS def test_exists_mgr_ok_multi(self): """XROOTD (RSE/PROTOCOLS): Check multiple files on storage (Success)""" self.mtc.test_exists_mgr_ok_multi() def test_exists_mgr_ok_single_lfn(self): """XROOTD (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)""" self.mtc.test_exists_mgr_ok_single_lfn() def test_exists_mgr_ok_single_pfn(self): """XROOTD (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)""" self.mtc.test_exists_mgr_ok_single_pfn() def test_exists_mgr_false_multi(self): """XROOTD (RSE/PROTOCOLS): Check multiple files on storage (Fail)""" self.mtc.test_exists_mgr_false_multi() def test_exists_mgr_false_single(self): """XROOTD (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)""" self.mtc.test_exists_mgr_false_single_lfn() def test_exists_mgr_false_single_pfn(self): """XROOTD (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)""" self.mtc.test_exists_mgr_false_single_pfn() # MGR-Tests: RENAME def test_rename_mgr_ok_multi(self): """XROOTD (RSE/PROTOCOLS): Rename multiple files on storage (Success)""" self.mtc.test_rename_mgr_ok_multi() def test_rename_mgr_ok_single_lfn(self): """XROOTD (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)""" self.mtc.test_rename_mgr_ok_single_lfn() def test_rename_mgr_ok_single_pfn(self): """XROOTD (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)""" self.mtc.test_rename_mgr_ok_single_pfn() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_multi(self): """XROOTD (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self): """XROOTD (RSE/PROTOCOLS): Rename a single file on storage using LFN(FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self): """XROOTD (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_multi(self): """XROOTD (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_lfn(self): """XROOTD (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_pfn(self): """XROOTD (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_pfn() def test_change_scope_mgr_ok_single_lfn(self): """XROOTD (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)""" self.mtc.test_change_scope_mgr_ok_single_lfn() def test_change_scope_mgr_ok_single_pfn(self): """XROOTD (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)""" self.mtc.test_change_scope_mgr_ok_single_pfn()
class TestRseSRM(): tmpdir = None user = None @classmethod def setupClass(cls): """SRM (RSE/PROTOCOLS): Creating necessary directories and files """ # Creating local files cls.tmpdir = tempfile.mkdtemp() cls.user = uuid() with open("%s/data.raw" % cls.tmpdir, "wb") as out: out.seek((1024 * 1024) - 1) # 1 MB out.write('\0') for f in MgrTestCases.files_local: shutil.copy('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) with open('etc/rse_repository.json') as f: data = json.load(f) prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix'] hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname'] if hostname.count("://"): hostname = hostname.split("://")[1] if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys(): port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['port']) else: port = 0 if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys() and 'web_service_path' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes'].keys(): web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes']['web_service_path'] else: web_service_path = '' os.system('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % cls.tmpdir) if port > 0: cls.static_file = 'srm://%s:%s%s%s/data.raw' % (hostname, port, web_service_path, prefix) else: cls.static_file = 'srm://%s%s%s/data.raw' % (hostname, web_service_path, prefix) cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % (cls.tmpdir, cls.static_file) execute(cmd) for f in MgrTestCases.files_remote: tmp = mgr.lfns2pfns(mgr.get_rse_info('FZK-LCG2_SCRATCHDISK'), {'name': f, 'scope': 'user.%s' % cls.user}, scheme='srm').values()[0] cmd = 'srmcp --debug=false -retry_num=0 file:///%s/data.raw %s' % (cls.tmpdir, tmp) execute(cmd) @classmethod def teardownClass(cls): """SRM (RSE/PROTOCOLS): Removing created directorie s and files""" with open('etc/rse_repository.json') as f: data = json.load(f) prefix = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['prefix'] hostname = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['hostname'] if hostname.count("://"): hostname = hostname.split("://")[1] if 'port' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys(): port = int(data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['port']) else: port = 0 if 'extended_attributes' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm'].keys() and 'web_service_path' in data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes'].keys(): web_service_path = data['FZK-LCG2_SCRATCHDISK']['protocols']['supported']['srm']['extended_attributes']['web_service_path'] else: web_service_path = '' shutil.rmtree(cls.tmpdir) clean_raw = '%s/data.raw' % prefix if int(port) > 0: srm_path = ''.join(["srm://", hostname, ":", port, web_service_path]) else: srm_path = ''.join(["srm://", hostname, web_service_path]) list_files_cmd_user = '******' % (srm_path, prefix, cls.user) clean_files = str(execute(list_files_cmd_user)[1]).split('\n') list_files_cmd_user = '******' % (srm_path, prefix, cls.user) clean_files += str(execute(list_files_cmd_user)[1]).split('\n') clean_files.append("1024 " + clean_raw) for files in clean_files: if len(files.strip()) > 0: file = files.split()[1] if not file.endswith("/"): clean_cmd = 'srmrm --debug=false -retry_num=0 %s/%s' % (srm_path, file) execute(clean_cmd) clean_directory = ['user', 'group'] for directory in clean_directory: clean_cmd = 'srmrmdir --debug=false -retry_num=0 -recursive %s%s/%s/%s' % (srm_path, prefix, directory, cls.user) execute(clean_cmd) def setup(self): """SRM (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseSRM.tmpdir self.rse_id = 'FZK-LCG2_SCRATCHDISK' self.mtc = MgrTestCases(self.tmpdir, 'FZK-LCG2_SCRATCHDISK', TestRseSRM.user, TestRseSRM.static_file) # self.mtc = MgrTestCases(self.tmpdir, 'FZK-LCG2_SCRATCHDISK', TestRseSRM.user, "srm://atlassrm-fzk.gridka.de/pnfs/gridka.de/atlas/disk-only/atlasscratchdisk/user/wguan/rucio.test.2") self.mtc.setup_scheme('srm') # Mgr-Tests: GET def test_multi_get_mgr_ok(self): """SRM (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)""" self.mtc.test_multi_get_mgr_ok() def test_get_mgr_ok_single_lfn(self): """SRM (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)""" self.mtc.test_get_mgr_ok_single_lfn() def test_get_mgr_ok_single_pfn(self): """SRM (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)""" self.mtc.test_get_mgr_ok_single_pfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_multi(self): """SRM (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_lfn(self): """SRM (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_pfn(self): """SRM (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_single_pfn() # Mgr-Tests: PUT def test_put_mgr_ok_multi(self): """SRM (RSE/PROTOCOLS): Put multiple files to storage providing LFNs and PFNs (Success)""" self.mtc.test_put_mgr_ok_multi() def test_put_mgr_ok_single(self): """SRM (RSE/PROTOCOLS): Put a single file to storage (Success)""" self.mtc.test_put_mgr_ok_single() @raises(exception.SourceNotFound) def test_put_mgr_SourceNotFound_multi(self): """SRM (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)""" self.mtc.test_put_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_put_mgr_SourceNotFound_single(self): """SRM (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)""" self.mtc.test_put_mgr_SourceNotFound_single() @raises(exception.FileReplicaAlreadyExists) def test_put_mgr_FileReplicaAlreadyExists_multi(self): """SRM (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)""" self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi() @raises(exception.FileReplicaAlreadyExists) def test_put_mgr_FileReplicaAlreadyExists_single(self): """SRM (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)""" self.mtc.test_put_mgr_FileReplicaAlreadyExists_single() # MGR-Tests: DELETE def test_delete_mgr_ok_multi(self): """SRM (RSE/PROTOCOLS): Delete multiple files from storage (Success)""" self.mtc.test_delete_mgr_ok_multi() def test_delete_mgr_ok_single(self): """SRM (RSE/PROTOCOLS): Delete a single file from storage (Success)""" self.mtc.test_delete_mgr_ok_single() @raises(exception.SourceNotFound) def test_delete_mgr_SourceNotFound_multi(self): """SRM (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)""" self.mtc.test_delete_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_delete_mgr_SourceNotFound_single(self): """SRM (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)""" self.mtc.test_delete_mgr_SourceNotFound_single() # MGR-Tests: EXISTS def test_exists_mgr_ok_multi(self): """SRM (RSE/PROTOCOLS): Check multiple files on storage (Success)""" self.mtc.test_exists_mgr_ok_multi() def test_exists_mgr_ok_single_lfn(self): """SRM (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)""" self.mtc.test_exists_mgr_ok_single_lfn() def test_exists_mgr_ok_single_pfn(self): """SRM (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)""" self.mtc.test_exists_mgr_ok_single_pfn() def test_exists_mgr_false_multi(self): """SRM (RSE/PROTOCOLS): Check multiple files on storage (Fail)""" self.mtc.test_exists_mgr_false_multi() def test_exists_mgr_false_single(self): """SRM (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)""" self.mtc.test_exists_mgr_false_single_lfn() def test_exists_mgr_false_single_pfn(self): """SRM (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)""" self.mtc.test_exists_mgr_false_single_pfn() # MGR-Tests: RENAME def test_rename_mgr_ok_multi(self): """SRM (RSE/PROTOCOLS): Rename multiple files on storage (Success)""" self.mtc.test_rename_mgr_ok_multi() def test_rename_mgr_ok_single_lfn(self): """SRM (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)""" self.mtc.test_rename_mgr_ok_single_lfn() def test_rename_mgr_ok_single_pfn(self): """SRM (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)""" self.mtc.test_rename_mgr_ok_single_pfn() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_multi(self): """SRM (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self): """SRM (RSE/PROTOCOLS): Rename a single file on storage using LFN(FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self): """SRM (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_multi(self): """SRM (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_lfn(self): """SRM (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_pfn(self): """SRM (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_pfn() def test_change_scope_mgr_ok_single_lfn(self): """SRM (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)""" self.mtc.test_change_scope_mgr_ok_single_lfn() def test_change_scope_mgr_ok_single_pfn(self): """SRM (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)""" self.mtc.test_change_scope_mgr_ok_single_pfn()
class TestRseS3(): tmpdir = None user = None @classmethod def setupClass(cls): """S3 (RSE/PROTOCOLS): Creating necessary directories and files """ # Creating local files cls.tmpdir = tempfile.mkdtemp() cls.user = uuid() # cls.user = '******' # use again when latency issue with S3 storage is resolved with open("%s/data.raw" % cls.tmpdir, "wb") as out: out.seek((1024 * 1024) - 1) # 1 MB out.write('\0') for f in MgrTestCases.files_local: os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, f)) fnull = open(os.devnull, 'w') # Create test files on storage try: subprocess.call(["s3cmd", "mb", "s3://USER"], stdout=fnull, stderr=fnull, shell=False) subprocess.call(["s3cmd", "mb", "s3://GROUP"], stdout=fnull, stderr=fnull, shell=False) subprocess.call(["s3cmd", "mb", "s3://NONDETERMINISTIC"], stdout=fnull, stderr=fnull, shell=False) except S3Error: pass cls.static_file = 's3://NONDETERMINISTIC/data.raw' subprocess.call([ "s3cmd", "put", "%s/data.raw" % cls.tmpdir, cls.static_file, "--no-progress" ], stdout=fnull, stderr=fnull) for f in MgrTestCases.files_remote: subprocess.call([ "s3cmd", "cp", cls.static_file, mgr.lfns2pfns(mgr.get_rse_info('SWIFT'), { 'name': f, 'scope': 'user.%s' % cls.user }).values()[0] ], stdout=fnull, stderr=fnull) fnull.close() def setup(self): """S3 (RSE/PROTOCOLS): Creating Mgr-instance """ self.tmpdir = TestRseS3.tmpdir self.mtc = MgrTestCases(self.tmpdir, 'SWIFT', TestRseS3.user, TestRseS3.static_file) @classmethod def tearDownClass(cls): """S3 (RSE/PROTOCOLS): Removing created directories and files """ # Remove test files from storage fnull = open(os.devnull, 'w') subprocess.call(["s3cmd", "rb", "s3://USER", "--recursive"], stdout=fnull, stderr=fnull) subprocess.call(["s3cmd", "rb", "s3://GROUP", "--recursive"], stdout=fnull, stderr=fnull) shutil.rmtree(cls.tmpdir) fnull.close() # Mgr-Tests: GET def test_multi_get_mgr_ok(self): """S3 (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (Success)""" self.mtc.test_multi_get_mgr_ok() def test_get_mgr_ok_single_lfn(self): """S3 (RSE/PROTOCOLS): Get a single file from storage providing LFN (Success)""" self.mtc.test_get_mgr_ok_single_lfn() def test_get_mgr_ok_single_pfn(self): """S3 (RSE/PROTOCOLS): Get a single file from storage providing PFN (Success)""" self.mtc.test_get_mgr_ok_single_pfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_multi(self): """S3 (RSE/PROTOCOLS): Get multiple files from storage providing LFNs and PFNs (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_lfn(self): """S3 (RSE/PROTOCOLS): Get a single file from storage providing LFN (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_get_mgr_SourceNotFound_single_pfn(self): """S3 (RSE/PROTOCOLS): Get a single file from storage providing PFN (SourceNotFound)""" self.mtc.test_get_mgr_SourceNotFound_single_pfn() # Mgr-Tests: PUT def test_put_mgr_ok_multi(self): """S3 (RSE/PROTOCOLS): Put multiple files to storage (Success)""" self.mtc.test_put_mgr_ok_multi() def test_put_mgr_ok_single(self): """S3 (RSE/PROTOCOLS): Put a single file to storage (Success)""" self.mtc.test_put_mgr_ok_single() @raises(exception.SourceNotFound) def test_put_mgr_SourceNotFound_multi(self): """S3 (RSE/PROTOCOLS): Put multiple files to storage (SourceNotFound)""" self.mtc.test_put_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_put_mgr_SourceNotFound_single(self): """S3 (RSE/PROTOCOLS): Put a single file to storage (SourceNotFound)""" self.mtc.test_put_mgr_SourceNotFound_single() @raises(exception.FileReplicaAlreadyExists) def test_put_mgr_FileReplicaAlreadyExists_multi(self): """S3 (RSE/PROTOCOLS): Put multiple files to storage (FileReplicaAlreadyExists)""" self.mtc.test_put_mgr_FileReplicaAlreadyExists_multi() @raises(exception.FileReplicaAlreadyExists) def test_put_mgr_FileReplicaAlreadyExists_single(self): """S3 (RSE/PROTOCOLS): Put a single file to storage (FileReplicaAlreadyExists)""" self.mtc.test_put_mgr_FileReplicaAlreadyExists_single() # MGR-Tests: DELETE def test_delete_mgr_ok_multi(self): """S3 (RSE/PROTOCOLS): Delete multiple files from storage (Success)""" self.mtc.test_delete_mgr_ok_multi() def test_delete_mgr_ok_single(self): """S3 (RSE/PROTOCOLS): Delete a single file from storage (Success)""" self.mtc.test_delete_mgr_ok_single() @raises(exception.SourceNotFound) def test_delete_mgr_SourceNotFound_multi(self): """S3 (RSE/PROTOCOLS): Delete multiple files from storage (SourceNotFound)""" self.mtc.test_delete_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_delete_mgr_SourceNotFound_single(self): """S3 (RSE/PROTOCOLS): Delete a single file from storage (SourceNotFound)""" self.mtc.test_delete_mgr_SourceNotFound_single() # MGR-Tests: EXISTS def test_exists_mgr_ok_multi(self): """S3 (RSE/PROTOCOLS): Check multiple files on storage (Success)""" self.mtc.test_exists_mgr_ok_multi() def test_exists_mgr_ok_single_lfn(self): """S3 (RSE/PROTOCOLS): Check a single file on storage using LFN (Success)""" self.mtc.test_exists_mgr_ok_single_lfn() def test_exists_mgr_ok_single_pfn(self): """S3 (RSE/PROTOCOLS): Check a single file on storage using PFN (Success)""" self.mtc.test_exists_mgr_ok_single_pfn() def test_exists_mgr_false_multi(self): """S3 (RSE/PROTOCOLS): Check multiple files on storage (Fail)""" self.mtc.test_exists_mgr_false_multi() def test_exists_mgr_false_single_lfn(self): """S3 (RSE/PROTOCOLS): Check a single file on storage using LFN (Fail)""" self.mtc.test_exists_mgr_false_single_lfn() def test_exists_mgr_false_single_pfn(self): """S3 (RSE/PROTOCOLS): Check a single file on storage using PFN (Fail)""" self.mtc.test_exists_mgr_false_single_pfn() # MGR-Tests: RENAME def test_rename_mgr_ok_multi(self): """S3 (RSE/PROTOCOLS): Rename multiple files on storage (Success)""" self.mtc.test_rename_mgr_ok_multi() def test_rename_mgr_ok_single_lfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (Success)""" self.mtc.test_rename_mgr_ok_single_lfn() def test_rename_mgr_ok_single_pfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (Success)""" self.mtc.test_rename_mgr_ok_single_pfn() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_multi(self): """S3 (RSE/PROTOCOLS): Rename multiple files on storage (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_multi() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_single_lfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_lfn() @raises(exception.FileReplicaAlreadyExists) def test_rename_mgr_FileReplicaAlreadyExists_single_pfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (FileReplicaAlreadyExists)""" self.mtc.test_rename_mgr_FileReplicaAlreadyExists_single_pfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_multi(self): """S3 (RSE/PROTOCOLS): Rename multiple files on storage (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_multi() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_lfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using LFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_lfn() @raises(exception.SourceNotFound) def test_rename_mgr_SourceNotFound_single_pfn(self): """S3 (RSE/PROTOCOLS): Rename a single file on storage using PFN (SourceNotFound)""" self.mtc.test_rename_mgr_SourceNotFound_single_pfn() def test_change_scope_mgr_ok_single_lfn(self): """S3 (RSE/PROTOCOLS): Change the scope of a single file on storage using LFN (Success)""" self.mtc.test_change_scope_mgr_ok_single_lfn() def test_change_scope_mgr_ok_single_pfn(self): """S3 (RSE/PROTOCOLS): Change the scope of a single file on storage using PFN (Success)""" self.mtc.test_change_scope_mgr_ok_single_pfn()