class TestDIDClients(object): def __init__(self): self.did_client = DIDClient() self.replica_client = ReplicaClient() def test_add_and_list_archive(self): """ ARCHIVE (CLIENT): Add files to archive and list the content.""" scope, rse = 'mock', 'MOCK' archive_file = 'file_' + generate_uuid() + '.zip' files = [] for i in range(10): files.append({ 'scope': scope, 'name': 'lfn.%s' % str(generate_uuid()), 'bytes': 724963570, 'adler32': '0cc737eb', 'type': 'FILE', 'meta': { 'guid': str(generate_uuid()) } }) self.replica_client.add_replicas(rse=rse, files=[{ 'scope': scope, 'name': archive_file, 'bytes': 1, 'adler32': '0cc737eb' }]) self.did_client.add_files_to_archive(scope=scope, name=archive_file, files=files) content = [ fil for fil in self.did_client.list_archive_content(scope=scope, name=archive_file) ] assert_equal(len(content), 10)
def test_list_dataset_replicas_archive(self): """ REPLICA (CLIENT): List dataset replicas with archives. """ replica_client = ReplicaClient() did_client = DIDClient() rule_client = RuleClient() scope = 'mock' rse = 'APERTURE_%s' % rse_name_generator() rse_id = add_rse(rse, **self.vo) add_protocol(rse_id=rse_id, parameter={ 'scheme': 'root', 'hostname': 'root.aperture.com', 'port': 1409, 'prefix': '//test/chamber/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } }) rse2 = 'BLACKMESA_%s' % rse_name_generator() rse2_id = add_rse(rse2, **self.vo) add_protocol(rse_id=rse2_id, parameter={ 'scheme': 'root', 'hostname': 'root.blackmesa.com', 'port': 1409, 'prefix': '//underground/facility', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } }) # register archive archive = { 'scope': scope, 'name': 'another.%s.zip' % generate_uuid(), 'type': 'FILE', 'bytes': 2596, 'adler32': 'deedbeaf' } replica_client.add_replicas(rse=rse, files=[archive]) replica_client.add_replicas(rse=rse2, files=[archive]) archived_files = [{ 'scope': scope, 'name': 'zippedfile-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE', 'bytes': 4322, 'adler32': 'deaddead' } for i in range(2)] replica_client.add_replicas(rse=rse2, files=archived_files) did_client.add_files_to_archive(scope=scope, name=archive['name'], files=archived_files) dataset_name = 'find_me.' + str(generate_uuid()) did_client.add_dataset(scope=scope, name=dataset_name) did_client.attach_dids(scope=scope, name=dataset_name, dids=archived_files) rule_client.add_replication_rule(dids=[{ 'scope': scope, 'name': dataset_name }], account='root', copies=1, rse_expression=rse, grouping='DATASET') res = [ r for r in replica_client.list_dataset_replicas(scope=scope, name=dataset_name) ] assert len(res) == 1 assert res[0]['state'] == 'UNAVAILABLE' res = [ r for r in replica_client.list_dataset_replicas( scope=scope, name=dataset_name, deep=True) ] assert len(res) == 3 assert res[0]['state'] == 'AVAILABLE' assert res[1]['state'] == 'AVAILABLE' assert res[2]['state'] == 'AVAILABLE' del_rse(rse_id)
class TestArchive(object): def __init__(self): self.dc = DIDClient() self.rc = ReplicaClient() if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = {'vo': 'tst'} else: self.vo = {} def test_add_and_list_archive(self): """ ARCHIVE (CLIENT): Add files to archive and list the content """ scope, rse = 'mock', 'MOCK' archive_files = ['file_' + generate_uuid() + '.zip' for _ in range(2)] files = [] for i in range(10): files.append({ 'scope': scope, 'name': 'lfn.%s' % str(generate_uuid()), 'bytes': 724963570, 'adler32': '0cc737eb', 'type': 'FILE', 'meta': { 'guid': str(generate_uuid()) } }) for archive_file in archive_files: self.rc.add_replicas(rse=rse, files=[{ 'scope': scope, 'name': archive_file, 'bytes': 1, 'adler32': '0cc737eb' }]) self.dc.add_files_to_archive(scope=scope, name=archive_file, files=files) content = [ f for f in self.dc.list_archive_content(scope=scope, name=archive_file) ] assert_equal(len(content), 10) def test_list_archive_contents_transparently(self): """ ARCHIVE (CORE): Transparent archive listing """ scope = InternalScope('mock', **self.vo) rse = 'APERTURE_%s' % rse_name_generator() rse_id = add_rse(rse, **self.vo) root = InternalAccount('root', **self.vo) add_protocol( rse_id, { 'scheme': 'root', 'hostname': 'root.aperture.com', 'port': 1409, 'prefix': '//test/chamber/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } }) # register archive archive = { 'scope': scope, 'name': 'weighted.storage.cube.zip', 'type': 'FILE', 'bytes': 2596, 'adler32': 'beefdead' } archive_client = archive.copy() archive_client['scope'] = archive_client['scope'].external add_replicas(rse_id=rse_id, files=[archive], account=root) # archived files with replicas files_with_replicas = [{ 'scope': scope, 'name': 'witrep-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE', 'bytes': 1234, 'adler32': 'deadbeef' } for i in range(2)] files_with_replicas_client = [] for f in files_with_replicas: new_file = f.copy() new_file['scope'] = new_file['scope'].external files_with_replicas_client.append(new_file) add_replicas(rse_id=rse_id, files=files_with_replicas, account=root) self.dc.add_files_to_archive(scope=scope.external, name=archive_client['name'], files=files_with_replicas_client) res = [ r['pfns'] for r in self.rc.list_replicas(dids=[{ 'scope': scope.external, 'name': f['name'] } for f in files_with_replicas_client], resolve_archives=True) ] assert_equal(len(res), 2) assert_equal(len(res[0]), 2) assert_equal(len(res[1]), 2) for r in res: for p in r: if r[p]['domain'] == 'zip': assert_in('weighted.storage.cube.zip?xrdcl.unzip=witrep-', p) else: assert_not_in( 'weighted.storage.cube.zip?xrdcl.unzip=witrep-', p) # archived files without replicas files = [{ 'scope': scope.external, 'name': 'norep-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE', 'bytes': 1234, 'adler32': 'deadbeef' } for i in range(2)] self.dc.add_files_to_archive(scope=scope.external, name=archive_client['name'], files=files) res = [ r['pfns'] for r in self.rc.list_replicas(dids=[{ 'scope': scope.external, 'name': f['name'] } for f in files], resolve_archives=True) ] assert_equal(len(res), 2) for r in res: assert_in('weighted.storage.cube.zip?xrdcl.unzip=norep-', r.keys()[0]) def test_list_archive_contents_at_rse(self): """ ARCHIVE (CORE): Transparent archive listing at RSE """ scope = InternalScope('mock', **self.vo) root = InternalAccount('root', **self.vo) rse1 = 'APERTURE_%s' % rse_name_generator() rse1_id = add_rse(rse1, **self.vo) add_protocol( rse1_id, { 'scheme': 'root', 'hostname': 'root.aperture.com', 'port': 1409, 'prefix': '//test/chamber/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } }) rse2 = 'BLACKMESA_%s' % rse_name_generator() rse2_id = add_rse(rse2, **self.vo) add_protocol( rse2_id, { 'scheme': 'root', 'hostname': 'root.blackmesa.com', 'port': 1409, 'prefix': '//lambda/complex/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } }) # register archive archive1 = { 'scope': scope, 'name': 'cube.1.zip', 'type': 'FILE', 'bytes': 2596, 'adler32': 'beefdead' } archive2 = { 'scope': scope, 'name': 'cube.2.zip', 'type': 'FILE', 'bytes': 5432, 'adler32': 'deadbeef' } add_replicas(rse_id=rse1_id, files=[archive1], account=root) add_replicas(rse_id=rse2_id, files=[archive2], account=root) # archived files with replicas archived_file = [{ 'scope': scope.external, 'name': 'zippedfile-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE', 'bytes': 4322, 'adler32': 'beefbeef' } for i in range(2)] self.dc.add_files_to_archive(scope=scope.external, name=archive1['name'], files=archived_file) self.dc.add_files_to_archive(scope=scope.external, name=archive2['name'], files=archived_file) res = [ r['pfns'] for r in self.rc.list_replicas(dids=[{ 'scope': f['scope'], 'name': f['name'] } for f in archived_file], rse_expression=rse1, resolve_archives=True) ] res = self.rc.list_replicas(dids=[{ 'scope': f['scope'], 'name': f['name'] } for f in archived_file], metalink=True, rse_expression=rse1, resolve_archives=True) assert_in('APERTURE', res) assert_not_in('BLACKMESA', res) res = self.rc.list_replicas(dids=[{ 'scope': f['scope'], 'name': f['name'] } for f in archived_file], metalink=True, rse_expression=rse2, resolve_archives=True) assert_in('BLACKMESA', res) assert_not_in('APERTURE', res)
class TestDownloadClient(unittest.TestCase): def setUp(self): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): self.vo = { 'vo': config_get('client', 'vo', raise_exception=False, default='tst') } else: self.vo = {} logger = logging.getLogger('dlul_client') logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.DEBUG) self.client = Client() self.did_client = DIDClient() self.upload_client = UploadClient(_client=self.client, logger=logger) self.download_client = DownloadClient(client=self.client, logger=logger) def _upoad_test_file(self, rse, scope, name, path=None): item = { 'path': path if path else file_generator(), 'rse': rse, 'did_scope': scope, 'did_name': name, 'guid': generate_uuid(), } assert self.upload_client.upload([item]) == 0 return item @staticmethod def _check_download_result(actual_result, expected_result): assert len(expected_result) == len(actual_result) expected_result = sorted(expected_result, key=lambda x: x['did']) actual_result = sorted(actual_result, key=lambda x: x['did']) for i, expected in enumerate(expected_result): for param_name, expected_value in expected.items(): assert param_name and actual_result[i][param_name] == expected[ param_name] def test_download_without_base_dir(self): rse = 'MOCK4' scope = 'mock' item = self._upoad_test_file(rse, scope, 'testDownloadNoBasedir' + generate_uuid()) did = '%s:%s' % (scope, item['did_name']) try: # download to the default location, i.e. to ./ result = self.download_client.download_dids([{'did': did}]) self._check_download_result( actual_result=result, expected_result=[{ 'did': did, 'clientState': 'DONE', }], ) # re-downloading the same file again should not overwrite it result = self.download_client.download_dids([{'did': did}]) self._check_download_result( actual_result=result, expected_result=[{ 'did': did, 'clientState': 'ALREADY_DONE', }], ) finally: shutil.rmtree(scope) def test_download_multiple(self): rse = 'MOCK4' scope = 'mock' base_name = 'testDownloadItem' + generate_uuid() item000 = self._upoad_test_file(rse, scope, base_name + '.000') item001 = self._upoad_test_file(rse, scope, base_name + '.001') item100 = self._upoad_test_file(rse, scope, base_name + '.100') with TemporaryDirectory() as tmp_dir: # Download specific DID result = self.download_client.download_dids([{ 'did': '%s:%s' % (scope, item000['did_name']), 'base_dir': tmp_dir }]) self._check_download_result( actual_result=result, expected_result=[{ 'did': '%s:%s' % (scope, item000['did_name']), 'clientState': 'DONE', }], ) # Download multiple files with wildcard. One file already exists on the file system. Will not be re-downloaded. result = self.download_client.download_dids([{ 'did': '%s:%s.0*' % (scope, base_name), 'base_dir': tmp_dir }]) self._check_download_result( actual_result=result, expected_result=[ { 'did': '%s:%s' % (scope, item000['did_name']), 'clientState': 'ALREADY_DONE', }, { 'did': '%s:%s' % (scope, item001['did_name']), 'clientState': 'DONE', }, ], ) # Download with filter result = self.download_client.download_dids([{ 'filters': { 'guid': item000['guid'], 'scope': scope }, 'base_dir': tmp_dir }]) self._check_download_result( actual_result=result, expected_result=[{ 'did': '%s:%s' % (scope, item000['did_name']), }], ) # Download with wildcard and name result = self.download_client.download_dids([{ 'did': '%s:*' % scope, 'filters': { 'guid': item100['guid'] }, 'base_dir': tmp_dir }]) self._check_download_result( actual_result=result, expected_result=[{ 'did': '%s:%s' % (scope, item100['did_name']), 'clientState': 'DONE', }], ) # Don't create subdirectories by scope result = self.download_client.download_dids([{ 'did': '%s:%s.*' % (scope, base_name), 'base_dir': tmp_dir, 'no_subdir': True }]) self._check_download_result( actual_result=result, expected_result=[ { 'did': '%s:%s' % (scope, item000['did_name']), 'clientState': 'DONE', 'dest_file_paths': ['%s/%s' % (tmp_dir, item000['did_name'])], }, { 'did': '%s:%s' % (scope, item001['did_name']), 'clientState': 'DONE', 'dest_file_paths': ['%s/%s' % (tmp_dir, item001['did_name'])], }, { 'did': '%s:%s' % (scope, item100['did_name']), 'clientState': 'DONE', 'dest_file_paths': ['%s/%s' % (tmp_dir, item100['did_name'])], }, ], ) # Re-download file existing on the file system with no-subdir set. It must be overwritten. result = self.download_client.download_dids([{ 'did': '%s:%s' % (scope, item100['did_name']), 'base_dir': tmp_dir, 'no_subdir': True }]) self._check_download_result( actual_result=result, expected_result=[{ 'did': '%s:%s' % (scope, item100['did_name']), 'clientState': 'ALREADY_DONE', 'dest_file_paths': ['%s/%s' % (tmp_dir, item100['did_name'])], }], ) @pytest.mark.xfail( reason= 'XRD1 must be initialized https://github.com/rucio/rucio/pull/4165/') def test_download_from_archive_on_xrd(self): scope = 'test' rse = 'XRD1' base_name = 'testDownloadArchive' + generate_uuid() with TemporaryDirectory() as tmp_dir: # Create a zip archive with two files and upload it name000 = base_name + '.000' data000 = '000' adler000 = '01230091' name001 = base_name + '.001' data001 = '001' adler001 = '01240092' zip_name = base_name + '.zip' zip_path = '%s/%s' % (tmp_dir, zip_name) with ZipFile(zip_path, 'w') as myzip: myzip.writestr(name000, data=data000) myzip.writestr(name001, data=data001) self._upoad_test_file(rse, scope, zip_name, path=zip_path) self.did_client.add_files_to_archive( scope, zip_name, [ { 'scope': scope, 'name': name000, 'bytes': len(data000), 'type': 'FILE', 'adler32': adler000, 'meta': { 'guid': str(generate_uuid()) } }, { 'scope': scope, 'name': name001, 'bytes': len(data001), 'type': 'FILE', 'adler32': adler001, 'meta': { 'guid': str(generate_uuid()) } }, ], ) # Download one file from the archive result = self.download_client.download_dids([{ 'did': '%s:%s' % (scope, name000), 'base_dir': tmp_dir }]) self._check_download_result( actual_result=result, expected_result=[ { 'did': '%s:%s' % (scope, name000), 'clientState': 'DONE', }, ], ) with open('%s/%s/%s' % (tmp_dir, scope, name000), 'r') as file: assert file.read() == data000 # Download both files from the archive result = self.download_client.download_dids([{ 'did': '%s:%s.00*' % (scope, base_name), 'base_dir': tmp_dir }]) self._check_download_result( actual_result=result, expected_result=[ { 'did': '%s:%s' % (scope, name000), 'clientState': 'ALREADY_DONE', }, { 'did': '%s:%s' % (scope, name001), 'clientState': 'DONE', }, ], ) with open('%s/%s/%s' % (tmp_dir, scope, name001), 'r') as file: assert file.read() == data001 pfn = next(filter(lambda r: name001 in r['did'], result))['sources'][0]['pfn'] # Download by pfn from the archive result = self.download_client.download_pfns([{ 'did': '%s:%s' % (scope, name001), 'pfn': pfn, 'rse': rse, 'base_dir': tmp_dir, 'no_subdir': True }]) self._check_download_result( actual_result=result, expected_result=[ { 'did': '%s:%s' % (scope, name001), 'clientState': 'DONE', }, ], ) def test_trace_copy_out_and_checksum_validation(self): rse = 'MOCK4' scope = 'mock' name = 'testDownloadTraces' + generate_uuid() self._upoad_test_file(rse, scope, name) with TemporaryDirectory() as tmp_dir: # Try downloading non-existing did traces = [] with pytest.raises(NoFilesDownloaded): self.download_client.download_dids([{ 'did': 'some:randomNonExistingDid', 'base_dir': tmp_dir }], traces_copy_out=traces) assert len( traces) == 1 and traces[0]['clientState'] == 'FILE_NOT_FOUND' # Download specific DID traces = [] self.download_client.download_dids([{ 'did': '%s:%s' % (scope, name), 'base_dir': tmp_dir }], traces_copy_out=traces) assert len(traces) == 1 and traces[0]['clientState'] == 'DONE' # Download same DID again traces = [] result = self.download_client.download_dids( [{ 'did': '%s:%s' % (scope, name), 'base_dir': tmp_dir }], traces_copy_out=traces) assert len( traces) == 1 and traces[0]['clientState'] == 'ALREADY_DONE' # Change the local file and download the same file again. Checksum validation should fail and it must be re-downloaded with open(result[0]['dest_file_paths'][0], 'a') as f: f.write("more data") traces = [] result = self.download_client.download_dids( [{ 'did': '%s:%s' % (scope, name), 'base_dir': tmp_dir }], traces_copy_out=traces) assert len(traces) == 1 and traces[0]['clientState'] == 'DONE' pfn = result[0]['sources'][0]['pfn'] # Switch to a new empty directory with TemporaryDirectory() as tmp_dir: # Wildcards in did name are not allowed on pfn downloads traces = [] with pytest.raises(InputValidationError): self.download_client.download_pfns([{ 'did': '%s:*' % scope, 'pfn': pfn, 'rse': rse, 'base_dir': tmp_dir }], traces_copy_out=traces) assert not traces # Same pfn, but without wildcard in the did should work traces = [] self.download_client.download_pfns([{ 'did': '%s:%s' % (scope, name), 'pfn': pfn, 'rse': rse, 'base_dir': tmp_dir }], traces_copy_out=traces) assert len(traces) == 1 and traces[0]['clientState'] == 'DONE' # Same pfn. Local file already present. Shouldn't be overwritten. traces = [] self.download_client.download_pfns([{ 'did': '%s:%s' % (scope, name), 'pfn': pfn, 'rse': rse, 'base_dir': tmp_dir }], traces_copy_out=traces) assert len( traces) == 1 and traces[0]['clientState'] == 'ALREADY_DONE' # Provide wrong checksum for validation, the file will be re-downloaded but checksum validation fails traces = [] with pytest.raises(NoFilesDownloaded): self.download_client.download_pfns( [{ 'did': '%s:%s' % (scope, name), 'pfn': pfn, 'rse': rse, 'adler32': 'wrong', 'base_dir': tmp_dir }], traces_copy_out=traces) assert len( traces) == 1 and traces[0]['clientState'] == 'FAIL_VALIDATE' # Switch to a new empty directory with TemporaryDirectory() as tmp_dir: # Simulate checksum corruption by changing the source file. We rely on the particularity # that the MOCK4 rse uses the posix protocol: files are stored on the local file system protocol = rsemgr.create_protocol(rsemgr.get_rse_info( rse, vo=self.client.vo), operation='read') assert isinstance(protocol, PosixProtocol) mock_rse_local_path = protocol.pfn2path(pfn) with open(mock_rse_local_path, 'w') as f: f.write('some completely other data') # Download fails checksum validation traces = [] with pytest.raises(NoFilesDownloaded): self.download_client.download_dids( [{ 'did': '%s:%s' % (scope, name), 'base_dir': tmp_dir }], traces_copy_out=traces) assert len( traces) == 1 and traces[0]['clientState'] == 'FAIL_VALIDATE' # Ignore_checksum set. Download works. traces = [] self.download_client.download_dids([{ 'did': '%s:%s' % (scope, name), 'base_dir': tmp_dir, 'ignore_checksum': True }], traces_copy_out=traces) assert len(traces) == 1 and traces[0]['clientState'] == 'DONE'
class TestArchive(object): def __init__(self): self.dc = DIDClient() self.rc = ReplicaClient() def test_add_and_list_archive(self): """ ARCHIVE (CLIENT): Add files to archive and list the content """ scope, rse = 'mock', 'MOCK' archive_files = ['file_' + generate_uuid() + '.zip' for _ in range(2)] files = [] for i in range(10): files.append({ 'scope': scope, 'name': 'lfn.%s' % str(generate_uuid()), 'bytes': 724963570, 'adler32': '0cc737eb', 'type': 'FILE', 'meta': { 'guid': str(generate_uuid()) } }) for archive_file in archive_files: self.rc.add_replicas(rse=rse, files=[{ 'scope': scope, 'name': archive_file, 'bytes': 1, 'adler32': '0cc737eb' }]) self.dc.add_files_to_archive(scope=scope, name=archive_file, files=files) content = [ f for f in self.dc.list_archive_content(scope=scope, name=archive_file) ] assert_equal(len(content), 10) def test_list_archive_contents_transparently(self): """ ARCHIVE (CORE): Transparent archive listing """ scope = 'mock' rse = 'APERTURE_%s' % rse_name_generator() add_rse(rse) add_protocol( rse, { 'scheme': 'root', 'hostname': 'root.aperture.com', 'port': 1409, 'prefix': '//test/chamber/', 'impl': 'rucio.rse.protocols.xrootd.Default', 'domains': { 'lan': { 'read': 1, 'write': 1, 'delete': 1 }, 'wan': { 'read': 1, 'write': 1, 'delete': 1 } } }) # register archive archive = { 'scope': scope, 'name': 'weighted.storage.cube.zip', 'type': 'FILE', 'bytes': 2596, 'adler32': 'beefdead' } add_replicas(rse=rse, files=[archive], account='root') # archived files with replicas files_with_replicas = [{ 'scope': scope, 'name': 'witrep-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE', 'bytes': 1234, 'adler32': 'deadbeef' } for i in xrange(2)] add_replicas(rse=rse, files=files_with_replicas, account='root') self.dc.add_files_to_archive(scope=scope, name=archive['name'], files=files_with_replicas) res = [ r['pfns'] for r in self.rc.list_replicas(dids=[{ 'scope': scope, 'name': f['name'] } for f in files_with_replicas]) ] assert_equal(len(res), 2) assert_equal(len(res[0]), 2) assert_equal(len(res[1]), 2) for r in res: for p in r: if r[p]['domain'] == 'zip': assert_in('weighted.storage.cube.zip?xrdcl.unzip=witrep-', p) else: assert_not_in( 'weighted.storage.cube.zip?xrdcl.unzip=witrep-', p) # archived files without replicas files = [{ 'scope': scope, 'name': 'norep-%i-%s' % (i, str(generate_uuid())), 'type': 'FILE', 'bytes': 1234, 'adler32': 'deadbeef' } for i in xrange(2)] self.dc.add_files_to_archive(scope=scope, name=archive['name'], files=files) res = [ r['pfns'] for r in self.rc.list_replicas(dids=[{ 'scope': scope, 'name': f['name'] } for f in files]) ] assert_equal(len(res), 2) for r in res: assert_in('weighted.storage.cube.zip?xrdcl.unzip=norep-', r.keys()[0]) del_rse(rse)