class ObjectStorageApiTestBase(BaseTestCase): def setUp(self): super(ObjectStorageApiTestBase, self).setUp() self.api = ObjectStorageApi(self.ns, endpoint=self.uri) self.created = list() def tearDown(self): super(ObjectStorageApiTestBase, self).tearDown() for ct, name in self.created: try: self.api.object_delete(self.account, ct, name) except Exception: logging.exception("Failed to delete %s/%s/%s//%s", self.ns, self.account, ct, name) def _create(self, name, metadata=None): return self.api.container_create(self.account, name, properties=metadata) def _delete(self, name): self.api.container_delete(self.account, name) def _clean(self, name, clear=False): if clear: # must clean properties before self.api.container_del_properties(self.account, name, []) self._delete(name) def _get_properties(self, name, properties=None): return self.api.container_get_properties(self.account, name, properties=properties) def _set_properties(self, name, properties=None): return self.api.container_set_properties(self.account, name, properties=properties) def _upload_empty(self, container, *objs, **kwargs): """Upload empty objects to `container`""" for obj in objs: self.api.object_create(self.account, container, obj_name=obj, data="", **kwargs) self.created.append((container, obj))
class TestBlobConverter(BaseTestCase): def setUp(self): super(TestBlobConverter, self).setUp() self.container = random_str(16) self.path = random_str(16) self.api = ObjectStorageApi(self.ns) self.api.container_create(self.account, self.container) _, chunks = self.api.container.content_prepare( self.account, self.container, self.path, size=1) services = self.conscience.all_services('rawx') self.rawx_volumes = dict() for rawx in services: tags = rawx['tags'] service_id = tags.get('tag.service_id', None) if service_id is None: service_id = rawx['addr'] volume = tags.get('tag.vol', None) self.rawx_volumes[service_id] = volume self.api.object_create( self.account, self.container, obj_name=self.path, data="chunk") meta, self.chunks = self.api.object_locate( self.account, self.container, self.path) self.version = meta['version'] self.content_id = meta['id'] self.container_id = cid_from_name(self.account, self.container) def tearDown(self): try: self.api.object_delete(self.account, self.container, self.path) except Exception: pass super(TestBlobConverter, self).tearDown() def _chunk_path(self, chunk): url = chunk['url'] chunk_id = url.split('/', 3)[3] volume = self.rawx_volumes[self._chunk_volume_id(chunk)] return volume + '/' + chunk_id[:3] + '/' + chunk_id def _chunk_volume_id(self, chunk): return chunk['url'].split('/', 3)[2] def _deindex_chunk(self, chunk): rdir = RdirClient(self.conf, pool_manager=self.conscience.pool_manager) url = chunk['url'] volume_id = url.split('/', 3)[2] chunk_id = url.split('/', 3)[3] rdir.chunk_delete(volume_id, self.container_id, self.content_id, chunk_id) def _convert_and_check(self, chunk_volume, chunk_path, chunk_id_info, expected_raw_meta=None, expected_errors=0): conf = self.conf conf['volume'] = self.rawx_volumes[chunk_volume] converter = BlobConverter(conf, logger=self.logger) converter.safe_convert_chunk(chunk_path) self.assertEqual(1, converter.total_chunks_processed) self.assertEqual(1, converter.passes) self.assertEqual(expected_errors, converter.errors) checker = Checker(self.ns) for chunk_id, info in chunk_id_info.items(): account, container, path, version, content_id = info fullpath = encode_fullpath( account, container, path, version, content_id) cid = cid_from_name(account, container) meta, raw_meta = read_chunk_metadata(chunk_path, chunk_id) self.assertEqual(meta.get('chunk_id'), chunk_id) self.assertEqual(meta.get('container_id'), cid) self.assertEqual(meta.get('content_path'), path) self.assertEqual(meta.get('content_version'), version) self.assertEqual(meta.get('content_id'), content_id) self.assertEqual(meta.get('full_path'), fullpath) checker.check(Target( account, container=container, obj=path, chunk='http://' + converter.volume_id + '/' + chunk_id)) for _ in checker.run(): pass self.assertTrue(checker.report()) if expected_raw_meta: self.assertDictEqual(expected_raw_meta, raw_meta) continue self.assertNotIn(chunk_xattr_keys['chunk_id'], raw_meta) self.assertNotIn(chunk_xattr_keys['container_id'], raw_meta) self.assertNotIn(chunk_xattr_keys['content_path'], raw_meta) self.assertNotIn(chunk_xattr_keys['content_version'], raw_meta) self.assertNotIn(chunk_xattr_keys['content_id'], raw_meta) self.assertIn(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + chunk_id, raw_meta) for k in raw_meta.keys(): if k.startswith('oio:'): self.fail('old fullpath always existing') self.assertEqual(raw_meta[chunk_xattr_keys['oio_version']], OIO_VERSION) def _test_converter_single_chunk(self, chunk, expected_errors=0): chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._convert_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}, expected_errors=expected_errors) def test_converter(self): chunk = random.choice(self.chunks) self._test_converter_single_chunk(chunk) def test_converter_old_chunk(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._convert_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_old_chunk_with_wrong_path(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path + '+', self.version, self.content_id) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._convert_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_old_chunk_with_wrong_content_id(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, '0123456789ABCDEF0123456789ABCDEF') chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._convert_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_old_chunk_with_old_fullpath(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id, add_old_fullpath=True) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._convert_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_old_chunk_with_old_fullpath_and_wrong_path(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id, add_old_fullpath=True) convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path + '+', self.version, self.content_id) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._convert_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_old_chunk_with_wrong_fullpath(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, 'None', '0123456789ABCDEF0123456789ABCDEF', add_old_fullpath=True) convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._convert_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_linked_chunk(self): self.api.object_link( self.account, self.container, self.path, self.account, self.container, self.path + '.link') linked_meta, linked_chunks = self.api.object_locate( self.account, self.container, self.path + '.link') self.assertNotEqual(self.content_id, linked_meta['id']) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) for c in linked_chunks: if chunk_volume == c['url'].split('/')[2]: linked_chunk_id2 = c['url'].split('/')[3] break linked_chunk = random.choice(linked_chunks) linked_chunk_volume = linked_chunk['url'].split('/')[2] linked_chunk_id = linked_chunk['url'].split('/')[3] linked_chunk_path = self._chunk_path(linked_chunk) for c in self.chunks: if linked_chunk_volume == c['url'].split('/')[2]: chunk_id2 = c['url'].split('/')[3] break self._convert_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id), linked_chunk_id2: (self.account, self.container, self.path + '.link', linked_meta['version'], linked_meta['id'])}) self._convert_and_check( linked_chunk_volume, linked_chunk_path, {chunk_id2: (self.account, self.container, self.path, self.version, self.content_id), linked_chunk_id: (self.account, self.container, self.path + '.link', linked_meta['version'], linked_meta['id'])}) def test_converter_old_linked_chunk(self): self.api.object_link( self.account, self.container, self.path, self.account, self.container, self.path + '.link') linked_meta, linked_chunks = self.api.object_locate( self.account, self.container, self.path + '.link') self.assertNotEqual(self.content_id, linked_meta['id']) for c in linked_chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path + '.link', 'None', '0123456789ABCDEF0123456789ABCDEF', add_old_fullpath=True) for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) for c in linked_chunks: if chunk_volume == c['url'].split('/')[2]: linked_chunk_id2 = c['url'].split('/')[3] break linked_chunk = random.choice(linked_chunks) linked_chunk_volume = linked_chunk['url'].split('/')[2] linked_chunk_id = linked_chunk['url'].split('/')[3] linked_chunk_path = self._chunk_path(linked_chunk) for c in self.chunks: if linked_chunk_volume == c['url'].split('/')[2]: chunk_id2 = c['url'].split('/')[3] break self._convert_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id), linked_chunk_id2: (self.account, self.container, self.path + '.link', linked_meta['version'], linked_meta['id'])}) self._convert_and_check( linked_chunk_volume, linked_chunk_path, {chunk_id2: (self.account, self.container, self.path, self.version, self.content_id), linked_chunk_id: (self.account, self.container, self.path + '.link', linked_meta['version'], linked_meta['id'])}) def test_converter_old_chunk_with_link_on_same_object(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) self.api.object_link( self.account, self.container, self.path, self.account, self.container, self.path) linked_meta, linked_chunks = self.api.object_locate( self.account, self.container, self.path) self.assertNotEqual(self.content_id, linked_meta['id']) linked_chunk = random.choice(linked_chunks) linked_chunk_volume = linked_chunk['url'].split('/')[2] linked_chunk_id = linked_chunk['url'].split('/')[3] linked_chunk_path = self._chunk_path(linked_chunk) # old xattr not removed _, expected_raw_meta = read_chunk_metadata(linked_chunk_path, linked_chunk_id) expected_raw_meta[chunk_xattr_keys['oio_version']] = OIO_VERSION self._convert_and_check( linked_chunk_volume, linked_chunk_path, {linked_chunk_id: (self.account, self.container, self.path, linked_meta['version'], linked_meta['id'])}, expected_raw_meta=expected_raw_meta, expected_errors=1) def test_converter_old_linked_chunk_with_link_on_same_object(self): self.api.object_link( self.account, self.container, self.path, self.account, self.container, self.path + '.link') linked_meta, linked_chunks = self.api.object_locate( self.account, self.container, self.path + '.link') self.assertNotEqual(self.content_id, linked_meta['id']) for c in linked_chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path + '.link', 'None', '0123456789ABCDEF0123456789ABCDEF', add_old_fullpath=True) for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id, add_old_fullpath=True) self.api.object_link( self.account, self.container, self.path + '.link', self.account, self.container, self.path + '.link') linked_meta, linked_chunks = self.api.object_locate( self.account, self.container, self.path + '.link') self.assertNotEqual(self.content_id, linked_meta['id']) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) for c in linked_chunks: if chunk_volume == c['url'].split('/')[2]: linked_chunk_id2 = c['url'].split('/')[3] break linked_chunk = random.choice(linked_chunks) linked_chunk_volume = linked_chunk['url'].split('/')[2] linked_chunk_id = linked_chunk['url'].split('/')[3] linked_chunk_path = self._chunk_path(linked_chunk) for c in self.chunks: if linked_chunk_volume == c['url'].split('/')[2]: chunk_id2 = c['url'].split('/')[3] break self._convert_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id), linked_chunk_id2: (self.account, self.container, self.path + '.link', linked_meta['version'], linked_meta['id'])}) self._convert_and_check( linked_chunk_volume, linked_chunk_path, {chunk_id2: (self.account, self.container, self.path, self.version, self.content_id), linked_chunk_id: (self.account, self.container, self.path + '.link', linked_meta['version'], linked_meta['id'])}) def test_converter_with_versioning(self): self.api.container_set_properties( self.account, self.container, system={'sys.m2.policy.version': '2'}) self.api.object_create( self.account, self.container, obj_name=self.path, data='version') versioned_meta, versioned_chunks = self.api.object_locate( self.account, self.container, self.path) self.assertNotEqual(self.content_id, versioned_meta['id']) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) versioned_chunk = random.choice(versioned_chunks) versioned_chunk_volume = versioned_chunk['url'].split('/')[2] versioned_chunk_id = versioned_chunk['url'].split('/')[3] versioned_chunk_path = self._chunk_path(versioned_chunk) self._convert_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) self._convert_and_check( versioned_chunk_volume, versioned_chunk_path, {versioned_chunk_id: (self.account, self.container, self.path, versioned_meta['version'], versioned_meta['id'])}) def test_converter_old_chunk_with_versioning(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) self.api.container_set_properties( self.account, self.container, system={'sys.m2.policy.version': '2'}) self.api.object_create( self.account, self.container, obj_name=self.path, data='version') versioned_meta, versioned_chunks = self.api.object_locate( self.account, self.container, self.path) self.assertNotEqual(self.content_id, versioned_meta['id']) for c in versioned_chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, versioned_meta['version'], versioned_meta['id']) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) versioned_chunk = random.choice(versioned_chunks) versioned_chunk_volume = versioned_chunk['url'].split('/')[2] versioned_chunk_id = versioned_chunk['url'].split('/')[3] versioned_chunk_path = self._chunk_path(versioned_chunk) self._convert_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) self._convert_and_check( versioned_chunk_volume, versioned_chunk_path, {versioned_chunk_id: (self.account, self.container, self.path, versioned_meta['version'], versioned_meta['id'])}) def test_converter_file_not_found(self): """ Test what happens when the BlobConverter encounters a chunk with neither a fullpath extended attribute, not any of the legacy attributes. """ victim = random.choice(self.chunks) path = self._chunk_path(victim) chunk_volume = victim['url'].split('/')[2] os.remove(path) with patch('oio.blob.converter.BlobConverter.recover_chunk_fullpath') \ as recover: self._convert_and_check(chunk_volume, path, {}, expected_errors=1) recover.assert_not_called() def test_recover_missing_old_fullpath(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) victim = random.choice(self.chunks) self._test_converter_single_chunk(victim) def test_recover_missing_content_path(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id, add_old_fullpath=True) victim = random.choice(self.chunks) path = self._chunk_path(victim) remove_xattr(path, chunk_xattr_keys['content_path']) self._test_converter_single_chunk(victim) def test_recover_missing_old_fullpath_and_content_path(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) victim = random.choice(self.chunks) path = self._chunk_path(victim) remove_xattr(path, chunk_xattr_keys['content_path']) self._test_converter_single_chunk(victim) def test_recover_missing_fullpath(self): """ Test what happens when the BlobConverter encounters a chunk with neither a fullpath extended attribute, not any of the legacy attributes. """ victim = random.choice(self.chunks) path = self._chunk_path(victim) remove_fullpath_xattr(path) self._test_converter_single_chunk(victim) def test_recover_missing_fullpath_not_indexed(self): """ Test what happens when the BlobConverter encounters a chunk with neither a fullpath extended attribute, not any of the legacy attributes, and the chunk does not appear in rdir. """ victim = random.choice(self.chunks) path = self._chunk_path(victim) remove_fullpath_xattr(path) self._deindex_chunk(victim) conf = dict(self.conf) conf['volume'] = self.rawx_volumes[self._chunk_volume_id(victim)] converter = BlobConverter(conf) self.assertRaises(KeyError, converter.recover_chunk_fullpath, path) def test_recover_missing_fullpath_orphan_chunk(self): """ Test what happens when the BlobConverter encounters a chunk with neither a fullpath extended attribute, not any of the legacy attributes, and the chunk does not appear in object description. """ victim = random.choice(self.chunks) path = self._chunk_path(victim) remove_fullpath_xattr(path) cbean = { 'content': self.content_id, 'hash': victim['hash'], 'id': victim['url'], 'size': victim['size'], 'pos': victim['pos'], 'type': 'chunk' } self.api.container.container_raw_delete( self.account, self.container, data=[cbean]) conf = dict(self.conf) conf['volume'] = self.rawx_volumes[self._chunk_volume_id(victim)] converter = BlobConverter(conf) self.assertRaises(OrphanChunk, converter.recover_chunk_fullpath, path)
class TestContentVersioning(BaseTestCase): def setUp(self): super(TestContentVersioning, self).setUp() self.api = ObjectStorageApi(self.conf['namespace']) self.container = random_str(8) system = {'sys.m2.policy.version': '3'} self.api.container_create(self.account, self.container, system=system) def test_versioning_enabled(self): props = self.api.container_get_properties( self.account, self.container) self.assertEqual('3', props['system']['sys.m2.policy.version']) def test_list_versions(self): self.api.object_create(self.account, self.container, obj_name="versioned", data="content0") self.api.object_create(self.account, self.container, obj_name="versioned", data="content1") listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(2, len(objects)) self.assertNotEqual(objects[0]['version'], objects[1]['version']) def test_container_purge(self): # many contents for i in range(0, 4): self.api.object_create(self.account, self.container, obj_name="versioned", data="content") listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(4, len(objects)) oldest_version = min(objects, key=lambda x: x['version']) # use the maxvers of the container configuration self.api.container_purge(self.account, self.container) listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(3, len(objects)) self.assertNotIn(oldest_version, [x['version'] for x in objects]) oldest_version = min(objects, key=lambda x: x['version']) # use the maxvers of the request self.api.container_purge(self.account, self.container, maxvers=1) listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(1, len(objects)) self.assertNotIn(oldest_version, [x['version'] for x in objects]) def test_content_purge(self): # many contents for i in range(0, 4): self.api.object_create(self.account, self.container, obj_name="versioned", data="content") listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(4, len(objects)) oldest_version = min(objects, key=lambda x: x['version']) # use the maxvers of the container configuration self.api.container.content_purge(self.account, self.container, "versioned") listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(3, len(objects)) self.assertNotIn(oldest_version, [x['version'] for x in objects]) oldest_version = min(objects, key=lambda x: x['version']) # use the maxvers of the request self.api.container.content_purge(self.account, self.container, "versioned", maxvers=1) listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(1, len(objects)) self.assertNotIn(oldest_version, [x['version'] for x in objects]) # other contents for i in range(0, 4): self.api.object_create(self.account, self.container, obj_name="versioned2", data="content"+str(i)) listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(5, len(objects)) # use the maxvers of the container configuration self.api.container.content_purge(self.account, self.container, "versioned") listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(5, len(objects)) def test_delete_exceeding_version(self): def check_num_objects_and_get_oldest_version(expected): listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(expected, len(objects)) return min(objects, key=lambda x: x['version']) system = {'sys.m2.policy.version.delete_exceeding': '1'} self.api.container_set_properties(self.account, self.container, system=system) self.api.object_create(self.account, self.container, obj_name="versioned", data="content0") self.api.object_create(self.account, self.container, obj_name="versioned", data="content1") self.api.object_create(self.account, self.container, obj_name="versioned", data="content2") oldest_version = check_num_objects_and_get_oldest_version(3) self.api.object_create(self.account, self.container, obj_name="versioned", data="content3") new_oldest_version = check_num_objects_and_get_oldest_version(3) self.assertLess(oldest_version['version'], new_oldest_version['version']) def test_change_flag_delete_exceeding_versions(self): def check_num_objects(expected): listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(expected, len(objects)) for i in range(5): self.api.object_create(self.account, self.container, obj_name="versioned", data="content"+str(i)) check_num_objects(5) system = {'sys.m2.policy.version.delete_exceeding': '1'} self.api.container_set_properties(self.account, self.container, system=system) self.api.object_create(self.account, self.container, obj_name="versioned", data="content5") check_num_objects(3) for i in range(6, 10): self.api.object_create(self.account, self.container, obj_name="versioned", data="content"+str(i)) check_num_objects(3) system['sys.m2.policy.version.delete_exceeding'] = '0' self.api.container_set_properties(self.account, self.container, system=system) self.api.object_create(self.account, self.container, obj_name="versioned", data="content11") check_num_objects(4)
class TestBlobAuditor(BaseTestCase): def setUp(self): super(TestBlobAuditor, self).setUp() self.container = random_str(16) self.cid = cid_from_name(self.account, self.container) self.path = random_str(16) self.api = ObjectStorageApi(self.ns) self.blob_client = BlobClient(self.conf) self.api.container_create(self.account, self.container) _, chunks = self.api.container.content_prepare( self.account, self.container, self.path, 1) services = self.conscience.all_services('rawx') self.rawx_volumes = dict() for rawx in services: tags = rawx['tags'] service_id = tags.get('tag.service_id', None) if service_id is None: service_id = rawx['addr'] volume = tags.get('tag.vol', None) self.rawx_volumes[service_id] = volume self.api.object_create( self.account, self.container, obj_name=self.path, data="chunk") meta, self.chunks = self.api.object_locate( self.account, self.container, self.path) self.version = meta['version'] self.content_id = meta['id'] def _chunk_path(self, chunk): url = chunk['url'] volume_id = url.split('/', 3)[2] chunk_id = url.split('/', 3)[3] volume = self.rawx_volumes[volume_id] return volume + '/' + chunk_id[:3] + '/' + chunk_id def test_audit(self): chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] auditor = BlobAuditorWorker(self.conf, None, self.rawx_volumes[chunk_volume]) auditor.chunk_audit(self._chunk_path(chunk), chunk_id) def test_audit_old_chunk(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] auditor = BlobAuditorWorker(self.conf, None, self.rawx_volumes[chunk_volume]) auditor.chunk_audit(self._chunk_path(chunk), chunk_id) def test_audit_linked_chunk(self): self.api.object_link( self.account, self.container, self.path, self.account, self.container, self.path + '.link') chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) auditor = BlobAuditorWorker(self.conf, None, self.rawx_volumes[chunk_volume]) auditor.chunk_audit(chunk_path, chunk_id) linked_meta, linked_chunks = self.api.object_locate( self.account, self.container, self.path + '.link') self.assertNotEqual(self.content_id, linked_meta['id']) linked_chunk = random.choice(linked_chunks) linked_chunk_id = linked_chunk['url'].split('/')[3] linked_chunk_path = self._chunk_path(linked_chunk) auditor.chunk_audit(linked_chunk_path, linked_chunk_id) auditor.chunk_audit(chunk_path, chunk_id) copy_chunk(chunk_path, chunk_path + '.copy') auditor.chunk_audit(chunk_path + '.copy', chunk_id) self.api.object_delete( self.account, self.container, self.path) auditor.chunk_audit(linked_chunk_path, linked_chunk_id) self.assertRaises(OrphanChunk, auditor.chunk_audit, chunk_path + '.copy', chunk_id) def test_audit_with_versioning(self): self.api.container_set_properties( self.account, self.container, system={'sys.m2.policy.version': '2'}) self.api.object_create( self.account, self.container, obj_name=self.path, data="version") chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) auditor = BlobAuditorWorker(self.conf, None, self.rawx_volumes[chunk_volume]) auditor.chunk_audit(chunk_path, chunk_id) versioned_meta, versioned_chunks = self.api.object_locate( self.account, self.container, self.path) self.assertNotEqual(self.content_id, versioned_meta['id']) versioned_chunk = random.choice(versioned_chunks) versioned_chunk_volume = versioned_chunk['url'].split('/')[2] versioned_chunk_id = versioned_chunk['url'].split('/')[3] versioned_chunk_path = self._chunk_path(versioned_chunk) versioned_auditor = BlobAuditorWorker( self.conf, None, self.rawx_volumes[versioned_chunk_volume]) versioned_auditor.chunk_audit(versioned_chunk_path, versioned_chunk_id) auditor.chunk_audit(chunk_path, chunk_id) copy_chunk(chunk_path, chunk_path + '.copy') auditor.chunk_audit(chunk_path + '.copy', chunk_id) self.api.object_delete( self.account, self.container, self.path, version=self.version) versioned_auditor.chunk_audit(versioned_chunk_path, versioned_chunk_id) self.assertRaises(OrphanChunk, auditor.chunk_audit, chunk_path + '.copy', chunk_id)
class TestContentVersioning(BaseTestCase): def setUp(self): super(TestContentVersioning, self).setUp() self.api = ObjectStorageApi(self.conf['namespace']) self.container = random_str(8) system = {'sys.m2.policy.version': '3'} self.wait_for_score(('meta2', )) self.api.container_create(self.account, self.container, system=system) def test_versioning_enabled(self): props = self.api.container_get_properties(self.account, self.container) self.assertEqual('3', props['system']['sys.m2.policy.version']) def test_list_versions(self): self.api.object_create(self.account, self.container, obj_name="versioned", data="content0") self.api.object_create(self.account, self.container, obj_name="versioned", data="content1") listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(2, len(objects)) self.assertNotEqual(objects[0]['version'], objects[1]['version']) def test_container_purge(self): # many contents for i in range(0, 4): self.api.object_create(self.account, self.container, obj_name="versioned", data="content") listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(4, len(objects)) oldest_version = min(objects, key=lambda x: x['version']) # use the maxvers of the container configuration self.api.container_purge(self.account, self.container) listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(3, len(objects)) self.assertNotIn(oldest_version, [x['version'] for x in objects]) oldest_version = min(objects, key=lambda x: x['version']) # use the maxvers of the request self.api.container_purge(self.account, self.container, maxvers=1) listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(1, len(objects)) self.assertNotIn(oldest_version, [x['version'] for x in objects]) def test_content_purge(self): # many contents for i in range(0, 4): self.api.object_create(self.account, self.container, obj_name="versioned", data="content") listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(4, len(objects)) oldest_version = min(objects, key=lambda x: x['version']) # use the maxvers of the container configuration self.api.container.content_purge(self.account, self.container, "versioned") listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(3, len(objects)) self.assertNotIn(oldest_version, [x['version'] for x in objects]) oldest_version = min(objects, key=lambda x: x['version']) # use the maxvers of the request self.api.container.content_purge(self.account, self.container, "versioned", maxvers=1) listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(1, len(objects)) self.assertNotIn(oldest_version, [x['version'] for x in objects]) # other contents for i in range(0, 4): self.api.object_create(self.account, self.container, obj_name="versioned2", data="content" + str(i)) listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(5, len(objects)) # use the maxvers of the container configuration self.api.container.content_purge(self.account, self.container, "versioned") listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(5, len(objects)) def test_delete_exceeding_version(self): def check_num_objects_and_get_oldest_version(expected_objects, expected_deleted_aliases, oldest_version): listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] nb_objects = 0 nb_deleted = 0 new_oldest_version = 0 for obj in objects: if obj['deleted']: nb_deleted += 1 else: nb_objects += 1 if new_oldest_version == 0 \ or new_oldest_version > obj['version']: new_oldest_version = obj['version'] self.assertEqual(expected_objects, nb_objects) self.assertEqual(expected_deleted_aliases, nb_deleted) if oldest_version is not None: self.assertLess(oldest_version, new_oldest_version) return new_oldest_version system = {'sys.m2.policy.version.delete_exceeding': '1'} self.api.container_set_properties(self.account, self.container, system=system) self.api.object_create(self.account, self.container, obj_name="versioned", data="content0") oldest_version = check_num_objects_and_get_oldest_version(1, 0, None) self.api.object_create(self.account, self.container, obj_name="versioned", data="content1") self.assertEqual(oldest_version, check_num_objects_and_get_oldest_version(2, 0, None)) self.api.object_create(self.account, self.container, obj_name="versioned", data="content2") self.assertEqual(oldest_version, check_num_objects_and_get_oldest_version(3, 0, None)) self.api.object_create(self.account, self.container, obj_name="versioned", data="content3") oldest_version = check_num_objects_and_get_oldest_version( 3, 0, oldest_version) self.api.object_delete(self.account, self.container, "versioned") self.assertEqual(oldest_version, check_num_objects_and_get_oldest_version(3, 1, None)) self.api.object_create(self.account, self.container, obj_name="versioned", data="content4") oldest_version = check_num_objects_and_get_oldest_version( 3, 1, oldest_version) self.api.object_create(self.account, self.container, obj_name="versioned", data="content5") oldest_version = check_num_objects_and_get_oldest_version( 3, 1, oldest_version) self.api.object_create(self.account, self.container, obj_name="versioned", data="content6") # FIXME(adu) The deleted alias should be deleted at the same time oldest_version = check_num_objects_and_get_oldest_version( 3, 1, oldest_version) self.api.object_create(self.account, self.container, obj_name="versioned", data="content7") oldest_version = check_num_objects_and_get_oldest_version( 3, 1, oldest_version) def test_change_flag_delete_exceeding_versions(self): def check_num_objects(expected): listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(expected, len(objects)) for i in range(5): self.api.object_create(self.account, self.container, obj_name="versioned", data="content" + str(i)) check_num_objects(5) system = {'sys.m2.policy.version.delete_exceeding': '1'} self.api.container_set_properties(self.account, self.container, system=system) self.api.object_create(self.account, self.container, obj_name="versioned", data="content5") check_num_objects(3) for i in range(6, 10): self.api.object_create(self.account, self.container, obj_name="versioned", data="content" + str(i)) check_num_objects(3) system['sys.m2.policy.version.delete_exceeding'] = '0' self.api.container_set_properties(self.account, self.container, system=system) self.api.object_create(self.account, self.container, obj_name="versioned", data="content11") check_num_objects(4) def test_purge_objects_with_delete_marker(self): def check_num_objects(expected): listing = self.api.object_list(self.account, self.container, versions=True) objects = listing['objects'] self.assertEqual(expected, len(objects)) for i in range(5): self.api.object_create(self.account, self.container, obj_name="versioned", data="content" + str(i)) check_num_objects(5) self.api.object_delete(self.account, self.container, "versioned") self.assertRaises(NoSuchObject, self.api.object_locate, self.account, self.container, "versioned") check_num_objects(6) self.api.container.content_purge(self.account, self.container, "versioned") self.assertRaises(NoSuchObject, self.api.object_locate, self.account, self.container, "versioned") check_num_objects(4) system = {'sys.m2.keep_deleted_delay': '1'} self.api.container_set_properties(self.account, self.container, system=system) time.sleep(2) self.api.container.content_purge(self.account, self.container, "versioned") check_num_objects(0) def test_list_objects(self): resp = self.api.object_list(self.account, self.container) self.assertEqual(0, len(list(resp['objects']))) self.assertFalse(resp.get('truncated')) def _check_objects(expected_objects, objects): self.assertEqual(len(expected_objects), len(objects)) for i in range(len(expected_objects)): self.assertEqual(expected_objects[i]['name'], objects[i]['name']) self.assertEqual(int(expected_objects[i]['version']), int(objects[i]['version'])) self.assertEqual(true_value(expected_objects[i]['deleted']), true_value(objects[i]['deleted'])) all_versions = dict() def _create_object(obj_name, all_versions): self.api.object_create(self.account, self.container, obj_name=obj_name, data="test") versions = all_versions.get(obj_name, list()) versions.append( self.api.object_show(self.account, self.container, obj_name)) all_versions[obj_name] = versions def _delete_object(obj_name, all_versions): self.api.object_delete(self.account, self.container, obj_name) versions = all_versions.get(obj_name, list()) versions.append( self.api.object_show(self.account, self.container, obj_name)) all_versions[obj_name] = versions def _get_current_objects(all_versions): current_objects = list() obj_names = sorted(all_versions.keys()) for obj_name in obj_names: obj = all_versions[obj_name][-1] if not true_value(obj['deleted']): current_objects.append(obj) return current_objects def _get_object_versions(all_versions): object_versions = list() obj_names = sorted(all_versions.keys()) for obj_name in obj_names: versions = all_versions[obj_name] versions.reverse() object_versions += versions versions.reverse() return object_versions # 0 object expected_current_objects = _get_current_objects(all_versions) expected_object_versions = _get_object_versions(all_versions) resp = self.api.object_list(self.account, self.container, limit=3) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=2) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=1) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, versions=True) _check_objects(expected_object_versions, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, versions=True, limit=3) _check_objects(expected_object_versions, list(resp['objects'])) self.assertFalse(resp.get('truncated')) # 3 objects with 1 version for i in range(3): _create_object("versioned" + str(i), all_versions) expected_current_objects = _get_current_objects(all_versions) expected_object_versions = _get_object_versions(all_versions) resp = self.api.object_list(self.account, self.container) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=3) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=2) _check_objects(expected_current_objects[:2], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned1', resp['next_marker']) resp = self.api.object_list(self.account, self.container, limit=1) _check_objects(expected_current_objects[:1], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned0', resp['next_marker']) resp = self.api.object_list(self.account, self.container, versions=True) _check_objects(expected_object_versions, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, versions=True, limit=3) _check_objects(expected_object_versions[:3], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0') _check_objects(expected_current_objects[1:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', limit=1) _check_objects(expected_current_objects[1:2], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned1', resp['next_marker']) resp = self.api.object_list(self.account, self.container, marker='versioned0', versions=True) _check_objects(expected_object_versions[1:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', versions=True, limit=3) _check_objects(expected_object_versions[1:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) # 3 objects with 2 versions for i in range(3): _create_object("versioned" + str(i), all_versions) expected_current_objects = _get_current_objects(all_versions) expected_object_versions = _get_object_versions(all_versions) resp = self.api.object_list(self.account, self.container) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=3) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=2) _check_objects(expected_current_objects[:2], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned1', resp['next_marker']) resp = self.api.object_list(self.account, self.container, limit=1) _check_objects(expected_current_objects[:1], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned0', resp['next_marker']) resp = self.api.object_list(self.account, self.container, versions=True) _check_objects(expected_object_versions, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, versions=True, limit=3) _check_objects(expected_object_versions[:3], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned1', resp['next_marker']) resp = self.api.object_list(self.account, self.container, marker='versioned0') _check_objects(expected_current_objects[1:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', limit=1) _check_objects(expected_current_objects[1:2], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned1', resp['next_marker']) resp = self.api.object_list(self.account, self.container, marker='versioned0', versions=True) _check_objects(expected_object_versions[2:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', versions=True, limit=3) _check_objects(expected_object_versions[2:5], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned2', resp['next_marker']) # 3 objects with 2 versions and 1 object with delete marker _delete_object("versioned1", all_versions) expected_current_objects = _get_current_objects(all_versions) expected_object_versions = _get_object_versions(all_versions) resp = self.api.object_list(self.account, self.container) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=3) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=2) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=1) _check_objects(expected_current_objects[:1], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned0', resp['next_marker']) resp = self.api.object_list(self.account, self.container, versions=True) _check_objects(expected_object_versions, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, versions=True, limit=3) _check_objects(expected_object_versions[:3], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned1', resp['next_marker']) resp = self.api.object_list(self.account, self.container, marker='versioned0') _check_objects(expected_current_objects[1:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', limit=1) _check_objects(expected_current_objects[1:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', versions=True) _check_objects(expected_object_versions[2:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', versions=True, limit=3) _check_objects(expected_object_versions[2:5], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned1', resp['next_marker']) # 3 objects with 2 versions and 2 objects with delete marker _delete_object("versioned0", all_versions) expected_current_objects = _get_current_objects(all_versions) expected_object_versions = _get_object_versions(all_versions) resp = self.api.object_list(self.account, self.container) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=3) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=2) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=1) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, versions=True) _check_objects(expected_object_versions, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, versions=True, limit=3) _check_objects(expected_object_versions[:3], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned0', resp['next_marker']) resp = self.api.object_list(self.account, self.container, marker='versioned0') _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', limit=1) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', versions=True) _check_objects(expected_object_versions[3:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', versions=True, limit=3) _check_objects(expected_object_versions[3:6], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned1', resp['next_marker']) # 3 objects with 2 versions and 3 objects with delete marker _delete_object("versioned2", all_versions) expected_current_objects = _get_current_objects(all_versions) expected_object_versions = _get_object_versions(all_versions) resp = self.api.object_list(self.account, self.container) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=3) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=2) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=1) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, versions=True) _check_objects(expected_object_versions, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, versions=True, limit=3) _check_objects(expected_object_versions[:3], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned0', resp['next_marker']) resp = self.api.object_list(self.account, self.container, marker='versioned0') _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', limit=1) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', versions=True) _check_objects(expected_object_versions[3:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', versions=True, limit=3) _check_objects(expected_object_versions[3:6], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned1', resp['next_marker']) # 3 objects with 2 versions and 3 objects with delete marker # (1 current version and 2 non current versions) _create_object("versioned0", all_versions) expected_current_objects = _get_current_objects(all_versions) expected_object_versions = _get_object_versions(all_versions) resp = self.api.object_list(self.account, self.container) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=3) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=2) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, limit=1) _check_objects(expected_current_objects, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, versions=True) _check_objects(expected_object_versions, list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, versions=True, limit=3) _check_objects(expected_object_versions[:3], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned0', resp['next_marker']) resp = self.api.object_list(self.account, self.container, marker='versioned0') _check_objects(expected_current_objects[1:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', limit=1) _check_objects(expected_current_objects[1:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', versions=True) _check_objects(expected_object_versions[4:], list(resp['objects'])) self.assertFalse(resp.get('truncated')) resp = self.api.object_list(self.account, self.container, marker='versioned0', versions=True, limit=3) _check_objects(expected_object_versions[4:7], list(resp['objects'])) self.assertTrue(resp.get('truncated')) self.assertEqual('versioned1', resp['next_marker'])
class TestObjectStorageAPI(BaseTestCase): def setUp(self): super(TestObjectStorageAPI, self).setUp() self.api = ObjectStorageApi(self.ns, endpoint=self.uri) self.created = list() def tearDown(self): super(TestObjectStorageAPI, self).tearDown() for ct, name in self.created: try: self.api.object_delete(self.account, ct, name) except Exception: logging.exception("Failed to delete %s/%s/%s//%s", self.ns, self.account, ct, name) def _create(self, name, metadata=None): return self.api.container_create(self.account, name, properties=metadata) def _delete(self, name): self.api.container_delete(self.account, name) def _clean(self, name, clear=False): if clear: # must clean properties before self.api.container_del_properties(self.account, name, []) self._delete(name) def _get_properties(self, name, properties=None): return self.api.container_get_properties(self.account, name, properties=properties) def _set_properties(self, name, properties=None): return self.api.container_set_properties(self.account, name, properties=properties) def test_container_show(self): # container_show on unknown container name = random_str(32) self.assertRaises(exc.NoSuchContainer, self.api.container_show, self.account, name) self._create(name) # container_show on existing container res = self.api.container_show(self.account, name) self.assertIsNot(res['properties'], None) self._delete(name) # container_show on deleted container self.assertRaises(exc.NoSuchContainer, self.api.container_show, self.account, name) def test_container_create(self): name = random_str(32) res = self._create(name) self.assertEqual(res, True) # second create res = self._create(name) self.assertEqual(res, False) # clean self._delete(name) def test_create_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } res = self._create(name, metadata) self.assertEqual(res, True) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) def test_container_delete(self): name = random_str(32) # container_delete on unknown container self.assertRaises(exc.NoSuchContainer, self.api.container_delete, self.account, name) res = self._create(name) self.assertEqual(res, True) # container_delete on existing container self._delete(name) # verify deleted self.assertRaises(exc.NoSuchContainer, self.api.container_show, self.account, name) # second delete self.assertRaises(exc.NoSuchContainer, self.api.container_delete, self.account, name) # verify deleted self.assertRaises(exc.NoSuchContainer, self.api.container_show, self.account, name) def test_container_get_properties(self): name = random_str(32) # container_get_properties on unknown container self.assertRaises(exc.NoSuchContainer, self.api.container_get_properties, self.account, name) res = self._create(name) self.assertEqual(res, True) # container_get_properties on existing container data = self.api.container_get_properties(self.account, name) self.assertEqual(data['properties'], {}) self.assertIsNot(data['system'], None) self.assertIn("sys.user.name", data['system']) # container_get_properties metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } self._set_properties(name, metadata) data = self.api.container_get_properties(self.account, name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # container_get_properties on deleted container self.assertRaises(exc.NoSuchContainer, self.api.container_get_properties, self.account, name) def test_container_get_properties_filtered(self): self.skipTest("Server side properties filtering not implemented") name = random_str(32) res = self._create(name) self.assertEqual(res, True) # container_get_properties on existing container data = self.api.container_get_properties(self.account, name) self.assertEqual(data['properties'], {}) # container_get_properties metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } self._set_properties(name, metadata) # container_get_properties specify key key = metadata.keys().pop(0) data = self.api.container_get_properties(self.account, name, [key]) self.assertEqual({key: metadata[key]}, data['properties']) # clean self._clean(name, True) def test_container_set_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } # container_set_properties on unknown container self.assertRaises(exc.NoSuchContainer, self.api.container_set_properties, self.account, name, metadata) res = self._create(name) self.assertEqual(res, True) # container_set_properties on existing container self.api.container_set_properties(self.account, name, metadata) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # container_set_properties key = random_str(32) value = random_str(32) metadata2 = {key: value} self._set_properties(name, metadata2) metadata.update(metadata2) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # container_set_properties overwrite key key = metadata.keys().pop(0) value = random_str(32) metadata3 = {key: value} metadata.update(metadata3) self.api.container_set_properties(self.account, name, metadata3) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # container_set_properties on deleted container self.assertRaises(exc.NoSuchContainer, self.api.container_set_properties, self.account, name, metadata) def test_del_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } # container_del_properties on unknown container self.assertRaises(exc.NoSuchContainer, self.api.container_del_properties, self.account, name, []) res = self._create(name, metadata) self.assertEqual(res, True) key = metadata.keys().pop() del metadata[key] # container_del_properties on existing container self.api.container_del_properties(self.account, name, [key]) data = self._get_properties(name) self.assertNotIn(key, data['properties']) key = random_str(32) # We do not check if a property exists before deleting it # self.assertRaises( # exc.NoSuchContainer, self.api.container_del_properties, # self.account, name, [key]) self.api.container_del_properties(self.account, name, [key]) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # container_del_properties on deleted container self.assertRaises(exc.NoSuchContainer, self.api.container_del_properties, self.account, name, metadata.keys()) def test_object_create_mime_type(self): name = random_str(32) self.api.object_create(self.account, name, data="data", obj_name=name, mime_type='text/custom') meta, _ = self.api.object_locate(self.account, name, name) self.assertEqual(meta['mime_type'], 'text/custom') def _upload_data(self, name): chunksize = int(self.conf["chunk_size"]) size = int(chunksize * 12) data = random_data(int(size)) self.api.object_create(self.account, name, obj_name=name, data=data) self.created.append((name, name)) _, chunks = self.api.object_locate(self.account, name, name) logging.debug("Chunks: %s", chunks) return sort_chunks(chunks, False), data def _fetch_range(self, name, range_): if not isinstance(range_[0], tuple): ranges = (range_, ) else: ranges = range_ stream = self.api.object_fetch(self.account, name, name, ranges=ranges)[1] data = "" for chunk in stream: data += chunk return data def test_object_fetch_range_start(self): """From 0 to somewhere""" name = random_str(16) _, data = self._upload_data(name) end = 666 fdata = self._fetch_range(name, (0, end)) self.assertEqual(len(fdata), end + 1) self.assertEqual(fdata, data[0:end + 1]) def test_object_fetch_range_end(self): """From somewhere to end""" name = random_str(16) chunks, data = self._upload_data(name) start = 666 last = max(chunks.keys()) end = chunks[last][0]['offset'] + chunks[last][0]['size'] fdata = self._fetch_range(name, (start, end)) self.assertEqual(len(fdata), len(data) - start) self.assertEqual(fdata, data[start:]) def test_object_fetch_range_metachunk_start(self): """From the start of the second metachunk to somewhere""" name = random_str(16) chunks, data = self._upload_data(name) start = chunks[1][0]['offset'] end = start + 666 fdata = self._fetch_range(name, (start, end)) self.assertEqual(len(fdata), end - start + 1) self.assertEqual(fdata, data[start:end + 1]) def test_object_fetch_range_metachunk_end(self): """From somewhere to end of the first metachunk""" name = random_str(16) chunks, data = self._upload_data(name) start = 666 end = chunks[0][0]['size'] - 1 fdata = self._fetch_range(name, (start, end)) self.assertEqual(len(fdata), end - start + 1) self.assertEqual(fdata, data[start:end + 1]) def test_object_fetch_range_2_metachunks(self): """ From somewhere in the first metachunk to somewhere in the second metachunk """ name = random_str(16) chunks, data = self._upload_data(name) start = 666 end = start + chunks[0][0]['size'] - 1 fdata = self._fetch_range(name, (start, end)) self.assertEqual(len(fdata), end - start + 1) self.assertEqual(fdata, data[start:end + 1]) def test_object_fetch_several_ranges(self): """ Download several ranges at once. """ name = random_str(16) chunks, data = self._upload_data(name) start = 666 end = start + chunks[0][0]['size'] - 1 fdata = self._fetch_range(name, ((start, end), (end + 1, end + 2))) self.assertEqual(len(fdata), end - start + 3) self.assertEqual(fdata, data[start:end + 3]) # Notice that we download some bytes from the second metachunk # before some from the first. fdata = self._fetch_range( name, ((chunks[0][0]['size'], chunks[0][0]['size'] + 2), (0, 1), (1, 2), (4, 6))) self.assertEqual(len(fdata), 10) self.assertEqual( fdata, data[chunks[0][0]['size']:chunks[0][0]['size'] + 3] + data[0:2] + data[1:3] + data[4:7]) def test_object_create_then_append(self): """Create an object then append data""" name = random_str(16) self.api.object_create(self.account, name, data="1" * 128, obj_name=name) _, size, _ = self.api.object_create(self.account, name, data="2" * 128, obj_name=name, append=True) self.assertEqual(size, 128) _, data = self.api.object_fetch(self.account, name, name) data = "".join(data) self.assertEqual(len(data), 256) self.assertEqual(data, "1" * 128 + "2" * 128) def test_object_create_from_append(self): """Create an object with append operation""" name = random_str(16) self.api.container_create(self.account, name) self.api.object_create(self.account, name, data="1" * 128, obj_name=name, append=True) _, data = self.api.object_fetch(self.account, name, name) data = "".join(data) self.assertEqual(len(data), 128) self.assertEqual(data, "1" * 128) def test_container_object_create_from_append(self): """Try to create container and object with append operation""" name = random_str(16) _chunks, size, checksum = self.api.object_create(self.account, name, data="1" * 128, obj_name=name, append=True) self.assertEqual(size, 128) meta = self.api.object_get_properties(self.account, name, name) self.assertEqual(meta.get('hash', "").lower(), checksum.lower()) def test_container_refresh(self): account = random_str(32) # container_refresh on unknown container name = random_str(32) self.assertRaises(exc.NoSuchContainer, self.api.container_refresh, account, name) self.api.container_create(account, name) time.sleep(0.5) # ensure container event have been processed # container_refresh on existing container self.api.container_refresh(account, name) time.sleep(0.5) # ensure container event have been processed res = self.api.container_list(account, prefix=name) name_container, nb_objects, nb_bytes, _ = res[0] self.assertEqual(name_container, name) self.assertEqual(nb_objects, 0) self.assertEqual(nb_bytes, 0) self.api.object_create(account, name, data="data", obj_name=name) time.sleep(0.5) # ensure container event have been processed # container_refresh on existing container with data self.api.container_refresh(account, name) time.sleep(0.5) # ensure container event have been processed res = self.api.container_list(account, prefix=name) name_container, nb_objects, nb_bytes, _ = res[0] self.assertEqual(name_container, name) self.assertEqual(nb_objects, 1) self.assertEqual(nb_bytes, 4) self.api.object_delete(account, name, name) time.sleep(0.5) # ensure container event have been processed self.api.container_delete(account, name) time.sleep(0.5) # ensure container event have been processed # container_refresh on deleted container self.assertRaises(exc.NoSuchContainer, self.api.container_refresh, account, name) self.api.account_delete(account) def test_container_refresh_user_not_found(self): name = random_str(32) self.api.account.container_update(name, name, {"mtime": time.time()}) self.api.container_refresh(name, name) containers = self.api.container_list(name) self.assertEqual(len(containers), 0) self.api.account_delete(name) def test_account_refresh(self): # account_refresh on unknown account account = random_str(32) self.assertRaises(exc.NoSuchAccount, self.api.account_refresh, account) # account_refresh on existing account self.api.account_create(account) self.api.account_refresh(account) time.sleep(0.5) # ensure container event have been processed res = self.api.account_show(account) self.assertEqual(res["bytes"], 0) self.assertEqual(res["objects"], 0) self.assertEqual(res["containers"], 0) name = random_str(32) self.api.object_create(account, name, data="data", obj_name=name) time.sleep(0.5) # ensure container event have been processed self.api.account_refresh(account) time.sleep(0.5) # ensure container event have been processed res = self.api.account_show(account) self.assertEqual(res["bytes"], 4) self.assertEqual(res["objects"], 1) self.assertEqual(res["containers"], 1) self.api.object_delete(account, name, name) time.sleep(0.5) # ensure container event have been processed self.api.container_delete(account, name) time.sleep(0.5) # ensure container event have been processed self.api.account_delete(account) # account_refresh on deleted account self.assertRaises(exc.NoSuchAccount, self.api.account_refresh, account) def test_all_accounts_refresh(self): # clear accounts accounts = self.api.account_list() for account in accounts: try: self.api.account_flush(account) self.api.account_delete(account) except exc.NoSuchAccount: # account remove in the meantime pass # all_accounts_refresh with 0 account self.api.all_accounts_refresh() # all_accounts_refresh with 2 account account1 = random_str(32) self.api.account_create(account1) account2 = random_str(32) self.api.account_create(account2) self.api.all_accounts_refresh() res = self.api.account_show(account1) self.assertEqual(res["bytes"], 0) self.assertEqual(res["objects"], 0) self.assertEqual(res["containers"], 0) res = self.api.account_show(account2) self.assertEqual(res["bytes"], 0) self.assertEqual(res["objects"], 0) self.assertEqual(res["containers"], 0) self.api.account_delete(account1) self.api.account_delete(account2) def test_account_flush(self): # account_flush on unknown account account = random_str(32) self.assertRaises(exc.NoSuchAccount, self.api.account_flush, account) # account_flush on existing account name1 = random_str(32) self.api.container_create(account, name1) name2 = random_str(32) self.api.container_create(account, name2) time.sleep(0.5) # ensure container event have been processed self.api.account_flush(account) containers = self.api.container_list(account) self.assertEqual(len(containers), 0) res = self.api.account_show(account) self.assertEqual(res["bytes"], 0) self.assertEqual(res["objects"], 0) self.assertEqual(res["containers"], 0) self.api.container_delete(account, name1) self.api.container_delete(account, name2) time.sleep(0.5) # ensure container event have been processed self.api.account_delete(account) # account_flush on deleted account self.assertRaises(exc.NoSuchAccount, self.api.account_flush, account) def test_object_create_then_truncate(self): """Create an object then truncate data""" name = random_str(16) self.api.object_create(self.account, name, data="1" * 128, obj_name=name) self.api.object_truncate(self.account, name, name, size=64) _, data = self.api.object_fetch(self.account, name, name) data = "".join(data) self.assertEqual(len(data), 64) self.assertEqual(data, "1" * 64) def test_object_create_append_then_truncate(self): """Create an object, append data then truncate on chunk boundary""" name = random_str(16) self.api.object_create(self.account, name, data="1" * 128, obj_name=name) _, size, _ = self.api.object_create(self.account, name, data="2" * 128, obj_name=name, append=True) self.assertEqual(size, 128) self.api.object_truncate(self.account, name, name, size=128) _, data = self.api.object_fetch(self.account, name, name) data = "".join(data) self.assertEqual(len(data), 128) self.assertEqual(data, "1" * 128) self.api.object_truncate(self.account, name, name, size=128) def test_object_create_then_invalid_truncate(self): """Create an object, append data then try to truncate outside object range""" name = random_str(16) self.api.object_create(self.account, name, data="1" * 128, obj_name=name) self.assertRaises(exc.OioException, self.api.object_truncate, self.account, name, name, size=-1) self.assertRaises(exc.OioException, self.api.object_truncate, self.account, name, name, size=129) def test_container_snapshot(self): name = random_str(16) self.api.container_create(self.account, name) test_object = "test_object" self.api.object_create(self.account, name, data="0" * 128, obj_name=test_object) # Snapshot cannot have same name and same account self.assertRaises(exc.ClientException, self.api.container_snapshot, self.account, name, self.account, name) snapshot_name = random_str(16) self.assertNotEqual(snapshot_name, name) # Non existing snapshot should work self.api.container_snapshot(self.account, name, self.account, snapshot_name) # Already taken snapshot name should failed self.assertRaises(exc.ClientException, self.api.container_snapshot, self.account, name, self.account, snapshot_name) # Check Container Frozen so create should failed self.assertRaises(exc.ServiceBusy, self.api.object_create, self.account, snapshot_name, data="1" * 128, obj_name="should_not_be_created") # fullpath is set on every chunk chunk_list = self.api.object_locate(self.account, name, test_object)[1] # check that every chunk is different from the target snapshot_list = self.api.object_locate(self.account, snapshot_name, test_object)[1] for c, t in zip(chunk_list, snapshot_list): self.assertNotEqual(c['url'], t['url']) # check target can be used self.api.object_create(self.account, name, data="0" * 128, obj_name="should_be_created") # Create and send copy of a object url_list = [c['url'] for c in chunk_list] copy_list = self.api._generate_copy(url_list) # every chunks should have the fullpath fullpath = self.api._generate_fullpath(self.account, snapshot_name, 'copy', 12456) self.api._send_copy(url_list, copy_list, fullpath[0]) # check that every copy exists pool_manager = get_pool_manager() for c in copy_list: r = pool_manager.request('HEAD', c) self.assertEqual(r.status, 200) self.assertIn(fullpath[0], r.headers["X-oio-chunk-meta-full-path"].split(',')) # Snapshot on non existing container should failed self.assertRaises(exc.NoSuchContainer, self.api.container_snapshot, random_str(16), random_str(16), random_str(16), random_str(16)) # Snapshot need to have a account self.assertRaises(exc.ClientException, self.api.container_snapshot, self.account, name, None, random_str(16)) # Snapshot need to have a name self.assertRaises(exc.ClientException, self.api.container_snapshot, self.account, name, random_str(16), None)
class TestBlobConverter(BaseTestCase): def setUp(self): super(TestBlobConverter, self).setUp() self.container = random_str(16) self.path = random_str(16) self.api = ObjectStorageApi(self.ns) self.api.container_create(self.account, self.container) _, chunks = self.api.container.content_prepare( self.account, self.container, self.path, 1) services = self.conscience.all_services('rawx') self.rawx_volumes = dict() for rawx in services: tags = rawx['tags'] service_id = tags.get('tag.service_id', None) if service_id is None: service_id = rawx['addr'] volume = tags.get('tag.vol', None) self.rawx_volumes[service_id] = volume self.api.object_create( self.account, self.container, obj_name=self.path, data="chunk") meta, self.chunks = self.api.object_locate( self.account, self.container, self.path) self.version = meta['version'] self.content_id = meta['id'] def _chunk_path(self, chunk): url = chunk['url'] volume_id = url.split('/', 3)[2] chunk_id = url.split('/', 3)[3] volume = self.rawx_volumes[volume_id] return volume + '/' + chunk_id[:3] + '/' + chunk_id def _converter_and_check(self, chunk_volume, chunk_path, chunk_id_info, expected_raw_meta=None, expected_errors=0): conf = self.conf conf['volume'] = self.rawx_volumes[chunk_volume] converter = BlobConverter(conf) converter.safe_convert_chunk(chunk_path) self.assertEqual(1, converter.total_chunks_processed) self.assertEqual(1, converter.passes) self.assertEqual(expected_errors, converter.errors) checker = Checker(self.ns) for chunk_id, info in chunk_id_info.iteritems(): account, container, path, version, content_id = info fullpath = encode_fullpath( account, container, path, version, content_id) cid = cid_from_name(account, container) meta, raw_meta = read_chunk_metadata(chunk_path, chunk_id) self.assertEqual(meta.get('chunk_id'), chunk_id) self.assertEqual(meta.get('container_id'), cid) self.assertEqual(meta.get('content_path'), path) self.assertEqual(meta.get('content_version'), version) self.assertEqual(meta.get('content_id'), content_id) self.assertEqual(meta.get('full_path'), fullpath) checker.check(Target( account, container=container, obj=path, chunk='http://' + converter.volume_id + '/' + chunk_id)) checker.wait() self.assertTrue(checker.report()) if expected_raw_meta: self.assertDictEqual(expected_raw_meta, raw_meta) continue self.assertNotIn(chunk_xattr_keys['chunk_id'], raw_meta) self.assertNotIn(chunk_xattr_keys['container_id'], raw_meta) self.assertNotIn(chunk_xattr_keys['content_path'], raw_meta) self.assertNotIn(chunk_xattr_keys['content_version'], raw_meta) self.assertNotIn(chunk_xattr_keys['content_id'], raw_meta) self.assertIn(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + chunk_id, raw_meta) for k, v in raw_meta.iteritems(): if k.startswith('oio:'): self.fail('old fullpath always existing') self.assertEqual(raw_meta[chunk_xattr_keys['oio_version']], OIO_VERSION) def test_converter(self): chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._converter_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_old_chunk(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._converter_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_old_chunk_with_wrong_path(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path + '+', self.version, self.content_id) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._converter_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_old_chunk_with_wrong_content_id(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, '0123456789ABCDEF0123456789ABCDEF') chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._converter_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_old_chunk_with_old_fullpath(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id, add_old_fullpath=True) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._converter_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_old_chunk_with_old_fullpath_and_wrong_path(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id, add_old_fullpath=True) convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path + '+', self.version, self.content_id) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._converter_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_old_chunk_with_wrong_fullpath(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, 'None', '0123456789ABCDEF0123456789ABCDEF', add_old_fullpath=True) convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) self._converter_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) def test_converter_linked_chunk(self): self.api.object_link( self.account, self.container, self.path, self.account, self.container, self.path + '.link') linked_meta, linked_chunks = self.api.object_locate( self.account, self.container, self.path + '.link') self.assertNotEqual(self.content_id, linked_meta['id']) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) for c in linked_chunks: if chunk_volume == c['url'].split('/')[2]: linked_chunk_id2 = c['url'].split('/')[3] break linked_chunk = random.choice(linked_chunks) linked_chunk_volume = linked_chunk['url'].split('/')[2] linked_chunk_id = linked_chunk['url'].split('/')[3] linked_chunk_path = self._chunk_path(linked_chunk) for c in self.chunks: if linked_chunk_volume == c['url'].split('/')[2]: chunk_id2 = c['url'].split('/')[3] break self._converter_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id), linked_chunk_id2: (self.account, self.container, self.path + '.link', linked_meta['version'], linked_meta['id'])}) self._converter_and_check( linked_chunk_volume, linked_chunk_path, {chunk_id2: (self.account, self.container, self.path, self.version, self.content_id), linked_chunk_id: (self.account, self.container, self.path + '.link', linked_meta['version'], linked_meta['id'])}) def test_converter_old_linked_chunk(self): self.api.object_link( self.account, self.container, self.path, self.account, self.container, self.path + '.link') linked_meta, linked_chunks = self.api.object_locate( self.account, self.container, self.path + '.link') self.assertNotEqual(self.content_id, linked_meta['id']) for c in linked_chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path + '.link', 'None', '0123456789ABCDEF0123456789ABCDEF', add_old_fullpath=True) for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) for c in linked_chunks: if chunk_volume == c['url'].split('/')[2]: linked_chunk_id2 = c['url'].split('/')[3] break linked_chunk = random.choice(linked_chunks) linked_chunk_volume = linked_chunk['url'].split('/')[2] linked_chunk_id = linked_chunk['url'].split('/')[3] linked_chunk_path = self._chunk_path(linked_chunk) for c in self.chunks: if linked_chunk_volume == c['url'].split('/')[2]: chunk_id2 = c['url'].split('/')[3] break self._converter_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id), linked_chunk_id2: (self.account, self.container, self.path + '.link', linked_meta['version'], linked_meta['id'])}) self._converter_and_check( linked_chunk_volume, linked_chunk_path, {chunk_id2: (self.account, self.container, self.path, self.version, self.content_id), linked_chunk_id: (self.account, self.container, self.path + '.link', linked_meta['version'], linked_meta['id'])}) def test_converter_old_chunk_with_link_on_same_object(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) self.api.object_link( self.account, self.container, self.path, self.account, self.container, self.path) linked_meta, linked_chunks = self.api.object_locate( self.account, self.container, self.path) self.assertNotEqual(self.content_id, linked_meta['id']) linked_chunk = random.choice(linked_chunks) linked_chunk_volume = linked_chunk['url'].split('/')[2] linked_chunk_id = linked_chunk['url'].split('/')[3] linked_chunk_path = self._chunk_path(linked_chunk) # old xattr not removed _, expected_raw_meta = read_chunk_metadata(linked_chunk_path, linked_chunk_id) expected_raw_meta[chunk_xattr_keys['oio_version']] = OIO_VERSION self._converter_and_check( linked_chunk_volume, linked_chunk_path, {linked_chunk_id: (self.account, self.container, self.path, linked_meta['version'], linked_meta['id'])}, expected_raw_meta=expected_raw_meta, expected_errors=1) def test_converter_old_linked_chunk_with_link_on_same_object(self): self.api.object_link( self.account, self.container, self.path, self.account, self.container, self.path + '.link') linked_meta, linked_chunks = self.api.object_locate( self.account, self.container, self.path + '.link') self.assertNotEqual(self.content_id, linked_meta['id']) for c in linked_chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path + '.link', 'None', '0123456789ABCDEF0123456789ABCDEF', add_old_fullpath=True) for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id, add_old_fullpath=True) self.api.object_link( self.account, self.container, self.path + '.link', self.account, self.container, self.path + '.link') linked_meta, linked_chunks = self.api.object_locate( self.account, self.container, self.path + '.link') self.assertNotEqual(self.content_id, linked_meta['id']) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) for c in linked_chunks: if chunk_volume == c['url'].split('/')[2]: linked_chunk_id2 = c['url'].split('/')[3] break linked_chunk = random.choice(linked_chunks) linked_chunk_volume = linked_chunk['url'].split('/')[2] linked_chunk_id = linked_chunk['url'].split('/')[3] linked_chunk_path = self._chunk_path(linked_chunk) for c in self.chunks: if linked_chunk_volume == c['url'].split('/')[2]: chunk_id2 = c['url'].split('/')[3] break self._converter_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id), linked_chunk_id2: (self.account, self.container, self.path + '.link', linked_meta['version'], linked_meta['id'])}) self._converter_and_check( linked_chunk_volume, linked_chunk_path, {chunk_id2: (self.account, self.container, self.path, self.version, self.content_id), linked_chunk_id: (self.account, self.container, self.path + '.link', linked_meta['version'], linked_meta['id'])}) def test_converter_with_versioning(self): self.api.container_set_properties( self.account, self.container, system={'sys.m2.policy.version': '2'}) self.api.object_create( self.account, self.container, obj_name=self.path, data='version') versioned_meta, versioned_chunks = self.api.object_locate( self.account, self.container, self.path) self.assertNotEqual(self.content_id, versioned_meta['id']) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) versioned_chunk = random.choice(versioned_chunks) versioned_chunk_volume = versioned_chunk['url'].split('/')[2] versioned_chunk_id = versioned_chunk['url'].split('/')[3] versioned_chunk_path = self._chunk_path(versioned_chunk) self._converter_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) self._converter_and_check( versioned_chunk_volume, versioned_chunk_path, {versioned_chunk_id: (self.account, self.container, self.path, versioned_meta['version'], versioned_meta['id'])}) def test_converter_old_chunk_with_versioning(self): for c in self.chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, self.version, self.content_id) self.api.container_set_properties( self.account, self.container, system={'sys.m2.policy.version': '2'}) self.api.object_create( self.account, self.container, obj_name=self.path, data='version') versioned_meta, versioned_chunks = self.api.object_locate( self.account, self.container, self.path) self.assertNotEqual(self.content_id, versioned_meta['id']) for c in versioned_chunks: convert_to_old_chunk( self._chunk_path(c), self.account, self.container, self.path, versioned_meta['version'], versioned_meta['id']) chunk = random.choice(self.chunks) chunk_volume = chunk['url'].split('/')[2] chunk_id = chunk['url'].split('/')[3] chunk_path = self._chunk_path(chunk) versioned_chunk = random.choice(versioned_chunks) versioned_chunk_volume = versioned_chunk['url'].split('/')[2] versioned_chunk_id = versioned_chunk['url'].split('/')[3] versioned_chunk_path = self._chunk_path(versioned_chunk) self._converter_and_check( chunk_volume, chunk_path, {chunk_id: (self.account, self.container, self.path, self.version, self.content_id)}) self._converter_and_check( versioned_chunk_volume, versioned_chunk_path, {versioned_chunk_id: (self.account, self.container, self.path, versioned_meta['version'], versioned_meta['id'])})