class TestBlobAuditorFunctional(BaseTestCase): def setUp(self): super(TestBlobAuditorFunctional, self).setUp() self.namespace = self.conf['namespace'] self.account = self.conf['account'] self.test_dir = self.conf['sds_path'] rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx') self.rawx = 'http://' + rawx_addr self.h = hashlib.new('md5') conf = {"namespace": self.namespace} self.auditor = BlobAuditorWorker(conf, get_logger(None), None) self.container_c = ContainerClient(conf) self.blob_c = BlobClient() self.ref = random_str(8) self.container_c.container_create(self.account, self.ref) self.url_rand = random_id(64) self.data = random_str(1280) self.h.update(self.data) self.hash_rand = self.h.hexdigest().lower() self.content = TestContent(random_str(6), len(self.data), self.url_rand, 1) self.content.id_container = cid_from_name(self.account, self.ref).upper() self.chunk = TestChunk(self.content.size, self.url_rand, 0, self.hash_rand) self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk) self.chunk_proxy = { "hash": self.chunk.md5, "pos": "0", "size": self.chunk.size, "url": self.chunk_url } chunk_meta = { 'content_path': self.content.path, 'container_id': self.content.id_container, 'chunk_method': 'plain/nb_copy=3', 'policy': 'TESTPOLICY', 'id': '0000', 'version': 1, 'chunk_id': self.chunk.id_chunk, 'chunk_pos': self.chunk.pos, 'chunk_hash': self.chunk.md5, 'full_path': ['%s/%s/%s' % (self.account, self.ref, self.content.path)], 'oio_version': OIO_VERSION } self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data) self.chunk_path = self.test_dir + '/data/' + self.namespace + \ '-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk self.bad_container_id = '0' * 64 def tearDown(self): super(TestBlobAuditorFunctional, self).tearDown() try: self.container_c.content_delete(self.account, self.ref, self.content.path) except Exception: pass try: self.container_c.container_destroy(self.account, self.ref) except Exception: pass try: os.remove(self.chunk_path) except Exception: pass def init_content(self): self.container_c.content_create(self.account, self.ref, self.content.path, self.chunk.size, self.hash_rand, data=[self.chunk_proxy]) def test_chunk_audit(self): self.init_content() self.auditor.chunk_audit(self.chunk_path) def test_content_deleted(self): self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_container_deleted(self): self.container_c.container_delete(self.account, self.ref) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_corrupted(self): self.init_content() with open(self.chunk_path, "w") as f: f.write(random_str(1280)) self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_size(self): self.init_content() with open(self.chunk_path, "w") as f: f.write(random_str(320)) self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_size(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.' + chunk_xattr_keys['chunk_size'], '-1') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_hash(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.' + chunk_xattr_keys['chunk_hash'], 'WRONG_HASH') self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_path(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.' + chunk_xattr_keys['content_path'], 'WRONG_PATH') self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_id(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.' + chunk_xattr_keys['chunk_id'], 'WRONG_ID') self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_container(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.' + chunk_xattr_keys['container_id'], self.bad_container_id) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_position(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.chunk.position', '42') xattr.setxattr(self.chunk_path, 'user.' + chunk_xattr_keys['chunk_pos'], '42') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_hash(self): self.h.update(self.data) self.hash_rand = self.h.hexdigest().lower() self.chunk.md5 = self.hash_rand self.chunk_proxy['hash'] = self.chunk.md5 self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_length(self): self.chunk.size = 320 self.chunk_proxy['size'] = self.chunk.size self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_chunk_size(self): self.chunk.size = 320 self.chunk_proxy['size'] = self.chunk.size self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_url(self): self.chunk_proxy['url'] = '%s/WRONG_ID' % self.rawx self.init_content() self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)
class TestBlobIndexer(BaseTestCase): def setUp(self): super(TestBlobIndexer, self).setUp() self.rdir_client = RdirClient(self.conf) self.blob_client = BlobClient(self.conf) _, self.rawx_path, rawx_addr, _ = \ self.get_service_url('rawx') services = self.conscience.all_services('rawx') self.rawx_id = None for rawx in services: if rawx_addr == rawx['addr']: self.rawx_id = rawx['tags'].get('tag.service_id', None) if self.rawx_id is None: self.rawx_id = rawx_addr conf = self.conf.copy() conf['volume'] = self.rawx_path self.blob_indexer = BlobIndexer(conf) # clear rawx/rdir chunk_files = paths_gen(self.rawx_path) for chunk_file in chunk_files: os.remove(chunk_file) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) def _put_chunk(self): account = random_str(16) container = random_str(16) cid = cid_from_name(account, container) content_path = random_str(16) content_version = 1234567890 content_id = random_id(32) fullpath = encode_fullpath(account, container, content_path, content_version, content_id) chunk_id = random_chunk_id() data = random_buffer(string.printable, 100) meta = { 'full_path': fullpath, 'container_id': cid, 'content_path': content_path, 'version': content_version, 'id': content_id, 'chunk_method': 'ec/algo=liberasurecode_rs_vand,k=6,m=3', 'policy': 'TESTPOLICY', 'chunk_hash': md5(data).hexdigest().upper(), 'oio_version': OIO_VERSION, 'chunk_pos': 0, 'metachunk_hash': md5().hexdigest(), 'metachunk_size': 1024 } self.blob_client.chunk_put('http://' + self.rawx_id + '/' + chunk_id, meta, data) sleep(1) # ensure chunk event have been processed return account, container, cid, content_path, content_version, \ content_id, chunk_id def _delete_chunk(self, chunk_id): self.blob_client.chunk_delete('http://' + self.rawx_id + '/' + chunk_id) sleep(1) # ensure chunk event have been processed def _link_chunk(self, target_chunk_id): account = random_str(16) container = random_str(16) cid = cid_from_name(account, container) content_path = random_str(16) content_version = 1234567890 content_id = random_id(32) fullpath = encode_fullpath(account, container, content_path, content_version, content_id) _, link = self.blob_client.chunk_link( 'http://' + self.rawx_id + '/' + target_chunk_id, None, fullpath) chunk_id = link.split('/')[-1] sleep(1) # ensure chunk event have been processed return account, container, cid, content_path, content_version, \ content_id, chunk_id def _chunk_path(self, chunk_id): return self.rawx_path + '/' + chunk_id[:3] + '/' + chunk_id def test_blob_indexer(self): _, _, expected_cid, _, _, expected_content_id, expected_chunk_id = \ self._put_chunk() chunks = list(self.rdir_client.chunk_fetch(self.rawx_id)) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) self.blob_indexer.index_pass() self.assertEqual(1, self.blob_indexer.successes) self.assertEqual(0, self.blob_indexer.errors) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) self._delete_chunk(expected_chunk_id) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(0, len(chunks)) def test_blob_indexer_with_old_chunk(self): expected_account, expected_container, expected_cid, \ expected_content_path, expected_content_version, \ expected_content_id, expected_chunk_id = self._put_chunk() chunks = list(self.rdir_client.chunk_fetch(self.rawx_id)) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) convert_to_old_chunk(self._chunk_path(chunk_id), expected_account, expected_container, expected_content_path, expected_content_version, expected_content_id) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) self.blob_indexer.index_pass() self.assertEqual(1, self.blob_indexer.successes) self.assertEqual(0, self.blob_indexer.errors) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) self._delete_chunk(expected_chunk_id) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(0, len(chunks)) def test_blob_indexer_with_linked_chunk(self): _, _, expected_cid, _, _, expected_content_id, expected_chunk_id = \ self._put_chunk() chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) self.blob_indexer.index_pass() self.assertEqual(1, self.blob_indexer.successes) self.assertEqual(0, self.blob_indexer.errors) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) self.assertEqual(expected_chunk_id, chunk_id) _, _, linked_cid, _, _, linked_content_id, linked_chunk_id = \ self._link_chunk(expected_chunk_id) self.rdir_client.admin_clear(self.rawx_id, clear_all=True) self.blob_indexer.index_pass() self.assertEqual(2, self.blob_indexer.successes) self.assertEqual(0, self.blob_indexer.errors) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(2, len(chunks)) self.assertNotEqual(chunks[0][2], chunks[1][2]) for chunk in chunks: cid, content_id, chunk_id, _ = chunk if chunk_id == expected_chunk_id: self.assertEqual(expected_cid, cid) self.assertEqual(expected_content_id, content_id) else: self.assertEqual(linked_cid, cid) self.assertEqual(linked_content_id, content_id) self.assertEqual(linked_chunk_id, chunk_id) self._delete_chunk(expected_chunk_id) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(1, len(chunks)) cid, content_id, chunk_id, _ = chunks[0] self.assertEqual(linked_cid, cid) self.assertEqual(linked_content_id, content_id) self.assertEqual(linked_chunk_id, chunk_id) self._delete_chunk(linked_chunk_id) chunks = self.rdir_client.chunk_fetch(self.rawx_id) chunks = list(chunks) self.assertEqual(0, len(chunks))
class TestBlobAuditorFunctional(BaseTestCase): def setUp(self): super(TestBlobAuditorFunctional, self).setUp() self.namespace = self.conf['namespace'] self.account = self.conf['account'] self.test_dir = self.conf['sds_path'] self.chars = string.ascii_lowercase + string.ascii_uppercase +\ string.digits self.chars_id = string.digits + 'ABCDEF' self.rawx = 'http://' + self.conf["rawx"][0]['addr'] self.h = hashlib.new('md5') conf = {"namespace": self.namespace} self.auditor = BlobAuditorWorker(conf, get_logger(None), None) self.container_c = ContainerClient(conf) self.blob_c = BlobClient() self.ref = rand_generator(self.chars, 8) self.container_c.container_create(self.account, self.ref) self.url_rand = rand_generator(self.chars_id, 64) self.data = rand_generator(self.chars, 1280) self.h.update(self.data) self.hash_rand = self.h.hexdigest().lower() self.content = TestContent( rand_generator(self.chars, 6), len(self.data), self.url_rand, 1) self.content.id_container = cid_from_name( self.account, self.ref).upper() self.chunk = TestChunk(self.content.size, self.url_rand, 0, self.hash_rand) self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk) self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0", "size": self.chunk.size, "url": self.chunk_url} chunk_meta = {'content_size': self.content.size, 'content_chunksnb': self.content.nb_chunks, 'content_path': self.content.path, 'content_cid': self.content.id_container, 'content_id': '0000', 'content_version': 1, 'chunk_id': self.chunk.id_chunk, 'chunk_pos': self.chunk.pos} self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data) self.chunk_path = self.test_dir + '/data/NS-rawx-1/' +\ self.chunk.id_chunk[0:2] + "/" + self.chunk.id_chunk self.bad_container_id = '0'*64 def tearDown(self): super(TestBlobAuditorFunctional, self).tearDown() try: self.container_c.content_delete( self.account, self.ref, self.content.path) except Exception: pass try: self.container_c.container_destroy(self.account, self.ref) except Exception: pass try: os.remove(self.chunk_path) except Exception: pass def init_content(self): self.container_c.content_create( self.account, self.ref, self.content.path, self.chunk.size, self.hash_rand, data=[self.chunk_proxy]) def test_chunk_audit(self): self.init_content() self.auditor.chunk_audit(self.chunk_path) def test_content_deleted(self): self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_container_deleted(self): self.container_c.container_destroy(self.account, self.ref) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_corrupted(self): self.init_content() with open(self.chunk_path, "w") as f: f.write(rand_generator(self.chars, 1280)) self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_size(self): self.init_content() with open(self.chunk_path, "w") as f: f.write(rand_generator(self.chars, 320)) self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_nbchunk(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.content.nbchunk', '42') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_size(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.chunk.size', '-1') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_hash(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.chunk.hash', 'WRONG_HASH') self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_size(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.content.size', '-1') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_path(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.content.path', 'WRONG_PATH') self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_id(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.chunk.id', 'WRONG_ID') self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_container(self): self.init_content() xattr.setxattr( self.chunk_path, 'user.grid.content.container', self.bad_container_id) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_position(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.chunk.position', '42') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_hash(self): self.h.update(self.data) self.hash_rand = self.h.hexdigest().lower() self.chunk.md5 = self.hash_rand self.chunk_proxy['hash'] = self.chunk.md5 self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_length(self): self.chunk.size = 320 self.chunk_proxy['size'] = self.chunk.size self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_chunk_size(self): self.chunk.size = 320 self.chunk_proxy['size'] = self.chunk.size self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_url(self): self.chunk_proxy['url'] = '%s/WRONG_ID' % self.rawx self.init_content() self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_content_bad_path(self): self.content.path = 'BAD_PATH' self.init_content() self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)
class TestRebuilderCrawler(BaseTestCase): def setUp(self): super(TestRebuilderCrawler, self).setUp() self.namespace = self.conf['namespace'] self.account = self.conf['account'] self.gridconf = {"namespace": self.namespace} self.container_client = ContainerClient(self.gridconf) self.blob_client = BlobClient() self.container_name = "TestRebuilderCrawler%d" % int(time.time()) self.container_client.container_create(acct=self.account, ref=self.container_name) def _push_content(self, content): for c in content.chunks: self.blob_client.chunk_put(c.url, c.get_create_xattr(), c.data) self.container_client.content_create(acct=content.account, ref=content.container_name, path=content.content_name, size=content.size, checksum=content.hash, content_id=content.content_id, stgpol=content.stgpol, data=content.get_create_meta2()) def tearDown(self): super(TestRebuilderCrawler, self).tearDown() def test_rebuild_chunk(self): # push a new content content = TestContent(self.conf, self.account, self.container_name, "mycontent", "TWOCOPIES") data = "azerty" content.add_chunk(data, pos='0', rawx=0) content.add_chunk(data, pos='0', rawx=1) self._push_content(content) # rebuild the first rawx rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) rebuilder.chunk_rebuild(content.container_id, content.content_id, content.chunks[0].id) # check meta2 information _, res = self.container_client.content_show(acct=content.account, ref=content.container_name, content=content.content_id) new_chunk_info = None for c in res: if (c['url'] != content.chunks[0].url and c['url'] != content.chunks[1].url): new_chunk_info = c new_chunk_id = new_chunk_info['url'].split('/')[-1] self.assertEqual(new_chunk_info['hash'], content.chunks[0].hash) self.assertEqual(new_chunk_info['pos'], content.chunks[0].pos) self.assertEqual(new_chunk_info['size'], content.chunks[0].size) # check chunk information meta, stream = self.blob_client.chunk_get(new_chunk_info['url']) self.assertEqual(meta['content_size'], str(content.chunks[0].size)) self.assertEqual(meta['content_path'], content.content_name) self.assertEqual(meta['content_cid'], content.container_id) self.assertEqual(meta['content_id'], content.content_id) self.assertEqual(meta['chunk_id'], new_chunk_id) self.assertEqual(meta['chunk_pos'], content.chunks[0].pos) self.assertEqual(meta['content_version'], content.version) self.assertEqual(meta['chunk_hash'], content.chunks[0].hash) self.assertEqual(stream.next(), content.chunks[0].data) # check rtime flag in rdir rdir_client = RdirClient(self.gridconf) res = rdir_client.chunk_fetch(self.conf['rawx'][0]['addr']) key = (content.container_id, content.content_id, content.chunks[0].id) for i_container, i_content, i_chunk, i_value in res: if (i_container, i_content, i_chunk) == key: check_value = i_value self.assertIsNotNone(check_value.get('rtime')) @unittest.skipIf(len(get_config()['rawx']) != 3, "The number of rawx must be 3") def test_rebuild_no_spare(self): # push a new content content = TestContent(self.conf, self.account, self.container_name, "mycontent", "THREECOPIES") data = "azerty" content.add_chunk(data, pos='0', rawx=0) content.add_chunk(data, pos='0', rawx=1) content.add_chunk(data, pos='0', rawx=2) self._push_content(content) # rebuild the first rawx rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) self.assertRaises(SpareChunkException, rebuilder.chunk_rebuild, content.container_id, content.content_id, content.chunks[0].id) def test_rebuild_upload_failed(self): # push a new content content = TestContent(self.conf, self.account, self.container_name, "mycontent", "TWOCOPIES") data = "azerty" content.add_chunk(data, pos='0', rawx=0) content.add_chunk(data, pos='0', rawx=1) self._push_content(content) # rebuild the first rawx rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) # Force upload to raise an exception with patch('oio.content.content.BlobClient') as MockClass: instance = MockClass.return_value instance.chunk_copy.side_effect = Exception("xx") self.assertRaises(UnrecoverableContent, rebuilder.chunk_rebuild, content.container_id, content.content_id, content.chunks[0].id) def test_rebuild_nonexistent_chunk(self): rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) # try to rebuild an nonexistant chunk self.assertRaises(OrphanChunk, rebuilder.chunk_rebuild, 64 * '0', 32 * '0', 64 * '0') def test_rebuild_orphan_chunk(self): # push a new content content = TestContent(self.conf, self.account, self.container_name, "mycontent", "TWOCOPIES") data = "azerty" content.add_chunk(data, pos='0', rawx=0) content.add_chunk(data, pos='0', rawx=1) self._push_content(content) # rebuild the first rawx rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) # try to rebuild an nonexistant chunk self.assertRaises(OrphanChunk, rebuilder.chunk_rebuild, content.container_id, content.content_id, 64 * '0') def test_rebuild_with_no_copy(self): # push a new content content = TestContent(self.conf, self.account, self.container_name, "mycontent", "SINGLE") data = "azerty" content.add_chunk(data, pos='0', rawx=0) self._push_content(content) # rebuild the first rawx rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) # try to rebuild chunk without copy self.assertRaises(UnrecoverableContent, rebuilder.chunk_rebuild, content.container_id, content.content_id, content.chunks[0].id)
class TestBlobAuditorFunctional(BaseTestCase): def setUp(self): super(TestBlobAuditorFunctional, self).setUp() self.namespace = self.conf['namespace'] self.account = self.conf['account'] self.test_dir = self.conf['sds_path'] self.chars = string.ascii_lowercase + string.ascii_uppercase +\ string.digits self.chars_id = string.digits + 'ABCDEF' self.rawx = 'http://' + self.conf["rawx"][0]['addr'] self.h = hashlib.new('md5') conf = {"namespace": self.namespace} self.auditor = BlobAuditorWorker(conf, get_logger(None), None) self.container_c = ContainerClient(conf) self.blob_c = BlobClient() self.ref = rand_generator(self.chars, 8) self.container_c.container_create(self.account, self.ref) self.url_rand = rand_generator(self.chars_id, 64) self.data = rand_generator(self.chars, 1280) self.h.update(self.data) self.hash_rand = self.h.hexdigest().lower() self.content = TestContent(rand_generator(self.chars, 6), len(self.data), self.url_rand, 1) self.content.id_container = cid_from_name(self.account, self.ref).upper() self.chunk = TestChunk(self.content.size, self.url_rand, 0, self.hash_rand) self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk) self.chunk_proxy = { "hash": self.chunk.md5, "pos": "0", "size": self.chunk.size, "url": self.chunk_url } chunk_meta = { 'content_size': self.content.size, 'content_chunksnb': self.content.nb_chunks, 'content_path': self.content.path, 'content_cid': self.content.id_container, 'content_mimetype': 'application/octet-stream', 'content_chunkmethod': 'bytes', 'content_policy': 'TESTPOLICY', 'content_id': '0000', 'content_version': 1, 'chunk_id': self.chunk.id_chunk, 'chunk_pos': self.chunk.pos } self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data) self.chunk_path = self.test_dir + '/data/' + self.namespace + \ '-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk self.bad_container_id = '0' * 64 def tearDown(self): super(TestBlobAuditorFunctional, self).tearDown() try: self.container_c.content_delete(self.account, self.ref, self.content.path) except Exception: pass try: self.container_c.container_destroy(self.account, self.ref) except Exception: pass try: os.remove(self.chunk_path) except Exception: pass def init_content(self): self.container_c.content_create(self.account, self.ref, self.content.path, self.chunk.size, self.hash_rand, data=[self.chunk_proxy]) def test_chunk_audit(self): self.init_content() self.auditor.chunk_audit(self.chunk_path) def test_content_deleted(self): self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_container_deleted(self): self.container_c.container_destroy(self.account, self.ref) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_corrupted(self): self.init_content() with open(self.chunk_path, "w") as f: f.write(rand_generator(self.chars, 1280)) self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_size(self): self.init_content() with open(self.chunk_path, "w") as f: f.write(rand_generator(self.chars, 320)) self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_nbchunk(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.content.nbchunk', '42') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_size(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.chunk.size', '-1') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_hash(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.chunk.hash', 'WRONG_HASH') self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_size(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.content.size', '-1') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_path(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.content.path', 'WRONG_PATH') self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_id(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.chunk.id', 'WRONG_ID') self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_container(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.content.container', self.bad_container_id) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_position(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.chunk.position', '42') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_hash(self): self.h.update(self.data) self.hash_rand = self.h.hexdigest().lower() self.chunk.md5 = self.hash_rand self.chunk_proxy['hash'] = self.chunk.md5 self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_length(self): self.chunk.size = 320 self.chunk_proxy['size'] = self.chunk.size self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_chunk_size(self): self.chunk.size = 320 self.chunk_proxy['size'] = self.chunk.size self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_url(self): self.chunk_proxy['url'] = '%s/WRONG_ID' % self.rawx self.init_content() self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_content_bad_path(self): self.content.path = 'BAD_PATH' self.init_content() self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)
class TestBlobAuditorFunctional(BaseTestCase): def setUp(self): super(TestBlobAuditorFunctional, self).setUp() self.namespace = self.conf['namespace'] self.account = self.conf['account'] self.ref = random_str(8) _, rawx_loc, rawx_addr, rawx_uuid = \ self.get_service_url('rawx') self.rawx_id = 'http://' + (rawx_uuid if rawx_uuid else rawx_addr) self.auditor = BlobAuditorWorker(self.conf, get_logger(None), None) self.container_client = ContainerClient(self.conf) self.blob_client = BlobClient(conf=self.conf) self.container_client.container_create(self.account, self.ref) self.content = TestContent(self.account, self.ref) self.chunk = TestChunk(self.rawx_id, rawx_loc, self.content.size, self.content.hash) chunk_meta = { 'container_id': self.content.cid, 'content_path': self.content.path, 'version': self.content.version, 'id': self.content.id, 'full_path': self.content.fullpath, 'chunk_method': 'plain/nb_copy=3', 'policy': 'TESTPOLICY', 'chunk_id': self.chunk.id, 'chunk_pos': self.chunk.pos, 'chunk_hash': self.chunk.metachunk_hash, 'chunk_size': self.chunk.metachunk_size, 'metachunk_hash': self.chunk.metachunk_hash, 'metachunk_size': self.chunk.metachunk_size, 'oio_version': OIO_VERSION } self.blob_client.chunk_put(self.chunk.url, chunk_meta, self.content.data) def tearDown(self): super(TestBlobAuditorFunctional, self).tearDown() try: self.container_client.content_delete(self.account, self.ref, self.content.path) except Exception: pass try: self.container_client.container_delete(self.account, self.ref) except Exception: pass try: os.remove(self.chunk.path) except Exception: pass def init_content(self): chunk_proxy = { "url": self.chunk.url, "pos": str(self.chunk.pos), "hash": self.chunk.metachunk_hash, "size": self.chunk.metachunk_size } self.container_client.content_create(self.account, self.ref, self.content.path, version=int(time.time() * 1000000), content_id=self.content.id, size=self.content.size, checksum=self.content.hash, data={'chunks': [chunk_proxy]}, stgpol="SINGLE") def test_chunk_audit(self): self.init_content() self.auditor.chunk_audit(self.chunk.path, self.chunk.id) def test_content_deleted(self): self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk.path, self.chunk.id) def test_container_deleted(self): self.container_client.container_delete(self.account, self.ref) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk.path, self.chunk.id) def test_chunk_corrupted(self): self.init_content() with open(self.chunk.path, "wb") as outf: outf.write(os.urandom(1280)) self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk.path, self.chunk.id) def test_chunk_bad_chunk_size(self): self.init_content() with open(self.chunk.path, "wb") as outf: outf.write(os.urandom(320)) exc_class = (exc.FaultyChunk, exc.CorruptedChunk) self.assertRaises(exc_class, self.auditor.chunk_audit, self.chunk.path, self.chunk.id) def test_xattr_bad_xattr_metachunk_size(self): self.init_content() xattr.setxattr(self.chunk.path, 'user.' + CHUNK_XATTR_KEYS['metachunk_size'], b'320') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk.path, self.chunk.id) def test_xattr_bad_xattr_metachunk_hash(self): self.init_content() xattr.setxattr(self.chunk.path, 'user.' + CHUNK_XATTR_KEYS['metachunk_hash'], b'0123456789ABCDEF0123456789ABCDEF') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk.path, self.chunk.id) def test_xattr_bad_xattr_chunk_id(self): self.init_content() xattr.removexattr( self.chunk.path, 'user.' + CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + str(self.chunk.id)) xattr.setxattr( self.chunk.path, 'user.' + CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + 'WRONG_ID', self.content.fullpath.encode('utf-8')) self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk.path, self.chunk.id) def test_xattr_bad_xattr_content_container(self): self.init_content() xattr.setxattr( self.chunk.path, 'user.' + CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + str(self.chunk.id), encode_fullpath(self.account, 'WRONG_REF', self.content.path, self.content.version, self.content.id).encode('utf-8')) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk.path, self.chunk.id) def test_xattr_bad_xattr_content_id(self): self.init_content() xattr.setxattr( self.chunk.path, 'user.' + CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + str(self.chunk.id), encode_fullpath(self.account, self.ref, self.content.path, self.content.version, '0123456789ABCDEF').encode('utf-8')) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk.path, self.chunk.id) def test_xattr_bad_xattr_chunk_position(self): self.init_content() xattr.setxattr(self.chunk.path, 'user.' + CHUNK_XATTR_KEYS['chunk_pos'], b'42') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk.path, self.chunk.id) def test_chunk_bad_meta2_metachunk_size(self): self.content.size = 320 self.chunk.metachunk_size = 320 self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk.path, self.chunk.id) def test_chunk_bad_meta2_metachunk_hash(self): self.chunk.metachunk_hash = '0123456789ABCDEF0123456789ABCDEF' self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk.path, self.chunk.id) def test_chunk_bad_meta2_chunk_url(self): self.chunk.url = '%s/0123456789ABCDEF' % self.rawx_id self.init_content() self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk.path, self.chunk.id)
class TestRebuilderCrawler(BaseTestCase): def setUp(self): super(TestRebuilderCrawler, self).setUp() self.namespace = self.conf['namespace'] self.account = self.conf['account'] self.gridconf = {"namespace": self.namespace} self.container_client = ContainerClient(self.gridconf) self.blob_client = BlobClient() self.container_name = "TestRebuilderCrawler%d" % int(time.time()) self.container_client.container_create(acct=self.account, ref=self.container_name) def _push_content(self, content): for c in content.chunks: self.blob_client.chunk_put(c.url, c.get_create_xattr(), c.data) self.container_client.content_create(acct=content.account, ref=content.container_name, path=content.content_name, size=content.size, checksum=content.hash, content_id=content.content_id, stgpol=content.stgpol, data=content.get_create_meta2()) def tearDown(self): super(TestRebuilderCrawler, self).tearDown() def test_rebuild_chunk(self): # push a new content content = TestContent(self.conf, self.account, self.container_name, "mycontent", "TWOCOPIES") data = "azerty" content.add_chunk(data, pos='0', rawx=0) content.add_chunk(data, pos='0', rawx=1) self._push_content(content) # rebuild the first rawx rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) rebuilder.chunk_rebuild(content.container_id, content.content_id, content.chunks[0].id) # check meta2 information _, res = self.container_client.content_show(acct=content.account, ref=content.container_name, content=content.content_id) new_chunk_info = None for c in res: if (c['url'] != content.chunks[0].url and c['url'] != content.chunks[1].url): new_chunk_info = c new_chunk_id = new_chunk_info['url'].split('/')[-1] self.assertEqual(new_chunk_info['hash'], content.chunks[0].hash) self.assertEqual(new_chunk_info['pos'], content.chunks[0].pos) self.assertEqual(new_chunk_info['size'], content.chunks[0].size) # check chunk information meta, stream = self.blob_client.chunk_get(new_chunk_info['url']) self.assertEqual(meta['content_size'], str(content.chunks[0].size)) self.assertEqual(meta['content_path'], content.content_name) self.assertEqual(meta['content_cid'], content.container_id) self.assertEqual(meta['content_id'], content.content_id) self.assertEqual(meta['chunk_id'], new_chunk_id) self.assertEqual(meta['chunk_pos'], content.chunks[0].pos) self.assertEqual(meta['content_version'], content.version) self.assertEqual(meta['chunk_hash'], content.chunks[0].hash) self.assertEqual(stream.next(), content.chunks[0].data) # check rtime flag in rdir rdir_client = RdirClient(self.gridconf) res = rdir_client.chunk_fetch(self.conf['rawx'][0]['addr']) key = (content.container_id, content.content_id, content.chunks[0].id) for i_container, i_content, i_chunk, i_value in res: if (i_container, i_content, i_chunk) == key: check_value = i_value self.assertIsNotNone(check_value.get('rtime')) @unittest.skipIf( len(get_config()['rawx']) != 3, "The number of rawx must be 3") def test_rebuild_no_spare(self): # push a new content content = TestContent(self.conf, self.account, self.container_name, "mycontent", "THREECOPIES") data = "azerty" content.add_chunk(data, pos='0', rawx=0) content.add_chunk(data, pos='0', rawx=1) content.add_chunk(data, pos='0', rawx=2) self._push_content(content) # rebuild the first rawx rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) self.assertRaises(SpareChunkException, rebuilder.chunk_rebuild, content.container_id, content.content_id, content.chunks[0].id) def test_rebuild_upload_failed(self): # push a new content content = TestContent(self.conf, self.account, self.container_name, "mycontent", "TWOCOPIES") data = "azerty" content.add_chunk(data, pos='0', rawx=0) content.add_chunk(data, pos='0', rawx=1) self._push_content(content) # rebuild the first rawx rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) # Force upload to raise an exception with patch('oio.content.content.BlobClient') as MockClass: instance = MockClass.return_value instance.chunk_copy.side_effect = Exception("xx") self.assertRaises(UnrecoverableContent, rebuilder.chunk_rebuild, content.container_id, content.content_id, content.chunks[0].id) def test_rebuild_nonexistent_chunk(self): rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) # try to rebuild an nonexistant chunk self.assertRaises(OrphanChunk, rebuilder.chunk_rebuild, 64 * '0', 32 * '0', 64 * '0') def test_rebuild_orphan_chunk(self): # push a new content content = TestContent(self.conf, self.account, self.container_name, "mycontent", "TWOCOPIES") data = "azerty" content.add_chunk(data, pos='0', rawx=0) content.add_chunk(data, pos='0', rawx=1) self._push_content(content) # rebuild the first rawx rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) # try to rebuild an nonexistant chunk self.assertRaises(OrphanChunk, rebuilder.chunk_rebuild, content.container_id, content.content_id, 64 * '0') def test_rebuild_with_no_copy(self): # push a new content content = TestContent(self.conf, self.account, self.container_name, "mycontent", "SINGLE") data = "azerty" content.add_chunk(data, pos='0', rawx=0) self._push_content(content) # rebuild the first rawx rebuilder = BlobRebuilderWorker(self.gridconf, None, self.conf['rawx'][0]['addr']) # try to rebuild chunk without copy self.assertRaises(UnrecoverableContent, rebuilder.chunk_rebuild, content.container_id, content.content_id, content.chunks[0].id)
class TestBlobAuditorFunctional(BaseTestCase): def setUp(self): super(TestBlobAuditorFunctional, self).setUp() self.namespace = self.conf['namespace'] self.account = self.conf['account'] self.test_dir = self.conf['sds_path'] rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx') self.rawx = 'http://' + rawx_addr self.h = hashlib.new('md5') conf = {"namespace": self.namespace} self.auditor = BlobAuditorWorker(conf, get_logger(None), None) self.container_c = ContainerClient(conf) self.blob_c = BlobClient() self.ref = random_str(8) self.container_c.container_create(self.account, self.ref) self.url_rand = random_id(64) self.data = random_str(1280) self.h.update(self.data) self.hash_rand = self.h.hexdigest().lower() self.content = TestContent( random_str(6), len(self.data), self.url_rand, 1) self.content.id_container = cid_from_name( self.account, self.ref).upper() self.chunk = TestChunk(self.content.size, self.url_rand, 0, self.hash_rand) self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk) self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0", "size": self.chunk.size, "url": self.chunk_url} chunk_meta = {'content_path': self.content.path, 'container_id': self.content.id_container, 'content_chunkmethod': 'plain/nb_copy=3', 'content_policy': 'TESTPOLICY', 'content_id': '0000', 'content_version': 1, 'chunk_id': self.chunk.id_chunk, 'chunk_pos': self.chunk.pos} self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data) self.chunk_path = self.test_dir + '/data/' + self.namespace + \ '-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk self.bad_container_id = '0'*64 def tearDown(self): super(TestBlobAuditorFunctional, self).tearDown() try: self.container_c.content_delete( self.account, self.ref, self.content.path) except Exception: pass try: self.container_c.container_destroy(self.account, self.ref) except Exception: pass try: os.remove(self.chunk_path) except Exception: pass def init_content(self): self.container_c.content_create( self.account, self.ref, self.content.path, self.chunk.size, self.hash_rand, data=[self.chunk_proxy]) def test_chunk_audit(self): self.init_content() self.auditor.chunk_audit(self.chunk_path) def test_content_deleted(self): self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_container_deleted(self): self.container_c.container_destroy(self.account, self.ref) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_corrupted(self): self.init_content() with open(self.chunk_path, "w") as f: f.write(random_str(1280)) self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_size(self): self.init_content() with open(self.chunk_path, "w") as f: f.write(random_str(320)) self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_size(self): self.init_content() xattr.setxattr( self.chunk_path, 'user.' + chunk_xattr_keys['chunk_size'], '-1') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_hash(self): self.init_content() xattr.setxattr( self.chunk_path, 'user.' + chunk_xattr_keys['chunk_hash'], 'WRONG_HASH') self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_path(self): self.init_content() xattr.setxattr( self.chunk_path, 'user.' + chunk_xattr_keys['content_path'], 'WRONG_PATH') self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_id(self): self.init_content() xattr.setxattr( self.chunk_path, 'user.' + chunk_xattr_keys['chunk_id'], 'WRONG_ID') self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_container(self): self.init_content() xattr.setxattr( self.chunk_path, 'user.' + chunk_xattr_keys['container_id'], self.bad_container_id) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_position(self): self.init_content() xattr.setxattr(self.chunk_path, 'user.grid.chunk.position', '42') xattr.setxattr( self.chunk_path, 'user.' + chunk_xattr_keys['chunk_pos'], '42') self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_hash(self): self.h.update(self.data) self.hash_rand = self.h.hexdigest().lower() self.chunk.md5 = self.hash_rand self.chunk_proxy['hash'] = self.chunk.md5 self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_length(self): self.chunk.size = 320 self.chunk_proxy['size'] = self.chunk.size self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_chunk_size(self): self.chunk.size = 320 self.chunk_proxy['size'] = self.chunk.size self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_url(self): self.chunk_proxy['url'] = '%s/WRONG_ID' % self.rawx self.init_content() self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)
class TestBlobAuditorFunctional(BaseTestCase): def setUp(self): super(TestBlobAuditorFunctional, self).setUp() self.namespace = self.conf["namespace"] self.account = self.conf["account"] self.test_dir = self.conf["sds_path"] rawx_num, rawx_path, rawx_addr = self.get_service_url("rawx") self.rawx = "http://" + rawx_addr self.h = hashlib.new("md5") conf = {"namespace": self.namespace} self.auditor = BlobAuditorWorker(conf, get_logger(None), None) self.container_c = ContainerClient(conf) self.blob_c = BlobClient() self.ref = random_str(8) self.container_c.container_create(self.account, self.ref) self.url_rand = random_id(64) self.data = random_str(1280) self.h.update(self.data) self.hash_rand = self.h.hexdigest().lower() self.content = TestContent(random_str(6), len(self.data), self.url_rand, 1) self.content.id_container = cid_from_name(self.account, self.ref).upper() self.chunk = TestChunk(self.content.size, self.url_rand, 0, self.hash_rand) self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk) self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0", "size": self.chunk.size, "url": self.chunk_url} chunk_meta = { "content_path": self.content.path, "container_id": self.content.id_container, "chunk_method": "plain/nb_copy=3", "policy": "TESTPOLICY", "id": "0000", "version": 1, "chunk_id": self.chunk.id_chunk, "chunk_pos": self.chunk.pos, "chunk_hash": self.chunk.md5, } self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data) self.chunk_path = ( self.test_dir + "/data/" + self.namespace + "-rawx-1/" + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk ) self.bad_container_id = "0" * 64 def tearDown(self): super(TestBlobAuditorFunctional, self).tearDown() try: self.container_c.content_delete(self.account, self.ref, self.content.path) except Exception: pass try: self.container_c.container_destroy(self.account, self.ref) except Exception: pass try: os.remove(self.chunk_path) except Exception: pass def init_content(self): self.container_c.content_create( self.account, self.ref, self.content.path, self.chunk.size, self.hash_rand, data=[self.chunk_proxy] ) def test_chunk_audit(self): self.init_content() self.auditor.chunk_audit(self.chunk_path) def test_content_deleted(self): self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_container_deleted(self): self.container_c.container_destroy(self.account, self.ref) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_corrupted(self): self.init_content() with open(self.chunk_path, "w") as f: f.write(random_str(1280)) self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_size(self): self.init_content() with open(self.chunk_path, "w") as f: f.write(random_str(320)) self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_size(self): self.init_content() xattr.setxattr(self.chunk_path, "user." + chunk_xattr_keys["chunk_size"], "-1") self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_hash(self): self.init_content() xattr.setxattr(self.chunk_path, "user." + chunk_xattr_keys["chunk_hash"], "WRONG_HASH") self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_path(self): self.init_content() xattr.setxattr(self.chunk_path, "user." + chunk_xattr_keys["content_path"], "WRONG_PATH") self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_id(self): self.init_content() xattr.setxattr(self.chunk_path, "user." + chunk_xattr_keys["chunk_id"], "WRONG_ID") self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_content_container(self): self.init_content() xattr.setxattr(self.chunk_path, "user." + chunk_xattr_keys["container_id"], self.bad_container_id) self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path) def test_xattr_bad_chunk_position(self): self.init_content() xattr.setxattr(self.chunk_path, "user.grid.chunk.position", "42") xattr.setxattr(self.chunk_path, "user." + chunk_xattr_keys["chunk_pos"], "42") self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_hash(self): self.h.update(self.data) self.hash_rand = self.h.hexdigest().lower() self.chunk.md5 = self.hash_rand self.chunk_proxy["hash"] = self.chunk.md5 self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_length(self): self.chunk.size = 320 self.chunk_proxy["size"] = self.chunk.size self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_chunk_size(self): self.chunk.size = 320 self.chunk_proxy["size"] = self.chunk.size self.init_content() self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit, self.chunk_path) def test_chunk_bad_url(self): self.chunk_proxy["url"] = "%s/WRONG_ID" % self.rawx self.init_content() self.assertRaises(exc.OrphanChunk, self.auditor.chunk_audit, self.chunk_path)