def test_move_old_chunk(self): for chunk in self.chunks: convert_to_old_chunk(self._chunk_path(chunk), self.account, self.container, self.path, self.version, self.content_id) orig_chunk = random.choice(self.chunks) chunk_volume = orig_chunk['url'].split('/')[2] chunk_id = orig_chunk['url'].split('/')[3] chunk_headers, chunk_stream = self.blob_client.chunk_get( orig_chunk['url'], check_headers=False) chunks_kept = list(self.chunks) chunks_kept.remove(orig_chunk) mover = BlobMoverWorker(self.conf, None, self.rawx_volumes[chunk_volume]) mover.chunk_move(self._chunk_path(orig_chunk), chunk_id) _, new_chunks = self.api.object_locate(self.account, self.container, self.path) new_chunk = list(new_chunks) self.assertEqual(len(new_chunks), len(chunks_kept) + 1) url_kept = [c['url'] for c in chunks_kept] new_chunk = None for chunk in new_chunks: if chunk['url'] not in url_kept: self.assertIsNone(new_chunk) new_chunk = chunk self.assertNotEqual(orig_chunk['real_url'], new_chunk['real_url']) self.assertNotEqual(orig_chunk['url'], new_chunk['url']) self.assertEqual(orig_chunk['pos'], new_chunk['pos']) self.assertEqual(orig_chunk['size'], new_chunk['size']) self.assertEqual(orig_chunk['hash'], new_chunk['hash']) new_chunk_headers, new_chunk_stream = self.blob_client.chunk_get( new_chunk['url']) chunk_data = b''.join(chunk_stream) new_chunk_data = b''.join(new_chunk_stream) self.assertEqual(chunk_data, new_chunk_data) fullpath = encode_fullpath(self.account, self.container, self.path, self.version, self.content_id) self.assertEqual(fullpath, new_chunk_headers['full_path']) del new_chunk_headers['full_path'] self.assertNotEqual(chunk_headers['chunk_id'], new_chunk_headers['chunk_id']) new_chunk_id = new_chunk['url'].split('/')[3] self.assertEqual(new_chunk_id, new_chunk_headers['chunk_id']) del chunk_headers['chunk_id'] del new_chunk_headers['chunk_id'] self.assertEqual(OIO_VERSION, new_chunk_headers['oio_version']) del chunk_headers['oio_version'] del new_chunk_headers['oio_version'] self.assertEqual(chunk_headers, new_chunk_headers)
def setUp(self): conf = {'namespace': 'NS'} with patch('oio.blob.mover.ContainerClient'): self.mover = BlobMoverWorker(conf, None, '/tmp') self.mover.address = '127.0.0.1:6000' self.path = '/tmp/toto' self.content_path = 'a' self.content_cid = 'CID' self.chunk_hash = '00000000000000000000000000000000' self.chunk_size = 1 self.chunk_id = 'AAAA' self.chunk_id2 = 'CCCC' self.chunk_url = 'http://127.0.0.1:6000/' + self.chunk_id self.chunk_url2 = 'http://127.0.0.1:6002/' + self.chunk_id2 self.chunk_pos = '0' self.metadata = { 'content_cid': self.content_cid, 'content_path': self.content_path, 'content_size': 1, 'content_chunksnb': 1, 'chunk_id': self.chunk_id, 'chunk_pos': self.chunk_pos, 'chunk_size': self.chunk_size, 'chunk_hash': self.chunk_hash } self.chunk = {'url': self.chunk_url, 'pos': self.chunk_pos} self.chunk2 = {'url': self.chunk_url2, 'pos': self.chunk_pos} self.chunks = [self.chunk, self.chunk2] self.spare_data = { 'notin': [self.chunk2], 'broken': [self.chunk], 'size': 0 } self.new_chunk_id = 'BBBB' self.new_chunk_url = 'http://127.0.0.1:6001/' + self.new_chunk_id self.new_chunks = {'chunks': [{'id': self.new_chunk_url}]} self.new_metadata = self.metadata.copy() self.new_metadata['chunk_id'] = self.new_chunk_id self.update_data = { 'old': [{ 'type': 'chunk', 'id': self.chunk_url, 'size': self.chunk_size, 'hash': self.chunk_hash }], 'new': [{ 'type': 'chunk', 'id': self.new_chunk_url, 'size': self.chunk_size, 'hash': self.chunk_hash }] }
def test_move_with_wrong_size(self): if not self.chunk_method.startswith('ec'): self.skipTest('Only works with EC') orig_chunk = random.choice(self.chunks) chunk_volume = orig_chunk['url'].split('/')[2] chunk_id = orig_chunk['url'].split('/')[3] mover = BlobMoverWorker(self.conf, None, self.rawx_volumes[chunk_volume]) meta, stream = mover.blob_client.chunk_get(orig_chunk['url']) data = stream.read() stream.close() data = data[:-1] del meta['chunk_hash'] wrong_stream = GeneratorIO(data) mover.blob_client.chunk_get = Mock(return_value=(meta, wrong_stream)) self.assertRaises(ChunkException, mover.chunk_move, self._chunk_path(orig_chunk), chunk_id)