def test_move_old_chunk(self):
for chunk in self.chunks:
convert_to_old_chunk(self._chunk_path(chunk), self.account,
self.container, self.path, self.version,
self.content_id)
orig_chunk = random.choice(self.chunks)
chunk_volume = orig_chunk['url'].split('/')[2]
chunk_id = orig_chunk['url'].split('/')[3]
chunk_headers, chunk_stream = self.blob_client.chunk_get(
orig_chunk['url'], check_headers=False)
chunks_kept = list(self.chunks)
chunks_kept.remove(orig_chunk)
mover = BlobMoverWorker(self.conf, None,
self.rawx_volumes[chunk_volume])
mover.chunk_move(self._chunk_path(orig_chunk), chunk_id)
_, new_chunks = self.api.object_locate(self.account, self.container,
self.path)
new_chunk = list(new_chunks)
self.assertEqual(len(new_chunks), len(chunks_kept) + 1)
url_kept = [c['url'] for c in chunks_kept]
new_chunk = None
for chunk in new_chunks:
if chunk['url'] not in url_kept:
self.assertIsNone(new_chunk)
new_chunk = chunk
self.assertNotEqual(orig_chunk['real_url'], new_chunk['real_url'])
self.assertNotEqual(orig_chunk['url'], new_chunk['url'])
self.assertEqual(orig_chunk['pos'], new_chunk['pos'])
self.assertEqual(orig_chunk['size'], new_chunk['size'])
self.assertEqual(orig_chunk['hash'], new_chunk['hash'])
new_chunk_headers, new_chunk_stream = self.blob_client.chunk_get(
new_chunk['url'])
chunk_data = b''.join(chunk_stream)
new_chunk_data = b''.join(new_chunk_stream)
self.assertEqual(chunk_data, new_chunk_data)
fullpath = encode_fullpath(self.account, self.container, self.path,
self.version, self.content_id)
self.assertEqual(fullpath, new_chunk_headers['full_path'])
del new_chunk_headers['full_path']
self.assertNotEqual(chunk_headers['chunk_id'],
new_chunk_headers['chunk_id'])
new_chunk_id = new_chunk['url'].split('/')[3]
self.assertEqual(new_chunk_id, new_chunk_headers['chunk_id'])
del chunk_headers['chunk_id']
del new_chunk_headers['chunk_id']
self.assertEqual(OIO_VERSION, new_chunk_headers['oio_version'])
del chunk_headers['oio_version']
del new_chunk_headers['oio_version']
self.assertEqual(chunk_headers, new_chunk_headers)
def setUp(self):
conf = {'namespace': 'NS'}
with patch('oio.blob.mover.ContainerClient'):
self.mover = BlobMoverWorker(conf, None, '/tmp')
self.mover.address = '127.0.0.1:6000'
self.path = '/tmp/toto'
self.content_path = 'a'
self.content_cid = 'CID'
self.chunk_hash = '00000000000000000000000000000000'
self.chunk_size = 1
self.chunk_id = 'AAAA'
self.chunk_id2 = 'CCCC'
self.chunk_url = 'http://127.0.0.1:6000/' + self.chunk_id
self.chunk_url2 = 'http://127.0.0.1:6002/' + self.chunk_id2
self.chunk_pos = '0'
self.metadata = {
'content_cid': self.content_cid,
'content_path': self.content_path,
'content_size': 1,
'content_chunksnb': 1,
'chunk_id': self.chunk_id,
'chunk_pos': self.chunk_pos,
'chunk_size': self.chunk_size,
'chunk_hash': self.chunk_hash
}
self.chunk = {'url': self.chunk_url, 'pos': self.chunk_pos}
self.chunk2 = {'url': self.chunk_url2, 'pos': self.chunk_pos}
self.chunks = [self.chunk, self.chunk2]
self.spare_data = {
'notin': [self.chunk2],
'broken': [self.chunk],
'size': 0
}
self.new_chunk_id = 'BBBB'
self.new_chunk_url = 'http://127.0.0.1:6001/' + self.new_chunk_id
self.new_chunks = {'chunks': [{'id': self.new_chunk_url}]}
self.new_metadata = self.metadata.copy()
self.new_metadata['chunk_id'] = self.new_chunk_id
self.update_data = {
'old': [{
'type': 'chunk',
'id': self.chunk_url,
'size': self.chunk_size,
'hash': self.chunk_hash
}],
'new': [{
'type': 'chunk',
'id': self.new_chunk_url,
'size': self.chunk_size,
'hash': self.chunk_hash
}]
}
def test_move_with_wrong_size(self):
if not self.chunk_method.startswith('ec'):
self.skipTest('Only works with EC')
orig_chunk = random.choice(self.chunks)
chunk_volume = orig_chunk['url'].split('/')[2]
chunk_id = orig_chunk['url'].split('/')[3]
mover = BlobMoverWorker(self.conf, None,
self.rawx_volumes[chunk_volume])
meta, stream = mover.blob_client.chunk_get(orig_chunk['url'])
data = stream.read()
stream.close()
data = data[:-1]
del meta['chunk_hash']
wrong_stream = GeneratorIO(data)
mover.blob_client.chunk_get = Mock(return_value=(meta, wrong_stream))
self.assertRaises(ChunkException, mover.chunk_move,
self._chunk_path(orig_chunk), chunk_id)