def test_download_with_missing_chunks(self):
container = random_str(8)
obj = random_str(8)
expected_data = random_data(10)
chunks, _, _, meta = self.storage.object_create_ext(self.account,
container,
obj_name=obj,
data=expected_data,
policy='EC')
storage_method = STORAGE_METHODS.load(meta['chunk_method'])
sorted_chunks = _sort_chunks(chunks, storage_method.ec)
for i in range(storage_method.ec_nb_parity):
data = b''
for pos in range(len(sorted_chunks)):
chunk = random.choice(sorted_chunks[pos])
sorted_chunks[pos].remove(chunk)
resp = self._download_metachunk(meta, sorted_chunks[pos])
self.assertEqual(200, resp.status)
data += resp.data
self.assertEqual(expected_data, data)
for pos in range(len(sorted_chunks)):
chunk = random.choice(sorted_chunks[pos])
sorted_chunks[pos].remove(chunk)
resp = self._download_metachunk(meta, sorted_chunks[pos])
self.assertEqual(500, resp.status)
def test_download_with_stopped_rawx(self):
container = random_str(8)
obj = random_str(8)
expected_data = random_data(10)
chunks, _, _, meta = self.storage.object_create_ext(
self.account, container, obj_name=obj, data=expected_data,
policy='EC')
storage_method = STORAGE_METHODS.load(meta['chunk_method'])
sorted_chunks = _sort_chunks(chunks, storage_method.ec)
sorted_present_chunks = sorted_chunks.copy()
try:
for i in range(storage_method.ec_nb_parity):
data = ''
for pos in range(len(sorted_chunks)):
if pos == 0:
chunk = random.choice(sorted_present_chunks[pos])
sorted_present_chunks[pos].remove(chunk)
gridinit_key = self.service_to_gridinit_key(
urlparse(chunk['url']).netloc, 'rawx')
self._service(gridinit_key, 'stop')
resp = self._download_metachunk(meta, sorted_chunks[pos])
self.assertEqual(200, resp.status)
data += resp.data
self.assertEqual(expected_data, data)
chunk = random.choice(sorted_present_chunks[0])
sorted_present_chunks[0].remove(chunk)
gridinit_key = self.service_to_gridinit_key(
urlparse(chunk['url']).netloc, 'rawx')
self._service(gridinit_key, 'stop')
resp = self._download_metachunk(meta, sorted_chunks[pos])
self.assertEqual(500, resp.status)
finally:
self._service('@rawx', 'start')
def _upload_data(self, name):
chunksize = int(self.conf["chunk_size"])
size = int(chunksize * 12)
data = random_data(int(size))
self.api.object_create(self.account, name, obj_name=name, data=data)
self.created.append((name, name))
_, chunks = self.api.object_locate(self.account, name, name)
logging.debug("Chunks: %s", chunks)
return sort_chunks(chunks, False), data
def _upload_data(self, name):
# FIXME: find chunk_size
size = int(1048576 * 12)
data = random_data(int(size))
self.api.object_create(self.account, name, obj_name=name, data=data)
self.created.append((name, name))
_, chunks = self.api.object_analyze(self.account, name, name)
logging.debug("Chunks: %s", chunks)
return sort_chunks(chunks, False), data
def _test_download(self, length):
container = random_str(8)
obj = random_str(8)
expected_data = random_data(length)
chunks, _, _, meta = self.storage.object_create_ext(
self.account, container, obj_name=obj, data=expected_data,
policy='EC')
storage_method = STORAGE_METHODS.load(meta['chunk_method'])
sorted_chunks = _sort_chunks(chunks, storage_method.ec)
data = ''
for pos in range(len(sorted_chunks)):
resp = self._download_metachunk(meta, sorted_chunks[pos])
self.assertEqual(200, resp.status)
data += resp.data
self.assertEqual(expected_data, data)