def test_write_timeout(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE
meta_chunk = self.meta_chunk()
resps = [201] * (len(meta_chunk) - 1)
resps.append(Timeout(1.0))
with set_http_connect(*resps):
handler = ReplicatedChunkWriteHandler(self.sysmeta, meta_chunk,
checksum,
self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), len(meta_chunk) - 1)
for i in range(len(meta_chunk) - 1):
self.assertEqual(chunks[i].get('error'), None)
# # JFS: starting at branche 3.x, it has been preferred to save only
# # the chunks that succeeded.
# self.assertEqual(
# chunks[len(meta_chunk) - 1].get('error'), '1.0 second')
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def test_write_response_error(self):
test_cases = [
{
'error': green.ChunkWriteTimeout(1.0),
'msg': 'resp: Chunk write timeout 1.0 second'
},
{
'error': Exception('failure'),
'msg': 'resp: failure'
},
]
for test in test_cases:
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + \
self.storage_method.ec_nb_parity
resps = [201] * (nb - 1)
resps.append((100, test['error']))
with set_http_connect(*resps):
handler = EcMetachunkWriter(self.sysmeta,
self.meta_chunk_copy(), checksum,
self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(
source, size)
self.assertEqual(len(chunks), nb)
for i in range(nb - 1):
self.assertEqual(chunks[i].get('error'), None)
self.assertEqual(chunks[nb - 1].get('error'), test['msg'])
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_MD5)
def test_write_connect_errors(self):
test_cases = [
{
'error': green.ConnectionTimeout(1.0),
'msg': 'connect: Connection timeout 1.0 second'
},
{
'error': Exception('failure'),
'msg': 'connect: failure'
},
]
for test in test_cases:
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + \
self.storage_method.ec_nb_parity
resps = [201] * (nb - 1)
# Put the error in the middle to mess with chunk indices
err_pos = random.randint(0, nb)
resps.insert(err_pos, test['error'])
with set_http_connect(*resps):
handler = EcMetachunkWriter(self.sysmeta,
self.meta_chunk_copy(), checksum,
self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(
source, size)
self.assertEqual(len(chunks), nb)
for i in range(nb - 1):
self.assertEqual(chunks[i].get('error'), None)
self.assertEqual(chunks[nb - 1].get('error'), test['msg'])
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_MD5)
def test_write_quorum_success(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE
meta_chunk = self.meta_chunk()
quorum_size = self.storage_method.quorum
resps = [201] * quorum_size
resps += [500] * (len(meta_chunk) - quorum_size)
with set_http_connect(*resps):
handler = ReplicatedMetachunkWriter(
self.sysmeta, meta_chunk, checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), len(meta_chunk)-1)
for i in range(quorum_size):
self.assertEqual(chunks[i].get('error'), None)
# # JFS: starting at branche 3.x, it has been preferred to save
# # only the chunks that succeeded.
# for i in xrange(quorum_size, len(meta_chunk)):
# self.assertEqual(chunks[i].get('error'), 'HTTP 500')
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_MD5)
def test_write_exception(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [500] * nb
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
self.assertRaises(exc.OioException, handler.stream, source, size)
def test_write_exception(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [500] * nb
with set_http_connect(*resps):
handler = EcMetachunkWriter(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
self.assertRaises(exc.OioException, handler.stream, source, size)
def test_write_exception(self):
checksum = self.checksum()
source = empty_stream()
meta_chunk = self.meta_chunk()
size = CHUNK_SIZE
resps = [500] * len(meta_chunk)
with set_http_connect(*resps):
handler = ReplicatedMetachunkWriter(
self.sysmeta, meta_chunk, checksum, self.storage_method)
self.assertRaises(exc.OioException, handler.stream, source, size)
def test_write_exception(self):
checksum = self.checksum()
source = empty_stream()
meta_chunk = self.meta_chunk()
size = CHUNK_SIZE
resps = [500] * len(meta_chunk)
with set_http_connect(*resps):
handler = ReplicatedChunkWriteHandler(
self.sysmeta, meta_chunk, checksum, self.storage_method)
self.assertRaises(exc.OioException, handler.stream, source, size)
def test_write_quorum_error(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE
meta_chunk = self.meta_chunk()
quorum_size = self.storage_method.quorum
resps = [500] * quorum_size
resps += [201] * (len(meta_chunk) - quorum_size)
with set_http_connect(*resps):
handler = ReplicatedChunkWriteHandler(
self.sysmeta, meta_chunk, checksum, self.storage_method)
self.assertRaises(exc.OioException, handler.stream, source, size)
def test_write_quorum_error(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE
meta_chunk = self.meta_chunk()
quorum_size = self.storage_method.quorum
resps = [500] * quorum_size
resps += [201] * (len(meta_chunk) - quorum_size)
with set_http_connect(*resps):
handler = ReplicatedMetachunkWriter(
self.sysmeta, meta_chunk, checksum, self.storage_method)
self.assertRaises(exc.OioException, handler.stream, source, size)
def test_write_quorum_error(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
quorum_size = self.storage_method.quorum
resps = [500] * quorum_size
resps += [201] * (nb - quorum_size)
with set_http_connect(*resps):
handler = EcMetachunkWriter(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
self.assertRaises(exc.ServiceBusy, handler.stream, source, size)
def test_write_simple(self):
checksum = self.checksum()
source = empty_stream()
meta_chunk = self.meta_chunk()
size = CHUNK_SIZE
resps = [201] * len(meta_chunk)
with set_http_connect(*resps):
handler = ReplicatedMetachunkWriter(self.sysmeta, meta_chunk,
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), len(meta_chunk))
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def test_write_quorum_error(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
quorum_size = self.storage_method.quorum
resps = [500] * quorum_size
resps += [201] * (nb - quorum_size)
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
# TODO use specialized Exception
self.assertRaises(exc.OioException, handler.stream, source, size)
def test_write_simple(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [201] * nb
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), nb)
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def test_write_simple(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [201] * nb
with set_http_connect(*resps):
handler = EcMetachunkWriter(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), nb)
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_MD5)
def test_write_quorum_error(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
quorum_size = self.storage_method.quorum
resps = [500] * quorum_size
resps += [201] * (nb - quorum_size)
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
# TODO use specialized Exception
self.assertRaises(exc.OioException, handler.stream, source, size)
def test_write_exception(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [500] * nb
with set_http_connect(*resps):
handler = EcMetachunkWriter(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
# From now on, exceptions happening during chunk upload are
# considered retryable, and thus will tell the caller the
# service was just too busy.
self.assertRaises(exc.ServiceBusy, handler.stream, source, size)
def test_write_simple(self):
checksum = self.checksum()
source = empty_stream()
meta_chunk = self.meta_chunk()
size = CHUNK_SIZE
resps = [201] * len(meta_chunk)
with set_http_connect(*resps):
handler = ReplicatedChunkWriteHandler(
self.sysmeta, meta_chunk, checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(
source, size)
self.assertEqual(len(chunks), len(meta_chunk))
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def _test_write_checksum_algo(self, expected_checksum, **kwargs):
global_checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [201] * nb
with set_http_connect(*resps):
handler = EcMetachunkWriter(self.sysmeta, self.meta_chunk(),
global_checksum, self.storage_method,
**kwargs)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(nb, len(chunks))
self.assertEqual(0, bytes_transferred)
self.assertEqual(expected_checksum, checksum)
def _test_write_checksum_algo(self, expected_checksum, **kwargs):
global_checksum = self.checksum()
source = empty_stream()
meta_chunk = self.meta_chunk()
size = CHUNK_SIZE
resps = [201] * len(meta_chunk)
with set_http_connect(*resps, headers=kwargs.get('headers')):
handler = ReplicatedMetachunkWriter(
self.sysmeta, self.meta_chunk(), global_checksum,
self.storage_method, **kwargs)
bytes_transferred, checksum, chunks = \
handler.stream(source, size)
self.assertEqual(len(meta_chunk), len(chunks))
self.assertEqual(0, bytes_transferred)
self.assertEqual(expected_checksum, checksum)
def test_write_timeout(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [201] * (nb - 1)
resps.append(Timeout(1.0))
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), nb)
for i in range(nb - 1):
self.assertEqual(chunks[i].get('error'), None)
self.assertEqual(chunks[nb - 1].get('error'), '1.0 second')
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def test_write_partial_exception(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [201] * (nb - 1)
resps.append(Exception("failure"))
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), nb)
for i in range(nb - 1):
self.assertEqual(chunks[i].get('error'), None)
self.assertEqual(chunks[nb - 1].get('error'), 'failure')
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def test_write_partial_exception(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE
meta_chunk = self.meta_chunk()
resps = [201] * (len(meta_chunk) - 1)
resps.append(Exception("failure"))
with set_http_connect(*resps):
handler = ReplicatedChunkWriteHandler(
self.sysmeta, meta_chunk, checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), len(meta_chunk))
for i in range(len(meta_chunk) - 1):
self.assertEqual(chunks[i].get('error'), None)
self.assertEqual(chunks[len(meta_chunk) - 1].get('error'), 'failure')
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def test_write_quorum_success(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
quorum_size = self.storage_method.quorum
resps = [201] * quorum_size
resps += [500] * (nb - quorum_size)
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), nb)
for i in range(quorum_size):
self.assertEqual(chunks[i].get('error'), None)
for i in xrange(quorum_size, nb):
self.assertEqual(chunks[i].get('error'), 'HTTP 500')
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def test_write_quorum_success(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
quorum_size = self.storage_method.quorum
resps = [201] * quorum_size
resps += [500] * (nb - quorum_size)
with set_http_connect(*resps):
handler = EcMetachunkWriter(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), nb)
for i in range(quorum_size):
self.assertEqual(chunks[i].get('error'), None)
for i in range(quorum_size, nb):
self.assertEqual(chunks[i].get('error'), 'resp: HTTP 500')
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_MD5)
def test_write_quorum_success(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE
meta_chunk = self.meta_chunk()
quorum_size = self.storage_method.quorum
resps = [201] * quorum_size
resps += [500] * (len(meta_chunk) - quorum_size)
with set_http_connect(*resps):
handler = ReplicatedChunkWriteHandler(self.sysmeta, meta_chunk,
checksum,
self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), len(meta_chunk))
for i in range(quorum_size):
self.assertEqual(chunks[i].get('error'), None)
for i in xrange(quorum_size, len(meta_chunk)):
self.assertEqual(chunks[i].get('error'), 'HTTP 500')
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)