def write_ec_meta_chunk(self, source, size, storage_method, sysmeta,
meta_chunk):
meta_checksum = md5()
handler = ECChunkWriteHandler(sysmeta, meta_chunk, meta_checksum,
storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
return Response("OK")
def test_write_response_error(self):
test_cases = [
{
'error': Timeout(1.0),
'msg': 'resp: Timeout 1.0 second'
},
{
'error': Exception('failure'),
'msg': 'resp: failure'
},
]
for test in test_cases:
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + \
self.storage_method.ec_nb_parity
resps = [201] * (nb - 1)
resps.append((100, test['error']))
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta,
self.meta_chunk_copy(), checksum,
self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(
source, size)
self.assertEqual(len(chunks), nb)
for i in range(nb - 1):
self.assertEqual(chunks[i].get('error'), None)
self.assertEqual(chunks[nb - 1].get('error'), test['msg'])
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def write_ec_meta_chunk(self, source, size, storage_method, sysmeta,
meta_chunk):
meta_checksum = md5()
handler = ECChunkWriteHandler(sysmeta, meta_chunk, meta_checksum,
storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
return Response("OK")
def test_write_transfer(self):
checksum = self.checksum()
segment_size = self.storage_method.ec_segment_size
test_data = ('1234' * segment_size)[:-10]
size = len(test_data)
test_data_checksum = self.checksum(test_data).hexdigest()
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [201] * nb
source = BytesIO(test_data)
put_reqs = defaultdict(lambda: {'parts': []})
def cb_body(conn_id, part):
put_reqs[conn_id]['parts'].append(part)
# TODO test headers
with set_http_connect(*resps, cb_body=cb_body):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(test_data), bytes_transferred)
self.assertEqual(checksum, self.checksum(test_data).hexdigest())
fragments = []
for conn_id, info in put_reqs.items():
body, trailers = decode_chunked_body(''.join(info['parts']))
fragments.append(body)
metachunk_size = int(trailers[chunk_headers['metachunk_size']])
metachunk_hash = trailers[chunk_headers['metachunk_hash']]
self.assertEqual(metachunk_size, size)
self.assertEqual(metachunk_hash, test_data_checksum)
self.assertEqual(len(fragments), nb)
fragment_size = self.storage_method.ec_fragment_size
# retrieve segments
frags = []
for frag in fragments:
data = [
frag[x:x + fragment_size]
for x in range(0, len(frag), fragment_size)
]
frags.append(data)
fragments = zip(*frags)
final_data = ''
for frag in fragments:
self.assertEqual(len(frag), nb)
frag = list(frag)
final_data += self.storage_method.driver.decode(frag)
self.assertEqual(len(test_data), len(final_data))
self.assertEqual(test_data_checksum,
self.checksum(final_data).hexdigest())
def test_write_transfer(self):
checksum = self.checksum()
segment_size = self.storage_method.ec_segment_size
test_data = ('1234' * segment_size)[:-10]
size = len(test_data)
test_data_checksum = self.checksum(test_data).hexdigest()
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [201] * nb
source = StringIO(test_data)
put_reqs = defaultdict(lambda: {'parts': []})
def cb_body(conn_id, part):
put_reqs[conn_id]['parts'].append(part)
# TODO test headers
with set_http_connect(*resps, cb_body=cb_body):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(test_data), bytes_transferred)
self.assertEqual(checksum, self.checksum(test_data).hexdigest())
fragments = []
for conn_id, info in put_reqs.items():
body, trailers = decode_chunked_body(''.join(info['parts']))
fragments.append(body)
metachunk_size = int(trailers[chunk_headers['metachunk_size']])
metachunk_hash = trailers[chunk_headers['metachunk_hash']]
self.assertEqual(metachunk_size, size)
self.assertEqual(metachunk_hash, test_data_checksum)
self.assertEqual(len(fragments), nb)
fragment_size = self.storage_method.ec_fragment_size
# retrieve segments
frags = []
for frag in fragments:
data = [frag[x:x + fragment_size]
for x in range(0, len(frag), fragment_size)]
frags.append(data)
fragments = zip(*frags)
final_data = ''
for frag in fragments:
self.assertEqual(len(frag), nb)
frag = list(frag)
final_data += self.storage_method.driver.decode(frag)
self.assertEqual(len(test_data), len(final_data))
self.assertEqual(
test_data_checksum, self.checksum(final_data).hexdigest())
def test_write_simple(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [201] * nb
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), nb)
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def test_write_simple(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [201] * nb
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), nb)
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def test_write_partial_exception(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [201] * (nb - 1)
resps.append(Exception("failure"))
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), nb)
for i in range(nb - 1):
self.assertEqual(chunks[i].get('error'), None)
self.assertEqual(chunks[nb - 1].get('error'), 'failure')
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def test_write_partial_exception(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
resps = [201] * (nb - 1)
resps.append(Exception("failure"))
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), nb)
for i in range(nb - 1):
self.assertEqual(chunks[i].get('error'), None)
self.assertEqual(chunks[nb - 1].get('error'), 'failure')
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def test_write_quorum_success(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
quorum_size = self.storage_method.quorum
resps = [201] * quorum_size
resps += [500] * (nb - quorum_size)
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), nb)
for i in range(quorum_size):
self.assertEqual(chunks[i].get('error'), None)
for i in xrange(quorum_size, nb):
self.assertEqual(chunks[i].get('error'), 'resp: HTTP 500')
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)
def test_write_quorum_success(self):
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + self.storage_method.ec_nb_parity
quorum_size = self.storage_method.quorum
resps = [201] * quorum_size
resps += [500] * (nb - quorum_size)
with set_http_connect(*resps):
handler = ECChunkWriteHandler(self.sysmeta, self.meta_chunk(),
checksum, self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(source, size)
self.assertEqual(len(chunks), nb)
for i in range(quorum_size):
self.assertEqual(chunks[i].get('error'), None)
for i in xrange(quorum_size, nb):
self.assertEqual(chunks[i].get('error'), 'HTTP 500')
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_CHECKSUM)