def test_padded_segment(self):
"""test a segment that needs padding"""
segment_size = incoming_slice_size - 1
test_data = os.urandom(segment_size)
segmenter = ZfecSegmenter(_min_segments, _num_segments)
padding_size = segmenter.padding_size(test_data)
encoded_segments = segmenter.encode(block_generator(test_data))
segment_numbers = range(1, _num_segments + 1)
test_segment_numbers = random.sample(segment_numbers, _min_segments)
test_segments = [encoded_segments[n - 1] for n in test_segment_numbers]
decoded_segments = segmenter.decode(test_segments,
test_segment_numbers, padding_size)
decoded_data = "".join(decoded_segments)
self.assertTrue(decoded_data == test_data, len(decoded_data))
def test_padded_segment(self):
"""test a segment that needs padding"""
segment_size = incoming_slice_size - 1
test_data = os.urandom(segment_size)
segmenter = ZfecSegmenter(_min_segments, _num_segments)
padding_size = segmenter.padding_size(test_data)
encoded_segments = segmenter.encode(block_generator(test_data))
segment_numbers = range(1, _num_segments+1)
test_segment_numbers = random.sample(segment_numbers, _min_segments)
test_segments = [encoded_segments[n-1] for n in test_segment_numbers]
decoded_segments = segmenter.decode(
test_segments, test_segment_numbers, padding_size
)
decoded_data = "".join(decoded_segments)
self.assertTrue(decoded_data == test_data, len(decoded_data))
def _run_test(req_socket, segment_size):
"""test a segment that doesn't need padding"""
log = logging.getLogger("_run_test_{0}".format(segment_size))
test_data = os.urandom(segment_size)
encoded_blocks = list()
for raw_block in block_generator(test_data):
request = {
"message-type": "zfec-encode",
}
reply, reply_data = _contact_server(req_socket, request, [
raw_block,
])
if reply["result"] != "success":
log.error("{0} failed {1}".format(request, reply["error-message"]))
return False
encoded_blocks.append((
reply["padding-size"],
reply_data,
))
segment_numbers = range(1, _num_segments + 1)
test_segment_numbers = random.sample(segment_numbers, _min_segments)
decoded_segments = list()
for padding_size, encoded_block in encoded_blocks:
request = {
"message-type": "zfec-decode",
"segment-numbers": test_segment_numbers,
"padding-size": padding_size
}
test_segments = [encoded_block[n - 1] for n in test_segment_numbers]
reply, reply_data = _contact_server(req_socket, request, test_segments)
if reply["result"] != "success":
log.error("{0} failed {1}".format(request, reply["error-message"]))
return False
decoded_segments.append(reply_data[0])
decoded_data = b"".join(decoded_segments)
if decoded_data != test_data:
log.error("decoded data does not match test data")
return False
for padding_size, encoded_block in encoded_blocks:
good_segment_numbers = random.sample(segment_numbers, _min_segments)
bad_segment_numbers = \
list(set(segment_numbers) - set(good_segment_numbers))
request = {
"message-type": "zfec-rebuild-encoded-shares",
"segment-numbers": good_segment_numbers,
"needed-segment-numbers": bad_segment_numbers,
"padding-size": padding_size
}
good_segments = [encoded_block[n - 1] for n in good_segment_numbers]
reply, reply_data = _contact_server(req_socket, request, good_segments)
if reply["result"] != "success":
log.error("{0} failed {1}".format(request, reply))
return False
zip_object = zip(reply["rebuilt-segment-numbers"], reply_data)
for segment_num, rebuilt_segment in zip_object:
if rebuilt_segment != encoded_block[segment_num - 1]:
log.error("{0} failed rebuilt block mismatch".format(request))
return False
return True
file_md5 = hashlib.md5()
file_size = 0
segments = None
zfec_padding_size = None
try:
while True:
slice_item = \
data_queue.get(block=True,
timeout=_max_sequence_upload_interval)
if slice_item is None:
break
actual_content_length += len(slice_item)
file_adler32 = zlib.adler32(slice_item, file_adler32)
file_md5.update(slice_item)
file_size += len(slice_item)
segments = segmenter.encode(block_generator(slice_item))
zfec_padding_size = segmenter.padding_size(slice_item)
if actual_content_length == expected_content_length:
archiver.archive_final(
file_size,
file_adler32,
file_md5.digest(),
segments,
zfec_padding_size,
_reply_timeout
)
else:
archiver.archive_slice(
segments, zfec_padding_size, _reply_timeout
)
except gevent.queue.Empty, instance:
file_md5 = hashlib.md5()
file_size = 0
segments = None
zfec_padding_size = None
try:
while True:
slice_item = \
data_queue.get(block=True,
timeout=_max_sequence_upload_interval)
if slice_item is None:
break
actual_content_length += len(slice_item)
file_adler32 = zlib.adler32(slice_item, file_adler32)
file_md5.update(slice_item)
file_size += len(slice_item)
segments = segmenter.encode(block_generator(slice_item))
zfec_padding_size = segmenter.padding_size(slice_item)
if actual_content_length == expected_content_length:
archiver.archive_final(file_size, file_adler32,
file_md5.digest(), segments,
zfec_padding_size, _reply_timeout)
else:
archiver.archive_slice(segments, zfec_padding_size,
_reply_timeout)
except gevent.queue.Empty, instance:
# Ticket #69 Protection in Web Writer from Slow Uploads
self._log.error("archive failed: {0} timeout {1}".format(
description, instance))
_send_archive_cancel(user_request_id, unified_id, conjoined_part,
self._data_writer_clients)
queue_entry = \
