def time_encode(self, num_data, num_parity, ec_type, hd, file_size,
iterations):
"""
:return average encode time
"""
timer = Timer()
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
timer.start()
for l in range(iterations):
pyeclib_c.encode(handle, whole_file_bytes)
tsum = timer.stop_and_return()
return tsum / iterations
def time_encode(self, num_data, num_parity, ec_type, hd,
file_size, iterations):
"""
:return average encode time
"""
timer = Timer()
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
timer.start()
for l in range(iterations):
pyeclib_c.encode(handle, whole_file_bytes)
tsum = timer.stop_and_return()
return tsum / iterations
def time_decode(self, num_data, num_parity, ec_type, hd, file_size,
iterations):
"""
:return 2-tuple, (success, average decode time)
"""
timer = Timer()
tsum = 0
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
success = True
fragments = pyeclib_c.encode(handle, whole_file_bytes)
orig_fragments = fragments[:]
for i in range(iterations):
missing_idxs = []
num_missing = hd - 1
for j in range(num_missing):
num_frags_left = len(fragments)
idx = random.randint(0, num_frags_left - 1)
fragments.pop(idx)
timer.start()
decoded_file_bytes = pyeclib_c.decode(handle, fragments,
len(fragments[0]))
tsum += timer.stop_and_return()
fragments = orig_fragments[:]
if whole_file_bytes != decoded_file_bytes:
success = False
return success, tsum / iterations
def time_decode(self,
num_data, num_parity, ec_type, hd,
file_size, iterations):
"""
:return 2-tuple, (success, average decode time)
"""
timer = Timer()
tsum = 0
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
success = True
fragments = pyeclib_c.encode(handle, whole_file_bytes)
orig_fragments = fragments[:]
for i in range(iterations):
missing_idxs = []
num_missing = hd - 1
for j in range(num_missing):
num_frags_left = len(fragments)
idx = random.randint(0, num_frags_left - 1)
fragments.pop(idx)
timer.start()
decoded_file_bytes = pyeclib_c.decode(handle,
fragments,
len(fragments[0]))
tsum += timer.stop_and_return()
fragments = orig_fragments[:]
if whole_file_bytes != decoded_file_bytes:
success = False
return success, tsum / iterations
def time_range_decode(self, num_data, num_parity, ec_type, hd, file_size,
iterations):
"""
:return 2-tuple, (success, average decode time)
"""
timer = Timer()
tsum = 0
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
success = True
begins = [
long(random.randint(0,
len(whole_file_bytes) - 1)) for i in range(3)
]
ends = [
long(random.randint(begins[i], len(whole_file_bytes)))
for i in range(3)
]
ranges = zip(begins, ends)
fragments = pyeclib_c.encode(handle, whole_file_bytes)
orig_fragments = fragments[:]
for i in range(iterations):
missing_idxs = []
num_missing = hd - 1
for j in range(num_missing):
num_frags_left = len(fragments)
idx = random.randint(0, num_frags_left - 1)
fragments.pop(idx)
timer.start()
decoded_file_ranges = pyeclib_c.decode(handle, fragments,
len(fragments[0]), ranges)
tsum += timer.stop_and_return()
fragments = orig_fragments[:]
range_offset = 0
for r in ranges:
if whole_file_bytes[r[0]:r[1] +
1] != decoded_file_ranges[range_offset]:
success = False
range_offset += 1
return success, tsum / iterations
def time_range_decode(self,
num_data, num_parity, ec_type, hd,
file_size, iterations):
"""
:return 2-tuple, (success, average decode time)
"""
timer = Timer()
tsum = 0
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
success = True
begins = [int(random.randint(0, len(whole_file_bytes) - 1))
for i in range(3)]
ends = [int(random.randint(begins[i], len(whole_file_bytes)))
for i in range(3)]
ranges = list(zip(begins, ends))
fragments = pyeclib_c.encode(handle, whole_file_bytes)
orig_fragments = fragments[:]
for i in range(iterations):
num_missing = hd - 1
for j in range(num_missing):
num_frags_left = len(fragments)
idx = random.randint(0, num_frags_left - 1)
fragments.pop(idx)
timer.start()
decoded_file_ranges = pyeclib_c.decode(handle,
fragments,
len(fragments[0]),
ranges)
tsum += timer.stop_and_return()
fragments = orig_fragments[:]
range_offset = 0
for r in ranges:
if whole_file_bytes[
r[0]: r[1] + 1] != decoded_file_ranges[range_offset]:
success = False
range_offset += 1
return success, tsum / iterations
def time_reconstruct(self,
num_data, num_parity, ec_type, hd,
file_size, iterations):
"""
:return 2-tuple, (success, average reconstruct time)
"""
timer = Timer()
tsum = 0
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
success = True
orig_fragments = pyeclib_c.encode(handle, whole_file_bytes)
for i in range(iterations):
fragments = orig_fragments[:]
num_missing = 1
missing_idxs = []
for j in range(num_missing):
num_frags_left = len(fragments)
idx = random.randint(0, num_frags_left - 1)
while idx in missing_idxs:
idx = random.randint(0, num_frags_left - 1)
missing_idxs.append(idx)
fragments.pop(idx)
timer.start()
reconstructed_fragment = pyeclib_c.reconstruct(handle,
fragments,
len(fragments[0]),
missing_idxs[0])
tsum += timer.stop_and_return()
if orig_fragments[missing_idxs[0]] != reconstructed_fragment:
success = False
# Output the fragments for debugging
with open("orig_fragments", "wb") as fd_orig:
fd_orig.write(orig_fragments[missing_idxs[0]])
with open("decoded_fragments", "wb") as fd_decoded:
fd_decoded.write(reconstructed_fragment)
print(("Fragment %d was not reconstructed!!!" % missing_idxs[0]))
sys.exit(2)
return success, tsum / iterations
def time_reconstruct(self, num_data, num_parity, ec_type, hd, file_size,
iterations):
"""
:return 2-tuple, (success, average reconstruct time)
"""
timer = Timer()
tsum = 0
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
success = True
orig_fragments = pyeclib_c.encode(handle, whole_file_bytes)
for i in range(iterations):
fragments = orig_fragments[:]
num_missing = 1
missing_idxs = []
for j in range(num_missing):
num_frags_left = len(fragments)
idx = random.randint(0, num_frags_left - 1)
while idx in missing_idxs:
idx = random.randint(0, num_frags_left - 1)
missing_idxs.append(idx)
fragments.pop(idx)
timer.start()
reconstructed_fragment = pyeclib_c.reconstruct(
handle, fragments, len(fragments[0]), missing_idxs[0])
tsum += timer.stop_and_return()
if orig_fragments[missing_idxs[0]] != reconstructed_fragment:
success = False
# Output the fragments for debugging
with open("orig_fragments", "wb") as fd_orig:
fd_orig.write(orig_fragments[missing_idxs[0]])
with open("decoded_fragments", "wb") as fd_decoded:
fd_decoded.write(reconstructed_fragment)
print(("Fragment %d was not reconstructed!!!" %
missing_idxs[0]))
sys.exit(2)
return success, tsum / iterations
def encode(self, data_bytes):
return pyeclib_c.encode(self.handle, data_bytes)
