def time_encode(self, num_data, num_parity, ec_type, hd, file_size, iterations): """ :return average encode time """ timer = Timer() handle = pyeclib_c.init(num_data, num_parity, ec_type, hd) whole_file_bytes = self.get_tmp_file(file_size).read() timer.start() for l in range(iterations): pyeclib_c.encode(handle, whole_file_bytes) tsum = timer.stop_and_return() return tsum / iterations
def time_decode(self, num_data, num_parity, ec_type, hd, file_size, iterations): """ :return 2-tuple, (success, average decode time) """ timer = Timer() tsum = 0 handle = pyeclib_c.init(num_data, num_parity, ec_type, hd) whole_file_bytes = self.get_tmp_file(file_size).read() success = True fragments = pyeclib_c.encode(handle, whole_file_bytes) orig_fragments = fragments[:] for i in range(iterations): missing_idxs = [] num_missing = hd - 1 for j in range(num_missing): num_frags_left = len(fragments) idx = random.randint(0, num_frags_left - 1) fragments.pop(idx) timer.start() decoded_file_bytes = pyeclib_c.decode(handle, fragments, len(fragments[0])) tsum += timer.stop_and_return() fragments = orig_fragments[:] if whole_file_bytes != decoded_file_bytes: success = False return success, tsum / iterations
def time_range_decode(self, num_data, num_parity, ec_type, hd, file_size, iterations): """ :return 2-tuple, (success, average decode time) """ timer = Timer() tsum = 0 handle = pyeclib_c.init(num_data, num_parity, ec_type, hd) whole_file_bytes = self.get_tmp_file(file_size).read() success = True begins = [ long(random.randint(0, len(whole_file_bytes) - 1)) for i in range(3) ] ends = [ long(random.randint(begins[i], len(whole_file_bytes))) for i in range(3) ] ranges = zip(begins, ends) fragments = pyeclib_c.encode(handle, whole_file_bytes) orig_fragments = fragments[:] for i in range(iterations): missing_idxs = [] num_missing = hd - 1 for j in range(num_missing): num_frags_left = len(fragments) idx = random.randint(0, num_frags_left - 1) fragments.pop(idx) timer.start() decoded_file_ranges = pyeclib_c.decode(handle, fragments, len(fragments[0]), ranges) tsum += timer.stop_and_return() fragments = orig_fragments[:] range_offset = 0 for r in ranges: if whole_file_bytes[r[0]:r[1] + 1] != decoded_file_ranges[range_offset]: success = False range_offset += 1 return success, tsum / iterations
def time_range_decode(self, num_data, num_parity, ec_type, hd, file_size, iterations): """ :return 2-tuple, (success, average decode time) """ timer = Timer() tsum = 0 handle = pyeclib_c.init(num_data, num_parity, ec_type, hd) whole_file_bytes = self.get_tmp_file(file_size).read() success = True begins = [int(random.randint(0, len(whole_file_bytes) - 1)) for i in range(3)] ends = [int(random.randint(begins[i], len(whole_file_bytes))) for i in range(3)] ranges = list(zip(begins, ends)) fragments = pyeclib_c.encode(handle, whole_file_bytes) orig_fragments = fragments[:] for i in range(iterations): num_missing = hd - 1 for j in range(num_missing): num_frags_left = len(fragments) idx = random.randint(0, num_frags_left - 1) fragments.pop(idx) timer.start() decoded_file_ranges = pyeclib_c.decode(handle, fragments, len(fragments[0]), ranges) tsum += timer.stop_and_return() fragments = orig_fragments[:] range_offset = 0 for r in ranges: if whole_file_bytes[ r[0]: r[1] + 1] != decoded_file_ranges[range_offset]: success = False range_offset += 1 return success, tsum / iterations
def time_reconstruct(self, num_data, num_parity, ec_type, hd, file_size, iterations): """ :return 2-tuple, (success, average reconstruct time) """ timer = Timer() tsum = 0 handle = pyeclib_c.init(num_data, num_parity, ec_type, hd) whole_file_bytes = self.get_tmp_file(file_size).read() success = True orig_fragments = pyeclib_c.encode(handle, whole_file_bytes) for i in range(iterations): fragments = orig_fragments[:] num_missing = 1 missing_idxs = [] for j in range(num_missing): num_frags_left = len(fragments) idx = random.randint(0, num_frags_left - 1) while idx in missing_idxs: idx = random.randint(0, num_frags_left - 1) missing_idxs.append(idx) fragments.pop(idx) timer.start() reconstructed_fragment = pyeclib_c.reconstruct(handle, fragments, len(fragments[0]), missing_idxs[0]) tsum += timer.stop_and_return() if orig_fragments[missing_idxs[0]] != reconstructed_fragment: success = False # Output the fragments for debugging with open("orig_fragments", "wb") as fd_orig: fd_orig.write(orig_fragments[missing_idxs[0]]) with open("decoded_fragments", "wb") as fd_decoded: fd_decoded.write(reconstructed_fragment) print(("Fragment %d was not reconstructed!!!" % missing_idxs[0])) sys.exit(2) return success, tsum / iterations
def time_reconstruct(self, num_data, num_parity, ec_type, hd, file_size, iterations): """ :return 2-tuple, (success, average reconstruct time) """ timer = Timer() tsum = 0 handle = pyeclib_c.init(num_data, num_parity, ec_type, hd) whole_file_bytes = self.get_tmp_file(file_size).read() success = True orig_fragments = pyeclib_c.encode(handle, whole_file_bytes) for i in range(iterations): fragments = orig_fragments[:] num_missing = 1 missing_idxs = [] for j in range(num_missing): num_frags_left = len(fragments) idx = random.randint(0, num_frags_left - 1) while idx in missing_idxs: idx = random.randint(0, num_frags_left - 1) missing_idxs.append(idx) fragments.pop(idx) timer.start() reconstructed_fragment = pyeclib_c.reconstruct( handle, fragments, len(fragments[0]), missing_idxs[0]) tsum += timer.stop_and_return() if orig_fragments[missing_idxs[0]] != reconstructed_fragment: success = False # Output the fragments for debugging with open("orig_fragments", "wb") as fd_orig: fd_orig.write(orig_fragments[missing_idxs[0]]) with open("decoded_fragments", "wb") as fd_decoded: fd_decoded.write(reconstructed_fragment) print(("Fragment %d was not reconstructed!!!" % missing_idxs[0])) sys.exit(2) return success, tsum / iterations
def encode(self, data_bytes): return pyeclib_c.encode(self.handle, data_bytes)