def reconstruct(self, fragment_payloads, indexes_to_reconstruct): _fragment_payloads = list(fragment_payloads) fragment_len = self._validate_and_return_fragment_size( _fragment_payloads) if fragment_len < 0: raise ECDriverError( "Invalid fragment payload in ECPyECLibDriver.reconstruct") reconstructed_data = [] # Reconstruct the data, then the parity # The parity cannot be reconstructed until # after all data is reconstructed indexes_to_reconstruct.sort() _indexes_to_reconstruct = indexes_to_reconstruct[:] while len(_indexes_to_reconstruct) > 0: index = _indexes_to_reconstruct.pop(0) reconstructed = pyeclib_c.reconstruct(self.handle, _fragment_payloads, fragment_len, index) reconstructed_data.append(reconstructed) _fragment_payloads.append(reconstructed) return reconstructed_data
def time_reconstruct(self, num_data, num_parity, ec_type, hd, file_size, iterations): """ :return 2-tuple, (success, average reconstruct time) """ timer = Timer() tsum = 0 handle = pyeclib_c.init(num_data, num_parity, ec_type, hd) whole_file_bytes = self.get_tmp_file(file_size).read() success = True orig_fragments = pyeclib_c.encode(handle, whole_file_bytes) for i in range(iterations): fragments = orig_fragments[:] num_missing = 1 missing_idxs = [] for j in range(num_missing): num_frags_left = len(fragments) idx = random.randint(0, num_frags_left - 1) while idx in missing_idxs: idx = random.randint(0, num_frags_left - 1) missing_idxs.append(idx) fragments.pop(idx) timer.start() reconstructed_fragment = pyeclib_c.reconstruct(handle, fragments, len(fragments[0]), missing_idxs[0]) tsum += timer.stop_and_return() if orig_fragments[missing_idxs[0]] != reconstructed_fragment: success = False # Output the fragments for debugging with open("orig_fragments", "wb") as fd_orig: fd_orig.write(orig_fragments[missing_idxs[0]]) with open("decoded_fragments", "wb") as fd_decoded: fd_decoded.write(reconstructed_fragment) print(("Fragment %d was not reconstructed!!!" % missing_idxs[0])) sys.exit(2) return success, tsum / iterations
def time_reconstruct(self, num_data, num_parity, ec_type, hd, file_size, iterations): """ :return 2-tuple, (success, average reconstruct time) """ timer = Timer() tsum = 0 handle = pyeclib_c.init(num_data, num_parity, ec_type, hd) whole_file_bytes = self.get_tmp_file(file_size).read() success = True orig_fragments = pyeclib_c.encode(handle, whole_file_bytes) for i in range(iterations): fragments = orig_fragments[:] num_missing = 1 missing_idxs = [] for j in range(num_missing): num_frags_left = len(fragments) idx = random.randint(0, num_frags_left - 1) while idx in missing_idxs: idx = random.randint(0, num_frags_left - 1) missing_idxs.append(idx) fragments.pop(idx) timer.start() reconstructed_fragment = pyeclib_c.reconstruct( handle, fragments, len(fragments[0]), missing_idxs[0]) tsum += timer.stop_and_return() if orig_fragments[missing_idxs[0]] != reconstructed_fragment: success = False # Output the fragments for debugging with open("orig_fragments", "wb") as fd_orig: fd_orig.write(orig_fragments[missing_idxs[0]]) with open("decoded_fragments", "wb") as fd_decoded: fd_decoded.write(reconstructed_fragment) print(("Fragment %d was not reconstructed!!!" % missing_idxs[0])) sys.exit(2) return success, tsum / iterations
def reconstruct(self, fragment_payloads, indexes_to_reconstruct): _fragment_payloads = list(fragment_payloads) fragment_len = self._validate_and_return_fragment_size( _fragment_payloads) if fragment_len < 0: raise ECDriverError( "Invalid fragment payload in ECPyECLibDriver.reconstruct") reconstructed_data = [] # Reconstruct the data, then the parity # The parity cannot be reconstructed until # after all data is reconstructed indexes_to_reconstruct.sort() _indexes_to_reconstruct = indexes_to_reconstruct[:] while len(_indexes_to_reconstruct) > 0: index = _indexes_to_reconstruct.pop(0) reconstructed = pyeclib_c.reconstruct( self.handle, _fragment_payloads, fragment_len, index) reconstructed_data.append(reconstructed) _fragment_payloads.append(reconstructed) return reconstructed_data