def collect_available_backends():
available_backends = []
for ec_type in PyECLib_EC_Types:
try:
if ec_type == PyECLib_EC_Types.flat_xor_hd:
handle = pyeclib_c.init(10, 5, ec_type.value, 3)
else:
handle = pyeclib_c.init(10, 4, ec_type.value)
available_backends.append(ec_type.name)
print ec_type.name
except:
pass
return available_backends
def collect_available_backends():
available_backends = []
for ec_type in PyECLib_EC_Types:
try:
if ec_type == PyECLib_EC_Types.flat_xor_hd:
handle = pyeclib_c.init(10, 5, ec_type.value, 3)
else:
handle = pyeclib_c.init(10, 4, ec_type.value)
available_backends.append(ec_type.name)
print(ec_type.name)
except:
pass
return available_backends
def _test_get_required_fragments(self, num_data, num_parity, ec_type):
"""
:return boolean, True if all tests passed
"""
handle = pyeclib_c.init(num_data, num_parity, ec_type.value)
success = True
#
# MDS codes need any k fragments
#
if ec_type in ["jerasure_rs_vand", "jerasure_rs_cauchy"]:
expected_fragments = [i for i in range(num_data + num_parity)]
missing_fragments = []
#
# Remove between 1 and num_parity
#
for i in range(random.randint(0, num_parity - 1)):
missing_fragment = random.sample(expected_fragments, 1)[0]
missing_fragments.append(missing_fragment)
expected_fragments.remove(missing_fragment)
expected_fragments = expected_fragments[:num_data]
required_fragments = pyeclib_c.get_required_fragments(
handle, missing_fragments, [])
if expected_fragments != required_fragments:
success = False
print(("Unexpected required fragments list "
"(exp != req): %s != %s" %
(expected_fragments, required_fragments)))
return success
def time_decode(self, num_data, num_parity, ec_type, hd, file_size,
iterations):
"""
:return 2-tuple, (success, average decode time)
"""
timer = Timer()
tsum = 0
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
success = True
fragments = pyeclib_c.encode(handle, whole_file_bytes)
orig_fragments = fragments[:]
for i in range(iterations):
missing_idxs = []
num_missing = hd - 1
for j in range(num_missing):
num_frags_left = len(fragments)
idx = random.randint(0, num_frags_left - 1)
fragments.pop(idx)
timer.start()
decoded_file_bytes = pyeclib_c.decode(handle, fragments,
len(fragments[0]))
tsum += timer.stop_and_return()
fragments = orig_fragments[:]
if whole_file_bytes != decoded_file_bytes:
success = False
return success, tsum / iterations
def _test_get_required_fragments(self, num_data, num_parity, ec_type):
"""
:return boolean, True if all tests passed
"""
handle = pyeclib_c.init(num_data, num_parity, ec_type.value)
success = True
#
# MDS codes need any k fragments
#
if ec_type in ["jerasure_rs_vand", "jerasure_rs_cauchy"]:
expected_fragments = [i for i in range(num_data + num_parity)]
missing_fragments = []
#
# Remove between 1 and num_parity
#
for i in range(random.randint(0, num_parity - 1)):
missing_fragment = random.sample(expected_fragments, 1)[0]
missing_fragments.append(missing_fragment)
expected_fragments.remove(missing_fragment)
expected_fragments = expected_fragments[:num_data]
required_fragments = pyeclib_c.get_required_fragments(
handle,
missing_fragments, [])
if expected_fragments != required_fragments:
success = False
print(("Unexpected required fragments list "
"(exp != req): %s != %s" %
(expected_fragments, required_fragments)))
return success
def time_decode(self,
num_data, num_parity, ec_type, hd,
file_size, iterations):
"""
:return 2-tuple, (success, average decode time)
"""
timer = Timer()
tsum = 0
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
success = True
fragments = pyeclib_c.encode(handle, whole_file_bytes)
orig_fragments = fragments[:]
for i in range(iterations):
missing_idxs = []
num_missing = hd - 1
for j in range(num_missing):
num_frags_left = len(fragments)
idx = random.randint(0, num_frags_left - 1)
fragments.pop(idx)
timer.start()
decoded_file_bytes = pyeclib_c.decode(handle,
fragments,
len(fragments[0]))
tsum += timer.stop_and_return()
fragments = orig_fragments[:]
if whole_file_bytes != decoded_file_bytes:
success = False
return success, tsum / iterations
def __init__(self, k, m, hd, ec_type,
chksum_type=PyECLib_FRAGHDRCHKSUM_Types.none,
validate=False):
self.k = k
self.m = m
self.hd = hd
self.ec_type = ec_type
self.chksum_type = chksum_type
self.inline_chksum = 0
self.algsig_chksum = 0
# crc32 is the only inline checksum type currently supported
if self.chksum_type is PyECLib_FRAGHDRCHKSUM_Types.inline_crc32:
self.inline_chksum = 1
name = self.ec_type.name
self.handle = pyeclib_c.init(
self.k,
self.m,
ec_type.value,
self.hd,
self.inline_chksum,
self.algsig_chksum,
validate)
def __init__(self,
k,
m,
ec_type,
chksum_type=PyECLib_FRAGHDRCHKSUM_Types.none):
self.k = k
self.m = m
self.ec_type = ec_type
self.chksum_type = chksum_type
hd = m
self.inline_chksum = 0
self.algsig_chksum = 0
# crc32 is the only inline checksum type currently supported
if self.chksum_type is PyECLib_FRAGHDRCHKSUM_Types.inline_crc32:
self.inline_chksum = 1
name = self.ec_type.name
if name == "flat_xor_hd" or name == "flat_xor_hd_3":
hd = 3
if name == "flat_xor_hd_4":
hd = 4
self.handle = pyeclib_c.init(self.k, self.m, ec_type.value, hd,
self.inline_chksum, self.algsig_chksum)
def time_encode(self, num_data, num_parity, ec_type, hd, file_size,
iterations):
"""
:return average encode time
"""
timer = Timer()
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
timer.start()
for l in range(iterations):
pyeclib_c.encode(handle, whole_file_bytes)
tsum = timer.stop_and_return()
return tsum / iterations
def time_encode(self, num_data, num_parity, ec_type, hd,
file_size, iterations):
"""
:return average encode time
"""
timer = Timer()
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
timer.start()
for l in range(iterations):
pyeclib_c.encode(handle, whole_file_bytes)
tsum = timer.stop_and_return()
return tsum / iterations
def time_range_decode(self, num_data, num_parity, ec_type, hd, file_size,
iterations):
"""
:return 2-tuple, (success, average decode time)
"""
timer = Timer()
tsum = 0
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
success = True
begins = [
long(random.randint(0,
len(whole_file_bytes) - 1)) for i in range(3)
]
ends = [
long(random.randint(begins[i], len(whole_file_bytes)))
for i in range(3)
]
ranges = zip(begins, ends)
fragments = pyeclib_c.encode(handle, whole_file_bytes)
orig_fragments = fragments[:]
for i in range(iterations):
missing_idxs = []
num_missing = hd - 1
for j in range(num_missing):
num_frags_left = len(fragments)
idx = random.randint(0, num_frags_left - 1)
fragments.pop(idx)
timer.start()
decoded_file_ranges = pyeclib_c.decode(handle, fragments,
len(fragments[0]), ranges)
tsum += timer.stop_and_return()
fragments = orig_fragments[:]
range_offset = 0
for r in ranges:
if whole_file_bytes[r[0]:r[1] +
1] != decoded_file_ranges[range_offset]:
success = False
range_offset += 1
return success, tsum / iterations
def time_range_decode(self,
num_data, num_parity, ec_type, hd,
file_size, iterations):
"""
:return 2-tuple, (success, average decode time)
"""
timer = Timer()
tsum = 0
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
success = True
begins = [int(random.randint(0, len(whole_file_bytes) - 1))
for i in range(3)]
ends = [int(random.randint(begins[i], len(whole_file_bytes)))
for i in range(3)]
ranges = list(zip(begins, ends))
fragments = pyeclib_c.encode(handle, whole_file_bytes)
orig_fragments = fragments[:]
for i in range(iterations):
num_missing = hd - 1
for j in range(num_missing):
num_frags_left = len(fragments)
idx = random.randint(0, num_frags_left - 1)
fragments.pop(idx)
timer.start()
decoded_file_ranges = pyeclib_c.decode(handle,
fragments,
len(fragments[0]),
ranges)
tsum += timer.stop_and_return()
fragments = orig_fragments[:]
range_offset = 0
for r in ranges:
if whole_file_bytes[
r[0]: r[1] + 1] != decoded_file_ranges[range_offset]:
success = False
range_offset += 1
return success, tsum / iterations
def time_reconstruct(self,
num_data, num_parity, ec_type, hd,
file_size, iterations):
"""
:return 2-tuple, (success, average reconstruct time)
"""
timer = Timer()
tsum = 0
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
success = True
orig_fragments = pyeclib_c.encode(handle, whole_file_bytes)
for i in range(iterations):
fragments = orig_fragments[:]
num_missing = 1
missing_idxs = []
for j in range(num_missing):
num_frags_left = len(fragments)
idx = random.randint(0, num_frags_left - 1)
while idx in missing_idxs:
idx = random.randint(0, num_frags_left - 1)
missing_idxs.append(idx)
fragments.pop(idx)
timer.start()
reconstructed_fragment = pyeclib_c.reconstruct(handle,
fragments,
len(fragments[0]),
missing_idxs[0])
tsum += timer.stop_and_return()
if orig_fragments[missing_idxs[0]] != reconstructed_fragment:
success = False
# Output the fragments for debugging
with open("orig_fragments", "wb") as fd_orig:
fd_orig.write(orig_fragments[missing_idxs[0]])
with open("decoded_fragments", "wb") as fd_decoded:
fd_decoded.write(reconstructed_fragment)
print(("Fragment %d was not reconstructed!!!" % missing_idxs[0]))
sys.exit(2)
return success, tsum / iterations
def time_reconstruct(self, num_data, num_parity, ec_type, hd, file_size,
iterations):
"""
:return 2-tuple, (success, average reconstruct time)
"""
timer = Timer()
tsum = 0
handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
whole_file_bytes = self.get_tmp_file(file_size).read()
success = True
orig_fragments = pyeclib_c.encode(handle, whole_file_bytes)
for i in range(iterations):
fragments = orig_fragments[:]
num_missing = 1
missing_idxs = []
for j in range(num_missing):
num_frags_left = len(fragments)
idx = random.randint(0, num_frags_left - 1)
while idx in missing_idxs:
idx = random.randint(0, num_frags_left - 1)
missing_idxs.append(idx)
fragments.pop(idx)
timer.start()
reconstructed_fragment = pyeclib_c.reconstruct(
handle, fragments, len(fragments[0]), missing_idxs[0])
tsum += timer.stop_and_return()
if orig_fragments[missing_idxs[0]] != reconstructed_fragment:
success = False
# Output the fragments for debugging
with open("orig_fragments", "wb") as fd_orig:
fd_orig.write(orig_fragments[missing_idxs[0]])
with open("decoded_fragments", "wb") as fd_decoded:
fd_decoded.write(reconstructed_fragment)
print(("Fragment %d was not reconstructed!!!" %
missing_idxs[0]))
sys.exit(2)
return success, tsum / iterations
def __init__(self, k, m, hd, ec_type,
chksum_type=PyECLib_FRAGHDRCHKSUM_Types.none,
validate=False):
self.k = k
self.m = m
self.hd = hd
self.ec_type = ec_type
self.chksum_type = chksum_type
self.inline_chksum = 0
self.algsig_chksum = 0
# crc32 is the only inline checksum type currently supported
if self.chksum_type is PyECLib_FRAGHDRCHKSUM_Types.inline_crc32:
self.inline_chksum = 1
self.handle = pyeclib_c.init(
self.k,
self.m,
ec_type.value,
self.hd,
self.inline_chksum,
self.algsig_chksum,
validate)