def __init__(self, name, ec_segment_size, ec_type, ec_nb_data,
ec_nb_parity):
super(ECStorageMethod, self).__init__(name=name, ec=True)
try:
self._ec_nb_data = int(ec_nb_data)
except (TypeError, ValueError):
raise exceptions.InvalidStorageMethod('Invalid %r ec_nb_data' %
ec_nb_data)
try:
self._ec_nb_parity = int(ec_nb_parity)
except (TypeError, ValueError):
raise exceptions.InvalidStorageMethod('Invalid %r ec_nb_parity' %
ec_nb_parity)
self._ec_segment_size = ec_segment_size
self._ec_type = ec_type
try:
self.driver = ECDriver(k=ec_nb_data,
m=ec_nb_parity,
ec_type=ec_type)
except ECDriverError as exc:
msg = "'%s' (%s: %s) Check erasure code packages." % (
ec_type, exc.__class__.__name__, exc)
reraise(exceptions.InvalidStorageMethod,
exceptions.InvalidStorageMethod(msg),
sys.exc_info()[2])
self._ec_quorum_size = \
self._ec_nb_data + self.driver.min_parity_fragments_needed()
class Eraser(object):
"""A wrapper for pyeclib erasure coding driver (ECDriver)"""
def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
self.k = k
self.m = m
self.ec_type = ec_type
if EC_TYPE == "pylonghair":
self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
elif EC_TYPE == "striping" or EC_TYPE == "bypass":
self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
else:
self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
def encode(self, data):
"""Encode a string of bytes in flattened string of byte strips"""
raw_strips = self.driver.encode(data)
strips = []
for raw_strip in raw_strips:
strip = Strip()
strip.data = raw_strip
strips.append(strip)
return strips
def decode(self, playcloud_strips):
"""Decode byte strips in a string of bytes"""
strips = convert_strips_to_bytes_list(playcloud_strips)
return self.driver.decode(strips)
def get_available_backend(self, k, m, ec_type, chksum_type = "inline_crc32"):
if ec_type[:11] == "flat_xor_hd":
return ECDriver(k=k, m=m, ec_type="flat_xor_hd", chksum_type=chksum_type)
elif ec_type in self.available_backends:
return ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type)
else:
return None
def __init__(self, cfgfile, verbose=False):
uid = self.get_hwaddr()
config = configparser.ConfigParser()
config.read(cfgfile)
N = int(config['General']['n'])
F = int(config['General']['f'])
addrs = list(config['Nodes'].values())
self.verbose = verbose
context = zmq.Context()
self.k = N-2*F
self.ec_driver = ECDriver(k=self.k, m=N, ec_type='liberasurecode_rs_vand')
self.socketid = {}
self.sockets = []
self.uid = uid
self.id = "Client-{}".format(uid)
self.poller = zmq.Poller()
self.quorum = int(math.ceil((N+self.k)/2))
print("Starting client {}".format(uid))
for addr in addrs:
socket = context.socket(zmq.DEALER)
self.socketid[socket] = "Server-%s" % addr
self.poller.register(socket, zmq.POLLIN)
socket.connect("tcp://%s" % addr);
self.vprint("Connecting to server with address {}".format(addr))
self.sockets.append(socket)
class ECStorageMethod(StorageMethod):
def __init__(self, name, ec_segment_size, ec_type, ec_nb_data,
ec_nb_parity):
super(ECStorageMethod, self).__init__(name=name, ec=True)
try:
self._ec_nb_data = int(ec_nb_data)
except (TypeError, ValueError):
raise exc.InvalidStorageMethod('Invalid %r ec_nb_data' %
ec_nb_data)
try:
self._ec_nb_parity = int(ec_nb_parity)
except (TypeError, ValueError):
raise exc.InvalidStorageMethod('Invalid %r ec_nb_parity' %
ec_nb_parity)
self._ec_segment_size = ec_segment_size
self._ec_type = ec_type
self.driver = ECDriver(k=ec_nb_data,
m=ec_nb_parity,
ec_type=ec_type_to_pyeclib_type[ec_type])
self._ec_quorum_size = \
self._ec_nb_data + self.driver.min_parity_fragments_needed()
@property
def quorum(self):
return self._ec_quorum_size
@classmethod
def build(cls, params):
ec_nb_data = params.pop('k')
ec_nb_parity = params.pop('m')
ec_type = params.pop('algo')
return cls('ec',
ec_segment_size=EC_SEGMENT_SIZE,
ec_type=ec_type,
ec_nb_data=ec_nb_data,
ec_nb_parity=ec_nb_parity)
@property
def ec_type(self):
return self._ec_type
@property
def ec_nb_data(self):
return self._ec_nb_data
@property
def ec_nb_parity(self):
return self._ec_nb_parity
@property
def ec_segment_size(self):
return self._ec_segment_size
@property
def ec_fragment_size(self):
return self.driver.get_segment_info(
self.ec_segment_size, self.ec_segment_size)['fragment_size']
def __init__(self, cfgfile):
self.hw_addr = self.get_hwaddr()
self.uid = (-1, self.hw_addr)
self.loop = asyncio.get_event_loop()
config = configparser.ConfigParser()
config.read(cfgfile)
nbr_of_servers = int(config['General']['n'])
chunks_size = int(config['General']['chunks_size'])
f = int(config['General']['f'])
e = int(config['General']['e'])
k = nbr_of_servers - 2 * (f + e)
if (k < 1):
raise Exception("Coded elements less than 1")
if (nbr_of_servers + k > 32):
raise Exception(
"[liberasurecode] Total number of fragments (k + m) must be at most 32"
)
quorum_size = math.ceil((nbr_of_servers + k + 2 * e) / 2)
self.majority = math.ceil((nbr_of_servers + 1) / 2)
self.ec_driver = ECDriver(k=k,
m=nbr_of_servers,
ec_type='liberasurecode_rs_vand')
self.p = QuorumSend(quorum_size, PingPongMessage)
t = Thread(target=self.start_event_loop,
args=(self.loop, config['Nodes'], chunks_size))
t.daemon = True
t.start()
def test_verify_fragment_inline_chksum_succeed(self):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
file_bytes = file_str.encode('utf-8')
for pyeclib_driver in pyeclib_drivers:
fragments = pyeclib_driver.encode(file_bytes)
fragment_metadata_list = []
for fragment in fragments:
fragment_metadata_list.append(
pyeclib_driver.get_metadata(fragment))
expected_ret_value = {"status": 0 }
self.assertTrue(
pyeclib_driver.verify_stripe_metadata(fragment_metadata_list) == expected_ret_value)
def __init__(self, name, ec_segment_size, ec_type, ec_nb_data,
ec_nb_parity):
super(ECStorageMethod, self).__init__(name=name, ec=True)
try:
self._ec_nb_data = int(ec_nb_data)
except (TypeError, ValueError):
raise exceptions.InvalidStorageMethod('Invalid %r ec_nb_data' %
ec_nb_data)
try:
self._ec_nb_parity = int(ec_nb_parity)
except (TypeError, ValueError):
raise exceptions.InvalidStorageMethod('Invalid %r ec_nb_parity' %
ec_nb_parity)
self._ec_segment_size = ec_segment_size
self._ec_type = ec_type
from pyeclib.ec_iface import ECDriver
self.driver = ECDriver(k=ec_nb_data,
m=ec_nb_parity,
ec_type=ec_type_to_pyeclib_type[ec_type])
self._ec_quorum_size = \
self._ec_nb_data + self.driver.min_parity_fragments_needed()
def __init__(self, source, s, t, p, ec_type="isa_l_rs_vand"):
"""
StepEntangler constructor
Args:
source(Source): Block source implementing the get_random_blocks and
get_block primitives
s(int): Number of source blocks or the number of chunks to make from
the original data
t(int): Number of old blocks to entangle with
p(int): Number of parity blocks to produce using Reed-Solomon
"""
if not source or \
not callable(getattr(source, "get_block", None)) or \
not callable(getattr(source, "get_random_blocks", None)):
raise ValueError(
"source argument must implement a get_random_blocks and a get_block method"
)
if s <= 0:
raise ValueError("s({:d}) must be greater or equal to 1".format(s))
if t < 0:
raise ValueError("t({:d}) must be greater or equal to 0".format(t))
if p < s:
raise ValueError(
"p({:d}) must be greater or equal to s({:d})".format(p, s))
self.s = s
self.t = t
self.p = p
self.e = self.p - self.s
self.k = s + t
self.source = source
self.driver = ECDriver(k=self.k, m=self.p, ec_type=ec_type)
def test_get_segment_byterange_info(self):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(
ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
file_size = 1024 * 1024
segment_size = 3 * 1024
ranges = [(0, 1), (1, 12), (10, 1000), (0, segment_size-1), (1, segment_size+1), (segment_size-1, 2*segment_size)]
expected_results = {}
expected_results[(0, 1)] = {0: (0, 1)}
expected_results[(1, 12)] = {0: (1, 12)}
expected_results[(10, 1000)] = {0: (10, 1000)}
expected_results[(0, segment_size-1)] = {0: (0, segment_size-1)}
expected_results[(1, segment_size+1)] = {0: (1, segment_size-1), 1: (0, 1)}
expected_results[(segment_size-1, 2*segment_size)] = {0: (segment_size-1, segment_size-1), 1: (0, segment_size-1), 2: (0, 0)}
results = pyeclib_drivers[0].get_segment_info_byterange(ranges, file_size, segment_size)
for exp_result_key in expected_results:
self.assertTrue(results.has_key(exp_result_key))
self.assertTrue(len(results[exp_result_key]) == len(expected_results[exp_result_key]))
exp_result_map = expected_results[exp_result_key]
for segment_key in exp_result_map:
self.assertTrue(results[exp_result_key].has_key(segment_key))
self.assertTrue(results[exp_result_key][segment_key] == expected_results[exp_result_key][segment_key])
def __init__(self,
ec_k,
ec_m,
ec_type="liberasurecode_rs_vand",
aes_enabled=True):
self.ec_type = ec_type
if aes_enabled:
self.aes = AESDriver()
logger.info("Eraser will use AES encryption")
else:
logger.info("Eraser will not use AES encryption")
expected_module_name = "drivers." + ec_type.lower() + "_driver"
expected_class_name = ec_type[0].upper() + ec_type[1:].lower(
) + "Driver"
try:
mod = __import__(expected_module_name,
fromlist=[expected_class_name])
driver_class = None
driver_class = getattr(mod, expected_class_name)
self.driver = driver_class(k=ec_k,
m=ec_m,
ec_type=ec_type,
hd=None)
except (ImportError, AttributeError):
logger.exception("Driver " + ec_type +
" could not be loaded as a custom driver")
try:
self.driver = ECDriver(k=ec_k, m=ec_m, ec_type=ec_type)
except Exception as error:
logger.exception("Driver " + ec_type +
" could not be loaded by pyeclib")
raise error
class Eraser(object):
"""A wrapper for pyeclib erasure coding driver (ECDriver)"""
def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
self.k = k
self.m = m
self.ec_type = ec_type
if EC_TYPE == "pylonghair":
self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
elif EC_TYPE == "striping" or EC_TYPE == "bypass":
self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
else:
self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
def encode(self, data):
"""Encode a string of bytes in flattened string of byte strips"""
raw_strips = self.driver.encode(data)
strips = []
for raw_strip in raw_strips:
strip = Strip()
strip.data = raw_strip
strips.append(strip)
return strips
def decode(self, playcloud_strips):
"""Decode byte strips in a string of bytes"""
strips = convert_strips_to_bytes_list(playcloud_strips)
return self.driver.decode(strips)
class ECStorageMethod(StorageMethod):
def __init__(self, name, ec_segment_size, ec_type, ec_nb_data,
ec_nb_parity):
super(ECStorageMethod, self).__init__(name=name, ec=True)
try:
self._ec_nb_data = int(ec_nb_data)
except (TypeError, ValueError):
raise exc.InvalidStorageMethod('Invalid %r ec_nb_data' %
ec_nb_data)
try:
self._ec_nb_parity = int(ec_nb_parity)
except (TypeError, ValueError):
raise exc.InvalidStorageMethod('Invalid %r ec_nb_parity' %
ec_nb_parity)
self._ec_segment_size = ec_segment_size
self._ec_type = ec_type
self.driver = ECDriver(k=ec_nb_data, m=ec_nb_parity,
ec_type=ec_type_to_pyeclib_type[ec_type])
self._ec_quorum_size = \
self._ec_nb_data + self.driver.min_parity_fragments_needed()
@property
def quorum(self):
return self._ec_quorum_size
@classmethod
def build(cls, params):
ec_nb_data = params.pop('k')
ec_nb_parity = params.pop('m')
ec_type = params.pop('algo')
return cls('ec', ec_segment_size=EC_SEGMENT_SIZE,
ec_type=ec_type, ec_nb_data=ec_nb_data,
ec_nb_parity=ec_nb_parity)
@property
def ec_type(self):
return self._ec_type
@property
def ec_nb_data(self):
return self._ec_nb_data
@property
def ec_nb_parity(self):
return self._ec_nb_parity
@property
def ec_segment_size(self):
return self._ec_segment_size
@property
def ec_fragment_size(self):
return self.driver.get_segment_info(
self.ec_segment_size, self.ec_segment_size)['fragment_size']
def __init__(self, clientId, isLocal=True):
self.clientId = clientId
self.isLocal = isLocal
self.N = 4
self.F = 1
self.metadataQ = queue.Queue()
self.ec_driver = ECDriver(k=2, m=2, ec_type='liberasurecode_rs_vand')
if self.isLocal:
self.createLocalClients()
def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
self.k = k
self.m = m
self.ec_type = ec_type
if EC_TYPE == "pylonghair":
self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
elif EC_TYPE == "striping" or EC_TYPE == "bypass":
self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
else:
self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
class TestNullDriver(unittest.TestCase):
def setUp(self):
self.null_driver = ECDriver(library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
def tearDown(self):
pass
def test_null_driver(self):
self.null_driver.encode("")
self.null_driver.decode([])
def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
self.k = k
self.m = m
self.ec_type = ec_type
if EC_TYPE == "pylonghair":
self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
elif EC_TYPE == "striping" or EC_TYPE == "bypass":
self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
else:
self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
def check_metadata_formatted(self, k, m, ec_type, chksum_type):
if ec_type not in VALID_EC_TYPES:
return
filesize = 1024 * 1024 * 3
file_str = ''.join(
random.choice(ascii_letters) for i in range(filesize))
file_bytes = file_str.encode('utf-8')
pyeclib_driver = ECDriver(k=k,
m=m,
ec_type=ec_type,
chksum_type=chksum_type)
fragments = pyeclib_driver.encode(file_bytes)
f = 0
for fragment in fragments:
metadata = pyeclib_driver.get_metadata(fragment, 1)
if 'index' in metadata:
self.assertEqual(metadata['index'], f)
else:
self.assertTrue(False)
if 'chksum_mismatch' in metadata:
self.assertEqual(metadata['chksum_mismatch'], 0)
else:
self.assertTrue(False)
if 'backend_id' in metadata:
self.assertEqual(metadata['backend_id'], ec_type)
else:
self.assertTrue(False)
if 'orig_data_size' in metadata:
self.assertEqual(metadata['orig_data_size'], 3145728)
else:
self.assertTrue(False)
if 'chksum_type' in metadata:
self.assertEqual(metadata['chksum_type'], 'crc32')
else:
self.assertTrue(False)
if 'backend_version' not in metadata:
self.assertTrue(False)
if 'chksum' not in metadata:
self.assertTrue(False)
if 'size' not in metadata:
self.assertTrue(False)
f += 1
class TestNullDriver(unittest.TestCase):
def setUp(self):
self.null_driver = ECDriver(
library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
def tearDown(self):
pass
def test_null_driver(self):
self.null_driver.encode('')
self.null_driver.decode([])
def decode(k, m, base_filename_to_decode, ec_type):
#print("k = %d, m = %d" % (k, m))
#print("ec_type = %s" % ec_type)
home = expanduser("~")
Disk_base_dir = home + '/Disk'
key_list = base_filename_to_decode.split('.')
jdecoder_types = [
"reed_sol_van", "reed_sol_r6_op", "cauchy_orig", "cauchy_good",
"liberation", "blaum_roth", "liber8tion"
]
if ec_type in jdecoder_types:
print "Decoding started "
jdecoder = ctypes.CDLL(
'/home/raven/Downloads/Jerasure-master/Examples/jdecoder.so')
key = key_list[0]
decoded_data = str(jdecoder.decode(key, Disk_base_dir))
print "decoded data = ", decoded_data
else:
ec_driver = ECDriver(k=k, m=m, ec_type=ec_type)
fragment_list = []
i = 0
while i <= k:
if os.path.exists(Disk_base_dir + "/" + "Disk_" + str(i)):
curr_dir = Disk_base_dir + "/" + "Disk_" + str(i)
if i == 0:
if os.path.exists(curr_dir + "/" +
base_filename_to_decode):
with open(curr_dir + "/" + base_filename_to_decode,
'rb') as fp:
decoded_data = fp.read()
fp.close()
return decoded_data
else:
if os.path.exists(curr_dir + "/" +
base_filename_to_decode + "_fragment_" +
str(i)):
with open(
curr_dir + "/" + base_filename_to_decode +
"_fragment_" + str(i), 'rb') as fp:
fragment = fp.read()
fp.close()
fragment_list.append(fragment)
i = i + 1
if len(fragment_list) < k:
return -1 # Not enough fragments to decode
else:
decoded_data = ec_driver.decode(fragment_list)
return decoded_data
def check_metadata_formatted(self, k, m, ec_type, chksum_type):
if ec_type not in VALID_EC_TYPES:
return
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters)
for i in range(filesize))
file_bytes = file_str.encode('utf-8')
pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type,
chksum_type=chksum_type)
fragments = pyeclib_driver.encode(file_bytes)
f = 0
for fragment in fragments:
metadata = pyeclib_driver.get_metadata(fragment, 1)
if 'index' in metadata:
self.assertEqual(metadata['index'], f)
else:
self.assertTrue(False)
if 'chksum_mismatch' in metadata:
self.assertEqual(metadata['chksum_mismatch'], 0)
else:
self.assertTrue(False)
if 'backend_id' in metadata:
self.assertEqual(metadata['backend_id'], ec_type)
else:
self.assertTrue(False)
if 'orig_data_size' in metadata:
self.assertEqual(metadata['orig_data_size'], 3145728)
else:
self.assertTrue(False)
if 'chksum_type' in metadata:
self.assertEqual(metadata['chksum_type'], 'crc32')
else:
self.assertTrue(False)
if 'backend_version' not in metadata:
self.assertTrue(False)
if 'chksum' not in metadata:
self.assertTrue(False)
if 'size' not in metadata:
self.assertTrue(False)
f += 1
def test_invalid_km_args(self):
for ec_type in VALID_EC_TYPES:
with self.assertRaises(ECDriverError) as err_context:
# k is smaller than 1
ECDriver(ec_type=ec_type, k=-100, m=1)
self.assertEqual(str(err_context.exception),
"Invalid number of data fragments (k)")
with self.assertRaises(ECDriverError) as err_context:
# m is smaller than 1
ECDriver(ec_type=ec_type, k=1, m=-100)
self.assertEqual(str(err_context.exception),
"Invalid number of parity fragments (m)")
def check_metadata_formatted(self, k, m, ec_type, chksum_type):
if ec_type not in VALID_EC_TYPES:
return
filesize = 1024 * 1024 * 3
file_str = "".join(random.choice(ascii_letters) for i in range(filesize))
file_bytes = file_str.encode("utf-8")
pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type)
fragments = pyeclib_driver.encode(file_bytes)
f = 0
for fragment in fragments:
metadata = pyeclib_driver.get_metadata(fragment, 1)
if "index" in metadata:
self.assertEqual(metadata["index"], f)
else:
self.assertTrue(False)
if "chksum_mismatch" in metadata:
self.assertEqual(metadata["chksum_mismatch"], 0)
else:
self.assertTrue(False)
if "backend_id" in metadata:
self.assertEqual(metadata["backend_id"], ec_type)
else:
self.assertTrue(False)
if "orig_data_size" in metadata:
self.assertEqual(metadata["orig_data_size"], 3145728)
else:
self.assertTrue(False)
if "chksum_type" in metadata:
self.assertEqual(metadata["chksum_type"], "crc32")
else:
self.assertTrue(False)
if "backend_version" not in metadata:
self.assertTrue(False)
if "chksum" not in metadata:
self.assertTrue(False)
if "size" not in metadata:
self.assertTrue(False)
f += 1
def test_valid_algo(self):
print("")
for _type in PyECLib_EC_Types.names():
# Check if this algo works
if _type not in _available_backends:
print("Skipping test for %s backend" % _type)
continue
try:
if _type is 'shss':
_instance = ECDriver(k=10, m=4, ec_type=_type)
else:
_instance = ECDriver(k=10, m=5, ec_type=_type)
except ECDriverError:
self.fail("%s algorithm not supported" % _type)
def test_small_encode(self):
pyeclib_drivers = []
pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type="jerasure_rs_vand"))
pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type="jerasure_rs_vand"))
encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"]
for pyeclib_driver in pyeclib_drivers:
for encode_str in encode_strs:
encoded_fragments = pyeclib_driver.encode(encode_str)
decoded_str = pyeclib_driver.decode(encoded_fragments)
self.assertTrue(decoded_str == encode_str)
def encode(k, m, data_to_encode, base_filename_to_store, ec_type):
home = expanduser("~")
key_list = base_filename_to_store.split('.')
Disk_base_dir = home + '/Disk'
jencoder_types = [
"reed_sol_van", "reed_sol_r6_op", "cauchy_orig", "cauchy_good",
"liberation", "blaum_roth", "liber8tion"
]
# Create a replicated copy first.
#replica_dir = Disk_base_dir + "/Disk_0"
#if not os.path.exists(replica_dir):
# os.makedirs(replica_dir)
#with open(replica_dir + "/" + base_filename_to_store,'wb') as fp :
# fp.write(data_to_encode)
#fp.close()
if ec_type in jencoder_types:
jencoder = ctypes.CDLL(
'/home/raven/Downloads/Jerasure-master/Examples/jencoder.so')
w = 8
key = key_list[0]
jencoder.encode(key, data_to_encode, Disk_base_dir, k, m, ec_type, w,
0)
else:
ec_driver = ECDriver(k=k, m=m, ec_type=ec_type)
# encode
fragments = ec_driver.encode(data_to_encode)
# store
i = 1
for fragment in fragments:
fragment_dir = Disk_base_dir + '/' + "Disk_" + str(i)
if not os.path.exists(fragment_dir):
os.makedirs(fragment_dir)
with open(
"%s/%s" %
(fragment_dir, base_filename_to_store + "_fragment_" + str(i)),
"wb") as fp:
fp.write(fragment)
fp.close()
i += 1
def test_min_parity_fragments_needed(self):
pyeclib_drivers = []
for ec_type in ['flat_xor_hd_3', 'liberasurecode_rs_vand']:
if ec_type in VALID_EC_TYPES:
pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=ec_type))
self.assertTrue(
pyeclib_drivers[0].min_parity_fragments_needed() == 1)
def test_get_metadata_formatted(self):
pyeclib_driver = ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
file_bytes = file_str.encode('utf-8')
fragments = pyeclib_driver.encode(file_bytes)
i = 0
for fragment in fragments:
metadata = pyeclib_driver.get_metadata(fragment, 1)
if metadata.has_key('index'):
self.assertEqual(metadata['index'], i)
else:
self.assertTrue(false)
if metadata.has_key('chksum_mismatch'):
self.assertEqual(metadata['chksum_mismatch'], 0)
else:
self.assertTrue(false)
if metadata.has_key('backend_id'):
self.assertEqual(metadata['backend_id'], 'jerasure_rs_vand')
else:
self.assertTrue(false)
if metadata.has_key('orig_data_size'):
self.assertEqual(metadata['orig_data_size'], 3145728)
else:
self.assertTrue(false)
if metadata.has_key('chksum_type'):
self.assertEqual(metadata['chksum_type'], 'crc32')
else:
self.assertTrue(false)
if not metadata.has_key('backend_version'):
self.assertTrue(false)
if not metadata.has_key('chksum'):
self.assertTrue(false)
if not metadata.has_key('size'):
self.assertTrue(false)
i += 1
def test_verify_fragment_inline_chksum_fail(self):
pyeclib_drivers = []
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32"))
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32"))
filesize = 1024 * 1024 * 3
file_str = ''.join(random.choice(ascii_letters) for i in range(filesize))
file_bytes = file_str.encode('utf-8')
for pyeclib_driver in pyeclib_drivers:
fragments = pyeclib_driver.encode(file_bytes)
fragment_metadata_list = []
first_fragment_to_corrupt = random.randint(0, len(fragments))
num_to_corrupt = 2
fragments_to_corrupt = [
(first_fragment_to_corrupt + i) % len(fragments) for i in range(num_to_corrupt + 1)
]
fragments_to_corrupt.sort()
i = 0
for fragment in fragments:
if i in fragments_to_corrupt:
corrupted_fragment = fragment[:100] +\
(str(chr((b2i(fragment[100]) + 0x1)
% 128))).encode('utf-8') + fragment[101:]
fragment_metadata_list.append(
pyeclib_driver.get_metadata(corrupted_fragment))
else:
fragment_metadata_list.append(
pyeclib_driver.get_metadata(fragment))
i += 1
expected_ret_value = {"status": -206,
"reason": "Bad checksum",
"bad_fragments": fragments_to_corrupt}
self.assertEqual(
pyeclib_driver.verify_stripe_metadata(fragment_metadata_list),
expected_ret_value)
def encode(k,m,data_to_encode,base_filename_to_store,ec_type) :
home = expanduser("~")
key_list = base_filename_to_store.split('.')
Disk_base_dir = home + '/Disk'
jencoder_types = ["reed_sol_van", "reed_sol_r6_op","cauchy_orig","cauchy_good","liberation","blaum_roth","liber8tion"]
# Create a replicated copy first.
#replica_dir = Disk_base_dir + "/Disk_0"
#if not os.path.exists(replica_dir):
# os.makedirs(replica_dir)
#with open(replica_dir + "/" + base_filename_to_store,'wb') as fp :
# fp.write(data_to_encode)
#fp.close()
if ec_type in jencoder_types :
jencoder = ctypes.CDLL('/home/raven/Downloads/Jerasure-master/Examples/jencoder.so')
w = 8
key = key_list[0]
jencoder.encode(key,data_to_encode,Disk_base_dir,k,m,ec_type,w,0)
else :
ec_driver = ECDriver(k = k, m = m, ec_type = ec_type)
# encode
fragments = ec_driver.encode(data_to_encode)
# store
i = 1
for fragment in fragments:
fragment_dir = Disk_base_dir + '/' + "Disk_" + str(i)
if not os.path.exists(fragment_dir):
os.makedirs(fragment_dir)
with open("%s/%s" % (fragment_dir, base_filename_to_store + "_fragment_" + str(i)), "wb") as fp:
fp.write(fragment)
fp.close()
i += 1
def decode(k,m,base_filename_to_decode,ec_type) :
#print("k = %d, m = %d" % (k, m))
#print("ec_type = %s" % ec_type)
home = expanduser("~")
Disk_base_dir = home + '/Disk'
key_list = base_filename_to_decode.split('.')
jdecoder_types = ["reed_sol_van", "reed_sol_r6_op","cauchy_orig","cauchy_good","liberation","blaum_roth","liber8tion"]
if ec_type in jdecoder_types :
print "Decoding started "
jdecoder = ctypes.CDLL('/home/raven/Downloads/Jerasure-master/Examples/jdecoder.so')
key = key_list[0]
decoded_data = str(jdecoder.decode(key,Disk_base_dir))
print "decoded data = ", decoded_data
else :
ec_driver = ECDriver(k = k, m = m, ec_type = ec_type)
fragment_list = []
i = 0
while i <=k :
if os.path.exists(Disk_base_dir + "/" + "Disk_" + str(i)) :
curr_dir = Disk_base_dir + "/" + "Disk_" + str(i)
if i == 0 :
if os.path.exists(curr_dir + "/" + base_filename_to_decode) :
with open(curr_dir + "/" + base_filename_to_decode,'rb') as fp :
decoded_data = fp.read()
fp.close()
return decoded_data
else :
if os.path.exists(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i)) :
with open(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i),'rb') as fp :
fragment = fp.read()
fp.close()
fragment_list.append(fragment)
i = i + 1
if len(fragment_list) < k :
return -1 # Not enough fragments to decode
else :
decoded_data = ec_driver.decode(fragment_list)
return decoded_data
def test_missing_required_args(self):
# missing ec_type
with self.assertRaises(ECDriverError) as err_context:
ECDriver(k=1, m=1)
self.assertEqual(
str(err_context.exception), "Invalid Argument: either ec_type or "
"library_import_str must be provided")
for ec_type in VALID_EC_TYPES:
# missing k
with self.assertRaises(ECDriverError) as err_context:
ECDriver(ec_type=ec_type, m=1)
self.assertEqual(str(err_context.exception),
"Invalid Argument: k is required")
# missing m
with self.assertRaises(ECDriverError) as err_context:
ECDriver(ec_type=ec_type, k=1)
self.assertEqual(str(err_context.exception),
"Invalid Argument: m is required")
def test_valid_algo(self):
print("")
for _type in ALL_EC_TYPES:
# Check if this algo works
if _type not in VALID_EC_TYPES:
print("Skipping test for %s backend" % _type)
continue
try:
if _type == 'shss':
ECDriver(k=10, m=4, ec_type=_type)
elif _type == 'libphazr':
ECDriver(k=4, m=4, ec_type=_type)
else:
ECDriver(k=10, m=5, ec_type=_type)
except ECDriverError:
self.fail("%s algorithm not supported" % _type)
self.assertRaises(ECBackendNotSupported,
ECDriver,
k=10,
m=5,
ec_type="invalid_algo")
def dummy(self, ec_type=ec_type):
if ec_type not in VALID_EC_TYPES:
raise unittest.SkipTest
if ec_type == 'shss':
k = 10
m = 4
elif ec_type == 'libphazr':
k = 4
m = 4
else:
k = 10
m = 5
ECDriver(k=k, m=m, ec_type=ec_type)
def __init__(self, factory, msg_wrapper_f=lambda _x: _x):
self._factory = factory
self._step = _BRACHA_STEP.one
self._init_count = 0
self._echo_count = 0
self._ready_count = 0
self._root = None
self._fragments = {}
self._v = None
self._done = False
self._msg_wrapper_f = msg_wrapper_f
self._n = self._factory.config.n
self._t = self._factory.config.t
self._sent_ready = False
# NOTE: #define EC_MAX_FRAGMENTS 32
# https://github.com/openstack/liberasurecode/blob/master/include/erasurecode/erasurecode.h
k = self._n - 2 * self._t
m = 2 * self._t
logging.debug("Bracha: erasure code params k={}, m={}".format(k, m))
self._ec_driver = ECDriver(k=k, m=m, ec_type='liberasurecode_rs_vand')
random.seed()
def test_ec_fragment_size_cached(self):
policy = ECStoragePolicy(
0, 'ec2-1', ec_type=DEFAULT_TEST_EC_TYPE,
ec_ndata=2, ec_nparity=1, object_ring=FakeRing(replicas=3),
ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE, is_default=True)
ec_driver = ECDriver(ec_type=DEFAULT_TEST_EC_TYPE,
k=2, m=1)
expected_fragment_size = ec_driver.get_segment_info(
DEFAULT_EC_OBJECT_SEGMENT_SIZE,
DEFAULT_EC_OBJECT_SEGMENT_SIZE)['fragment_size']
with mock.patch.object(
policy.pyeclib_driver, 'get_segment_info') as fake:
fake.return_value = {
'fragment_size': expected_fragment_size}
for x in range(10):
self.assertEqual(expected_fragment_size,
policy.fragment_size)
# pyeclib_driver.get_segment_info is called only once
self.assertEqual(1, fake.call_count)
def test_valid_ec_types(self):
# Build list of available types and compare to VALID_EC_TYPES
available_ec_types = []
for _type in ALL_EC_TYPES:
try:
if _type == 'shss':
_k = 10
_m = 4
elif _type == 'libphazr':
_k = 4
_m = 4
else:
_k = 10
_m = 5
ECDriver(k=_k, m=_m, ec_type=_type, validate=True)
available_ec_types.append(_type)
except Exception:
# ignore any errors, assume backend not available
pass
self.assertEqual(available_ec_types, VALID_EC_TYPES)
def __init__(self, name, ec_segment_size, ec_type, ec_nb_data,
ec_nb_parity):
super(ECStorageMethod, self).__init__(name=name, ec=True)
try:
self._ec_nb_data = int(ec_nb_data)
except (TypeError, ValueError):
raise exc.InvalidStorageMethod('Invalid %r ec_nb_data' %
ec_nb_data)
try:
self._ec_nb_parity = int(ec_nb_parity)
except (TypeError, ValueError):
raise exc.InvalidStorageMethod('Invalid %r ec_nb_parity' %
ec_nb_parity)
self._ec_segment_size = ec_segment_size
self._ec_type = ec_type
self.driver = ECDriver(k=ec_nb_data, m=ec_nb_parity,
ec_type=ec_type_to_pyeclib_type[ec_type])
self._ec_quorum_size = \
self._ec_nb_data + self.driver.min_parity_fragments_needed()
def __init__(self, idx, name='', aliases='', is_default=False,
is_deprecated=False, object_ring=None,
diskfile_module='egg:swift#erasure_coding.fs',
ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE,
ec_type=None, ec_ndata=None, ec_nparity=None,
ec_duplication_factor=1):
super(ECStoragePolicy, self).__init__(
idx=idx, name=name, aliases=aliases, is_default=is_default,
is_deprecated=is_deprecated, object_ring=object_ring,
diskfile_module=diskfile_module)
# Validate erasure_coding policy specific members
# ec_type is one of the EC implementations supported by PyEClib
if ec_type is None:
raise PolicyError('Missing ec_type')
if ec_type not in VALID_EC_TYPES:
raise PolicyError('Wrong ec_type %s for policy %s, should be one'
' of "%s"' % (ec_type, self.name,
', '.join(VALID_EC_TYPES)))
self._ec_type = ec_type
# Define _ec_ndata as the number of EC data fragments
# Accessible as the property "ec_ndata"
try:
value = int(ec_ndata)
if value <= 0:
raise ValueError
self._ec_ndata = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_data_fragments %r' %
ec_ndata, index=self.idx)
# Define _ec_nparity as the number of EC parity fragments
# Accessible as the property "ec_nparity"
try:
value = int(ec_nparity)
if value <= 0:
raise ValueError
self._ec_nparity = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_parity_fragments %r'
% ec_nparity, index=self.idx)
# Define _ec_segment_size as the encode segment unit size
# Accessible as the property "ec_segment_size"
try:
value = int(ec_segment_size)
if value <= 0:
raise ValueError
self._ec_segment_size = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_object_segment_size %r' %
ec_segment_size, index=self.idx)
if self._ec_type == 'isa_l_rs_vand' and self._ec_nparity >= 5:
logger = logging.getLogger("swift.common.storage_policy")
if not logger.handlers:
# If nothing else, log to stderr
logger.addHandler(logging.StreamHandler(sys.__stderr__))
logger.warning(
'Storage policy %s uses an EC configuration known to harm '
'data durability. Any data in this policy should be migrated. '
'See https://bugs.launchpad.net/swift/+bug/1639691 for '
'more information.' % self.name)
if not is_deprecated:
raise PolicyError(
'Storage policy %s uses an EC configuration known to harm '
'data durability. This policy MUST be deprecated.'
% self.name)
# Initialize PyECLib EC backend
try:
self.pyeclib_driver = \
ECDriver(k=self._ec_ndata, m=self._ec_nparity,
ec_type=self._ec_type)
except ECDriverError as e:
raise PolicyError("Error creating EC policy (%s)" % e,
index=self.idx)
# quorum size in the EC case depends on the choice of EC scheme.
self._ec_quorum_size = \
self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed()
self._fragment_size = None
self._ec_duplication_factor = \
config_positive_int_value(ec_duplication_factor)
def _get_ecdriver(threshold, total_shares):
return ECDriver(k=threshold,
m=total_shares - threshold,
ec_type='liberasurecode_rs_vand')
class Eraser(object):
"""A wrapper for pyeclib erasure coding driver (ECDriver)"""
def __init__(self,
ec_k,
ec_m,
ec_type="liberasurecode_rs_vand",
aes_enabled=True):
self.ec_type = ec_type
if aes_enabled:
self.aes = AESDriver()
logger.info("Eraser will use AES encryption")
else:
logger.info("Eraser will not use AES encryption")
expected_module_name = "drivers." + ec_type.lower() + "_driver"
expected_class_name = ec_type[0].upper() + ec_type[1:].lower(
) + "Driver"
try:
mod = __import__(expected_module_name,
fromlist=[expected_class_name])
driver_class = None
driver_class = getattr(mod, expected_class_name)
self.driver = driver_class(k=ec_k,
m=ec_m,
ec_type=ec_type,
hd=None)
except (ImportError, AttributeError):
logger.exception("Driver " + ec_type +
" could not be loaded as a custom driver")
try:
self.driver = ECDriver(k=ec_k, m=ec_m, ec_type=ec_type)
except Exception as error:
logger.exception("Driver " + ec_type +
" could not be loaded by pyeclib")
raise error
def encode(self, data):
"""Encode a string of bytes in flattened string of byte strips"""
payload = data
if hasattr(self, 'aes'):
payload = self.aes.encode(data)[0]
strips = self.driver.encode(payload)
return strips
def decode(self, strips):
"""Decode byte strips in a string of bytes"""
payload = self.driver.decode(strips)
if hasattr(self, 'aes'):
return self.aes.decode([payload])
return payload
def reconstruct(self, available_payload_fragments, missing_indices):
"""
Reconstruct missing fragments of data
Args:
available_payload_fragments(list(bytes)): Available fragments of data
missing_indices(list(int)): List of the indices of the missing blocks
Returns:
list(bytes): A list of the reconstructed fragments
"""
return self.driver.reconstruct(available_payload_fragments,
missing_indices)
def fragments_needed(self, missing_indices):
"""
Return a list of the fragments needed to recover the missing ones
Args:
missing_indices(list(int)): The list of indices of the fragments to recover
Returns:
list(int): A list of the indices of the fragments required to recover the missing ones
"""
return self.driver.fragments_needed(missing_indices)
def __init__(self, idx, name='', aliases='', is_default=False,
is_deprecated=False, object_ring=None,
ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE,
ec_type=None, ec_ndata=None, ec_nparity=None):
super(ECStoragePolicy, self).__init__(
idx=idx, name=name, aliases=aliases, is_default=is_default,
is_deprecated=is_deprecated, object_ring=object_ring)
# Validate erasure_coding policy specific members
# ec_type is one of the EC implementations supported by PyEClib
if ec_type is None:
raise PolicyError('Missing ec_type')
if ec_type not in VALID_EC_TYPES:
raise PolicyError('Wrong ec_type %s for policy %s, should be one'
' of "%s"' % (ec_type, self.name,
', '.join(VALID_EC_TYPES)))
self._ec_type = ec_type
# Define _ec_ndata as the number of EC data fragments
# Accessible as the property "ec_ndata"
try:
value = int(ec_ndata)
if value <= 0:
raise ValueError
self._ec_ndata = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_data_fragments %r' %
ec_ndata, index=self.idx)
# Define _ec_nparity as the number of EC parity fragments
# Accessible as the property "ec_nparity"
try:
value = int(ec_nparity)
if value <= 0:
raise ValueError
self._ec_nparity = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_parity_fragments %r'
% ec_nparity, index=self.idx)
# Define _ec_segment_size as the encode segment unit size
# Accessible as the property "ec_segment_size"
try:
value = int(ec_segment_size)
if value <= 0:
raise ValueError
self._ec_segment_size = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_object_segment_size %r' %
ec_segment_size, index=self.idx)
# Initialize PyECLib EC backend
try:
self.pyeclib_driver = \
ECDriver(k=self._ec_ndata, m=self._ec_nparity,
ec_type=self._ec_type)
except ECDriverError as e:
raise PolicyError("Error creating EC policy (%s)" % e,
index=self.idx)
# quorum size in the EC case depends on the choice of EC scheme.
self._ec_quorum_size = \
self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed()
class ECStoragePolicy(BaseStoragePolicy):
"""
Represents a storage policy of type 'erasure_coding'.
Not meant to be instantiated directly; use
:func:`~swift.common.storage_policy.reload_storage_policies` to load
POLICIES from ``swift.conf``.
"""
def __init__(self, idx, name='', aliases='', is_default=False,
is_deprecated=False, object_ring=None,
ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE,
ec_type=None, ec_ndata=None, ec_nparity=None):
super(ECStoragePolicy, self).__init__(
idx=idx, name=name, aliases=aliases, is_default=is_default,
is_deprecated=is_deprecated, object_ring=object_ring)
# Validate erasure_coding policy specific members
# ec_type is one of the EC implementations supported by PyEClib
if ec_type is None:
raise PolicyError('Missing ec_type')
if ec_type not in VALID_EC_TYPES:
raise PolicyError('Wrong ec_type %s for policy %s, should be one'
' of "%s"' % (ec_type, self.name,
', '.join(VALID_EC_TYPES)))
self._ec_type = ec_type
# Define _ec_ndata as the number of EC data fragments
# Accessible as the property "ec_ndata"
try:
value = int(ec_ndata)
if value <= 0:
raise ValueError
self._ec_ndata = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_data_fragments %r' %
ec_ndata, index=self.idx)
# Define _ec_nparity as the number of EC parity fragments
# Accessible as the property "ec_nparity"
try:
value = int(ec_nparity)
if value <= 0:
raise ValueError
self._ec_nparity = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_parity_fragments %r'
% ec_nparity, index=self.idx)
# Define _ec_segment_size as the encode segment unit size
# Accessible as the property "ec_segment_size"
try:
value = int(ec_segment_size)
if value <= 0:
raise ValueError
self._ec_segment_size = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_object_segment_size %r' %
ec_segment_size, index=self.idx)
# Initialize PyECLib EC backend
try:
self.pyeclib_driver = \
ECDriver(k=self._ec_ndata, m=self._ec_nparity,
ec_type=self._ec_type)
except ECDriverError as e:
raise PolicyError("Error creating EC policy (%s)" % e,
index=self.idx)
# quorum size in the EC case depends on the choice of EC scheme.
self._ec_quorum_size = \
self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed()
@property
def ec_type(self):
return self._ec_type
@property
def ec_ndata(self):
return self._ec_ndata
@property
def ec_nparity(self):
return self._ec_nparity
@property
def ec_segment_size(self):
return self._ec_segment_size
@property
def fragment_size(self):
"""
Maximum length of a fragment, including header.
NB: a fragment archive is a sequence of 0 or more max-length
fragments followed by one possibly-shorter fragment.
"""
# Technically pyeclib's get_segment_info signature calls for
# (data_len, segment_size) but on a ranged GET we don't know the
# ec-content-length header before we need to compute where in the
# object we should request to align with the fragment size. So we
# tell pyeclib a lie - from it's perspective, as long as data_len >=
# segment_size it'll give us the answer we want. From our
# perspective, because we only use this answer to calculate the
# *minimum* size we should read from an object body even if data_len <
# segment_size we'll still only read *the whole one and only last
# fragment* and pass than into pyeclib who will know what to do with
# it just as it always does when the last fragment is < fragment_size.
return self.pyeclib_driver.get_segment_info(
self.ec_segment_size, self.ec_segment_size)['fragment_size']
@property
def ec_scheme_description(self):
"""
This short hand form of the important parts of the ec schema is stored
in Object System Metadata on the EC Fragment Archives for debugging.
"""
return "%s %d+%d" % (self._ec_type, self._ec_ndata, self._ec_nparity)
def __repr__(self):
return ("%s, EC config(ec_type=%s, ec_segment_size=%d, "
"ec_ndata=%d, ec_nparity=%d)") % \
(super(ECStoragePolicy, self).__repr__(), self.ec_type,
self.ec_segment_size, self.ec_ndata, self.ec_nparity)
@classmethod
def _config_options_map(cls):
options = super(ECStoragePolicy, cls)._config_options_map()
options.update({
'ec_type': 'ec_type',
'ec_object_segment_size': 'ec_segment_size',
'ec_num_data_fragments': 'ec_ndata',
'ec_num_parity_fragments': 'ec_nparity',
})
return options
def get_info(self, config=False):
info = super(ECStoragePolicy, self).get_info(config=config)
if not config:
info.pop('ec_object_segment_size')
info.pop('ec_num_data_fragments')
info.pop('ec_num_parity_fragments')
info.pop('ec_type')
return info
def _validate_ring(self):
"""
EC specific validation
Replica count check - we need _at_least_ (#data + #parity) replicas
configured. Also if the replica count is larger than exactly that
number there's a non-zero risk of error for code that is considering
the number of nodes in the primary list from the ring.
"""
if not self.object_ring:
raise PolicyError('Ring is not loaded')
nodes_configured = self.object_ring.replica_count
if nodes_configured != (self.ec_ndata + self.ec_nparity):
raise RingValidationError(
'EC ring for policy %s needs to be configured with '
'exactly %d nodes. Got %d.' % (
self.name, self.ec_ndata + self.ec_nparity,
nodes_configured))
@property
def quorum(self):
"""
Number of successful backend requests needed for the proxy to consider
the client request successful.
The quorum size for EC policies defines the minimum number
of data + parity elements required to be able to guarantee
the desired fault tolerance, which is the number of data
elements supplemented by the minimum number of parity
elements required by the chosen erasure coding scheme.
For example, for Reed-Solomon, the minimum number parity
elements required is 1, and thus the quorum_size requirement
is ec_ndata + 1.
Given the number of parity elements required is not the same
for every erasure coding scheme, consult PyECLib for
min_parity_fragments_needed()
"""
return self._ec_quorum_size
from pyeclib.ec_iface import ECDriver
import pickle
from random import shuffle
if __name__ == '__main__':
with open('results.pkl', 'rb') as f:
results_dict = pickle.load(f)
for key, frags in results_dict.items():
k, m = key
driver = ECDriver(ec_type='jerasure_rs_vand', k=k, m=m)
for num in range(10):
shuffle(frags)
assert 'a'*100 == driver.decode(
frags[:k], force_metadata_checks=True)
print '%s passed' % str(key)
parser = argparse.ArgumentParser(description='Decoder for PyECLib.')
parser.add_argument('k', type=int, help='number of data elements')
parser.add_argument('m', type=int, help='number of parity elements')
parser.add_argument('ec_type', help='EC algorithm used')
parser.add_argument('fragments', metavar='fragment', nargs='+',
help='fragments to decode')
parser.add_argument('filename', help='output file')
args = parser.parse_args()
print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("fragments = %s" % args.fragments)
print("filename = %s" % args.filename)
ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
fragment_list = []
# read fragments
for fragment in args.fragments:
with open(("%s" % fragment), "rb") as fp:
fragment_list.append(fp.read())
# decode
decoded_file = ec_driver.decode(fragment_list)
# write
with open("%s.decoded" % args.filename, "wb") as fp:
fp.write(decoded_file)
from pyeclib.ec_iface import ECDriver
import pickle
if __name__ == '__main__':
ec_pattern = [(6, 3), (10, 4), (20, 4), (11, 7)]
results_dict = {}
for k, m in ec_pattern:
driver = ECDriver(ec_type='jerasure_rs_vand', k=k, m=m)
results_dict[(k, m)] = driver.encode('a' * 100)
with open('results.pkl', 'wb') as f:
pickle.dump(results_dict, f)
class ECStoragePolicy(BaseStoragePolicy):
"""
Represents a storage policy of type 'erasure_coding'.
Not meant to be instantiated directly; use
:func:`~swift.common.storage_policy.reload_storage_policies` to load
POLICIES from ``swift.conf``.
"""
def __init__(self, idx, name='', aliases='', is_default=False,
is_deprecated=False, object_ring=None,
diskfile_module='egg:swift#erasure_coding.fs',
ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE,
ec_type=None, ec_ndata=None, ec_nparity=None,
ec_duplication_factor=1):
super(ECStoragePolicy, self).__init__(
idx=idx, name=name, aliases=aliases, is_default=is_default,
is_deprecated=is_deprecated, object_ring=object_ring,
diskfile_module=diskfile_module)
# Validate erasure_coding policy specific members
# ec_type is one of the EC implementations supported by PyEClib
if ec_type is None:
raise PolicyError('Missing ec_type')
if ec_type not in VALID_EC_TYPES:
raise PolicyError('Wrong ec_type %s for policy %s, should be one'
' of "%s"' % (ec_type, self.name,
', '.join(VALID_EC_TYPES)))
self._ec_type = ec_type
# Define _ec_ndata as the number of EC data fragments
# Accessible as the property "ec_ndata"
try:
value = int(ec_ndata)
if value <= 0:
raise ValueError
self._ec_ndata = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_data_fragments %r' %
ec_ndata, index=self.idx)
# Define _ec_nparity as the number of EC parity fragments
# Accessible as the property "ec_nparity"
try:
value = int(ec_nparity)
if value <= 0:
raise ValueError
self._ec_nparity = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_parity_fragments %r'
% ec_nparity, index=self.idx)
# Define _ec_segment_size as the encode segment unit size
# Accessible as the property "ec_segment_size"
try:
value = int(ec_segment_size)
if value <= 0:
raise ValueError
self._ec_segment_size = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_object_segment_size %r' %
ec_segment_size, index=self.idx)
if self._ec_type == 'isa_l_rs_vand' and self._ec_nparity >= 5:
logger = logging.getLogger("swift.common.storage_policy")
if not logger.handlers:
# If nothing else, log to stderr
logger.addHandler(logging.StreamHandler(sys.__stderr__))
logger.warning(
'Storage policy %s uses an EC configuration known to harm '
'data durability. Any data in this policy should be migrated. '
'See https://bugs.launchpad.net/swift/+bug/1639691 for '
'more information.' % self.name)
if not is_deprecated:
raise PolicyError(
'Storage policy %s uses an EC configuration known to harm '
'data durability. This policy MUST be deprecated.'
% self.name)
# Initialize PyECLib EC backend
try:
self.pyeclib_driver = \
ECDriver(k=self._ec_ndata, m=self._ec_nparity,
ec_type=self._ec_type)
except ECDriverError as e:
raise PolicyError("Error creating EC policy (%s)" % e,
index=self.idx)
# quorum size in the EC case depends on the choice of EC scheme.
self._ec_quorum_size = \
self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed()
self._fragment_size = None
self._ec_duplication_factor = \
config_positive_int_value(ec_duplication_factor)
@property
def ec_type(self):
return self._ec_type
@property
def ec_ndata(self):
return self._ec_ndata
@property
def ec_nparity(self):
return self._ec_nparity
@property
def ec_n_unique_fragments(self):
return self._ec_ndata + self._ec_nparity
@property
def ec_segment_size(self):
return self._ec_segment_size
@property
def fragment_size(self):
"""
Maximum length of a fragment, including header.
NB: a fragment archive is a sequence of 0 or more max-length
fragments followed by one possibly-shorter fragment.
"""
# Technically pyeclib's get_segment_info signature calls for
# (data_len, segment_size) but on a ranged GET we don't know the
# ec-content-length header before we need to compute where in the
# object we should request to align with the fragment size. So we
# tell pyeclib a lie - from it's perspective, as long as data_len >=
# segment_size it'll give us the answer we want. From our
# perspective, because we only use this answer to calculate the
# *minimum* size we should read from an object body even if data_len <
# segment_size we'll still only read *the whole one and only last
# fragment* and pass than into pyeclib who will know what to do with
# it just as it always does when the last fragment is < fragment_size.
if self._fragment_size is None:
self._fragment_size = self.pyeclib_driver.get_segment_info(
self.ec_segment_size, self.ec_segment_size)['fragment_size']
return self._fragment_size
@property
def ec_scheme_description(self):
"""
This short hand form of the important parts of the ec schema is stored
in Object System Metadata on the EC Fragment Archives for debugging.
"""
return "%s %d+%d" % (self._ec_type, self._ec_ndata, self._ec_nparity)
@property
def ec_duplication_factor(self):
return self._ec_duplication_factor
def __repr__(self):
extra_info = ''
if self.ec_duplication_factor != 1:
extra_info = ', ec_duplication_factor=%d' % \
self.ec_duplication_factor
return ("%s, EC config(ec_type=%s, ec_segment_size=%d, "
"ec_ndata=%d, ec_nparity=%d%s)") % \
(super(ECStoragePolicy, self).__repr__(), self.ec_type,
self.ec_segment_size, self.ec_ndata, self.ec_nparity,
extra_info)
@classmethod
def _config_options_map(cls):
options = super(ECStoragePolicy, cls)._config_options_map()
options.update({
'ec_type': 'ec_type',
'ec_object_segment_size': 'ec_segment_size',
'ec_num_data_fragments': 'ec_ndata',
'ec_num_parity_fragments': 'ec_nparity',
'ec_duplication_factor': 'ec_duplication_factor',
})
return options
def get_info(self, config=False):
info = super(ECStoragePolicy, self).get_info(config=config)
if not config:
info.pop('ec_object_segment_size')
info.pop('ec_num_data_fragments')
info.pop('ec_num_parity_fragments')
info.pop('ec_type')
info.pop('ec_duplication_factor')
return info
@property
def quorum(self):
"""
Number of successful backend requests needed for the proxy to consider
the client PUT request successful.
The quorum size for EC policies defines the minimum number
of data + parity elements required to be able to guarantee
the desired fault tolerance, which is the number of data
elements supplemented by the minimum number of parity
elements required by the chosen erasure coding scheme.
For example, for Reed-Solomon, the minimum number parity
elements required is 1, and thus the quorum_size requirement
is ec_ndata + 1.
Given the number of parity elements required is not the same
for every erasure coding scheme, consult PyECLib for
min_parity_fragments_needed()
"""
return self._ec_quorum_size * self.ec_duplication_factor
def load_ring(self, swift_dir):
"""
Load the ring for this policy immediately.
:param swift_dir: path to rings
"""
if self.object_ring:
return
def validate_ring_data(ring_data):
"""
EC specific validation
Replica count check - we need _at_least_ (#data + #parity) replicas
configured. Also if the replica count is larger than exactly that
number there's a non-zero risk of error for code that is
considering the number of nodes in the primary list from the ring.
"""
configured_fragment_count = ring_data.replica_count
required_fragment_count = \
(self.ec_n_unique_fragments) * self.ec_duplication_factor
if configured_fragment_count != required_fragment_count:
raise RingLoadError(
'EC ring for policy %s needs to be configured with '
'exactly %d replicas. Got %s.' % (
self.name, required_fragment_count,
configured_fragment_count))
self.object_ring = Ring(
swift_dir, ring_name=self.ring_name,
validation_hook=validate_ring_data)
def get_backend_index(self, node_index):
"""
Backend index for PyECLib
:param node_index: integer of node index
:return: integer of actual fragment index. if param is not an integer,
return None instead
"""
try:
node_index = int(node_index)
except ValueError:
return None
return node_index % self.ec_n_unique_fragments
# Results will be List[(ec_type, throughput)]
results = []
# Num iterations
num_iterations = 10
for scheme in schemes:
print(scheme)
# Generate a new string for each test
file_str = ''.join(
random.choice(
string.ascii_uppercase + string.digits) for x in range(args.s))
try:
ec_driver = ECDriver(k=scheme.k, m=scheme.m, ec_type=scheme.ec_type)
except Exception as e:
print("Scheme %s is not defined (%s)." % (scheme, e))
continue
timer.start()
for i in range(num_iterations):
ec_driver.encode(file_str)
duration = timer.stop_and_return()
results.append((scheme, duration))
timer.reset()
def setUp(self):
self.null_driver = ECDriver(
library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
parser.add_argument('m', type=int, help='number of parity elements')
parser.add_argument('ec_type', help='EC algorithm used')
parser.add_argument('file_dir', help='directory with the file')
parser.add_argument('filename', help='file to encode')
parser.add_argument('fragment_dir', help='directory to drop encoded fragments')
args = parser.parse_args()
print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("filename = %s" % args.filename)
start = time.time()
ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
# read
with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp:
whole_file_str = fp.read()
# encode
fragments = ec_driver.encode(whole_file_str)
# store
i = 0
for fragment in fragments:
with open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "wb") as fp:
fp.write(fragment)
i += 1
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import pyeclib
from pyeclib.ec_iface import ECDriver
import random
import string
import sys
import os
import argparse
parser = argparse.ArgumentParser(
description='PyECLib tool to determine fragment indexes needed to '
'recover missing fragments.')
parser.add_argument('k', type=int, help='number of data elements')
parser.add_argument('m', type=int, help='number of parity elements')
parser.add_argument('ec_type', help='EC algorithm used')
parser.add_argument('missing_fragments', type=int, metavar='missing_fragment',
nargs='+', help='missing_fragments')
args = parser.parse_args()
ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
fragments_needed = ec_driver.fragments_needed(args.missing_fragments)
print(fragments_needed)
print ""
print "A script that benchmarks pyeclib libraries reconstruction capabilities"
print ""
print "Arguments:"
print "\tsize Size of the payload to encode in bytes"
if __name__ == "__main__":
if len(sys.argv) != 2:
print_usage()
sys.exit(0)
SIZE = int(sys.argv[1])
EC_K = int(os.environ.get("EC_K", 10))
EC_M = int(os.environ.get("EC_M", 4))
EC_TYPE = os.environ.get("EC_TYPE", "liberasurecode_rs_vand")
DRIVER = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
DATA = os.urandom(SIZE)
STRIPS = DRIVER.encode(DATA)
LENGTH = EC_K + EC_M
SUPPORTED_DISTANCE = LENGTH - EC_K + 1
print "About to reconstruct ", REQUESTS, " times a payload of size ", SIZE, " bytes (", \
(DRIVER.ec_type if hasattr(DRIVER, "ec_type") else EC_TYPE), ", k =", DRIVER.k, \
", m =", DRIVER.m, ") from 0 to", SUPPORTED_DISTANCE, "missing blocks"
random.seed(0)
for missing_blocks in range(SUPPORTED_DISTANCE):
for i in range(REQUESTS):
missing_indices = range(missing_blocks)
start = time.clock()
