def __init__(self, name, ec_segment_size, ec_type, ec_nb_data, ec_nb_parity): super(ECStorageMethod, self).__init__(name=name, ec=True) try: self._ec_nb_data = int(ec_nb_data) except (TypeError, ValueError): raise exceptions.InvalidStorageMethod('Invalid %r ec_nb_data' % ec_nb_data) try: self._ec_nb_parity = int(ec_nb_parity) except (TypeError, ValueError): raise exceptions.InvalidStorageMethod('Invalid %r ec_nb_parity' % ec_nb_parity) self._ec_segment_size = ec_segment_size self._ec_type = ec_type try: self.driver = ECDriver(k=ec_nb_data, m=ec_nb_parity, ec_type=ec_type) except ECDriverError as exc: msg = "'%s' (%s: %s) Check erasure code packages." % ( ec_type, exc.__class__.__name__, exc) reraise(exceptions.InvalidStorageMethod, exceptions.InvalidStorageMethod(msg), sys.exc_info()[2]) self._ec_quorum_size = \ self._ec_nb_data + self.driver.min_parity_fragments_needed()
class Eraser(object): """A wrapper for pyeclib erasure coding driver (ECDriver)""" def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"): self.k = k self.m = m self.ec_type = ec_type if EC_TYPE == "pylonghair": self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE) elif EC_TYPE == "striping" or EC_TYPE == "bypass": self.driver = ECStripingDriver(k=EC_K, m=0, hd=None) else: self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE) def encode(self, data): """Encode a string of bytes in flattened string of byte strips""" raw_strips = self.driver.encode(data) strips = [] for raw_strip in raw_strips: strip = Strip() strip.data = raw_strip strips.append(strip) return strips def decode(self, playcloud_strips): """Decode byte strips in a string of bytes""" strips = convert_strips_to_bytes_list(playcloud_strips) return self.driver.decode(strips)
def get_available_backend(self, k, m, ec_type, chksum_type = "inline_crc32"): if ec_type[:11] == "flat_xor_hd": return ECDriver(k=k, m=m, ec_type="flat_xor_hd", chksum_type=chksum_type) elif ec_type in self.available_backends: return ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type) else: return None
def __init__(self, cfgfile, verbose=False): uid = self.get_hwaddr() config = configparser.ConfigParser() config.read(cfgfile) N = int(config['General']['n']) F = int(config['General']['f']) addrs = list(config['Nodes'].values()) self.verbose = verbose context = zmq.Context() self.k = N-2*F self.ec_driver = ECDriver(k=self.k, m=N, ec_type='liberasurecode_rs_vand') self.socketid = {} self.sockets = [] self.uid = uid self.id = "Client-{}".format(uid) self.poller = zmq.Poller() self.quorum = int(math.ceil((N+self.k)/2)) print("Starting client {}".format(uid)) for addr in addrs: socket = context.socket(zmq.DEALER) self.socketid[socket] = "Server-%s" % addr self.poller.register(socket, zmq.POLLIN) socket.connect("tcp://%s" % addr); self.vprint("Connecting to server with address {}".format(addr)) self.sockets.append(socket)
class ECStorageMethod(StorageMethod): def __init__(self, name, ec_segment_size, ec_type, ec_nb_data, ec_nb_parity): super(ECStorageMethod, self).__init__(name=name, ec=True) try: self._ec_nb_data = int(ec_nb_data) except (TypeError, ValueError): raise exc.InvalidStorageMethod('Invalid %r ec_nb_data' % ec_nb_data) try: self._ec_nb_parity = int(ec_nb_parity) except (TypeError, ValueError): raise exc.InvalidStorageMethod('Invalid %r ec_nb_parity' % ec_nb_parity) self._ec_segment_size = ec_segment_size self._ec_type = ec_type self.driver = ECDriver(k=ec_nb_data, m=ec_nb_parity, ec_type=ec_type_to_pyeclib_type[ec_type]) self._ec_quorum_size = \ self._ec_nb_data + self.driver.min_parity_fragments_needed() @property def quorum(self): return self._ec_quorum_size @classmethod def build(cls, params): ec_nb_data = params.pop('k') ec_nb_parity = params.pop('m') ec_type = params.pop('algo') return cls('ec', ec_segment_size=EC_SEGMENT_SIZE, ec_type=ec_type, ec_nb_data=ec_nb_data, ec_nb_parity=ec_nb_parity) @property def ec_type(self): return self._ec_type @property def ec_nb_data(self): return self._ec_nb_data @property def ec_nb_parity(self): return self._ec_nb_parity @property def ec_segment_size(self): return self._ec_segment_size @property def ec_fragment_size(self): return self.driver.get_segment_info( self.ec_segment_size, self.ec_segment_size)['fragment_size']
def __init__(self, cfgfile): self.hw_addr = self.get_hwaddr() self.uid = (-1, self.hw_addr) self.loop = asyncio.get_event_loop() config = configparser.ConfigParser() config.read(cfgfile) nbr_of_servers = int(config['General']['n']) chunks_size = int(config['General']['chunks_size']) f = int(config['General']['f']) e = int(config['General']['e']) k = nbr_of_servers - 2 * (f + e) if (k < 1): raise Exception("Coded elements less than 1") if (nbr_of_servers + k > 32): raise Exception( "[liberasurecode] Total number of fragments (k + m) must be at most 32" ) quorum_size = math.ceil((nbr_of_servers + k + 2 * e) / 2) self.majority = math.ceil((nbr_of_servers + 1) / 2) self.ec_driver = ECDriver(k=k, m=nbr_of_servers, ec_type='liberasurecode_rs_vand') self.p = QuorumSend(quorum_size, PingPongMessage) t = Thread(target=self.start_event_loop, args=(self.loop, config['Nodes'], chunks_size)) t.daemon = True t.start()
def test_verify_fragment_inline_chksum_succeed(self): pyeclib_drivers = [] pyeclib_drivers.append( ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) pyeclib_drivers.append( ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) pyeclib_drivers.append( ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) pyeclib_drivers.append( ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32")) filesize = 1024 * 1024 * 3 file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) file_bytes = file_str.encode('utf-8') for pyeclib_driver in pyeclib_drivers: fragments = pyeclib_driver.encode(file_bytes) fragment_metadata_list = [] for fragment in fragments: fragment_metadata_list.append( pyeclib_driver.get_metadata(fragment)) expected_ret_value = {"status": 0 } self.assertTrue( pyeclib_driver.verify_stripe_metadata(fragment_metadata_list) == expected_ret_value)
def __init__(self, name, ec_segment_size, ec_type, ec_nb_data, ec_nb_parity): super(ECStorageMethod, self).__init__(name=name, ec=True) try: self._ec_nb_data = int(ec_nb_data) except (TypeError, ValueError): raise exceptions.InvalidStorageMethod('Invalid %r ec_nb_data' % ec_nb_data) try: self._ec_nb_parity = int(ec_nb_parity) except (TypeError, ValueError): raise exceptions.InvalidStorageMethod('Invalid %r ec_nb_parity' % ec_nb_parity) self._ec_segment_size = ec_segment_size self._ec_type = ec_type from pyeclib.ec_iface import ECDriver self.driver = ECDriver(k=ec_nb_data, m=ec_nb_parity, ec_type=ec_type_to_pyeclib_type[ec_type]) self._ec_quorum_size = \ self._ec_nb_data + self.driver.min_parity_fragments_needed()
def __init__(self, source, s, t, p, ec_type="isa_l_rs_vand"): """ StepEntangler constructor Args: source(Source): Block source implementing the get_random_blocks and get_block primitives s(int): Number of source blocks or the number of chunks to make from the original data t(int): Number of old blocks to entangle with p(int): Number of parity blocks to produce using Reed-Solomon """ if not source or \ not callable(getattr(source, "get_block", None)) or \ not callable(getattr(source, "get_random_blocks", None)): raise ValueError( "source argument must implement a get_random_blocks and a get_block method" ) if s <= 0: raise ValueError("s({:d}) must be greater or equal to 1".format(s)) if t < 0: raise ValueError("t({:d}) must be greater or equal to 0".format(t)) if p < s: raise ValueError( "p({:d}) must be greater or equal to s({:d})".format(p, s)) self.s = s self.t = t self.p = p self.e = self.p - self.s self.k = s + t self.source = source self.driver = ECDriver(k=self.k, m=self.p, ec_type=ec_type)
def test_get_segment_byterange_info(self): pyeclib_drivers = [] pyeclib_drivers.append( ECDriver(k=12, m=2, ec_type="jerasure_rs_vand")) pyeclib_drivers.append( ECDriver(k=11, m=2, ec_type="jerasure_rs_vand")) pyeclib_drivers.append( ECDriver(k=10, m=2, ec_type="jerasure_rs_vand")) file_size = 1024 * 1024 segment_size = 3 * 1024 ranges = [(0, 1), (1, 12), (10, 1000), (0, segment_size-1), (1, segment_size+1), (segment_size-1, 2*segment_size)] expected_results = {} expected_results[(0, 1)] = {0: (0, 1)} expected_results[(1, 12)] = {0: (1, 12)} expected_results[(10, 1000)] = {0: (10, 1000)} expected_results[(0, segment_size-1)] = {0: (0, segment_size-1)} expected_results[(1, segment_size+1)] = {0: (1, segment_size-1), 1: (0, 1)} expected_results[(segment_size-1, 2*segment_size)] = {0: (segment_size-1, segment_size-1), 1: (0, segment_size-1), 2: (0, 0)} results = pyeclib_drivers[0].get_segment_info_byterange(ranges, file_size, segment_size) for exp_result_key in expected_results: self.assertTrue(results.has_key(exp_result_key)) self.assertTrue(len(results[exp_result_key]) == len(expected_results[exp_result_key])) exp_result_map = expected_results[exp_result_key] for segment_key in exp_result_map: self.assertTrue(results[exp_result_key].has_key(segment_key)) self.assertTrue(results[exp_result_key][segment_key] == expected_results[exp_result_key][segment_key])
def __init__(self, ec_k, ec_m, ec_type="liberasurecode_rs_vand", aes_enabled=True): self.ec_type = ec_type if aes_enabled: self.aes = AESDriver() logger.info("Eraser will use AES encryption") else: logger.info("Eraser will not use AES encryption") expected_module_name = "drivers." + ec_type.lower() + "_driver" expected_class_name = ec_type[0].upper() + ec_type[1:].lower( ) + "Driver" try: mod = __import__(expected_module_name, fromlist=[expected_class_name]) driver_class = None driver_class = getattr(mod, expected_class_name) self.driver = driver_class(k=ec_k, m=ec_m, ec_type=ec_type, hd=None) except (ImportError, AttributeError): logger.exception("Driver " + ec_type + " could not be loaded as a custom driver") try: self.driver = ECDriver(k=ec_k, m=ec_m, ec_type=ec_type) except Exception as error: logger.exception("Driver " + ec_type + " could not be loaded by pyeclib") raise error
def __init__(self, clientId, isLocal=True): self.clientId = clientId self.isLocal = isLocal self.N = 4 self.F = 1 self.metadataQ = queue.Queue() self.ec_driver = ECDriver(k=2, m=2, ec_type='liberasurecode_rs_vand') if self.isLocal: self.createLocalClients()
def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"): self.k = k self.m = m self.ec_type = ec_type if EC_TYPE == "pylonghair": self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE) elif EC_TYPE == "striping" or EC_TYPE == "bypass": self.driver = ECStripingDriver(k=EC_K, m=0, hd=None) else: self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
class TestNullDriver(unittest.TestCase): def setUp(self): self.null_driver = ECDriver(library_import_str="pyeclib.core.ECNullDriver", k=8, m=2) def tearDown(self): pass def test_null_driver(self): self.null_driver.encode("") self.null_driver.decode([])
def check_metadata_formatted(self, k, m, ec_type, chksum_type): if ec_type not in VALID_EC_TYPES: return filesize = 1024 * 1024 * 3 file_str = ''.join( random.choice(ascii_letters) for i in range(filesize)) file_bytes = file_str.encode('utf-8') pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type) fragments = pyeclib_driver.encode(file_bytes) f = 0 for fragment in fragments: metadata = pyeclib_driver.get_metadata(fragment, 1) if 'index' in metadata: self.assertEqual(metadata['index'], f) else: self.assertTrue(False) if 'chksum_mismatch' in metadata: self.assertEqual(metadata['chksum_mismatch'], 0) else: self.assertTrue(False) if 'backend_id' in metadata: self.assertEqual(metadata['backend_id'], ec_type) else: self.assertTrue(False) if 'orig_data_size' in metadata: self.assertEqual(metadata['orig_data_size'], 3145728) else: self.assertTrue(False) if 'chksum_type' in metadata: self.assertEqual(metadata['chksum_type'], 'crc32') else: self.assertTrue(False) if 'backend_version' not in metadata: self.assertTrue(False) if 'chksum' not in metadata: self.assertTrue(False) if 'size' not in metadata: self.assertTrue(False) f += 1
class TestNullDriver(unittest.TestCase): def setUp(self): self.null_driver = ECDriver( library_import_str="pyeclib.core.ECNullDriver", k=8, m=2) def tearDown(self): pass def test_null_driver(self): self.null_driver.encode('') self.null_driver.decode([])
def decode(k, m, base_filename_to_decode, ec_type): #print("k = %d, m = %d" % (k, m)) #print("ec_type = %s" % ec_type) home = expanduser("~") Disk_base_dir = home + '/Disk' key_list = base_filename_to_decode.split('.') jdecoder_types = [ "reed_sol_van", "reed_sol_r6_op", "cauchy_orig", "cauchy_good", "liberation", "blaum_roth", "liber8tion" ] if ec_type in jdecoder_types: print "Decoding started " jdecoder = ctypes.CDLL( '/home/raven/Downloads/Jerasure-master/Examples/jdecoder.so') key = key_list[0] decoded_data = str(jdecoder.decode(key, Disk_base_dir)) print "decoded data = ", decoded_data else: ec_driver = ECDriver(k=k, m=m, ec_type=ec_type) fragment_list = [] i = 0 while i <= k: if os.path.exists(Disk_base_dir + "/" + "Disk_" + str(i)): curr_dir = Disk_base_dir + "/" + "Disk_" + str(i) if i == 0: if os.path.exists(curr_dir + "/" + base_filename_to_decode): with open(curr_dir + "/" + base_filename_to_decode, 'rb') as fp: decoded_data = fp.read() fp.close() return decoded_data else: if os.path.exists(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i)): with open( curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i), 'rb') as fp: fragment = fp.read() fp.close() fragment_list.append(fragment) i = i + 1 if len(fragment_list) < k: return -1 # Not enough fragments to decode else: decoded_data = ec_driver.decode(fragment_list) return decoded_data
def check_metadata_formatted(self, k, m, ec_type, chksum_type): if ec_type not in VALID_EC_TYPES: return filesize = 1024 * 1024 * 3 file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) file_bytes = file_str.encode('utf-8') pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type) fragments = pyeclib_driver.encode(file_bytes) f = 0 for fragment in fragments: metadata = pyeclib_driver.get_metadata(fragment, 1) if 'index' in metadata: self.assertEqual(metadata['index'], f) else: self.assertTrue(False) if 'chksum_mismatch' in metadata: self.assertEqual(metadata['chksum_mismatch'], 0) else: self.assertTrue(False) if 'backend_id' in metadata: self.assertEqual(metadata['backend_id'], ec_type) else: self.assertTrue(False) if 'orig_data_size' in metadata: self.assertEqual(metadata['orig_data_size'], 3145728) else: self.assertTrue(False) if 'chksum_type' in metadata: self.assertEqual(metadata['chksum_type'], 'crc32') else: self.assertTrue(False) if 'backend_version' not in metadata: self.assertTrue(False) if 'chksum' not in metadata: self.assertTrue(False) if 'size' not in metadata: self.assertTrue(False) f += 1
def test_invalid_km_args(self): for ec_type in VALID_EC_TYPES: with self.assertRaises(ECDriverError) as err_context: # k is smaller than 1 ECDriver(ec_type=ec_type, k=-100, m=1) self.assertEqual(str(err_context.exception), "Invalid number of data fragments (k)") with self.assertRaises(ECDriverError) as err_context: # m is smaller than 1 ECDriver(ec_type=ec_type, k=1, m=-100) self.assertEqual(str(err_context.exception), "Invalid number of parity fragments (m)")
def check_metadata_formatted(self, k, m, ec_type, chksum_type): if ec_type not in VALID_EC_TYPES: return filesize = 1024 * 1024 * 3 file_str = "".join(random.choice(ascii_letters) for i in range(filesize)) file_bytes = file_str.encode("utf-8") pyeclib_driver = ECDriver(k=k, m=m, ec_type=ec_type, chksum_type=chksum_type) fragments = pyeclib_driver.encode(file_bytes) f = 0 for fragment in fragments: metadata = pyeclib_driver.get_metadata(fragment, 1) if "index" in metadata: self.assertEqual(metadata["index"], f) else: self.assertTrue(False) if "chksum_mismatch" in metadata: self.assertEqual(metadata["chksum_mismatch"], 0) else: self.assertTrue(False) if "backend_id" in metadata: self.assertEqual(metadata["backend_id"], ec_type) else: self.assertTrue(False) if "orig_data_size" in metadata: self.assertEqual(metadata["orig_data_size"], 3145728) else: self.assertTrue(False) if "chksum_type" in metadata: self.assertEqual(metadata["chksum_type"], "crc32") else: self.assertTrue(False) if "backend_version" not in metadata: self.assertTrue(False) if "chksum" not in metadata: self.assertTrue(False) if "size" not in metadata: self.assertTrue(False) f += 1
def test_valid_algo(self): print("") for _type in PyECLib_EC_Types.names(): # Check if this algo works if _type not in _available_backends: print("Skipping test for %s backend" % _type) continue try: if _type is 'shss': _instance = ECDriver(k=10, m=4, ec_type=_type) else: _instance = ECDriver(k=10, m=5, ec_type=_type) except ECDriverError: self.fail("%s algorithm not supported" % _type)
def test_small_encode(self): pyeclib_drivers = [] pyeclib_drivers.append(ECDriver(k=12, m=2, ec_type="jerasure_rs_vand")) pyeclib_drivers.append(ECDriver(k=11, m=2, ec_type="jerasure_rs_vand")) pyeclib_drivers.append(ECDriver(k=10, m=2, ec_type="jerasure_rs_vand")) encode_strs = [b"a", b"hello", b"hellohyhi", b"yo"] for pyeclib_driver in pyeclib_drivers: for encode_str in encode_strs: encoded_fragments = pyeclib_driver.encode(encode_str) decoded_str = pyeclib_driver.decode(encoded_fragments) self.assertTrue(decoded_str == encode_str)
def encode(k, m, data_to_encode, base_filename_to_store, ec_type): home = expanduser("~") key_list = base_filename_to_store.split('.') Disk_base_dir = home + '/Disk' jencoder_types = [ "reed_sol_van", "reed_sol_r6_op", "cauchy_orig", "cauchy_good", "liberation", "blaum_roth", "liber8tion" ] # Create a replicated copy first. #replica_dir = Disk_base_dir + "/Disk_0" #if not os.path.exists(replica_dir): # os.makedirs(replica_dir) #with open(replica_dir + "/" + base_filename_to_store,'wb') as fp : # fp.write(data_to_encode) #fp.close() if ec_type in jencoder_types: jencoder = ctypes.CDLL( '/home/raven/Downloads/Jerasure-master/Examples/jencoder.so') w = 8 key = key_list[0] jencoder.encode(key, data_to_encode, Disk_base_dir, k, m, ec_type, w, 0) else: ec_driver = ECDriver(k=k, m=m, ec_type=ec_type) # encode fragments = ec_driver.encode(data_to_encode) # store i = 1 for fragment in fragments: fragment_dir = Disk_base_dir + '/' + "Disk_" + str(i) if not os.path.exists(fragment_dir): os.makedirs(fragment_dir) with open( "%s/%s" % (fragment_dir, base_filename_to_store + "_fragment_" + str(i)), "wb") as fp: fp.write(fragment) fp.close() i += 1
def test_min_parity_fragments_needed(self): pyeclib_drivers = [] for ec_type in ['flat_xor_hd_3', 'liberasurecode_rs_vand']: if ec_type in VALID_EC_TYPES: pyeclib_drivers.append(ECDriver(k=10, m=5, ec_type=ec_type)) self.assertTrue( pyeclib_drivers[0].min_parity_fragments_needed() == 1)
def test_get_metadata_formatted(self): pyeclib_driver = ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32") filesize = 1024 * 1024 * 3 file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) file_bytes = file_str.encode('utf-8') fragments = pyeclib_driver.encode(file_bytes) i = 0 for fragment in fragments: metadata = pyeclib_driver.get_metadata(fragment, 1) if metadata.has_key('index'): self.assertEqual(metadata['index'], i) else: self.assertTrue(false) if metadata.has_key('chksum_mismatch'): self.assertEqual(metadata['chksum_mismatch'], 0) else: self.assertTrue(false) if metadata.has_key('backend_id'): self.assertEqual(metadata['backend_id'], 'jerasure_rs_vand') else: self.assertTrue(false) if metadata.has_key('orig_data_size'): self.assertEqual(metadata['orig_data_size'], 3145728) else: self.assertTrue(false) if metadata.has_key('chksum_type'): self.assertEqual(metadata['chksum_type'], 'crc32') else: self.assertTrue(false) if not metadata.has_key('backend_version'): self.assertTrue(false) if not metadata.has_key('chksum'): self.assertTrue(false) if not metadata.has_key('size'): self.assertTrue(false) i += 1
def test_verify_fragment_inline_chksum_fail(self): pyeclib_drivers = [] pyeclib_drivers.append( ECDriver(k=12, m=2, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) pyeclib_drivers.append( ECDriver(k=12, m=3, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) pyeclib_drivers.append( ECDriver(k=12, m=4, ec_type="jerasure_rs_vand", chksum_type="inline_crc32")) pyeclib_drivers.append( ECDriver(k=12, m=2, ec_type="jerasure_rs_cauchy", chksum_type="inline_crc32")) filesize = 1024 * 1024 * 3 file_str = ''.join(random.choice(ascii_letters) for i in range(filesize)) file_bytes = file_str.encode('utf-8') for pyeclib_driver in pyeclib_drivers: fragments = pyeclib_driver.encode(file_bytes) fragment_metadata_list = [] first_fragment_to_corrupt = random.randint(0, len(fragments)) num_to_corrupt = 2 fragments_to_corrupt = [ (first_fragment_to_corrupt + i) % len(fragments) for i in range(num_to_corrupt + 1) ] fragments_to_corrupt.sort() i = 0 for fragment in fragments: if i in fragments_to_corrupt: corrupted_fragment = fragment[:100] +\ (str(chr((b2i(fragment[100]) + 0x1) % 128))).encode('utf-8') + fragment[101:] fragment_metadata_list.append( pyeclib_driver.get_metadata(corrupted_fragment)) else: fragment_metadata_list.append( pyeclib_driver.get_metadata(fragment)) i += 1 expected_ret_value = {"status": -206, "reason": "Bad checksum", "bad_fragments": fragments_to_corrupt} self.assertEqual( pyeclib_driver.verify_stripe_metadata(fragment_metadata_list), expected_ret_value)
def encode(k,m,data_to_encode,base_filename_to_store,ec_type) : home = expanduser("~") key_list = base_filename_to_store.split('.') Disk_base_dir = home + '/Disk' jencoder_types = ["reed_sol_van", "reed_sol_r6_op","cauchy_orig","cauchy_good","liberation","blaum_roth","liber8tion"] # Create a replicated copy first. #replica_dir = Disk_base_dir + "/Disk_0" #if not os.path.exists(replica_dir): # os.makedirs(replica_dir) #with open(replica_dir + "/" + base_filename_to_store,'wb') as fp : # fp.write(data_to_encode) #fp.close() if ec_type in jencoder_types : jencoder = ctypes.CDLL('/home/raven/Downloads/Jerasure-master/Examples/jencoder.so') w = 8 key = key_list[0] jencoder.encode(key,data_to_encode,Disk_base_dir,k,m,ec_type,w,0) else : ec_driver = ECDriver(k = k, m = m, ec_type = ec_type) # encode fragments = ec_driver.encode(data_to_encode) # store i = 1 for fragment in fragments: fragment_dir = Disk_base_dir + '/' + "Disk_" + str(i) if not os.path.exists(fragment_dir): os.makedirs(fragment_dir) with open("%s/%s" % (fragment_dir, base_filename_to_store + "_fragment_" + str(i)), "wb") as fp: fp.write(fragment) fp.close() i += 1
def decode(k,m,base_filename_to_decode,ec_type) : #print("k = %d, m = %d" % (k, m)) #print("ec_type = %s" % ec_type) home = expanduser("~") Disk_base_dir = home + '/Disk' key_list = base_filename_to_decode.split('.') jdecoder_types = ["reed_sol_van", "reed_sol_r6_op","cauchy_orig","cauchy_good","liberation","blaum_roth","liber8tion"] if ec_type in jdecoder_types : print "Decoding started " jdecoder = ctypes.CDLL('/home/raven/Downloads/Jerasure-master/Examples/jdecoder.so') key = key_list[0] decoded_data = str(jdecoder.decode(key,Disk_base_dir)) print "decoded data = ", decoded_data else : ec_driver = ECDriver(k = k, m = m, ec_type = ec_type) fragment_list = [] i = 0 while i <=k : if os.path.exists(Disk_base_dir + "/" + "Disk_" + str(i)) : curr_dir = Disk_base_dir + "/" + "Disk_" + str(i) if i == 0 : if os.path.exists(curr_dir + "/" + base_filename_to_decode) : with open(curr_dir + "/" + base_filename_to_decode,'rb') as fp : decoded_data = fp.read() fp.close() return decoded_data else : if os.path.exists(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i)) : with open(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i),'rb') as fp : fragment = fp.read() fp.close() fragment_list.append(fragment) i = i + 1 if len(fragment_list) < k : return -1 # Not enough fragments to decode else : decoded_data = ec_driver.decode(fragment_list) return decoded_data
def test_missing_required_args(self): # missing ec_type with self.assertRaises(ECDriverError) as err_context: ECDriver(k=1, m=1) self.assertEqual( str(err_context.exception), "Invalid Argument: either ec_type or " "library_import_str must be provided") for ec_type in VALID_EC_TYPES: # missing k with self.assertRaises(ECDriverError) as err_context: ECDriver(ec_type=ec_type, m=1) self.assertEqual(str(err_context.exception), "Invalid Argument: k is required") # missing m with self.assertRaises(ECDriverError) as err_context: ECDriver(ec_type=ec_type, k=1) self.assertEqual(str(err_context.exception), "Invalid Argument: m is required")
def test_valid_algo(self): print("") for _type in ALL_EC_TYPES: # Check if this algo works if _type not in VALID_EC_TYPES: print("Skipping test for %s backend" % _type) continue try: if _type == 'shss': ECDriver(k=10, m=4, ec_type=_type) elif _type == 'libphazr': ECDriver(k=4, m=4, ec_type=_type) else: ECDriver(k=10, m=5, ec_type=_type) except ECDriverError: self.fail("%s algorithm not supported" % _type) self.assertRaises(ECBackendNotSupported, ECDriver, k=10, m=5, ec_type="invalid_algo")
def dummy(self, ec_type=ec_type): if ec_type not in VALID_EC_TYPES: raise unittest.SkipTest if ec_type == 'shss': k = 10 m = 4 elif ec_type == 'libphazr': k = 4 m = 4 else: k = 10 m = 5 ECDriver(k=k, m=m, ec_type=ec_type)
def __init__(self, factory, msg_wrapper_f=lambda _x: _x): self._factory = factory self._step = _BRACHA_STEP.one self._init_count = 0 self._echo_count = 0 self._ready_count = 0 self._root = None self._fragments = {} self._v = None self._done = False self._msg_wrapper_f = msg_wrapper_f self._n = self._factory.config.n self._t = self._factory.config.t self._sent_ready = False # NOTE: #define EC_MAX_FRAGMENTS 32 # https://github.com/openstack/liberasurecode/blob/master/include/erasurecode/erasurecode.h k = self._n - 2 * self._t m = 2 * self._t logging.debug("Bracha: erasure code params k={}, m={}".format(k, m)) self._ec_driver = ECDriver(k=k, m=m, ec_type='liberasurecode_rs_vand') random.seed()
def test_ec_fragment_size_cached(self): policy = ECStoragePolicy( 0, 'ec2-1', ec_type=DEFAULT_TEST_EC_TYPE, ec_ndata=2, ec_nparity=1, object_ring=FakeRing(replicas=3), ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE, is_default=True) ec_driver = ECDriver(ec_type=DEFAULT_TEST_EC_TYPE, k=2, m=1) expected_fragment_size = ec_driver.get_segment_info( DEFAULT_EC_OBJECT_SEGMENT_SIZE, DEFAULT_EC_OBJECT_SEGMENT_SIZE)['fragment_size'] with mock.patch.object( policy.pyeclib_driver, 'get_segment_info') as fake: fake.return_value = { 'fragment_size': expected_fragment_size} for x in range(10): self.assertEqual(expected_fragment_size, policy.fragment_size) # pyeclib_driver.get_segment_info is called only once self.assertEqual(1, fake.call_count)
def test_valid_ec_types(self): # Build list of available types and compare to VALID_EC_TYPES available_ec_types = [] for _type in ALL_EC_TYPES: try: if _type == 'shss': _k = 10 _m = 4 elif _type == 'libphazr': _k = 4 _m = 4 else: _k = 10 _m = 5 ECDriver(k=_k, m=_m, ec_type=_type, validate=True) available_ec_types.append(_type) except Exception: # ignore any errors, assume backend not available pass self.assertEqual(available_ec_types, VALID_EC_TYPES)
def __init__(self, name, ec_segment_size, ec_type, ec_nb_data, ec_nb_parity): super(ECStorageMethod, self).__init__(name=name, ec=True) try: self._ec_nb_data = int(ec_nb_data) except (TypeError, ValueError): raise exc.InvalidStorageMethod('Invalid %r ec_nb_data' % ec_nb_data) try: self._ec_nb_parity = int(ec_nb_parity) except (TypeError, ValueError): raise exc.InvalidStorageMethod('Invalid %r ec_nb_parity' % ec_nb_parity) self._ec_segment_size = ec_segment_size self._ec_type = ec_type self.driver = ECDriver(k=ec_nb_data, m=ec_nb_parity, ec_type=ec_type_to_pyeclib_type[ec_type]) self._ec_quorum_size = \ self._ec_nb_data + self.driver.min_parity_fragments_needed()
def __init__(self, idx, name='', aliases='', is_default=False, is_deprecated=False, object_ring=None, diskfile_module='egg:swift#erasure_coding.fs', ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE, ec_type=None, ec_ndata=None, ec_nparity=None, ec_duplication_factor=1): super(ECStoragePolicy, self).__init__( idx=idx, name=name, aliases=aliases, is_default=is_default, is_deprecated=is_deprecated, object_ring=object_ring, diskfile_module=diskfile_module) # Validate erasure_coding policy specific members # ec_type is one of the EC implementations supported by PyEClib if ec_type is None: raise PolicyError('Missing ec_type') if ec_type not in VALID_EC_TYPES: raise PolicyError('Wrong ec_type %s for policy %s, should be one' ' of "%s"' % (ec_type, self.name, ', '.join(VALID_EC_TYPES))) self._ec_type = ec_type # Define _ec_ndata as the number of EC data fragments # Accessible as the property "ec_ndata" try: value = int(ec_ndata) if value <= 0: raise ValueError self._ec_ndata = value except (TypeError, ValueError): raise PolicyError('Invalid ec_num_data_fragments %r' % ec_ndata, index=self.idx) # Define _ec_nparity as the number of EC parity fragments # Accessible as the property "ec_nparity" try: value = int(ec_nparity) if value <= 0: raise ValueError self._ec_nparity = value except (TypeError, ValueError): raise PolicyError('Invalid ec_num_parity_fragments %r' % ec_nparity, index=self.idx) # Define _ec_segment_size as the encode segment unit size # Accessible as the property "ec_segment_size" try: value = int(ec_segment_size) if value <= 0: raise ValueError self._ec_segment_size = value except (TypeError, ValueError): raise PolicyError('Invalid ec_object_segment_size %r' % ec_segment_size, index=self.idx) if self._ec_type == 'isa_l_rs_vand' and self._ec_nparity >= 5: logger = logging.getLogger("swift.common.storage_policy") if not logger.handlers: # If nothing else, log to stderr logger.addHandler(logging.StreamHandler(sys.__stderr__)) logger.warning( 'Storage policy %s uses an EC configuration known to harm ' 'data durability. Any data in this policy should be migrated. ' 'See https://bugs.launchpad.net/swift/+bug/1639691 for ' 'more information.' % self.name) if not is_deprecated: raise PolicyError( 'Storage policy %s uses an EC configuration known to harm ' 'data durability. This policy MUST be deprecated.' % self.name) # Initialize PyECLib EC backend try: self.pyeclib_driver = \ ECDriver(k=self._ec_ndata, m=self._ec_nparity, ec_type=self._ec_type) except ECDriverError as e: raise PolicyError("Error creating EC policy (%s)" % e, index=self.idx) # quorum size in the EC case depends on the choice of EC scheme. self._ec_quorum_size = \ self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed() self._fragment_size = None self._ec_duplication_factor = \ config_positive_int_value(ec_duplication_factor)
def _get_ecdriver(threshold, total_shares): return ECDriver(k=threshold, m=total_shares - threshold, ec_type='liberasurecode_rs_vand')
class Eraser(object): """A wrapper for pyeclib erasure coding driver (ECDriver)""" def __init__(self, ec_k, ec_m, ec_type="liberasurecode_rs_vand", aes_enabled=True): self.ec_type = ec_type if aes_enabled: self.aes = AESDriver() logger.info("Eraser will use AES encryption") else: logger.info("Eraser will not use AES encryption") expected_module_name = "drivers." + ec_type.lower() + "_driver" expected_class_name = ec_type[0].upper() + ec_type[1:].lower( ) + "Driver" try: mod = __import__(expected_module_name, fromlist=[expected_class_name]) driver_class = None driver_class = getattr(mod, expected_class_name) self.driver = driver_class(k=ec_k, m=ec_m, ec_type=ec_type, hd=None) except (ImportError, AttributeError): logger.exception("Driver " + ec_type + " could not be loaded as a custom driver") try: self.driver = ECDriver(k=ec_k, m=ec_m, ec_type=ec_type) except Exception as error: logger.exception("Driver " + ec_type + " could not be loaded by pyeclib") raise error def encode(self, data): """Encode a string of bytes in flattened string of byte strips""" payload = data if hasattr(self, 'aes'): payload = self.aes.encode(data)[0] strips = self.driver.encode(payload) return strips def decode(self, strips): """Decode byte strips in a string of bytes""" payload = self.driver.decode(strips) if hasattr(self, 'aes'): return self.aes.decode([payload]) return payload def reconstruct(self, available_payload_fragments, missing_indices): """ Reconstruct missing fragments of data Args: available_payload_fragments(list(bytes)): Available fragments of data missing_indices(list(int)): List of the indices of the missing blocks Returns: list(bytes): A list of the reconstructed fragments """ return self.driver.reconstruct(available_payload_fragments, missing_indices) def fragments_needed(self, missing_indices): """ Return a list of the fragments needed to recover the missing ones Args: missing_indices(list(int)): The list of indices of the fragments to recover Returns: list(int): A list of the indices of the fragments required to recover the missing ones """ return self.driver.fragments_needed(missing_indices)
def __init__(self, idx, name='', aliases='', is_default=False, is_deprecated=False, object_ring=None, ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE, ec_type=None, ec_ndata=None, ec_nparity=None): super(ECStoragePolicy, self).__init__( idx=idx, name=name, aliases=aliases, is_default=is_default, is_deprecated=is_deprecated, object_ring=object_ring) # Validate erasure_coding policy specific members # ec_type is one of the EC implementations supported by PyEClib if ec_type is None: raise PolicyError('Missing ec_type') if ec_type not in VALID_EC_TYPES: raise PolicyError('Wrong ec_type %s for policy %s, should be one' ' of "%s"' % (ec_type, self.name, ', '.join(VALID_EC_TYPES))) self._ec_type = ec_type # Define _ec_ndata as the number of EC data fragments # Accessible as the property "ec_ndata" try: value = int(ec_ndata) if value <= 0: raise ValueError self._ec_ndata = value except (TypeError, ValueError): raise PolicyError('Invalid ec_num_data_fragments %r' % ec_ndata, index=self.idx) # Define _ec_nparity as the number of EC parity fragments # Accessible as the property "ec_nparity" try: value = int(ec_nparity) if value <= 0: raise ValueError self._ec_nparity = value except (TypeError, ValueError): raise PolicyError('Invalid ec_num_parity_fragments %r' % ec_nparity, index=self.idx) # Define _ec_segment_size as the encode segment unit size # Accessible as the property "ec_segment_size" try: value = int(ec_segment_size) if value <= 0: raise ValueError self._ec_segment_size = value except (TypeError, ValueError): raise PolicyError('Invalid ec_object_segment_size %r' % ec_segment_size, index=self.idx) # Initialize PyECLib EC backend try: self.pyeclib_driver = \ ECDriver(k=self._ec_ndata, m=self._ec_nparity, ec_type=self._ec_type) except ECDriverError as e: raise PolicyError("Error creating EC policy (%s)" % e, index=self.idx) # quorum size in the EC case depends on the choice of EC scheme. self._ec_quorum_size = \ self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed()
class ECStoragePolicy(BaseStoragePolicy): """ Represents a storage policy of type 'erasure_coding'. Not meant to be instantiated directly; use :func:`~swift.common.storage_policy.reload_storage_policies` to load POLICIES from ``swift.conf``. """ def __init__(self, idx, name='', aliases='', is_default=False, is_deprecated=False, object_ring=None, ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE, ec_type=None, ec_ndata=None, ec_nparity=None): super(ECStoragePolicy, self).__init__( idx=idx, name=name, aliases=aliases, is_default=is_default, is_deprecated=is_deprecated, object_ring=object_ring) # Validate erasure_coding policy specific members # ec_type is one of the EC implementations supported by PyEClib if ec_type is None: raise PolicyError('Missing ec_type') if ec_type not in VALID_EC_TYPES: raise PolicyError('Wrong ec_type %s for policy %s, should be one' ' of "%s"' % (ec_type, self.name, ', '.join(VALID_EC_TYPES))) self._ec_type = ec_type # Define _ec_ndata as the number of EC data fragments # Accessible as the property "ec_ndata" try: value = int(ec_ndata) if value <= 0: raise ValueError self._ec_ndata = value except (TypeError, ValueError): raise PolicyError('Invalid ec_num_data_fragments %r' % ec_ndata, index=self.idx) # Define _ec_nparity as the number of EC parity fragments # Accessible as the property "ec_nparity" try: value = int(ec_nparity) if value <= 0: raise ValueError self._ec_nparity = value except (TypeError, ValueError): raise PolicyError('Invalid ec_num_parity_fragments %r' % ec_nparity, index=self.idx) # Define _ec_segment_size as the encode segment unit size # Accessible as the property "ec_segment_size" try: value = int(ec_segment_size) if value <= 0: raise ValueError self._ec_segment_size = value except (TypeError, ValueError): raise PolicyError('Invalid ec_object_segment_size %r' % ec_segment_size, index=self.idx) # Initialize PyECLib EC backend try: self.pyeclib_driver = \ ECDriver(k=self._ec_ndata, m=self._ec_nparity, ec_type=self._ec_type) except ECDriverError as e: raise PolicyError("Error creating EC policy (%s)" % e, index=self.idx) # quorum size in the EC case depends on the choice of EC scheme. self._ec_quorum_size = \ self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed() @property def ec_type(self): return self._ec_type @property def ec_ndata(self): return self._ec_ndata @property def ec_nparity(self): return self._ec_nparity @property def ec_segment_size(self): return self._ec_segment_size @property def fragment_size(self): """ Maximum length of a fragment, including header. NB: a fragment archive is a sequence of 0 or more max-length fragments followed by one possibly-shorter fragment. """ # Technically pyeclib's get_segment_info signature calls for # (data_len, segment_size) but on a ranged GET we don't know the # ec-content-length header before we need to compute where in the # object we should request to align with the fragment size. So we # tell pyeclib a lie - from it's perspective, as long as data_len >= # segment_size it'll give us the answer we want. From our # perspective, because we only use this answer to calculate the # *minimum* size we should read from an object body even if data_len < # segment_size we'll still only read *the whole one and only last # fragment* and pass than into pyeclib who will know what to do with # it just as it always does when the last fragment is < fragment_size. return self.pyeclib_driver.get_segment_info( self.ec_segment_size, self.ec_segment_size)['fragment_size'] @property def ec_scheme_description(self): """ This short hand form of the important parts of the ec schema is stored in Object System Metadata on the EC Fragment Archives for debugging. """ return "%s %d+%d" % (self._ec_type, self._ec_ndata, self._ec_nparity) def __repr__(self): return ("%s, EC config(ec_type=%s, ec_segment_size=%d, " "ec_ndata=%d, ec_nparity=%d)") % \ (super(ECStoragePolicy, self).__repr__(), self.ec_type, self.ec_segment_size, self.ec_ndata, self.ec_nparity) @classmethod def _config_options_map(cls): options = super(ECStoragePolicy, cls)._config_options_map() options.update({ 'ec_type': 'ec_type', 'ec_object_segment_size': 'ec_segment_size', 'ec_num_data_fragments': 'ec_ndata', 'ec_num_parity_fragments': 'ec_nparity', }) return options def get_info(self, config=False): info = super(ECStoragePolicy, self).get_info(config=config) if not config: info.pop('ec_object_segment_size') info.pop('ec_num_data_fragments') info.pop('ec_num_parity_fragments') info.pop('ec_type') return info def _validate_ring(self): """ EC specific validation Replica count check - we need _at_least_ (#data + #parity) replicas configured. Also if the replica count is larger than exactly that number there's a non-zero risk of error for code that is considering the number of nodes in the primary list from the ring. """ if not self.object_ring: raise PolicyError('Ring is not loaded') nodes_configured = self.object_ring.replica_count if nodes_configured != (self.ec_ndata + self.ec_nparity): raise RingValidationError( 'EC ring for policy %s needs to be configured with ' 'exactly %d nodes. Got %d.' % ( self.name, self.ec_ndata + self.ec_nparity, nodes_configured)) @property def quorum(self): """ Number of successful backend requests needed for the proxy to consider the client request successful. The quorum size for EC policies defines the minimum number of data + parity elements required to be able to guarantee the desired fault tolerance, which is the number of data elements supplemented by the minimum number of parity elements required by the chosen erasure coding scheme. For example, for Reed-Solomon, the minimum number parity elements required is 1, and thus the quorum_size requirement is ec_ndata + 1. Given the number of parity elements required is not the same for every erasure coding scheme, consult PyECLib for min_parity_fragments_needed() """ return self._ec_quorum_size
from pyeclib.ec_iface import ECDriver import pickle from random import shuffle if __name__ == '__main__': with open('results.pkl', 'rb') as f: results_dict = pickle.load(f) for key, frags in results_dict.items(): k, m = key driver = ECDriver(ec_type='jerasure_rs_vand', k=k, m=m) for num in range(10): shuffle(frags) assert 'a'*100 == driver.decode( frags[:k], force_metadata_checks=True) print '%s passed' % str(key)
parser = argparse.ArgumentParser(description='Decoder for PyECLib.') parser.add_argument('k', type=int, help='number of data elements') parser.add_argument('m', type=int, help='number of parity elements') parser.add_argument('ec_type', help='EC algorithm used') parser.add_argument('fragments', metavar='fragment', nargs='+', help='fragments to decode') parser.add_argument('filename', help='output file') args = parser.parse_args() print("k = %d, m = %d" % (args.k, args.m)) print("ec_type = %s" % args.ec_type) print("fragments = %s" % args.fragments) print("filename = %s" % args.filename) ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type) fragment_list = [] # read fragments for fragment in args.fragments: with open(("%s" % fragment), "rb") as fp: fragment_list.append(fp.read()) # decode decoded_file = ec_driver.decode(fragment_list) # write with open("%s.decoded" % args.filename, "wb") as fp: fp.write(decoded_file)
from pyeclib.ec_iface import ECDriver import pickle if __name__ == '__main__': ec_pattern = [(6, 3), (10, 4), (20, 4), (11, 7)] results_dict = {} for k, m in ec_pattern: driver = ECDriver(ec_type='jerasure_rs_vand', k=k, m=m) results_dict[(k, m)] = driver.encode('a' * 100) with open('results.pkl', 'wb') as f: pickle.dump(results_dict, f)
class ECStoragePolicy(BaseStoragePolicy): """ Represents a storage policy of type 'erasure_coding'. Not meant to be instantiated directly; use :func:`~swift.common.storage_policy.reload_storage_policies` to load POLICIES from ``swift.conf``. """ def __init__(self, idx, name='', aliases='', is_default=False, is_deprecated=False, object_ring=None, diskfile_module='egg:swift#erasure_coding.fs', ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE, ec_type=None, ec_ndata=None, ec_nparity=None, ec_duplication_factor=1): super(ECStoragePolicy, self).__init__( idx=idx, name=name, aliases=aliases, is_default=is_default, is_deprecated=is_deprecated, object_ring=object_ring, diskfile_module=diskfile_module) # Validate erasure_coding policy specific members # ec_type is one of the EC implementations supported by PyEClib if ec_type is None: raise PolicyError('Missing ec_type') if ec_type not in VALID_EC_TYPES: raise PolicyError('Wrong ec_type %s for policy %s, should be one' ' of "%s"' % (ec_type, self.name, ', '.join(VALID_EC_TYPES))) self._ec_type = ec_type # Define _ec_ndata as the number of EC data fragments # Accessible as the property "ec_ndata" try: value = int(ec_ndata) if value <= 0: raise ValueError self._ec_ndata = value except (TypeError, ValueError): raise PolicyError('Invalid ec_num_data_fragments %r' % ec_ndata, index=self.idx) # Define _ec_nparity as the number of EC parity fragments # Accessible as the property "ec_nparity" try: value = int(ec_nparity) if value <= 0: raise ValueError self._ec_nparity = value except (TypeError, ValueError): raise PolicyError('Invalid ec_num_parity_fragments %r' % ec_nparity, index=self.idx) # Define _ec_segment_size as the encode segment unit size # Accessible as the property "ec_segment_size" try: value = int(ec_segment_size) if value <= 0: raise ValueError self._ec_segment_size = value except (TypeError, ValueError): raise PolicyError('Invalid ec_object_segment_size %r' % ec_segment_size, index=self.idx) if self._ec_type == 'isa_l_rs_vand' and self._ec_nparity >= 5: logger = logging.getLogger("swift.common.storage_policy") if not logger.handlers: # If nothing else, log to stderr logger.addHandler(logging.StreamHandler(sys.__stderr__)) logger.warning( 'Storage policy %s uses an EC configuration known to harm ' 'data durability. Any data in this policy should be migrated. ' 'See https://bugs.launchpad.net/swift/+bug/1639691 for ' 'more information.' % self.name) if not is_deprecated: raise PolicyError( 'Storage policy %s uses an EC configuration known to harm ' 'data durability. This policy MUST be deprecated.' % self.name) # Initialize PyECLib EC backend try: self.pyeclib_driver = \ ECDriver(k=self._ec_ndata, m=self._ec_nparity, ec_type=self._ec_type) except ECDriverError as e: raise PolicyError("Error creating EC policy (%s)" % e, index=self.idx) # quorum size in the EC case depends on the choice of EC scheme. self._ec_quorum_size = \ self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed() self._fragment_size = None self._ec_duplication_factor = \ config_positive_int_value(ec_duplication_factor) @property def ec_type(self): return self._ec_type @property def ec_ndata(self): return self._ec_ndata @property def ec_nparity(self): return self._ec_nparity @property def ec_n_unique_fragments(self): return self._ec_ndata + self._ec_nparity @property def ec_segment_size(self): return self._ec_segment_size @property def fragment_size(self): """ Maximum length of a fragment, including header. NB: a fragment archive is a sequence of 0 or more max-length fragments followed by one possibly-shorter fragment. """ # Technically pyeclib's get_segment_info signature calls for # (data_len, segment_size) but on a ranged GET we don't know the # ec-content-length header before we need to compute where in the # object we should request to align with the fragment size. So we # tell pyeclib a lie - from it's perspective, as long as data_len >= # segment_size it'll give us the answer we want. From our # perspective, because we only use this answer to calculate the # *minimum* size we should read from an object body even if data_len < # segment_size we'll still only read *the whole one and only last # fragment* and pass than into pyeclib who will know what to do with # it just as it always does when the last fragment is < fragment_size. if self._fragment_size is None: self._fragment_size = self.pyeclib_driver.get_segment_info( self.ec_segment_size, self.ec_segment_size)['fragment_size'] return self._fragment_size @property def ec_scheme_description(self): """ This short hand form of the important parts of the ec schema is stored in Object System Metadata on the EC Fragment Archives for debugging. """ return "%s %d+%d" % (self._ec_type, self._ec_ndata, self._ec_nparity) @property def ec_duplication_factor(self): return self._ec_duplication_factor def __repr__(self): extra_info = '' if self.ec_duplication_factor != 1: extra_info = ', ec_duplication_factor=%d' % \ self.ec_duplication_factor return ("%s, EC config(ec_type=%s, ec_segment_size=%d, " "ec_ndata=%d, ec_nparity=%d%s)") % \ (super(ECStoragePolicy, self).__repr__(), self.ec_type, self.ec_segment_size, self.ec_ndata, self.ec_nparity, extra_info) @classmethod def _config_options_map(cls): options = super(ECStoragePolicy, cls)._config_options_map() options.update({ 'ec_type': 'ec_type', 'ec_object_segment_size': 'ec_segment_size', 'ec_num_data_fragments': 'ec_ndata', 'ec_num_parity_fragments': 'ec_nparity', 'ec_duplication_factor': 'ec_duplication_factor', }) return options def get_info(self, config=False): info = super(ECStoragePolicy, self).get_info(config=config) if not config: info.pop('ec_object_segment_size') info.pop('ec_num_data_fragments') info.pop('ec_num_parity_fragments') info.pop('ec_type') info.pop('ec_duplication_factor') return info @property def quorum(self): """ Number of successful backend requests needed for the proxy to consider the client PUT request successful. The quorum size for EC policies defines the minimum number of data + parity elements required to be able to guarantee the desired fault tolerance, which is the number of data elements supplemented by the minimum number of parity elements required by the chosen erasure coding scheme. For example, for Reed-Solomon, the minimum number parity elements required is 1, and thus the quorum_size requirement is ec_ndata + 1. Given the number of parity elements required is not the same for every erasure coding scheme, consult PyECLib for min_parity_fragments_needed() """ return self._ec_quorum_size * self.ec_duplication_factor def load_ring(self, swift_dir): """ Load the ring for this policy immediately. :param swift_dir: path to rings """ if self.object_ring: return def validate_ring_data(ring_data): """ EC specific validation Replica count check - we need _at_least_ (#data + #parity) replicas configured. Also if the replica count is larger than exactly that number there's a non-zero risk of error for code that is considering the number of nodes in the primary list from the ring. """ configured_fragment_count = ring_data.replica_count required_fragment_count = \ (self.ec_n_unique_fragments) * self.ec_duplication_factor if configured_fragment_count != required_fragment_count: raise RingLoadError( 'EC ring for policy %s needs to be configured with ' 'exactly %d replicas. Got %s.' % ( self.name, required_fragment_count, configured_fragment_count)) self.object_ring = Ring( swift_dir, ring_name=self.ring_name, validation_hook=validate_ring_data) def get_backend_index(self, node_index): """ Backend index for PyECLib :param node_index: integer of node index :return: integer of actual fragment index. if param is not an integer, return None instead """ try: node_index = int(node_index) except ValueError: return None return node_index % self.ec_n_unique_fragments
# Results will be List[(ec_type, throughput)] results = [] # Num iterations num_iterations = 10 for scheme in schemes: print(scheme) # Generate a new string for each test file_str = ''.join( random.choice( string.ascii_uppercase + string.digits) for x in range(args.s)) try: ec_driver = ECDriver(k=scheme.k, m=scheme.m, ec_type=scheme.ec_type) except Exception as e: print("Scheme %s is not defined (%s)." % (scheme, e)) continue timer.start() for i in range(num_iterations): ec_driver.encode(file_str) duration = timer.stop_and_return() results.append((scheme, duration)) timer.reset()
def setUp(self): self.null_driver = ECDriver( library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
parser.add_argument('m', type=int, help='number of parity elements') parser.add_argument('ec_type', help='EC algorithm used') parser.add_argument('file_dir', help='directory with the file') parser.add_argument('filename', help='file to encode') parser.add_argument('fragment_dir', help='directory to drop encoded fragments') args = parser.parse_args() print("k = %d, m = %d" % (args.k, args.m)) print("ec_type = %s" % args.ec_type) print("filename = %s" % args.filename) start = time.time() ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type) # read with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp: whole_file_str = fp.read() # encode fragments = ec_driver.encode(whole_file_str) # store i = 0 for fragment in fragments: with open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "wb") as fp: fp.write(fragment) i += 1
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import pyeclib from pyeclib.ec_iface import ECDriver import random import string import sys import os import argparse parser = argparse.ArgumentParser( description='PyECLib tool to determine fragment indexes needed to ' 'recover missing fragments.') parser.add_argument('k', type=int, help='number of data elements') parser.add_argument('m', type=int, help='number of parity elements') parser.add_argument('ec_type', help='EC algorithm used') parser.add_argument('missing_fragments', type=int, metavar='missing_fragment', nargs='+', help='missing_fragments') args = parser.parse_args() ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type) fragments_needed = ec_driver.fragments_needed(args.missing_fragments) print(fragments_needed)
print "" print "A script that benchmarks pyeclib libraries reconstruction capabilities" print "" print "Arguments:" print "\tsize Size of the payload to encode in bytes" if __name__ == "__main__": if len(sys.argv) != 2: print_usage() sys.exit(0) SIZE = int(sys.argv[1]) EC_K = int(os.environ.get("EC_K", 10)) EC_M = int(os.environ.get("EC_M", 4)) EC_TYPE = os.environ.get("EC_TYPE", "liberasurecode_rs_vand") DRIVER = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE) DATA = os.urandom(SIZE) STRIPS = DRIVER.encode(DATA) LENGTH = EC_K + EC_M SUPPORTED_DISTANCE = LENGTH - EC_K + 1 print "About to reconstruct ", REQUESTS, " times a payload of size ", SIZE, " bytes (", \ (DRIVER.ec_type if hasattr(DRIVER, "ec_type") else EC_TYPE), ", k =", DRIVER.k, \ ", m =", DRIVER.m, ") from 0 to", SUPPORTED_DISTANCE, "missing blocks" random.seed(0) for missing_blocks in range(SUPPORTED_DISTANCE): for i in range(REQUESTS): missing_indices = range(missing_blocks) start = time.clock()