예제 #1
0
class TestNullDriver(unittest.TestCase):
    def setUp(self):
        self.null_driver = ECDriver(library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)

    def tearDown(self):
        pass

    def test_null_driver(self):
        self.null_driver.encode("")
        self.null_driver.decode([])
예제 #2
0
class TestNullDriver(unittest.TestCase):
    def setUp(self):
        self.null_driver = ECDriver(
            library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)

    def tearDown(self):
        pass

    def test_null_driver(self):
        self.null_driver.encode('')
        self.null_driver.decode([])
예제 #3
0
class Eraser(object):
    """A wrapper for pyeclib erasure coding driver (ECDriver)"""
    def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
        self.k = k
        self.m = m
        self.ec_type = ec_type
        if EC_TYPE == "pylonghair":
            self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
        elif EC_TYPE == "striping" or EC_TYPE == "bypass":
            self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
        else:
            self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)

    def encode(self, data):
        """Encode a string of bytes in flattened string of byte strips"""
        raw_strips = self.driver.encode(data)
        strips = []
        for raw_strip in raw_strips:
            strip = Strip()
            strip.data = raw_strip
            strips.append(strip)
        return strips

    def decode(self, playcloud_strips):
        """Decode byte strips in a string of bytes"""
        strips = convert_strips_to_bytes_list(playcloud_strips)
        return self.driver.decode(strips)
class Eraser(object):
    """A wrapper for pyeclib erasure coding driver (ECDriver)"""
    def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
        self.k = k
        self.m = m
        self.ec_type = ec_type
        if EC_TYPE == "pylonghair":
            self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
        elif EC_TYPE == "striping" or EC_TYPE == "bypass":
            self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
        else:
            self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)

    def encode(self, data):
        """Encode a string of bytes in flattened string of byte strips"""
        raw_strips = self.driver.encode(data)
        strips = []
        for raw_strip in raw_strips:
            strip = Strip()
            strip.data = raw_strip
            strips.append(strip)
        return strips

    def decode(self, playcloud_strips):
        """Decode byte strips in a string of bytes"""
        strips = convert_strips_to_bytes_list(playcloud_strips)
        return self.driver.decode(strips)
예제 #5
0
def decode(k, m, base_filename_to_decode, ec_type):

    #print("k = %d, m = %d" % (k, m))
    #print("ec_type = %s" % ec_type)
    home = expanduser("~")
    Disk_base_dir = home + '/Disk'
    key_list = base_filename_to_decode.split('.')
    jdecoder_types = [
        "reed_sol_van", "reed_sol_r6_op", "cauchy_orig", "cauchy_good",
        "liberation", "blaum_roth", "liber8tion"
    ]
    if ec_type in jdecoder_types:
        print "Decoding started "
        jdecoder = ctypes.CDLL(
            '/home/raven/Downloads/Jerasure-master/Examples/jdecoder.so')
        key = key_list[0]
        decoded_data = str(jdecoder.decode(key, Disk_base_dir))
        print "decoded data = ", decoded_data
    else:
        ec_driver = ECDriver(k=k, m=m, ec_type=ec_type)

        fragment_list = []

        i = 0
        while i <= k:
            if os.path.exists(Disk_base_dir + "/" + "Disk_" + str(i)):
                curr_dir = Disk_base_dir + "/" + "Disk_" + str(i)
                if i == 0:
                    if os.path.exists(curr_dir + "/" +
                                      base_filename_to_decode):
                        with open(curr_dir + "/" + base_filename_to_decode,
                                  'rb') as fp:
                            decoded_data = fp.read()
                        fp.close()
                        return decoded_data
                else:
                    if os.path.exists(curr_dir + "/" +
                                      base_filename_to_decode + "_fragment_" +
                                      str(i)):
                        with open(
                                curr_dir + "/" + base_filename_to_decode +
                                "_fragment_" + str(i), 'rb') as fp:
                            fragment = fp.read()
                        fp.close()
                        fragment_list.append(fragment)

            i = i + 1

        if len(fragment_list) < k:
            return -1  # Not enough fragments to decode
        else:

            decoded_data = ec_driver.decode(fragment_list)
    return decoded_data
예제 #6
0
def decode(k,m,base_filename_to_decode,ec_type) :

	#print("k = %d, m = %d" % (k, m))
	#print("ec_type = %s" % ec_type)
	home = expanduser("~")	
	Disk_base_dir = home + '/Disk'
	key_list = base_filename_to_decode.split('.')
	jdecoder_types = ["reed_sol_van", "reed_sol_r6_op","cauchy_orig","cauchy_good","liberation","blaum_roth","liber8tion"]
	if ec_type in jdecoder_types :
		print "Decoding started "
		jdecoder = ctypes.CDLL('/home/raven/Downloads/Jerasure-master/Examples/jdecoder.so')
		key = key_list[0]
		decoded_data = str(jdecoder.decode(key,Disk_base_dir))
		print "decoded data = ", decoded_data
	else :
		ec_driver = ECDriver(k = k, m = m, ec_type = ec_type)

		fragment_list = []
	
	
		i = 0
		while i <=k :
			if os.path.exists(Disk_base_dir + "/" + "Disk_" + str(i)) :
				curr_dir = Disk_base_dir + "/" + "Disk_" + str(i)
				if i == 0 :
					if os.path.exists(curr_dir + "/" + base_filename_to_decode) :
						with open(curr_dir + "/" + base_filename_to_decode,'rb') as fp :
							decoded_data = fp.read()					
						fp.close()
						return decoded_data
				else :
					if os.path.exists(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i)) :
						with open(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i),'rb') as fp :
							fragment = fp.read()					
						fp.close()
						fragment_list.append(fragment)
					
			i = i + 1
	
		if len(fragment_list) < k :
			return -1 # Not enough fragments to decode
		else :
			
			decoded_data = ec_driver.decode(fragment_list)	
	return decoded_data	
예제 #7
0
class Eraser(object):
    """A wrapper for pyeclib erasure coding driver (ECDriver)"""
    def __init__(self,
                 ec_k,
                 ec_m,
                 ec_type="liberasurecode_rs_vand",
                 aes_enabled=True):
        self.ec_type = ec_type
        if aes_enabled:
            self.aes = AESDriver()
            logger.info("Eraser will use AES encryption")
        else:
            logger.info("Eraser will not use AES encryption")
        expected_module_name = "drivers." + ec_type.lower() + "_driver"
        expected_class_name = ec_type[0].upper() + ec_type[1:].lower(
        ) + "Driver"
        try:
            mod = __import__(expected_module_name,
                             fromlist=[expected_class_name])
            driver_class = None
            driver_class = getattr(mod, expected_class_name)
            self.driver = driver_class(k=ec_k,
                                       m=ec_m,
                                       ec_type=ec_type,
                                       hd=None)
        except (ImportError, AttributeError):
            logger.exception("Driver " + ec_type +
                             " could not be loaded as a custom driver")
            try:
                self.driver = ECDriver(k=ec_k, m=ec_m, ec_type=ec_type)
            except Exception as error:
                logger.exception("Driver " + ec_type +
                                 " could not be loaded by pyeclib")
                raise error

    def encode(self, data):
        """Encode a string of bytes in flattened string of byte strips"""
        payload = data
        if hasattr(self, 'aes'):
            payload = self.aes.encode(data)[0]
        strips = self.driver.encode(payload)
        return strips

    def decode(self, strips):
        """Decode byte strips in a string of bytes"""
        payload = self.driver.decode(strips)
        if hasattr(self, 'aes'):
            return self.aes.decode([payload])
        return payload

    def reconstruct(self, available_payload_fragments, missing_indices):
        """
        Reconstruct missing fragments of data
        Args:
            available_payload_fragments(list(bytes)): Available fragments of data
            missing_indices(list(int)): List of the indices of the missing blocks
        Returns:
            list(bytes): A list of the reconstructed fragments
        """
        return self.driver.reconstruct(available_payload_fragments,
                                       missing_indices)

    def fragments_needed(self, missing_indices):
        """
        Return a list of the fragments needed to recover the missing ones
        Args:
            missing_indices(list(int)): The list of indices of the fragments to recover
        Returns:
            list(int): A list of the indices of the fragments required to recover the missing ones
        """
        return self.driver.fragments_needed(missing_indices)
예제 #8
0
k = 0

#for i in range(0, k + m, 1):

start_time = 0
end_time = 0

downed_nodes = []

for i in range(0, num_stripes, 1):
    frags = []
    for j in range(args.m+args.k):
        try:
            f = open("./nodes/%d/%s.%d.%d" % (j, args.filename, j, i), 'rb')
            #print j
            frags.append(f.read())
        except IOError:
            if j not in downed_nodes:
                print("Logging data loss at node %d, notify name_node of downed node" % j)
                downed_nodes.append(j)

    start_time = timeit.default_timer()

    dec_chunk = ec_driver.decode(frags)

    end_time += timeit.default_timer()-start_time

    new_file.write(dec_chunk)

print("Decoding time: %f" % end_time)
예제 #9
0
from pyeclib.ec_iface import ECDriver
import pickle
from random import shuffle 

if __name__ == '__main__':
    with open('results.pkl', 'rb') as f:
        results_dict = pickle.load(f)

    for key, frags in results_dict.items():
        k, m = key
        driver = ECDriver(ec_type='jerasure_rs_vand', k=k, m=m)
        for num in range(10):
            shuffle(frags)
            assert 'a'*100 == driver.decode(
                frags[:k], force_metadata_checks=True)
        print '%s passed' % str(key)
예제 #10
0
class Client:
    def __init__(self, cfgfile):

        self.hw_addr = self.get_hwaddr()
        self.uid = (-1, self.hw_addr)
        self.loop = asyncio.get_event_loop()
        config = configparser.ConfigParser()
        config.read(cfgfile)
        nbr_of_servers = int(config['General']['n'])
        chunks_size = int(config['General']['chunks_size'])
        f = int(config['General']['f'])
        e = int(config['General']['e'])
        k = nbr_of_servers - 2 * (f + e)
        if (k < 1):
            raise Exception("Coded elements less than 1")
        if (nbr_of_servers + k > 32):
            raise Exception(
                "[liberasurecode] Total number of fragments (k + m) must be at most 32"
            )
        quorum_size = math.ceil((nbr_of_servers + k + 2 * e) / 2)
        self.majority = math.ceil((nbr_of_servers + 1) / 2)
        self.ec_driver = ECDriver(k=k,
                                  m=nbr_of_servers,
                                  ec_type='liberasurecode_rs_vand')
        self.p = QuorumSend(quorum_size, PingPongMessage)
        t = Thread(target=self.start_event_loop,
                   args=(self.loop, config['Nodes'], chunks_size))
        t.daemon = True
        t.start()

    def get_hwaddr(self):
        with open('/sys/class/net/lo/address') as f:
            hw_addr = f.read().splitlines()[0]
            if (hw_addr == '00:00:00:00:00:00'):
                hw_addr = ':'.join(['%02x'] * 6) % (
                    r.randint(0, 255), r.randint(0, 255), r.randint(0, 255),
                    r.randint(0, 255), r.randint(0, 255), r.randint(0, 255))
        return hw_addr

    def start_event_loop(self, loop, nodes, chunks_size):
        asyncio.set_event_loop(loop)
        i = 0
        for node in nodes:
            ip, port = nodes[node].split(':')
            c = SenderChannel(i,
                              self.hw_addr,
                              0,
                              self.p,
                              ip,
                              port,
                              chunks_size=chunks_size)
            asyncio.ensure_future(c.start())
            print("Create channel to {}:{}".format(ip, port))
            i = i + 1
        loop.run_forever()

    def qrmAccess(self, msg, opt_size=None):
        fut = asyncio.run_coroutine_threadsafe(self.p.phaseInit(msg, opt_size),
                                               self.loop)
        return fut.result()

    def write(self, msg):
        elements = self.ec_driver.encode(msg)
        res = self.qrmAccess((None, None, 'qry', 'write'))
        max_tag = max([x.get_tag() for x in res])
        new_int = int(max_tag[0]) + 1
        new_tag = (new_int, self.uid)
        self.qrmAccess((new_tag, elements, 'pre', 'write'))
        self.qrmAccess((new_tag, None, 'fin', 'write'))
        self.qrmAccess((new_tag, None, 'FIN', 'write'))

    def read(self):
        res = self.qrmAccess((None, None, 'qry', 'read'))
        max_tag = max([x.get_tag() for x in res])
        res = self.qrmAccess((max_tag, None, 'fin', 'read'))
        elements = [x.get_data() for x in res if x.get_data()]
        try:
            decoded_msg = self.ec_driver.decode(elements)
        except:
            decoded_msg = None
        return decoded_msg

    def check_uid(self):
        res = self.qrmAccess((None, None, 'cntrQry', None),
                             opt_size=self.majority)
        max_cntr = max([struct.unpack("i", x.get_data()) for x in res])[0]
        if (max_cntr != self.uid[0]):
            new_inc_nbr = max(max_cntr, self.uid[0]) + 1
            new_cntr = struct.pack("i", new_inc_nbr)
            self.qrmAccess((None, new_cntr, 'incCntr', None),
                           opt_size=self.majority)
            self.uid = (new_inc_nbr, self.hw_addr)
예제 #11
0
parser.add_argument('m', type=int, help='number of parity elements')
parser.add_argument('ec_type', help='EC algorithm used')
parser.add_argument('fragments',
                    metavar='fragment',
                    nargs='+',
                    help='fragments to decode')
parser.add_argument('filename', help='output file')

args = parser.parse_args()

print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("fragments = %s" % args.fragments)
print("filename = %s" % args.filename)

ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)

fragment_list = []

# read fragments
for fragment in args.fragments:
    with open(("%s" % fragment), "rb") as fp:
        fragment_list.append(fp.read())

# decode
decoded_file = ec_driver.decode(fragment_list)

# write
with open("%s.decoded" % args.filename, "wb") as fp:
    fp.write(decoded_file)
예제 #12
0
class StepEntangler(object):
    """
    Basic implementation of STeP based entanglement
    """
    # Use the Group Separator from the ASCII table as the delimiter between the
    # entanglement header and the data itself
    # https://www.lammertbies.nl/comm/info/ascii-characters.html
    HEADER_DELIMITER = chr(29)

    def __init__(self, source, s, t, p, ec_type="isa_l_rs_vand"):
        """
        StepEntangler constructor
        Args:
            source(Source): Block source implementing the get_random_blocks and
                            get_block primitives
            s(int): Number of source blocks or the number of chunks to make from
                    the original data
            t(int): Number of old blocks to entangle with
            p(int): Number of parity blocks to produce using Reed-Solomon
        """
        if not source or \
           not callable(getattr(source, "get_block", None)) or \
           not callable(getattr(source, "get_random_blocks", None)):
            raise ValueError(
                "source argument must implement a get_random_blocks and a get_block method"
            )
        if s <= 0:
            raise ValueError("s({:d}) must be greater or equal to 1".format(s))
        if t < 0:
            raise ValueError("t({:d}) must be greater or equal to 0".format(t))
        if p < s:
            raise ValueError(
                "p({:d}) must be greater or equal to s({:d})".format(p, s))
        self.s = s
        self.t = t
        self.p = p
        self.e = self.p - self.s
        self.k = s + t

        self.source = source
        self.driver = ECDriver(k=self.k, m=self.p, ec_type=ec_type)

    @staticmethod
    def __get_original_size_from_strip(strip):
        """
        Returns the size of the original data located
        Args:
            strip(bytes): The bytes containing the header, size and data
        Returns:
            int: The size of the original data
        """
        start = strip.find(EntanglementDriver.HEADER_DELIMITER) +\
                len(EntanglementDriver.HEADER_DELIMITER)
        end = strip.find(EntanglementDriver.HEADER_DELIMITER, start)
        return int(strip[start:end])

    @staticmethod
    def __get_data_from_strip(strip):
        """
        Returns the data part of the bytes strip
        Args:
            strip(bytes): The bytes containing the header and the data
        Returns:
            bytes: The data part of the strip
        """
        first_pos = strip.find(EntanglementDriver.HEADER_DELIMITER) +\
                    len(EntanglementDriver.HEADER_DELIMITER)
        pos = strip.find(EntanglementDriver.HEADER_DELIMITER, first_pos) +\
              len(EntanglementDriver.HEADER_DELIMITER)
        return strip[pos:]

    @staticmethod
    def compute_fragment_size(document_size, fragments):
        """
        Computes the fragment size that will be used to process a document of size
        `document_size`.
        Args:
            document_size(int): Document size in bytes
            fragments(int): Number of fragments
        Returns:
            int: The required fragment size in bytes
        Raises:
            ValueError: if the document size argument is not an integer or lower than 0
        """
        if not isinstance(document_size, int) or document_size < 0:
            raise ValueError(
                "document_size argument must be an integer greater or equal to 0"
            )
        if not isinstance(fragments, int) or fragments <= 0:
            raise ValueError(
                "fragments argument must be an integer greater than 0")
        fragment_size = int(math.ceil(document_size / float(fragments)))
        if (fragment_size % 2) == 1:
            fragment_size += 1
        return fragment_size

    def encode(self, data):
        """
        Encodes data using combining entanglemend and Reed-Solomon(n, k)
        where k = s + t and n = k + p.
        Args:
            data(bytes): The original data to encode
        Returns:
            list(bytes): The encoded bytes to store
        """
        pointer_blocks = []
        if self.t > 0:
            pointer_blocks = self.source.get_random_blocks(self.t)
        block_header = serialize_entanglement_header(pointer_blocks)
        size = len(data)
        fragment_size = StepEntangler.compute_fragment_size(size, self.s)
        padded_size = fragment_size * self.s
        padded_data = pad(data, padded_size)
        pointer_blocks = [
            pad(self.__get_data_from_strip(block.data)[80:], fragment_size)
            for block in pointer_blocks
        ]
        encoded = self.entangle(padded_data, pointer_blocks)
        parity_blocks = [
            block_header + self.HEADER_DELIMITER + str(size) +
            self.HEADER_DELIMITER + parity_block
            for parity_block in encoded[self.k:]
        ]
        return parity_blocks

    def entangle(self, data, blocks):
        """
        Performs entanglement combining the data and the extra blocks using
        Reed-Solomon.
        Args:
            data(bytes): The original piece of data
            blocks(list(bytes)): The pointer blocks
        Returns:
            list(bytes): The parity blocks produced by combining the data and
                         pointer blocks and running them through Reed-Solomon
                         encoding
        """
        return self.driver.encode(data + "".join(blocks))

    def fetch_and_prep_pointer_blocks(self, pointers, fragment_size,
                                      original_data_size):
        """
        Fetches the pointer blocks and rewrites their liberasurecode header so
        that they can be reused for reconstruction or decoding
        Args:
            pointer_blocks(list(list)): A list of 2 elements lists namely the
                                        filename and the index of each pointer
                                        block
            fragment_size(int): Size of each fragment
            original_data_size(int): Size of the original piece of data
        Returns:
            list(bytes): A list of cleaned up liberasurecode fragments formatted
                         and padded to fit the code
        Raises:
            ValueError: If the pointers argument is not of type list,
                        The fragment_size argument is not an int or is lower or
                        equal to 0,
                        The original_data_size argument is not an int or is lower
                        or equal to 0
        """
        if not isinstance(pointers, list):
            raise ValueError("pointers argument must be of type list")
        if not isinstance(fragment_size, int) or fragment_size <= 0:
            raise ValueError(
                "fragment_size argument must be an integer greater than 0")
        if not isinstance(original_data_size, int) or original_data_size <= 0:
            raise ValueError(
                "original_data_size argument must be an integer greater than 0"
            )
        if original_data_size < fragment_size:
            raise ValueError(
                "original_data_size must be greater or equal to fragment_size")
        pointer_collection = {}
        fetchers = []
        for pointer_index, coordinates in enumerate(pointers):
            path = coordinates[0]
            index = coordinates[1]
            fragment_index = self.s + pointer_index
            fetcher = PointerHandler(self.source, path, index, fragment_index,
                                     fragment_size, original_data_size,
                                     pointer_collection)
            fetcher.start()
            fetchers.append(fetcher)
        for fetcher in fetchers:
            fetcher.join()
        return [
            pointer_collection[key]
            for key in sorted(pointer_collection.keys())
        ]

    def decode(self, strips, path=None):
        """
        Decodes data using the entangled blocks and Reed-Solomon.
        Args:
            strips(list(bytes)): The encoded strips of data
        Returns:
            bytes: The decoded data
        Raises:
            ECDriverError: if the number of fragments is too low for Reed-Solomon
                           decoding
        """
        logger = logging.getLogger("entangled_driver")
        model_fragment_header = FragmentHeader(
            self.__get_data_from_strip(strips[0])[:80])
        fragment_size = model_fragment_header.metadata.size
        orig_data_size = model_fragment_header.metadata.orig_data_size
        modified_pointer_blocks = []

        original_data_size = self.__get_original_size_from_strip(strips[0])
        block_header_text = get_entanglement_header_from_strip(strips[0])
        strips = [self.__get_data_from_strip(strip) for strip in strips]
        if self.t > 0:
            block_header = parse_entanglement_header(block_header_text)
            modified_pointer_blocks = self.fetch_and_prep_pointer_blocks(
                block_header, fragment_size, orig_data_size)
            # Filter to see what pointers we were able to fetch from the proxy
            initial_length = len(modified_pointer_blocks)
            modified_pointer_blocks = [
                mpb for mpb in modified_pointer_blocks if mpb
            ]
            filtered_length = len(modified_pointer_blocks)
            if filtered_length != initial_length:
                logger.warn("Only found {:d} pointers out of {:d}".format(
                    filtered_length, initial_length))
                biggest_index = max(
                    [FragmentHeader(s[:80]).metadata.index for s in strips])
                missing = initial_length - filtered_length
                if missing > self.e:
                    message = "Configuration of Step (s={:d}, t={:d}, e={:d}, p={:d}) " + \
                              "does not allow for reconstruction with {:d} missing fragments"
                    raise ECDriverError(
                        message.format(self.s, self.t, self.e, self.p,
                                       missing))
                extra_parities_needed = [
                    index - self.k
                    for index in xrange(biggest_index + 1, biggest_index + 1 +
                                        missing, 1)
                ]
                logger.info("We need blocks {} from {:s}".format(
                    sorted(extra_parities_needed), path))
                for index in extra_parities_needed:
                    strips.append(
                        self.__get_data_from_strip(
                            self.source.get_block(path, index).data))
        decoded = self.disentangle(strips, modified_pointer_blocks)
        return decoded[:original_data_size]

    def disentangle(self, parity_blocks, pointer_blocks):
        """
        Performs disentanglement in order to reconstruct and decode the original
        data.
        Args:
            parity_blocks(bytes): The parity blocks produced from the original encoding
            pointer_blocks(bytes): The blocks used in the original entanglement adjusted to the right size
        Returns:
            bytes: The data is it was originally mixed with the pointer blocks before encoding
        """
        available_blocks = pointer_blocks + parity_blocks
        return self.driver.decode(available_blocks)

    def fragments_needed(self, missing_fragment_indexes):
        """
        Returns the list of fragments necessary for decoding/reconstruction of data
        Args:
            missing_fragment_indexes(list(int)): The list of missing fragments
        Returns:
            list(int): The list of fragments required for decoding/reconstruction
        Raises:
            ECDriverError: If the number of missing indexes to  work around is
                           greater than self.p - self.s
            ValueError: if one of the missing indexes is out of scope
                           (index < 0 || (self.s + self.s + self.p) <= index)
        """
        if self.e == 0:
            message = (
                "Configuration of Step (s={:d}, t={:d}, e={:d}, p={:d}) does not allow for reconstruction"
                .format(self.s, self.t, self.e, self.p))
            raise ECDriverError(message)
        if self.e < len(missing_fragment_indexes):
            message = (
                "Configuration of Step (s={:d}, t={:d}, e={:d}, p={:d}) does not allow for reconstruction of {:d} missing blocks"
                .format(self.s, self.t, self.e, self.p,
                        len(missing_fragment_indexes)))
            raise ECDriverError(message)
        for index in missing_fragment_indexes:
            if index < 0 or (self.s + self.t + self.p) <= index:
                raise ValueError(
                    "Index {:d} is out of range(0 <= index < {:d})".format(
                        index, self.s + self.t + self.p))
        required_indices = []
        for index in xrange(self.k, self.k + self.p):
            if not index in missing_fragment_indexes:
                required_indices.append(index)
                if len(required_indices) == self.s:
                    break
        return required_indices

    def reconstruct(self, available_fragment_payloads,
                    missing_fragment_indexes):
        """
        Reconstruct the missing fragements
        Args:
            list(bytes): Avalaible fragments
            list(int): Indices of the missing fragments
        Returns:
            list(bytes): The list of reconstructed blocks
        """
        header_text = get_entanglement_header_from_strip(
            available_fragment_payloads[0])
        list_of_pointer_blocks = parse_entanglement_header(header_text)

        parity_header = FragmentHeader(
            self.__get_data_from_strip(available_fragment_payloads[0])[:80])
        data_size = self.__get_original_size_from_strip(
            available_fragment_payloads[0])

        parity_blocks = [
            self.__get_data_from_strip(block)
            for block in available_fragment_payloads
        ]
        missing_fragment_indexes = [
            index + self.s + self.t for index in missing_fragment_indexes
        ]
        # Get pointer blocks
        modified_pointer_blocks = self.fetch_and_prep_pointer_blocks(
            list_of_pointer_blocks, parity_header.metadata.size,
            parity_header.metadata.orig_data_size)
        # Filter to remove responses for pointers that are missing
        modified_pointer_blocks = [
            mpb for mpb in modified_pointer_blocks if mpb
        ]
        assembled = modified_pointer_blocks + parity_blocks
        reconstructed = self.driver.reconstruct(assembled,
                                                missing_fragment_indexes)

        requested_blocks = []
        for index, block in enumerate(reconstructed):
            requested_block = header_text + self.HEADER_DELIMITER + str(
                data_size) + self.HEADER_DELIMITER + block
            requested_blocks.append(requested_block)

        return requested_blocks

    def __repr__(self):
        return "StepEntangler(s=" + str(self.s) + ", t=" + str(
            self.t) + ", p=" + str(self.p) + ")"
예제 #13
0
class cas:

  def __init__(self, cfgfile, verbose=False):
    uid = self.get_hwaddr()
    config = configparser.ConfigParser()
    config.read(cfgfile)
    N = int(config['General']['n'])
    F = int(config['General']['f'])
    addrs = list(config['Nodes'].values())
    self.verbose = verbose
    context = zmq.Context()
    self.k = N-2*F
    self.ec_driver = ECDriver(k=self.k, m=N, ec_type='liberasurecode_rs_vand')
    self.socketid = {}
    self.sockets = []
    self.uid = uid
    self.id = "Client-{}".format(uid)
    self.poller = zmq.Poller()
    self.quorum = int(math.ceil((N+self.k)/2))
    print("Starting client {}".format(uid))
    for addr in addrs:
      socket = context.socket(zmq.DEALER)
      self.socketid[socket] = "Server-%s" % addr
      self.poller.register(socket, zmq.POLLIN)
      socket.connect("tcp://%s" % addr);
      self.vprint("Connecting to server with address {}".format(addr))
      self.sockets.append(socket)

  def get_hwaddr(self):
    with open('/sys/class/net/lo/address') as f:
      hw_addr = f.read().splitlines()[0]
      if (hw_addr == '00:00:00:00:00:00'):
        hw_addr = ':'.join(['%02x']*6) % (
                        r.randint(0, 255),
                        r.randint(0, 255),
                        r.randint(0, 255),
                        r.randint(0, 255),
                        r.randint(0, 255),
                        r.randint(0, 255)
                        )
    return hw_addr

  def vprint(self, *args, **kwargs):
    if self.verbose:
      print(*args, **kwargs)

  def createMessage(self, msg):
    elements = self.ec_driver.encode(msg)
    return elements

  def assembleMessage(self, elements):
    decoded_msg = self.ec_driver.decode(elements)
    return decoded_msg

  #Send different coded element to everyone
  def sendCodedElements(self, msg, tag, msg_id):
    elements = self.createMessage(msg)
    for i in range(len(self.sockets)):
      payload = (tag, 0, 'pre')
      msg = ["PREWRITE", str(payload), msg_id, str(self.uid)]
      bmsg = [e.encode() for e in [""]+msg] + [elements[i]]
      self.sockets[i].send_multipart(bmsg)
      # mon_msg = str(["PREWRITE", str(payload), msg_id])
      self.log(bmsg[1:], "s", self.socketid[self.sockets[i]])

  def send(self, msg_type, payload, msg_id):
    msg = [msg_type, str(payload), msg_id, str(self.uid)]
    bmsg = [e.encode() for e in [""]+msg]
    for s in self.sockets:
      s.send_multipart(bmsg)
      self.log(bmsg[1:], "s", self.socketid[s])

  def receive(self, socket, expected=None):
    msg = socket.recv_multipart()
    self.log(msg, "r", self.socketid[socket])
    if expected:
      if msg[2] == expected:
        return msg
      else:
        self.vprint("[{me}] Did not match message: {msg}".format(me=self.id, msg=msg))
        return None
    return msg
 
  def preWrite(self, msg, tag):
    msg_id = str(self.hash(tag, datetime.now()))
    self.sendCodedElements(msg, tag, msg_id)
    acks = 0
    while acks < self.quorum:
      reply = dict(self.poller.poll())
      while reply:
        socket, event = reply.popitem()
        message = self.receive(socket, expected=msg_id.encode())
        if message:
          self.vprint("[{me}] Received (pre-write) message: {msg}".format(
            me=self.id, msg=message[1].decode()))
          if message[0] == b"ACK":
            acks += 1

  def readerFinalize(self, tag):
    acks = 0
    listOfElements = []
    payload = (tag, None, 'fin')
    msg_id = str(self.hash(datetime.now(), self.uid))
    self.send("RFINALIZE", payload, msg_id)
    while acks < self.quorum:
      reply = dict(self.poller.poll())
      while reply:
        socket, event = reply.popitem()
        message = self.receive(socket, expected=msg_id.encode())
        if message:
          self.vprint("[{me}] Received (read-fin) message: {msg}".format(
            me=self.id, msg=message[1].decode()))
          msg_type = message[0]
          if(msg_type == b"RFINALIZE"):
            if(len(message) == 4):
              listOfElements.append(message[-1])
            acks += 1
    if (len(listOfElements) >= self.k):
      self.vprint("[{me}] Got enough elements to assemble message!".format(me=self.id))
      return self.assembleMessage(listOfElements)
    else:
      self.vprint("[{me}] Did not get enough elements to assemble message!".format(me=self.id))
      return None

  def query(self):
    maxtag = (0,0)
    acks = 0
    payload = (None, None, self.uid)
    msg_id = str(self.hash(datetime.now(), self.uid))
    self.send("QUERY", payload, msg_id)
    while acks < self.quorum:
      reply = dict(self.poller.poll())
      while reply:
        socket, event = reply.popitem()
        message = self.receive(socket, expected=msg_id.encode())
        self.vprint("[{me}] Received (query) message: {msg}".format(
          me=self.id, msg=message))
        if message:
          msg_type = message[0]
          if(msg_type == b"QUERY"):
            msg_maxtag = literal_eval(message[1].decode())
            self.vprint("[{me}] Received maxts: {m}".format(me=self.id, m=msg_maxtag))
            if(self.compareTags(msg_maxtag, maxtag)):
              maxtag = msg_maxtag
            acks += 1
    return maxtag

  def read(self):
    #QUERY PHASE
    ts = self.query()
    self.vprint("[{me}] maxts = {m}".format(me=self.id, m=ts))
    res = self.readerFinalize(ts)
    return res

  def write(self, message):
    #QUERY PHASE
    maxtag = self.query()
    newTag = (maxtag[0]+1, self.uid)

    #PRE-WRITE PHASE
    self.preWrite(message, newTag)

    #WRITER FINALIZE PHASE
    acks = 0
    payload = (newTag, None, 'fin')
    msg_id = str(self.hash(payload, datetime.now()))
    self.send("WFINALIZE", payload, msg_id)
    self.vprint("[{me}] sent: msg_id={mid}".format(
      me=self.id, mid=msg_id))
    while acks < self.quorum:
      reply = dict(self.poller.poll())
      while reply:
        socket, event = reply.popitem()
        message = self.receive(socket, expected=msg_id.encode())
        if message:
          msg_type = message[0]
          if(msg_type == b"WFINALIZE"):
            acks += 1

  #Return True if the first one is larger
  def compareTags(self, firstTuple, secondTuple):
    return firstTuple > secondTuple

  #Create a list of message payload
  def unpackPayload(self, payload):
    payloadList = literal_eval(payload)
    return payloadList

  # returns the string which is the hexdigest of args using hash funciton hf
  def hash(self, *args):
    h = hashlib.new('sha256')
    for arg in args:
      h.update(str(arg).encode('ascii'))
    return h.hexdigest()

  def log(self, msg, mode, dst):
    if self.verbose:
      uid = str(self.uid)
      mode = str(mode)
      time = str(datetime.now())
      msg = str([e.decode() for e in msg[:4]])
      log_str = "%26s %100s %2s %15s %15s" % (time, msg, mode, uid, dst)
      log_dir = 'log'
      log_file = log_dir + '/client_%s.log' % (uid)
      try:
        with open(log_file, 'a') as f:
          f.write(log_str + '\n')
      except OSError as e:
        print("Could not open logfile: {}".format(e), file=sys.stderr)
예제 #14
0
parser = argparse.ArgumentParser(description='Decoder for PyECLib.')
parser.add_argument('k', type=int, help='number of data elements')
parser.add_argument('m', type=int, help='number of parity elements')
parser.add_argument('ec_type', help='EC algorithm used')
parser.add_argument('fragments', metavar='fragment', nargs='+',
                    help='fragments to decode')
parser.add_argument('filename', help='output file')

args = parser.parse_args()

print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("fragments = %s" % args.fragments)
print("filename = %s" % args.filename)

ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)

fragment_list = []

# read fragments
for fragment in args.fragments:
    with open(("%s" % fragment), "rb") as fp:
        fragment_list.append(fp.read())

# decode
decoded_file = ec_driver.decode(fragment_list)

# write
with open("%s.decoded" % args.filename, "wb") as fp:
    fp.write(decoded_file)
예제 #15
0
파일: bracha.py 프로젝트: kc1212/checo
class Bracha(object):
    """
    Bracha broadcast '87
    Implemented using state machine (BrachaStep)
    """
    def __init__(self, factory, msg_wrapper_f=lambda _x: _x):
        self._factory = factory
        self._step = _BRACHA_STEP.one
        self._init_count = 0
        self._echo_count = 0
        self._ready_count = 0
        self._root = None
        self._fragments = {}
        self._v = None
        self._done = False
        self._msg_wrapper_f = msg_wrapper_f
        self._n = self._factory.config.n
        self._t = self._factory.config.t
        self._sent_ready = False

        # NOTE: #define EC_MAX_FRAGMENTS 32
        # https://github.com/openstack/liberasurecode/blob/master/include/erasurecode/erasurecode.h
        k = self._n - 2 * self._t
        m = 2 * self._t
        logging.debug("Bracha: erasure code params k={}, m={}".format(k, m))
        self._ec_driver = ECDriver(k=k, m=m, ec_type='liberasurecode_rs_vand')
        random.seed()

    def handle(self, msg, sender_vk):
        # type: (pb.Bracha) -> Handled
        """
        This function is called on a new incoming message, we expect the type is correct
        msg should be in the following format
        struct Msg {
            ty: u32,
            body: String,
        }
        :param msg: the input message to send
        :return: the delivered message when completed, otherwise None
        """
        if self._done:
            logging.debug("Bracha: done, doing nothing")
            return Handled()

        ty = msg.ty
        logging.debug("Bracha: received {}".format(msg))

        assert isinstance(ty, int)
        assert msg.digest is not None

        # initialisation
        if self._root is None:
            # if body is None, we must be in the initial state
            assert self._init_count == 0 and self._echo_count == 0 and self._ready_count == 0
            self._root = msg.digest

        if self._root != msg.digest:
            logging.debug("Bracha: self.root {} != root: {}, discarding"
                          .format(b64encode(self._root), b64encode(msg.digest)))
            return Handled()

        # here we update the state
        if ty == _INIT:
            self._init_count += 1

        elif ty == _ECHO:
            self._fragments[sender_vk] = msg.fragment
            self._echo_count += 1
            assert self._echo_count == len(self._fragments), \
                "echo_count {} != fragment count {}".format(self._echo_count, len(self._fragments))

        elif ty == _READY:
            self._ready_count += 1
            assert self._root == msg.digest, \
                "self.root {} != root: {}".format(self._root, msg.digest)

        else:
            raise AssertionError("Bracha: unexpected msg type")

        # we should only see one init message
        assert self._init_count == 0 or self._init_count == 1

        # everything below is the algorithm, acting on the current state
        if ty == _INIT:
            logging.debug("Bracha: got init value, root = {}".format(b64encode(msg.digest)))
            self._upon_init(msg)

        if ty == _ECHO:
            logging.debug("Bracha: got echo value, root = {}".format(b64encode(msg.digest)))
            # TODO check Merkle branch
            pass

        if ty == _ECHO and self._echo_count >= self._n - self._t:
            logging.debug("Bracha: got n - t echo values, root = {}".format(b64encode(msg.digest)))
            self._upon_n_minus_t_echo()

        if ty == _READY and self._ready_count >= self._t + 1:
            self._upon_t_plus_1_ready()

        if self._ready_count >= 2 * self._t + 1 and self._echo_count >= self._n - 2*self._t:
            res = self._upon_2t_plus_1_ready()

            # NOTE: we use a random value to trip up tests, since it shouldn't be viewed by tests
            logging.info("Bracha: DELIVER {}"
                         .format(random.random() if self._factory.config.from_instruction else b64encode(res)))

            self._done = True
            return Handled(res)

        return Handled()

    def _upon_init(self, msg):
        assert isinstance(msg, pb.Bracha)
        msg.ty = _ECHO
        self.bcast(msg)

    def _decode_fragments(self):
        fragments = random.sample(self._fragments.values(), self._n - 2 * self._t)
        v = self._ec_driver.decode(fragments)
        return v

    def _upon_n_minus_t_echo(self):
        v = self._decode_fragments()
        assert libnacl.crypto_hash_sha256(v) == self._root
        self._v = v
        logging.debug("Bracha: erasure decoded msg v {}".format(b64encode(v)))

        if not self._sent_ready:
            logging.debug("Bracha: broadcast ready 1, root = {}".format(b64encode(self._root)))
            self.bcast(pb.Bracha(ty=_READY, digest=self._root))
            self._sent_ready = True

    def _upon_t_plus_1_ready(self):
        if not self._sent_ready:
            logging.debug("Bracha: broadcast ready 2, root = {}".format(b64encode(self._root)))
            self.bcast(pb.Bracha(ty=_READY, digest=self._root))
            self._sent_ready = True

    def _upon_2t_plus_1_ready(self):
        if self._v is None:
            self._v = self._decode_fragments()
        return self._v

    def bcast_init(self, msg="some test msg!!"):
        assert isinstance(msg, str)
        self._bcast_init_fragments(msg)

    def _bcast_init_fragments(self, msg):
        """
        
        :param msg: some bytes
        :return: 
        """
        fragments = self._ec_driver.encode(msg)
        digest = libnacl.crypto_hash_sha256(msg)

        logging.info("Bracha: initiate erasure code with {} fragments, digest {}"
                     .format(len(fragments), b64encode(digest)))

        assert len(fragments) == len(self._factory.promoters)
        for fragment, promoter in zip(fragments, self._factory.promoters):
            m = pb.Bracha(ty=_INIT, digest=digest, fragment=fragment)
            self._factory.send(promoter, self._msg_wrapper_f(m))

    def bcast(self, msg):
        self._factory.promoter_cast(self._msg_wrapper_f(msg))
예제 #16
0
class DepSkyClient:
    def __init__(self, clientId, isLocal=True):
        self.clientId = clientId
        self.isLocal = isLocal
        self.N = 4
        self.F = 1
        self.metadataQ = queue.Queue()
        self.ec_driver = ECDriver(k=2, m=2, ec_type='liberasurecode_rs_vand')

        if self.isLocal:
            self.createLocalClients()

    def read(self, container):
        maxVersion, metadata_list = self.getMetadataFromClouds(
            filename=container.duId + 'metadata')
        count = 0
        # Flush the metadata queue - cancel the remaining requests
        try:
            while True:
                self.metadataQ.get_nowait()
        except Empty:
            pass
        # Flushing complete

        if maxVersion == 0:
            print('No data to read! Sorry!!!')
            return
        dataFileName = container.duId + 'value' + str(maxVersion)
        secret_unit_list = self.readDataFromClouds(metadata_list, dataFileName)
        shares = []
        fragments = []
        for unit in secret_unit_list:
            shares.append((unit.secret_id, unit.share))
            fragments.append(unit.value)

        key = Shamir.combine(shares)
        encrypted_data = self.ec_driver.decode(fragments)

        cipher = getAESCiphers(key)
        data = cipher.decrypt(encrypted_data)
        return data

    def readDataFromClouds(self, metadata_list, filename):
        results = self.readInParallel(metadata_list, filename)
        return results

    def readInParallel(self, metadata_list, filename):
        q = queue.Queue()
        cloudIds = [cloudMetadata.cloudId for cloudMetadata in metadata_list]
        threads = [
            threading.Thread(target=self.getFileFromCloud,
                             args=(i, filename, q)) for i in cloudIds
        ]
        metadata_list_to_dict = {
            metadata.cloudId: metadata
            for metadata in metadata_list
        }
        results = []
        verified_count = 0

        for thread in threads:
            thread.daemon = True
            thread.start()

        for _ in range(len(cloudIds)):
            result = q.get()
            result = json.loads(result)
            secretUnit = SecretUnitDecoder().decode(result)
            cloudId = secretUnit.cloudId
            metadata_of_secret_unit = metadata_list_to_dict[cloudId]

            latest_metadata = metadata_of_secret_unit.metadata_list[0]
            if getHash(SecretUnitEncoder().encode(
                    secretUnit)).hexdigest() == latest_metadata.hashedUnit:
                results.append(secretUnit)
                verified_count += 1
                print('Data integrity of the value in cloud', cloudId,
                      'succeeded')
            else:
                print('Data integrity of the value stored in cloud', cloudId,
                      'failed')

            if verified_count > self.F:
                # remove the remaining items from the queue
                try:
                    while True:
                        q.get_nowait()
                except Empty:
                    pass

                break
        if verified_count < self.F + 1:
            print('Data integrity of F+1 clouds failed. Cannot recover!!')
            exit(0)
        return results

    def createLocalClients(self):
        for i in range(4):
            requests.post('http://localhost:5555',
                          data=json.dumps({'cloudId': i}))

    def generateKeyShares(self, key):
        return Shamir.split(self.F + 1, self.N, key)

    def erasureCode(self, value):
        return self.ec_driver.encode(value)

    def write(self, container, value):
        maxVersion, metadata_list = self.getMetadataFromClouds(
            filename=container.duId + 'metadata')
        nextVersion = maxVersion + 1
        value_hash = getHash(value, return_hexdigest=True)
        key = get_random_bytes(16)

        cipher = getAESCiphers(key)
        encrypted_value = cipher.encrypt(value.encode('utf8'))
        shares = self.generateKeyShares(key)
        fragments = self.erasureCode(encrypted_value)

        units = []

        for cloudId, (share, fragment) in enumerate(zip(shares, fragments)):
            idx, secret = share
            unit = SecretUnit(idx, secret, fragment, cloudId)
            units.append(SecretUnitEncoder().encode(unit))

        self.writeToClouds(container.duId + 'value' + str(nextVersion), units)
        self.writeMetadata(container.duId + 'metadata', value_hash, units,
                           nextVersion, metadata_list)

    def writeToClouds(self, filename, units):
        results = self.writeInParallel(filename, units)

    def writeInParallel(self, filename, dataList):
        q = queue.Queue()
        threads = []
        for i, data in enumerate(dataList):
            threads.append(
                threading.Thread(target=self.writeToCloud,
                                 args=(i, data, filename, q)))
        for thread in threads:
            thread.daemon = True
            thread.start()

        results = []
        for _ in range(self.N - self.F):
            results.append(q.get())
        return results

    def writeToCloud(self, cloudId, data, filename, result_queue):
        r = requests.post('http://localhost:5555/write',
                          data=json.dumps({
                              'cloudId': cloudId,
                              'data2write': data,
                              'filename': filename
                          }))
        result_queue.put(r.text)

    def writeMetadata(self, filename, dataHash, units, version, metadata_list):
        metadata_of_clouds = []

        for _ in range(self.F):
            value = self.metadataQ.get()
            if value == 'None':
                metadata_list.append(None)
            else:
                result = json.loads(value)
                cloudMetadata = CloudMetadataDecoder().decode(result)
                metadata_list.append(cloudMetadata)

        # sort the metadata - or better store as dict :)
        cloudsPresent = any(metadata_list)
        if cloudsPresent:
            metadata_list_to_dict = {
                metadata.cloudId: metadata
                for metadata in metadata_list
            }

        for i in range(self.N):
            hashedUnit = getHash(units[i]).hexdigest()
            # got the cloud in ascending order

            metadata = Metadata(dataHash, hashedUnit, version,
                                getSignature(i, hashedUnit))
            if cloudsPresent:
                cloud = metadata_list_to_dict[i]
                cloudMetadata = cloud.add_new_metadata(metadata)
            else:
                cloudMetadata = CloudMetadata([metadata], i)

            cloudMetadata = cloudMetadata.return_writable_content()
            metadata_of_clouds.append(cloudMetadata)

        results = self.writeInParallel(filename, metadata_of_clouds)

    def getMetadataInParallel(self, function, filename):
        q = self.metadataQ
        threads = [
            threading.Thread(target=function, args=(i, filename, q))
            for i in range(4)
        ]
        versions_from_metadata = []

        results = []
        verified_count = 0

        for thread in threads:
            thread.daemon = True
            thread.start()

        for _ in range(self.N):
            result = q.get()
            if result == 'None':
                results.append(None)
            else:
                result = json.loads(result)
                cloudMetadata = CloudMetadataDecoder().decode(result)
                latest_metadata = cloudMetadata.metadata_list[0]
                list_of_dicts = cloudMetadata.return_metadata_list_in_dicts()

                if verifySignature(cloudMetadata.cloudId,
                                   getHash(list_of_dicts).hexdigest(),
                                   cloudMetadata.metadata_list_sig):
                    if verifySignature(cloudMetadata.cloudId,
                                       latest_metadata.hashedUnit,
                                       latest_metadata.signature):
                        verified_count += 1
                        versions_from_metadata.append(latest_metadata.version)
                        results.append(cloudMetadata)
                        print('Verified metadata of cloud',
                              cloudMetadata.cloudId)
                    else:
                        print("The verification of metadata of cloud",
                              cloudMetadata.cloudId, 'failed')
                else:
                    print(
                        'The verification of signature of metadata list of cloud',
                        cloudMetadata.cloudId, 'failed')

            # breaking condition
            if not any(results) and len(results) == self.N - self.F:
                break

            if verified_count == self.N - self.F:
                break
        if verified_count < self.N - self.F and any(results):
            print(
                'Metadata of N-F clouds have been corrupted. Cannot proceed further!!!'
            )
            exit(0)

        return results, max(
            versions_from_metadata) if len(versions_from_metadata) > 0 else 0

    def getMetadataFromClouds(self, filename):
        results, maxVersion = self.getMetadataInParallel(
            self.getFileFromCloud, filename)
        return maxVersion, results

    def getFileFromCloud(self, cloudId, filename, result_queue):
        r = requests.post('http://localhost:5555/read',
                          data=json.dumps({
                              'cloudId': cloudId,
                              'filename': filename
                          }))
        result_queue.put(r.text)