示例#1
0
def collect_available_backends():
    available_backends = []
    for ec_type in PyECLib_EC_Types:
        try:
            if ec_type == PyECLib_EC_Types.flat_xor_hd:
                handle = pyeclib_c.init(10, 5, ec_type.value, 3)
            else:
                handle = pyeclib_c.init(10, 4, ec_type.value)
            available_backends.append(ec_type.name)
            print ec_type.name
        except:
            pass
    return available_backends
示例#2
0
def collect_available_backends():
    available_backends = []
    for ec_type in PyECLib_EC_Types:
        try:
           if ec_type == PyECLib_EC_Types.flat_xor_hd:
                handle = pyeclib_c.init(10, 5, ec_type.value, 3)
           else:
                handle = pyeclib_c.init(10, 4, ec_type.value)
           available_backends.append(ec_type.name)
           print(ec_type.name)
        except:
            pass
    return available_backends
示例#3
0
    def _test_get_required_fragments(self, num_data, num_parity, ec_type):
        """
        :return boolean, True if all tests passed
        """
        handle = pyeclib_c.init(num_data, num_parity, ec_type.value)
        success = True

        #
        # MDS codes need any k fragments
        #
        if ec_type in ["jerasure_rs_vand", "jerasure_rs_cauchy"]:
            expected_fragments = [i for i in range(num_data + num_parity)]
            missing_fragments = []

            #
            # Remove between 1 and num_parity
            #
            for i in range(random.randint(0, num_parity - 1)):
                missing_fragment = random.sample(expected_fragments, 1)[0]
                missing_fragments.append(missing_fragment)
                expected_fragments.remove(missing_fragment)

            expected_fragments = expected_fragments[:num_data]
            required_fragments = pyeclib_c.get_required_fragments(
                handle, missing_fragments, [])

            if expected_fragments != required_fragments:
                success = False
                print(("Unexpected required fragments list "
                       "(exp != req): %s != %s" %
                       (expected_fragments, required_fragments)))

        return success
示例#4
0
    def time_decode(self, num_data, num_parity, ec_type, hd, file_size,
                    iterations):
        """
        :return 2-tuple, (success, average decode time)
        """
        timer = Timer()
        tsum = 0
        handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
        whole_file_bytes = self.get_tmp_file(file_size).read()
        success = True

        fragments = pyeclib_c.encode(handle, whole_file_bytes)
        orig_fragments = fragments[:]

        for i in range(iterations):
            missing_idxs = []
            num_missing = hd - 1
            for j in range(num_missing):
                num_frags_left = len(fragments)
                idx = random.randint(0, num_frags_left - 1)
                fragments.pop(idx)

            timer.start()
            decoded_file_bytes = pyeclib_c.decode(handle, fragments,
                                                  len(fragments[0]))
            tsum += timer.stop_and_return()

            fragments = orig_fragments[:]

            if whole_file_bytes != decoded_file_bytes:
                success = False

        return success, tsum / iterations
示例#5
0
    def _test_get_required_fragments(self, num_data, num_parity, ec_type):
        """
        :return boolean, True if all tests passed
        """
        handle = pyeclib_c.init(num_data, num_parity, ec_type.value)
        success = True

        #
        # MDS codes need any k fragments
        #
        if ec_type in ["jerasure_rs_vand", "jerasure_rs_cauchy"]:
            expected_fragments = [i for i in range(num_data + num_parity)]
            missing_fragments = []

            #
            # Remove between 1 and num_parity
            #
            for i in range(random.randint(0, num_parity - 1)):
                missing_fragment = random.sample(expected_fragments, 1)[0]
                missing_fragments.append(missing_fragment)
                expected_fragments.remove(missing_fragment)

            expected_fragments = expected_fragments[:num_data]
            required_fragments = pyeclib_c.get_required_fragments(
                handle,
                missing_fragments, [])

            if expected_fragments != required_fragments:
                success = False
                print(("Unexpected required fragments list "
                       "(exp != req): %s != %s" %
                       (expected_fragments, required_fragments)))

        return success
示例#6
0
    def time_decode(self,
                    num_data, num_parity, ec_type, hd,
                    file_size, iterations):
        """
        :return 2-tuple, (success, average decode time)
        """
        timer = Timer()
        tsum = 0
        handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
        whole_file_bytes = self.get_tmp_file(file_size).read()
        success = True

        fragments = pyeclib_c.encode(handle, whole_file_bytes)
        orig_fragments = fragments[:]

        for i in range(iterations):
            missing_idxs = []
            num_missing = hd - 1
            for j in range(num_missing):
                num_frags_left = len(fragments)
                idx = random.randint(0, num_frags_left - 1)
                fragments.pop(idx)

            timer.start()
            decoded_file_bytes = pyeclib_c.decode(handle,
                                                 fragments,
                                                 len(fragments[0]))
            tsum += timer.stop_and_return()

            fragments = orig_fragments[:]

            if whole_file_bytes != decoded_file_bytes:
              success = False

        return success, tsum / iterations
示例#7
0
    def __init__(self, k, m, hd, ec_type,
                 chksum_type=PyECLib_FRAGHDRCHKSUM_Types.none,
                 validate=False):
        self.k = k
        self.m = m
        self.hd = hd
        self.ec_type = ec_type
        self.chksum_type = chksum_type

        self.inline_chksum = 0
        self.algsig_chksum = 0
        # crc32 is the only inline checksum type currently supported
        if self.chksum_type is PyECLib_FRAGHDRCHKSUM_Types.inline_crc32:
            self.inline_chksum = 1

        name = self.ec_type.name

        self.handle = pyeclib_c.init(
            self.k,
            self.m,
            ec_type.value,
            self.hd,
            self.inline_chksum,
            self.algsig_chksum,
            validate)
示例#8
0
    def __init__(self,
                 k,
                 m,
                 ec_type,
                 chksum_type=PyECLib_FRAGHDRCHKSUM_Types.none):
        self.k = k
        self.m = m
        self.ec_type = ec_type
        self.chksum_type = chksum_type
        hd = m

        self.inline_chksum = 0
        self.algsig_chksum = 0
        # crc32 is the only inline checksum type currently supported
        if self.chksum_type is PyECLib_FRAGHDRCHKSUM_Types.inline_crc32:
            self.inline_chksum = 1

        name = self.ec_type.name

        if name == "flat_xor_hd" or name == "flat_xor_hd_3":
            hd = 3
        if name == "flat_xor_hd_4":
            hd = 4

        self.handle = pyeclib_c.init(self.k, self.m, ec_type.value, hd,
                                     self.inline_chksum, self.algsig_chksum)
示例#9
0
    def time_encode(self, num_data, num_parity, ec_type, hd, file_size,
                    iterations):
        """
        :return average encode time
        """
        timer = Timer()
        handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
        whole_file_bytes = self.get_tmp_file(file_size).read()

        timer.start()
        for l in range(iterations):
            pyeclib_c.encode(handle, whole_file_bytes)
        tsum = timer.stop_and_return()

        return tsum / iterations
示例#10
0
    def time_encode(self, num_data, num_parity, ec_type, hd,
                    file_size, iterations):
        """
        :return average encode time
        """
        timer = Timer()
        handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
        whole_file_bytes = self.get_tmp_file(file_size).read()

        timer.start()
        for l in range(iterations):
            pyeclib_c.encode(handle, whole_file_bytes)
        tsum = timer.stop_and_return()

        return tsum / iterations
示例#11
0
    def time_range_decode(self, num_data, num_parity, ec_type, hd, file_size,
                          iterations):
        """
        :return 2-tuple, (success, average decode time)
        """
        timer = Timer()
        tsum = 0
        handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
        whole_file_bytes = self.get_tmp_file(file_size).read()
        success = True

        begins = [
            long(random.randint(0,
                                len(whole_file_bytes) - 1)) for i in range(3)
        ]
        ends = [
            long(random.randint(begins[i], len(whole_file_bytes)))
            for i in range(3)
        ]

        ranges = zip(begins, ends)

        fragments = pyeclib_c.encode(handle, whole_file_bytes)
        orig_fragments = fragments[:]

        for i in range(iterations):
            missing_idxs = []
            num_missing = hd - 1
            for j in range(num_missing):
                num_frags_left = len(fragments)
                idx = random.randint(0, num_frags_left - 1)
                fragments.pop(idx)

            timer.start()
            decoded_file_ranges = pyeclib_c.decode(handle, fragments,
                                                   len(fragments[0]), ranges)
            tsum += timer.stop_and_return()

            fragments = orig_fragments[:]

            range_offset = 0
            for r in ranges:
                if whole_file_bytes[r[0]:r[1] +
                                    1] != decoded_file_ranges[range_offset]:
                    success = False
                range_offset += 1

        return success, tsum / iterations
示例#12
0
    def time_range_decode(self,
                          num_data, num_parity, ec_type, hd,
                          file_size, iterations):
        """
        :return 2-tuple, (success, average decode time)
        """
        timer = Timer()
        tsum = 0
        handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
        whole_file_bytes = self.get_tmp_file(file_size).read()
        success = True

        begins = [int(random.randint(0, len(whole_file_bytes) - 1))
                  for i in range(3)]
        ends = [int(random.randint(begins[i], len(whole_file_bytes)))
                for i in range(3)]

        ranges = list(zip(begins, ends))

        fragments = pyeclib_c.encode(handle, whole_file_bytes)
        orig_fragments = fragments[:]

        for i in range(iterations):
            num_missing = hd - 1
            for j in range(num_missing):
                num_frags_left = len(fragments)
                idx = random.randint(0, num_frags_left - 1)
                fragments.pop(idx)

            timer.start()
            decoded_file_ranges = pyeclib_c.decode(handle,
                                                   fragments,
                                                   len(fragments[0]),
                                                   ranges)
            tsum += timer.stop_and_return()

            fragments = orig_fragments[:]

            range_offset = 0
            for r in ranges:
                if whole_file_bytes[
                        r[0]: r[1] + 1] != decoded_file_ranges[range_offset]:
                    success = False
                range_offset += 1

        return success, tsum / iterations
示例#13
0
    def time_reconstruct(self,
                         num_data, num_parity, ec_type, hd,
                         file_size, iterations):
        """
        :return 2-tuple, (success, average reconstruct time)
        """
        timer = Timer()
        tsum = 0
        handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
        whole_file_bytes = self.get_tmp_file(file_size).read()
        success = True

        orig_fragments = pyeclib_c.encode(handle, whole_file_bytes)

        for i in range(iterations):
            fragments = orig_fragments[:]
            num_missing = 1
            missing_idxs = []
            for j in range(num_missing):
                num_frags_left = len(fragments)
                idx = random.randint(0, num_frags_left - 1)
                while idx in missing_idxs:
                  idx = random.randint(0, num_frags_left - 1)
                missing_idxs.append(idx)
                fragments.pop(idx)

            timer.start()
            reconstructed_fragment = pyeclib_c.reconstruct(handle,
                                                           fragments,
                                                           len(fragments[0]),
                                                           missing_idxs[0])
            tsum += timer.stop_and_return()

            if orig_fragments[missing_idxs[0]] != reconstructed_fragment:
                success = False
                # Output the fragments for debugging
                with open("orig_fragments", "wb") as fd_orig:
                    fd_orig.write(orig_fragments[missing_idxs[0]])
                with open("decoded_fragments", "wb") as fd_decoded:
                    fd_decoded.write(reconstructed_fragment)
                print(("Fragment %d was not reconstructed!!!" % missing_idxs[0]))
                sys.exit(2)

        return success, tsum / iterations
示例#14
0
    def time_reconstruct(self, num_data, num_parity, ec_type, hd, file_size,
                         iterations):
        """
        :return 2-tuple, (success, average reconstruct time)
        """
        timer = Timer()
        tsum = 0
        handle = pyeclib_c.init(num_data, num_parity, ec_type, hd)
        whole_file_bytes = self.get_tmp_file(file_size).read()
        success = True

        orig_fragments = pyeclib_c.encode(handle, whole_file_bytes)

        for i in range(iterations):
            fragments = orig_fragments[:]
            num_missing = 1
            missing_idxs = []
            for j in range(num_missing):
                num_frags_left = len(fragments)
                idx = random.randint(0, num_frags_left - 1)
                while idx in missing_idxs:
                    idx = random.randint(0, num_frags_left - 1)
                missing_idxs.append(idx)
                fragments.pop(idx)

            timer.start()
            reconstructed_fragment = pyeclib_c.reconstruct(
                handle, fragments, len(fragments[0]), missing_idxs[0])
            tsum += timer.stop_and_return()

            if orig_fragments[missing_idxs[0]] != reconstructed_fragment:
                success = False
                # Output the fragments for debugging
                with open("orig_fragments", "wb") as fd_orig:
                    fd_orig.write(orig_fragments[missing_idxs[0]])
                with open("decoded_fragments", "wb") as fd_decoded:
                    fd_decoded.write(reconstructed_fragment)
                print(("Fragment %d was not reconstructed!!!" %
                       missing_idxs[0]))
                sys.exit(2)

        return success, tsum / iterations
示例#15
0
文件: core.py 项目: yylluu/pyeclib
    def __init__(self, k, m, hd, ec_type,
                 chksum_type=PyECLib_FRAGHDRCHKSUM_Types.none,
                 validate=False):
        self.k = k
        self.m = m
        self.hd = hd
        self.ec_type = ec_type
        self.chksum_type = chksum_type

        self.inline_chksum = 0
        self.algsig_chksum = 0
        # crc32 is the only inline checksum type currently supported
        if self.chksum_type is PyECLib_FRAGHDRCHKSUM_Types.inline_crc32:
            self.inline_chksum = 1

        self.handle = pyeclib_c.init(
            self.k,
            self.m,
            ec_type.value,
            self.hd,
            self.inline_chksum,
            self.algsig_chksum,
            validate)