示例#1
0
文件: utils.py 项目: jiegev5/CE7490_2
def gen_q(data_ndarray, ndim):
    """
    generate q using GF^8
    :param data_ndarray: the data ndarray
    :param ndim: real dim
    :return: q_ndarray with shape=(1, byte_ndarray.shape[1])
    """
    transposed = np.transpose(data_ndarray)
    # print(data_ndarray.shape)
    get_logger().info('transposed\n{}'.format(transposed))
    gf = GF()
    q_list = []
    for _1darray in transposed:
        bv_list = []
        for i, arr_val in enumerate(_1darray):
            res_i = gf.multiply(gf.generator[i % gf.circle], int(arr_val))
            # print('i={}, arr_val={}, res_i={}'.format(i, arr_val, res_i))
            bv_list.append(res_i)
            # map(lambda i: print(i), bv_list)
        q_value = reduce(operator.xor, bv_list)
        q_list.append(q_value)
    arr = np.array(q_list, ndmin=ndim, dtype=config.BYTE_TYPE)
    get_logger().info("arr={}".format(arr))
    # assert arr.shape[1] == data_ndarray.shape[1]
    return arr
示例#2
0
def gen_q(data_ndarray, ndim):
    """
    generate q using GF^8
    :param data_ndarray: the data ndarray
    :param ndim: real dim
    :return: q_ndarray with shape=(1, byte_ndarray.shape[1])
    """
    transposed = np.transpose(data_ndarray)
    # print(data_ndarray.shape)
    get_logger().info('transposed\n{}'.format(transposed))
    gf = GF()
    q_list = []
    for _1darray in transposed:
        bv_list = []
        for i, arr_val in enumerate(_1darray):
            res_i = gf.multiply(gf.generator[i % gf.circle], int(arr_val))
            # print('i={}, arr_val={}, res_i={}'.format(i, arr_val, res_i))
            bv_list.append(res_i)
            # map(lambda i: print(i), bv_list)
        q_value = reduce(operator.xor, bv_list)
        q_list.append(q_value)
    arr = np.array(q_list, ndmin=ndim, dtype=config.BYTE_TYPE)
    get_logger().info("arr={}".format(arr))
    # assert arr.shape[1] == data_ndarray.shape[1]
    return arr
示例#3
0
文件: utils.py 项目: jiegev5/CE7490_2
def gf_a_multiply_list(a, l):
    """
    multiplication of a and l
    :param a: scala type
    :param l:
    :return: list of int
    """
    gf = GF()
    return [gf.multiply(int(i), a) for i in l]
示例#4
0
def gf_a_multiply_list(a, l):
    """
    multiplication of a and l
    :param a: scala type
    :param l:
    :return: list of int
    """
    gf = GF()
    return [gf.multiply(int(i), a) for i in l]
示例#5
0
 def __init__(self, pgf, args):
     self.voice = None
     iLang = args.input.capitalize()
     oLang = args.output.capitalize()
     self.gf1 = GF(pgf, iLang, 'Sage', 'Command', lexer=commandsLex)
     self.gf2 = GF(pgf, 'Sage', oLang, 'Answer')
     self.args = args
     self.environment = ContextStack()
     self.environment.new('toplevel')
     self.environment.gf1 = self.gf1  # From prompt to change
     self.environment.kernel = None  # to be set on evaluation
示例#6
0
 def __init__(self, pgf, args):
     self.voice = None
     iLang = args.input.capitalize()
     oLang = args.output.capitalize()
     self.gf1 = GF(pgf, iLang, "Sage", "Command", lexer=commandsLex)
     self.gf2 = GF(pgf, "Sage", oLang, "Answer")
     self.args = args
     self.environment = ContextStack()
     self.environment.new("toplevel")
     self.environment.gf1 = self.gf1  # From prompt to change
     self.environment.kernel = None  # to be set on evaluation
示例#7
0
 def __init__(self, N):
     assert 4 <= N
     super(RAID6, self).__init__(N)
     # gf object
     self.gf = GF()
示例#8
0
class RAID6(RAID):
    _logger = get_logger()

    def __init__(self, N):
        assert 4 <= N
        super(RAID6, self).__init__(N)
        # gf object
        self.gf = GF()

    def check(self, byte_ndarray):
        """
        check involves p and q
        :param byte_ndarray: all ndarray including p and q
        :return:
        """
        # check p
        data_p_ndarray = byte_ndarray[:-1]
        utils.check_data_p(data_p_ndarray)
        # check_q
        data_ndarray = byte_ndarray[:-2]
        q_ndarray = byte_ndarray[-1:]
        utils.check_q(data_ndarray, q_ndarray)

    def read(self, fname, size):
        """
        read size chunk from fname(RAID6 system)
        """
        byte_ndarray = self._read_n(fname, self.N)
        self.check(byte_ndarray)
        data_ndarray = byte_ndarray[:-2]
        flat_list = data_ndarray.ravel(1)[:size]
        flat_str_list = [chr(e) for e in flat_list]
        return ''.join(flat_str_list)

    def recover(self, fname, exclude):
        """
        since there are different cases, we raise an error indicating it should not be called
        """
        raise NotImplementedError("not implemented; split into several cases")

    def _get_corrupted_data_disk(self, P_star, Q_star):
        """
        from P_star and Q_star, find the corrupted data disk index
        procondition: P_star!={00}, Q_star!={00} (in vector sense)
        """
        p0 = int(P_star[0][0])
        q0 = int(Q_star[0][0])
        log_p0 = self.gf.log_generator(p0)
        log_q0 = self.gf.log_generator(q0)
        return (log_q0 - log_p0) % self.gf.circle

    def detect_corruption(self, fname):
        """
        single disk corruption detection
        :param fname: data name in RAID6 system
        :return: corrupted disk index; for p, self.N-2; for q, self.N-1
        """
        # all disks, including P, Q
        byte_ndarray = self._read_n(fname, self.N)
        data_ndarray = byte_ndarray[:-2]
        self._logger.info("byte_ndarray=\n{}".format(byte_ndarray))
        P = byte_ndarray[-2:-1]
        self._logger.info("p={}".format(P))
        Q = byte_ndarray[-1]
        self._logger.info("Q={}".format(Q))
        P_prime = utils.gen_p(data_ndarray, ndim=2)
        self._logger.info("P_prime={}".format(P_prime))
        Q_prime = utils.gen_q(data_ndarray, ndim=2)
        self._logger.info("Q_prime={}".format(Q_prime))
        P_star = np.bitwise_xor(P, P_prime)
        Q_star = np.bitwise_xor(Q, Q_prime)
        P_nonzero = np.count_nonzero(P_star)
        Q_nonzero = np.count_nonzero(Q_star)
        if P_nonzero == 0 and Q_nonzero == 0:
            print("no corruption")
            return None
        elif P_nonzero == 0 and Q_nonzero != 0:
            print("Q corruption")
            return self.N - 1
        elif P_nonzero != 0 and Q_nonzero == 0:
            print("P corruption")
            return self.N - 2
        else:
            index = self._get_corrupted_data_disk(P_star, Q_star)
            print("data disk {} corruption".format(index))
            return index

    def recover_d_or_p(self, fname, index):
        """
        recover data drive or 'p' drive, simply using XOR
        :param fname: data name
        :param index: data disk or p disk index
        :return:
        """
        assert 0 <= index < self.N - 1
        byte_ndarray = self._read_n(fname, self.N - 1, exclude=index)
        parity = utils.gen_p(byte_ndarray, ndim=1)
        content = self._1darray_to_str(parity)
        fpath = self.get_real_name(index, fname)
        utils.write_content(fpath, content)
        # check data or p
        read_ndarray = self._read_n(fname, self.N - 1)
        utils.check_data_p(read_ndarray)

    def recover_q(self, fname):
        """
        recover 'q' drive, recompute
        :param fname: data name
        :return:
        """
        byte_ndarray = self._read_n(fname, self.N - 2)
        q_ndarray = utils.gen_q(byte_ndarray, ndim=2)
        assert q_ndarray.ndim == 2
        new_num = q_ndarray.shape[1]
        q_ndarray.shape = (new_num, )
        content = self._1darray_to_str(q_ndarray)
        fpath = self.get_real_name(self.N - 1, fname)
        utils.write_content(fpath, content)

    def recover_d_q(self, fname, index):
        """
        recover data/'p' drive (index) and 'q' drive: firstly using XOR to recover data drive, then recompute q
        :param fname: data name
        :param index: corrupted data disk index
        :return:
        """
        self.recover_d_or_p(fname, index)
        self.recover_q(fname)

    def recover_2d(self, fname, x, y):
        """
        recover data drives (x and y)
        :param fname: data name
        :param x: corrupted data disk index
        :param y: corrupted data disk index
        :return:
        """
        assert 0 <= x < self.N - 2
        assert 0 <= y < self.N - 2
        assert x != y
        byte_ndarray = self._read_n(fname, self.N, exclude=[x, y])
        DD = byte_ndarray[:-2]
        P = byte_ndarray[-2:-1]
        Q = byte_ndarray[-1:]
        # Pxy
        Pxy = utils.gen_p(DD, ndim=2)
        # Qxy
        Qxy = utils.gen_q(DD, ndim=2)
        # Axy, Bxy
        A = self.gf.Axy(x, y)
        B = self.gf.Bxy(x, y)
        # Dx
        first = utils.gf_a_multiply_list(A, utils.gf_1darray_add(P, Pxy))
        second = utils.gf_a_multiply_list(B, utils.gf_1darray_add(Q, Qxy))
        Dx = utils.gf_1darray_add(np.array(first, dtype=config.BYTE_TYPE),
                                  np.array(second, dtype=config.BYTE_TYPE))
        Dx_content = self._1darray_to_str(Dx)
        x_fpath = self.get_real_name(x, fname)
        utils.write_content(x_fpath, Dx_content)
        # Dy
        Dy = utils.gf_1darray_add(P ^ Pxy, Dx)
        Dy_content = self._1darray_to_str(Dy)
        y_fpath = self.get_real_name(y, fname)
        utils.write_content(y_fpath, Dy_content)

    def recover_d_p(self, fname, index):
        """
        recover data drive (index) and 'p' drive
        :param fname: data name
        :param index: data disk index
        :return:
        """
        assert 0 <= index < self.N - 2
        byte_ndarray = self._read_n(fname, self.N, exclude=index)
        DD = byte_ndarray[:-2]
        Q = byte_ndarray[-1:]
        # Dx
        Qx = utils.gen_q(DD, ndim=2)
        g_x_inv = self.gf.generator[(self.gf.circle - index) % self.gf.circle]
        ###
        _add_list = utils.gf_1darray_add(Q, Qx)
        Dx_list = utils.gf_a_multiply_list(g_x_inv, _add_list)
        ###
        Dx_content = ''.join(chr(i) for i in Dx_list)
        x_fpath = self.get_real_name(index, fname)
        utils.write_content(x_fpath, Dx_content)
        # p
        Dx = np.array(Dx_list, ndmin=2)
        assert Dx.shape[1] == byte_ndarray.shape[1]
        # update firstly
        DD[index] = Dx
        P = utils.gen_p(DD, ndim=1)
        assert P.shape[0] == byte_ndarray.shape[1]
        # do not need to update DD
        P_content = self._1darray_to_str(P)
        P_path = self.get_real_name(self.N - 2, fname)
        utils.write_content(P_path, P_content)

    def write(self, content, fname):
        """
        write content to fname(in RAID6 system)
        """
        byte_ndarray = self._gen_ndarray_from_content(content, self.N - 2)
        p_ndarray = utils.gen_p(byte_ndarray, ndim=2)
        q_ndarray = utils.gen_q(byte_ndarray, ndim=2)
        write_ndarray = np.concatenate([byte_ndarray, p_ndarray, q_ndarray])
        self._write_n(fname, write_ndarray, self.N)
示例#9
0
class GFSage(object):
    def __init__(self, pgf, args):
        self.voice = None
        iLang = args.input.capitalize()
        oLang = args.output.capitalize()
        self.gf1 = GF(pgf, iLang, 'Sage', 'Command', lexer=commandsLex)
        self.gf2 = GF(pgf, 'Sage', oLang, 'Answer')
        self.args = args
        self.environment = ContextStack()
        self.environment.new('toplevel')
        self.environment.gf1 = self.gf1  # From prompt to change
        self.environment.kernel = None  # to be set on evaluation

    def evaluate(self, command):
        """Pass command to Sagecell"""
        mainCmd = command.rstrip(' ;')
        self.environment.kernel = Kernel('gfsage')
        return eval("self.environment." + mainCmd)

    def say(self, text):
        if self.voice:
            self.voice.say(text)

    def run(self):
        def loop(text):
            ps = self.parseQuery(text)
            if len(ps) > 1:
                for j, p in enumerate(ps):
                    logging.info("Parsing %d: %s", j, p)
                # raise ValueError, "Ambiguous command (%d parsings)" % len(ps)
                logging.info("! Ambiguous command (%d parsings)", len(ps))
            elif not ps:
                raise ValueError, "No parse for '%s'" % text
            query = ps.pop()
            cmd = self.gf1.linearize(query)
            lastQueryVal = computeQuery(query)
            logging.debug(cmd)
            try:
                result = self.evaluate(cmd)
            except ValueError:
                logging.error("Computation failed")
                return
            if result:
                logging.info("Result is %s", result)
                for a in self.parseAnswer(result, lastQueryVal):
                    logging.debug(a)
                    text = self.gf2.linearize(a)
                    print text
                    self.say(text)
            else:
                logging.info("No result for '%s'", text)

        self.gf1.interact(loop)

    def parseQuery(self, a):
        return {
            e
            for e in self.gf1.parse(
                a,
                compute=lambda t: mockCompute(purgeRationals(purgeFloats(t))))
        }

    def parseAnswer(self, a, queryVal):
        parsed = {
            e
            for e in self.gf2.parse(a)
            if e.unpack()[0] in ('Simple', 'Yes', 'No', 'YesApprox')
        }
        purged = {purgeFloats(e) for e in parsed}
        rationals = {purgeRationals(e) for e in purged}
        return {
            completeReturn(e, queryVal, self.args.feedback)
            for e in rationals
        }
示例#10
0
class GFSage(object):
    def __init__(self, pgf, args):
        self.voice = None
        iLang = args.input.capitalize()
        oLang = args.output.capitalize()
        self.gf1 = GF(pgf, iLang, "Sage", "Command", lexer=commandsLex)
        self.gf2 = GF(pgf, "Sage", oLang, "Answer")
        self.args = args
        self.environment = ContextStack()
        self.environment.new("toplevel")
        self.environment.gf1 = self.gf1  # From prompt to change
        self.environment.kernel = None  # to be set on evaluation

    def evaluate(self, command):
        """Pass command to Sagecell"""
        mainCmd = command.rstrip(" ;")
        self.environment.kernel = Kernel("gfsage")
        return eval("self.environment." + mainCmd)

    def say(self, text):
        if self.voice:
            self.voice.say(text)

    def run(self):
        def loop(text):
            ps = self.parseQuery(text)
            if len(ps) > 1:
                for j, p in enumerate(ps):
                    logging.info("Parsing %d: %s", j, p)
                    # raise ValueError, "Ambiguous command (%d parsings)" % len(ps)
                logging.info("! Ambiguous command (%d parsings)", len(ps))
            elif not ps:
                raise ValueError, "No parse for '%s'" % text
            query = ps.pop()
            cmd = self.gf1.linearize(query)
            lastQueryVal = computeQuery(query)
            logging.debug(cmd)
            try:
                result = self.evaluate(cmd)
            except ValueError:
                logging.error("Computation failed")
                return
            if result:
                logging.info("Result is %s", result)
                for a in self.parseAnswer(result, lastQueryVal):
                    logging.debug(a)
                    text = self.gf2.linearize(a)
                    print text
                    self.say(text)
            else:
                logging.info("No result for '%s'", text)

        self.gf1.interact(loop)

    def parseQuery(self, a):
        return {e for e in self.gf1.parse(a, compute=lambda t: mockCompute(purgeRationals(purgeFloats(t))))}

    def parseAnswer(self, a, queryVal):
        parsed = {e for e in self.gf2.parse(a) if e.unpack()[0] in ("Simple", "Yes", "No", "YesApprox")}
        purged = {purgeFloats(e) for e in parsed}
        rationals = {purgeRationals(e) for e in purged}
        return {completeReturn(e, queryVal, self.args.feedback) for e in rationals}
示例#11
0
 def __init__(self, N):
     assert 4 <= N
     super(RAID6, self).__init__(N)
     # gf object
     self.gf = GF()
示例#12
0
class RAID6(RAID):
    _logger = get_logger()

    def __init__(self, N):
        assert 4 <= N
        super(RAID6, self).__init__(N)
        # gf object
        self.gf = GF()

    def check(self, byte_ndarray):
        """
        check involves p and q
        :param byte_ndarray: all ndarray including p and q
        :return:
        """
        # check p
        data_p_ndarray = byte_ndarray[:-1]
        utils.check_data_p(data_p_ndarray)
        # check_q
        data_ndarray = byte_ndarray[:-2]
        q_ndarray = byte_ndarray[-1:]
        utils.check_q(data_ndarray, q_ndarray)

    def read(self, fname, size):
        """
        read size chunk from fname(RAID6 system)
        """
        byte_ndarray = self._read_n(fname, self.N)
        self.check(byte_ndarray)
        data_ndarray = byte_ndarray[:-2]
        flat_list = data_ndarray.ravel(1)[:size]
        flat_str_list = [chr(e) for e in flat_list]
        return ''.join(flat_str_list)

    def recover(self, fname, exclude):
        """
        since there are different cases, we raise an error indicating it should not be called
        """
        raise NotImplementedError("not implemented; split into several cases")

    def _get_corrupted_data_disk(self, P_star, Q_star):
        """
        from P_star and Q_star, find the corrupted data disk index
        procondition: P_star!={00}, Q_star!={00} (in vector sense)
        """
        p0 = int(P_star[0][0])
        q0 = int(Q_star[0][0])
        log_p0 = self.gf.log_generator(p0)
        log_q0 = self.gf.log_generator(q0)
        return (log_q0 - log_p0) % self.gf.circle

    def detect_corruption(self, fname):
        """
        single disk corruption detection
        :param fname: data name in RAID6 system
        :return: corrupted disk index; for p, self.N-2; for q, self.N-1
        """
        # all disks, including P, Q
        byte_ndarray = self._read_n(fname, self.N)
        data_ndarray = byte_ndarray[:-2]
        self._logger.info("byte_ndarray=\n{}".format(byte_ndarray))
        P = byte_ndarray[-2:-1]
        self._logger.info("p={}".format(P))
        Q = byte_ndarray[-1]
        self._logger.info("Q={}".format(Q))
        P_prime = utils.gen_p(data_ndarray, ndim=2)
        self._logger.info("P_prime={}".format(P_prime))
        Q_prime = utils.gen_q(data_ndarray, ndim=2)
        self._logger.info("Q_prime={}".format(Q_prime))
        P_star = np.bitwise_xor(P, P_prime)
        Q_star = np.bitwise_xor(Q, Q_prime)
        P_nonzero = np.count_nonzero(P_star)
        Q_nonzero = np.count_nonzero(Q_star)
        if P_nonzero == 0 and Q_nonzero == 0:
            print("no corruption")
            return None
        elif P_nonzero == 0 and Q_nonzero != 0:
            print("Q corruption")
            return self.N - 1
        elif P_nonzero != 0 and Q_nonzero == 0:
            print("P corruption")
            return self.N - 2
        else:
            index = self._get_corrupted_data_disk(P_star, Q_star)
            print("data disk {} corruption".format(index))
            return index

    def recover_d_or_p(self, fname, index):
        """
        recover data drive or 'p' drive, simply using XOR
        :param fname: data name
        :param index: data disk or p disk index
        :return:
        """
        assert 0 <= index < self.N - 1
        byte_ndarray = self._read_n(fname, self.N - 1, exclude=index)
        parity = utils.gen_p(byte_ndarray, ndim=1)
        content = self._1darray_to_str(parity)
        fpath = self.get_real_name(index, fname)
        utils.write_content(fpath, content)
        # check data or p
        read_ndarray = self._read_n(fname, self.N - 1)
        utils.check_data_p(read_ndarray)

    def recover_q(self, fname):
        """
        recover 'q' drive, recompute
        :param fname: data name
        :return:
        """
        byte_ndarray = self._read_n(fname, self.N - 2)
        q_ndarray = utils.gen_q(byte_ndarray, ndim=2)
        assert q_ndarray.ndim == 2
        new_num = q_ndarray.shape[1]
        q_ndarray.shape = (new_num,)
        content = self._1darray_to_str(q_ndarray)
        fpath = self.get_real_name(self.N - 1, fname)
        utils.write_content(fpath, content)

    def recover_d_q(self, fname, index):
        """
        recover data/'p' drive (index) and 'q' drive: firstly using XOR to recover data drive, then recompute q
        :param fname: data name
        :param index: corrupted data disk index
        :return:
        """
        self.recover_d_or_p(fname, index)
        self.recover_q(fname)

    def recover_2d(self, fname, x, y):
        """
        recover data drives (x and y)
        :param fname: data name
        :param x: corrupted data disk index
        :param y: corrupted data disk index
        :return:
        """
        assert 0 <= x < self.N - 2
        assert 0 <= y < self.N - 2
        assert x != y
        byte_ndarray = self._read_n(fname, self.N, exclude=[x, y])
        DD = byte_ndarray[:-2]
        P = byte_ndarray[-2:-1]
        Q = byte_ndarray[-1:]
        # Pxy
        Pxy = utils.gen_p(DD, ndim=2)
        # Qxy
        Qxy = utils.gen_q(DD, ndim=2)
        # Axy, Bxy
        A = self.gf.Axy(x, y)
        B = self.gf.Bxy(x, y)
        # Dx
        first = utils.gf_a_multiply_list(A, utils.gf_1darray_add(P, Pxy))
        second = utils.gf_a_multiply_list(B, utils.gf_1darray_add(Q, Qxy))
        Dx = utils.gf_1darray_add(
            np.array(
                first, dtype=config.BYTE_TYPE), np.array(
                second, dtype=config.BYTE_TYPE))
        Dx_content = self._1darray_to_str(Dx)
        x_fpath = self.get_real_name(x, fname)
        utils.write_content(x_fpath, Dx_content)
        # Dy
        Dy = utils.gf_1darray_add(P ^ Pxy, Dx)
        Dy_content = self._1darray_to_str(Dy)
        y_fpath = self.get_real_name(y, fname)
        utils.write_content(y_fpath, Dy_content)

    def recover_d_p(self, fname, index):
        """
        recover data drive (index) and 'p' drive
        :param fname: data name
        :param index: data disk index
        :return:
        """
        assert 0 <= index < self.N - 2
        byte_ndarray = self._read_n(fname, self.N, exclude=index)
        DD = byte_ndarray[:-2]
        Q = byte_ndarray[-1:]
        # Dx
        Qx = utils.gen_q(DD, ndim=2)
        g_x_inv = self.gf.generator[(self.gf.circle - index) % self.gf.circle]
        ###
        _add_list = utils.gf_1darray_add(Q, Qx)
        Dx_list = utils.gf_a_multiply_list(g_x_inv, _add_list)
        ###
        Dx_content = ''.join(chr(i) for i in Dx_list)
        x_fpath = self.get_real_name(index, fname)
        utils.write_content(x_fpath, Dx_content)
        # p
        Dx = np.array(Dx_list, ndmin=2)
        assert Dx.shape[1] == byte_ndarray.shape[1]
        # update firstly
        DD[index] = Dx
        P = utils.gen_p(DD, ndim=1)
        assert P.shape[0] == byte_ndarray.shape[1]
        # do not need to update DD
        P_content = self._1darray_to_str(P)
        P_path = self.get_real_name(self.N - 2, fname)
        utils.write_content(P_path, P_content)

    def write(self, content, fname):
        """
        write content to fname(in RAID6 system)
        """
        byte_ndarray = self._gen_ndarray_from_content(content, self.N - 2)
        p_ndarray = utils.gen_p(byte_ndarray, ndim=2)
        q_ndarray = utils.gen_q(byte_ndarray, ndim=2)
        write_ndarray = np.concatenate([byte_ndarray, p_ndarray, q_ndarray])
        self._write_n(fname, write_ndarray, self.N)