Ejemplo n.º 1
0
 def recover_d_p(self, fname, index):
     """
     recover data drive (index) and 'p' drive
     :param fname: data name
     :param index: data disk index
     :return:
     """
     assert 0 <= index < self.N - 2
     byte_ndarray = self._read_n(fname, self.N, exclude=index)
     DD = byte_ndarray[:-2]
     Q = byte_ndarray[-1:]
     # Dx
     Qx = utils.gen_q(DD, ndim=2)
     g_x_inv = self.gf.generator[(self.gf.circle - index) % self.gf.circle]
     ###
     _add_list = utils.gf_1darray_add(Q, Qx)
     Dx_list = utils.gf_a_multiply_list(g_x_inv, _add_list)
     ###
     Dx_content = ''.join(chr(i) for i in Dx_list)
     x_fpath = self.get_real_name(index, fname)
     utils.write_content(x_fpath, Dx_content)
     # p
     Dx = np.array(Dx_list, ndmin=2)
     assert Dx.shape[1] == byte_ndarray.shape[1]
     # update firstly
     DD[index] = Dx
     P = utils.gen_p(DD, ndim=1)
     assert P.shape[0] == byte_ndarray.shape[1]
     # do not need to update DD
     P_content = self._1darray_to_str(P)
     P_path = self.get_real_name(self.N - 2, fname)
     utils.write_content(P_path, P_content)
Ejemplo n.º 2
0
 def recover_2d(self, fname, x, y):
     """
     recover data drives (x and y)
     :param fname: data name
     :param x: corrupted data disk index
     :param y: corrupted data disk index
     :return:
     """
     assert 0 <= x < self.N - 2
     assert 0 <= y < self.N - 2
     assert x != y
     byte_ndarray = self._read_n(fname, self.N, exclude=[x, y])
     DD = byte_ndarray[:-2]
     P = byte_ndarray[-2:-1]
     Q = byte_ndarray[-1:]
     # Pxy
     Pxy = utils.gen_p(DD, ndim=2)
     # Qxy
     Qxy = utils.gen_q(DD, ndim=2)
     # Axy, Bxy
     A = self.gf.Axy(x, y)
     B = self.gf.Bxy(x, y)
     # Dx
     first = utils.gf_a_multiply_list(A, utils.gf_1darray_add(P, Pxy))
     second = utils.gf_a_multiply_list(B, utils.gf_1darray_add(Q, Qxy))
     Dx = utils.gf_1darray_add(np.array(first, dtype=config.BYTE_TYPE),
                               np.array(second, dtype=config.BYTE_TYPE))
     Dx_content = self._1darray_to_str(Dx)
     x_fpath = self.get_real_name(x, fname)
     utils.write_content(x_fpath, Dx_content)
     # Dy
     Dy = utils.gf_1darray_add(P ^ Pxy, Dx)
     Dy_content = self._1darray_to_str(Dy)
     y_fpath = self.get_real_name(y, fname)
     utils.write_content(y_fpath, Dy_content)
Ejemplo n.º 3
0
 def recover_d_p(self, fname, index):
     """
     recover data drive (index) and 'p' drive
     :param fname: data name
     :param index: data disk index
     :return:
     """
     assert 0 <= index < self.N - 2
     byte_ndarray = self._read_n(fname, self.N, exclude=index)
     DD = byte_ndarray[:-2]
     Q = byte_ndarray[-1:]
     # Dx
     Qx = utils.gen_q(DD, ndim=2)
     g_x_inv = self.gf.generator[(self.gf.circle - index) % self.gf.circle]
     ###
     _add_list = utils.gf_1darray_add(Q, Qx)
     Dx_list = utils.gf_a_multiply_list(g_x_inv, _add_list)
     ###
     Dx_content = ''.join(chr(i) for i in Dx_list)
     x_fpath = self.get_real_name(index, fname)
     utils.write_content(x_fpath, Dx_content)
     # p
     Dx = np.array(Dx_list, ndmin=2)
     assert Dx.shape[1] == byte_ndarray.shape[1]
     # update firstly
     DD[index] = Dx
     P = utils.gen_p(DD, ndim=1)
     assert P.shape[0] == byte_ndarray.shape[1]
     # do not need to update DD
     P_content = self._1darray_to_str(P)
     P_path = self.get_real_name(self.N - 2, fname)
     utils.write_content(P_path, P_content)
Ejemplo n.º 4
0
 def recover(self, fname, index):
     """recover corrupted disk (index)"""
     assert 0 <= index < self.N
     byte_ndarray = self._read_n(fname, self.N, exclude=index)
     parity = utils.gen_p(byte_ndarray, ndim=1)
     content = self._1darray_to_str(parity)
     fpath = self.get_real_name(index, fname)
     utils.write_content(fpath, content)
     # check
     read_ndarray = self._read_n(fname, self.N)
     self.check(read_ndarray)
Ejemplo n.º 5
0
    def run(self):
        print("Loading input_file from {}".format(self.input_fname))
        file_content = utils.load_file_content(self.input_fname)

        for ch in range(self.n_mods):
            prefix = "channel{:02}".format(ch)
            for key in self.keys_to_convert:
                file_content[prefix + "/" + key] = self.convert(
                    file_content[prefix + "/" + key], ch)

        print("Writing output_file to {}".format(self.output_fname))
        utils.write_content(self.output_fname, file_content)
Ejemplo n.º 6
0
 def recover_q(self, fname):
     """
     recover 'q' drive, recompute
     :param fname: data name
     :return:
     """
     byte_ndarray = self._read_n(fname, self.N - 2)
     q_ndarray = utils.gen_q(byte_ndarray, ndim=2)
     assert q_ndarray.ndim == 2
     new_num = q_ndarray.shape[1]
     q_ndarray.shape = (new_num, )
     content = self._1darray_to_str(q_ndarray)
     fpath = self.get_real_name(self.N - 1, fname)
     utils.write_content(fpath, content)
Ejemplo n.º 7
0
 def recover_q(self, fname):
     """
     recover 'q' drive, recompute
     :param fname: data name
     :return:
     """
     byte_ndarray = self._read_n(fname, self.N - 2)
     q_ndarray = utils.gen_q(byte_ndarray, ndim=2)
     assert q_ndarray.ndim == 2
     new_num = q_ndarray.shape[1]
     q_ndarray.shape = (new_num,)
     content = self._1darray_to_str(q_ndarray)
     fpath = self.get_real_name(self.N - 1, fname)
     utils.write_content(fpath, content)
Ejemplo n.º 8
0
    def create_header_file(self):

        os.mkdir(ENGYN_CONFIG.SRC_DIR + '/' + self.app_stem)

        filepath = os.path.join(
            ENGYN_CONFIG.SRC_DIR, self.app_stem,
            "{}.{}".format(self.app_stem, ENGYN_CONFIG.HEADER_EXT))

        # create empty
        status = utils.touch(filepath)
        if (status != 0):
            self.err = status
            return status

        # parse content
        content = utils.parse(ENGYN_CONFIG.DEFAULT_HEADER)

        # inject class name
        content = content.replace(ENGYN_CONFIG.DEFAULT_APP_STEM, self.app_stem)\
                         .replace(ENGYN_CONFIG.DEFAULT_APP_STEM.upper(), self.app_stem.upper())

        # parse content
        status = utils.write_content(content, filepath)
        if (status != 0):
            self.err = status

        return status
Ejemplo n.º 9
0
    def parse_conversion_layer(self):
        filepath = os.path.join(ENGYN_CONFIG.SRC_DIR,
                                ENGYN_CONFIG.CONVERSION_FILE)

        # parse
        content = utils.parse_content_to_list(filepath)

        # locate
        idx = utils.locate_element_with_substr(content,
                                               '#include',
                                               last_occ=True)

        # add header include
        content.insert(
            idx + 1,
            '#include "{}/{}.{}"'.format(ENGYN_CONFIG.SRC_DIR, self.app_stem,
                                         ENGYN_CONFIG.HEADER_EXT))

        # consolidate
        content = '\n'.join(map(str, content))

        # write back
        status = utils.write_content(content, filepath)
        if (status != 0):
            self.err = status

        return status
Ejemplo n.º 10
0
    def parse_handleacs(self):
        filepath = os.path.join(ENGYN_CONFIG.SRC_DIR,
                                ENGYN_CONFIG.HANDLEACS_FILE)

        # parse
        content = utils.parse_content_to_list(filepath)

        # locate closing of swith case
        idx = utils.locate_element_with_substr(content, 'default:')

        func_name = ENGYN_CONFIG.CONSTS_APP_PREFIX + self.app_stem.upper()

        # add switch case
        content.insert(idx - 1, "\tbreak;".expandtabs(8))
        content.insert(
            idx - 1, "\tconv_layer_obj.start_app<{}>();".format(
                self.app_name).expandtabs(8))
        content.insert(idx - 1, "\tcase {}:".format(func_name).expandtabs(4))
        content.insert(idx - 1, "")

        # consolidate
        content = '\n'.join(map(str, content))

        # write back
        status = utils.write_content(content, filepath)
        if (status != 0):
            self.err = status

        return status
Ejemplo n.º 11
0
 def recover_d_or_p(self, fname, index):
     """
     recover data drive or 'p' drive, simply using XOR
     :param fname: data name
     :param index: data disk or p disk index
     :return:
     """
     assert 0 <= index < self.N - 1
     byte_ndarray = self._read_n(fname, self.N - 1, exclude=index)
     parity = utils.gen_p(byte_ndarray, ndim=1)
     content = self._1darray_to_str(parity)
     fpath = self.get_real_name(index, fname)
     utils.write_content(fpath, content)
     # check data or p
     read_ndarray = self._read_n(fname, self.N - 1)
     utils.check_data_p(read_ndarray)
Ejemplo n.º 12
0
    def create_cpp_file(self):

        # create empty
        filepath = os.path.join(
            ENGYN_CONFIG.SRC_DIR, self.app_stem,
            "{}.{}".format(self.app_stem, ENGYN_CONFIG.CPP_EXT))
        status = utils.touch(filepath)

        if (status != 0):
            self.err = status
            return status

        # parse content
        content = utils.parse(ENGYN_CONFIG.DEFAULT_CPP)

        # inject class name
        content = content.replace(ENGYN_CONFIG.DEFAULT_APP_STEM, self.app_stem)

        # parse content
        status = utils.write_content(content, filepath)

        if (status != 0):
            self.err = status

        return status
Ejemplo n.º 13
0
 def recover_d_or_p(self, fname, index):
     """
     recover data drive or 'p' drive, simply using XOR
     :param fname: data name
     :param index: data disk or p disk index
     :return:
     """
     assert 0 <= index < self.N - 1
     byte_ndarray = self._read_n(fname, self.N - 1, exclude=index)
     parity = utils.gen_p(byte_ndarray, ndim=1)
     content = self._1darray_to_str(parity)
     fpath = self.get_real_name(index, fname)
     utils.write_content(fpath, content)
     # check data or p
     read_ndarray = self._read_n(fname, self.N - 1)
     utils.check_data_p(read_ndarray)
Ejemplo n.º 14
0
def main():

    # インスタンス生成済みかどうかのフラグ
    _isInitialize = False
    def __init__(self):
        """
            初期化処理
        """
        if not self._isInitialize:
            self._isInitialize = True
            self.clientList = {}

    items = []

    header = utils.display_file_content('./template/header.tpl')
    #print header
    items.append(header)

    js1 = utils.include_column_filter('view01','')
    #print js1
    items.append(js1)

    h1element = utils.create_block_element('h1', 'LVS List')
    #print h1element
    items.append(h1element)

    ipaddr = utils.extract_ipaddr('192.168.0.11_txt')
    #print ipaddr
    items.append(ipaddr)

    table = utils.create_table_with_file('view01', 'Server', './data/192.168.11.11_local.txt')
    # todo: add separator
    #print table
    items.append(table)

    footer = utils.display_file_content('./template/footer.tpl')
    #print footer
    items.append(footer)

    html_content = ''.join(items)
    utils.write_content('/tmp/output/test', html_content)
Ejemplo n.º 15
0
def gen_rnd_file(fname, size, content_type):
    """
    randomly generate data chunk
    :param fname: the name denoting the data
    :param size: the size of the data chunk
    :param content_type: can be 'text' or others; if it's 'text', make the chunk only contain ascii letters
    :return:
    """
    if content_type == 'text':
        # noinspection PyUnusedLocal
        content = ''.join([random.choice(string.ascii_letters) for i in xrange(size)])
    else:
        content = os.urandom(size)
    fpath = os.path.join(config.root, fname)
    if os.path.isfile(fpath):
        file_size = os.stat(fpath).st_size
        if file_size == size:
            get_logger().warning('fname={} with size={} exists'.format(fname, size))
            return
    get_logger().warning('generating fname={} with size={}'.format(fname, size))
    utils.write_content(fpath, content)
Ejemplo n.º 16
0
 def recover_2d(self, fname, x, y):
     """
     recover data drives (x and y)
     :param fname: data name
     :param x: corrupted data disk index
     :param y: corrupted data disk index
     :return:
     """
     assert 0 <= x < self.N - 2
     assert 0 <= y < self.N - 2
     assert x != y
     byte_ndarray = self._read_n(fname, self.N, exclude=[x, y])
     DD = byte_ndarray[:-2]
     P = byte_ndarray[-2:-1]
     Q = byte_ndarray[-1:]
     # Pxy
     Pxy = utils.gen_p(DD, ndim=2)
     # Qxy
     Qxy = utils.gen_q(DD, ndim=2)
     # Axy, Bxy
     A = self.gf.Axy(x, y)
     B = self.gf.Bxy(x, y)
     # Dx
     first = utils.gf_a_multiply_list(A, utils.gf_1darray_add(P, Pxy))
     second = utils.gf_a_multiply_list(B, utils.gf_1darray_add(Q, Qxy))
     Dx = utils.gf_1darray_add(
         np.array(
             first, dtype=config.BYTE_TYPE), np.array(
             second, dtype=config.BYTE_TYPE))
     Dx_content = self._1darray_to_str(Dx)
     x_fpath = self.get_real_name(x, fname)
     utils.write_content(x_fpath, Dx_content)
     # Dy
     Dy = utils.gf_1darray_add(P ^ Pxy, Dx)
     Dy_content = self._1darray_to_str(Dy)
     y_fpath = self.get_real_name(y, fname)
     utils.write_content(y_fpath, Dy_content)
Ejemplo n.º 17
0
def gen_rnd_file(fname, size, content_type):
    """
    randomly generate data chunk
    :param fname: the name denoting the data
    :param size: the size of the data chunk
    :param content_type: can be 'text' or others; if it's 'text', make the chunk only contain ascii letters
    :return:
    """
    if content_type == 'text':
        # noinspection PyUnusedLocal
        content = ''.join(
            [random.choice(string.ascii_letters) for i in xrange(size)])
    else:
        content = os.urandom(size)
    fpath = os.path.join(config.root, fname)
    if os.path.isfile(fpath):
        file_size = os.stat(fpath).st_size
        if file_size == size:
            get_logger().warning('fname={} with size={} exists'.format(
                fname, size))
            return
    get_logger().warning('generating fname={} with size={}'.format(
        fname, size))
    utils.write_content(fpath, content)
Ejemplo n.º 18
0
    def parse_consts(self):
        filepath = os.path.join(ENGYN_CONFIG.SRC_DIR, ENGYN_CONFIG.CONSTS_FILE)
        # parse
        content = utils.parse_content_to_list(filepath)

        # locate closing line of struct
        idx = utils.locate_element_with_substr(content, '};')

        func_name = ENGYN_CONFIG.CONSTS_APP_PREFIX + self.app_stem.upper()

        # add app number
        content.insert(idx - 1, "  {} = {},".format(func_name, self.app_id))

        # consolidate
        content = '\n'.join(map(str, content))

        # write back
        status = utils.write_content(content, filepath)
        if (status != 0):
            self.err = status

        return status
Ejemplo n.º 19
0
 def _corrupt(fname, index, size):
     get_logger().warning("corrupting disk {}".format(index))
     error_fpath = r6.get_real_name(index, fname)
     error_content = os.urandom(size)
     utils.write_content(error_fpath, error_content)
Ejemplo n.º 20
0
 def _corrupt(fname, index, size):
     get_logger().warning("corrupting disk {}".format(index))
     error_fpath = r6.get_real_name(index, fname)
     error_content = os.urandom(size)
     utils.write_content(error_fpath, error_content)
Ejemplo n.º 21
0
 def __write_impl(self, fpath, write_array):
     """
     single file write
     """
     content = self._1darray_to_str(write_array)
     utils.write_content(fpath, content)
Ejemplo n.º 22
0
 def __write_impl(self, fpath, write_array):
     """
     single file write
     """
     content = self._1darray_to_str(write_array)
     utils.write_content(fpath, content)