def recover_d_p(self, fname, index):
"""
recover data drive (index) and 'p' drive
:param fname: data name
:param index: data disk index
:return:
"""
assert 0 <= index < self.N - 2
byte_ndarray = self._read_n(fname, self.N, exclude=index)
DD = byte_ndarray[:-2]
Q = byte_ndarray[-1:]
# Dx
Qx = utils.gen_q(DD, ndim=2)
g_x_inv = self.gf.generator[(self.gf.circle - index) % self.gf.circle]
###
_add_list = utils.gf_1darray_add(Q, Qx)
Dx_list = utils.gf_a_multiply_list(g_x_inv, _add_list)
###
Dx_content = ''.join(chr(i) for i in Dx_list)
x_fpath = self.get_real_name(index, fname)
utils.write_content(x_fpath, Dx_content)
# p
Dx = np.array(Dx_list, ndmin=2)
assert Dx.shape[1] == byte_ndarray.shape[1]
# update firstly
DD[index] = Dx
P = utils.gen_p(DD, ndim=1)
assert P.shape[0] == byte_ndarray.shape[1]
# do not need to update DD
P_content = self._1darray_to_str(P)
P_path = self.get_real_name(self.N - 2, fname)
utils.write_content(P_path, P_content)
def recover_2d(self, fname, x, y):
"""
recover data drives (x and y)
:param fname: data name
:param x: corrupted data disk index
:param y: corrupted data disk index
:return:
"""
assert 0 <= x < self.N - 2
assert 0 <= y < self.N - 2
assert x != y
byte_ndarray = self._read_n(fname, self.N, exclude=[x, y])
DD = byte_ndarray[:-2]
P = byte_ndarray[-2:-1]
Q = byte_ndarray[-1:]
# Pxy
Pxy = utils.gen_p(DD, ndim=2)
# Qxy
Qxy = utils.gen_q(DD, ndim=2)
# Axy, Bxy
A = self.gf.Axy(x, y)
B = self.gf.Bxy(x, y)
# Dx
first = utils.gf_a_multiply_list(A, utils.gf_1darray_add(P, Pxy))
second = utils.gf_a_multiply_list(B, utils.gf_1darray_add(Q, Qxy))
Dx = utils.gf_1darray_add(np.array(first, dtype=config.BYTE_TYPE),
np.array(second, dtype=config.BYTE_TYPE))
Dx_content = self._1darray_to_str(Dx)
x_fpath = self.get_real_name(x, fname)
utils.write_content(x_fpath, Dx_content)
# Dy
Dy = utils.gf_1darray_add(P ^ Pxy, Dx)
Dy_content = self._1darray_to_str(Dy)
y_fpath = self.get_real_name(y, fname)
utils.write_content(y_fpath, Dy_content)
def recover_d_p(self, fname, index):
"""
recover data drive (index) and 'p' drive
:param fname: data name
:param index: data disk index
:return:
"""
assert 0 <= index < self.N - 2
byte_ndarray = self._read_n(fname, self.N, exclude=index)
DD = byte_ndarray[:-2]
Q = byte_ndarray[-1:]
# Dx
Qx = utils.gen_q(DD, ndim=2)
g_x_inv = self.gf.generator[(self.gf.circle - index) % self.gf.circle]
###
_add_list = utils.gf_1darray_add(Q, Qx)
Dx_list = utils.gf_a_multiply_list(g_x_inv, _add_list)
###
Dx_content = ''.join(chr(i) for i in Dx_list)
x_fpath = self.get_real_name(index, fname)
utils.write_content(x_fpath, Dx_content)
# p
Dx = np.array(Dx_list, ndmin=2)
assert Dx.shape[1] == byte_ndarray.shape[1]
# update firstly
DD[index] = Dx
P = utils.gen_p(DD, ndim=1)
assert P.shape[0] == byte_ndarray.shape[1]
# do not need to update DD
P_content = self._1darray_to_str(P)
P_path = self.get_real_name(self.N - 2, fname)
utils.write_content(P_path, P_content)
def recover(self, fname, index):
"""recover corrupted disk (index)"""
assert 0 <= index < self.N
byte_ndarray = self._read_n(fname, self.N, exclude=index)
parity = utils.gen_p(byte_ndarray, ndim=1)
content = self._1darray_to_str(parity)
fpath = self.get_real_name(index, fname)
utils.write_content(fpath, content)
# check
read_ndarray = self._read_n(fname, self.N)
self.check(read_ndarray)
def run(self):
print("Loading input_file from {}".format(self.input_fname))
file_content = utils.load_file_content(self.input_fname)
for ch in range(self.n_mods):
prefix = "channel{:02}".format(ch)
for key in self.keys_to_convert:
file_content[prefix + "/" + key] = self.convert(
file_content[prefix + "/" + key], ch)
print("Writing output_file to {}".format(self.output_fname))
utils.write_content(self.output_fname, file_content)
def recover_q(self, fname):
"""
recover 'q' drive, recompute
:param fname: data name
:return:
"""
byte_ndarray = self._read_n(fname, self.N - 2)
q_ndarray = utils.gen_q(byte_ndarray, ndim=2)
assert q_ndarray.ndim == 2
new_num = q_ndarray.shape[1]
q_ndarray.shape = (new_num, )
content = self._1darray_to_str(q_ndarray)
fpath = self.get_real_name(self.N - 1, fname)
utils.write_content(fpath, content)
def recover_q(self, fname):
"""
recover 'q' drive, recompute
:param fname: data name
:return:
"""
byte_ndarray = self._read_n(fname, self.N - 2)
q_ndarray = utils.gen_q(byte_ndarray, ndim=2)
assert q_ndarray.ndim == 2
new_num = q_ndarray.shape[1]
q_ndarray.shape = (new_num,)
content = self._1darray_to_str(q_ndarray)
fpath = self.get_real_name(self.N - 1, fname)
utils.write_content(fpath, content)
def create_header_file(self):
os.mkdir(ENGYN_CONFIG.SRC_DIR + '/' + self.app_stem)
filepath = os.path.join(
ENGYN_CONFIG.SRC_DIR, self.app_stem,
"{}.{}".format(self.app_stem, ENGYN_CONFIG.HEADER_EXT))
# create empty
status = utils.touch(filepath)
if (status != 0):
self.err = status
return status
# parse content
content = utils.parse(ENGYN_CONFIG.DEFAULT_HEADER)
# inject class name
content = content.replace(ENGYN_CONFIG.DEFAULT_APP_STEM, self.app_stem)\
.replace(ENGYN_CONFIG.DEFAULT_APP_STEM.upper(), self.app_stem.upper())
# parse content
status = utils.write_content(content, filepath)
if (status != 0):
self.err = status
return status
def parse_conversion_layer(self):
filepath = os.path.join(ENGYN_CONFIG.SRC_DIR,
ENGYN_CONFIG.CONVERSION_FILE)
# parse
content = utils.parse_content_to_list(filepath)
# locate
idx = utils.locate_element_with_substr(content,
'#include',
last_occ=True)
# add header include
content.insert(
idx + 1,
'#include "{}/{}.{}"'.format(ENGYN_CONFIG.SRC_DIR, self.app_stem,
ENGYN_CONFIG.HEADER_EXT))
# consolidate
content = '\n'.join(map(str, content))
# write back
status = utils.write_content(content, filepath)
if (status != 0):
self.err = status
return status
def parse_handleacs(self):
filepath = os.path.join(ENGYN_CONFIG.SRC_DIR,
ENGYN_CONFIG.HANDLEACS_FILE)
# parse
content = utils.parse_content_to_list(filepath)
# locate closing of swith case
idx = utils.locate_element_with_substr(content, 'default:')
func_name = ENGYN_CONFIG.CONSTS_APP_PREFIX + self.app_stem.upper()
# add switch case
content.insert(idx - 1, "\tbreak;".expandtabs(8))
content.insert(
idx - 1, "\tconv_layer_obj.start_app<{}>();".format(
self.app_name).expandtabs(8))
content.insert(idx - 1, "\tcase {}:".format(func_name).expandtabs(4))
content.insert(idx - 1, "")
# consolidate
content = '\n'.join(map(str, content))
# write back
status = utils.write_content(content, filepath)
if (status != 0):
self.err = status
return status
def recover_d_or_p(self, fname, index):
"""
recover data drive or 'p' drive, simply using XOR
:param fname: data name
:param index: data disk or p disk index
:return:
"""
assert 0 <= index < self.N - 1
byte_ndarray = self._read_n(fname, self.N - 1, exclude=index)
parity = utils.gen_p(byte_ndarray, ndim=1)
content = self._1darray_to_str(parity)
fpath = self.get_real_name(index, fname)
utils.write_content(fpath, content)
# check data or p
read_ndarray = self._read_n(fname, self.N - 1)
utils.check_data_p(read_ndarray)
def create_cpp_file(self):
# create empty
filepath = os.path.join(
ENGYN_CONFIG.SRC_DIR, self.app_stem,
"{}.{}".format(self.app_stem, ENGYN_CONFIG.CPP_EXT))
status = utils.touch(filepath)
if (status != 0):
self.err = status
return status
# parse content
content = utils.parse(ENGYN_CONFIG.DEFAULT_CPP)
# inject class name
content = content.replace(ENGYN_CONFIG.DEFAULT_APP_STEM, self.app_stem)
# parse content
status = utils.write_content(content, filepath)
if (status != 0):
self.err = status
return status
def recover_d_or_p(self, fname, index):
"""
recover data drive or 'p' drive, simply using XOR
:param fname: data name
:param index: data disk or p disk index
:return:
"""
assert 0 <= index < self.N - 1
byte_ndarray = self._read_n(fname, self.N - 1, exclude=index)
parity = utils.gen_p(byte_ndarray, ndim=1)
content = self._1darray_to_str(parity)
fpath = self.get_real_name(index, fname)
utils.write_content(fpath, content)
# check data or p
read_ndarray = self._read_n(fname, self.N - 1)
utils.check_data_p(read_ndarray)
def main():
# インスタンス生成済みかどうかのフラグ
_isInitialize = False
def __init__(self):
"""
初期化処理
"""
if not self._isInitialize:
self._isInitialize = True
self.clientList = {}
items = []
header = utils.display_file_content('./template/header.tpl')
#print header
items.append(header)
js1 = utils.include_column_filter('view01','')
#print js1
items.append(js1)
h1element = utils.create_block_element('h1', 'LVS List')
#print h1element
items.append(h1element)
ipaddr = utils.extract_ipaddr('192.168.0.11_txt')
#print ipaddr
items.append(ipaddr)
table = utils.create_table_with_file('view01', 'Server', './data/192.168.11.11_local.txt')
# todo: add separator
#print table
items.append(table)
footer = utils.display_file_content('./template/footer.tpl')
#print footer
items.append(footer)
html_content = ''.join(items)
utils.write_content('/tmp/output/test', html_content)
def gen_rnd_file(fname, size, content_type):
"""
randomly generate data chunk
:param fname: the name denoting the data
:param size: the size of the data chunk
:param content_type: can be 'text' or others; if it's 'text', make the chunk only contain ascii letters
:return:
"""
if content_type == 'text':
# noinspection PyUnusedLocal
content = ''.join([random.choice(string.ascii_letters) for i in xrange(size)])
else:
content = os.urandom(size)
fpath = os.path.join(config.root, fname)
if os.path.isfile(fpath):
file_size = os.stat(fpath).st_size
if file_size == size:
get_logger().warning('fname={} with size={} exists'.format(fname, size))
return
get_logger().warning('generating fname={} with size={}'.format(fname, size))
utils.write_content(fpath, content)
def recover_2d(self, fname, x, y):
"""
recover data drives (x and y)
:param fname: data name
:param x: corrupted data disk index
:param y: corrupted data disk index
:return:
"""
assert 0 <= x < self.N - 2
assert 0 <= y < self.N - 2
assert x != y
byte_ndarray = self._read_n(fname, self.N, exclude=[x, y])
DD = byte_ndarray[:-2]
P = byte_ndarray[-2:-1]
Q = byte_ndarray[-1:]
# Pxy
Pxy = utils.gen_p(DD, ndim=2)
# Qxy
Qxy = utils.gen_q(DD, ndim=2)
# Axy, Bxy
A = self.gf.Axy(x, y)
B = self.gf.Bxy(x, y)
# Dx
first = utils.gf_a_multiply_list(A, utils.gf_1darray_add(P, Pxy))
second = utils.gf_a_multiply_list(B, utils.gf_1darray_add(Q, Qxy))
Dx = utils.gf_1darray_add(
np.array(
first, dtype=config.BYTE_TYPE), np.array(
second, dtype=config.BYTE_TYPE))
Dx_content = self._1darray_to_str(Dx)
x_fpath = self.get_real_name(x, fname)
utils.write_content(x_fpath, Dx_content)
# Dy
Dy = utils.gf_1darray_add(P ^ Pxy, Dx)
Dy_content = self._1darray_to_str(Dy)
y_fpath = self.get_real_name(y, fname)
utils.write_content(y_fpath, Dy_content)
def gen_rnd_file(fname, size, content_type):
"""
randomly generate data chunk
:param fname: the name denoting the data
:param size: the size of the data chunk
:param content_type: can be 'text' or others; if it's 'text', make the chunk only contain ascii letters
:return:
"""
if content_type == 'text':
# noinspection PyUnusedLocal
content = ''.join(
[random.choice(string.ascii_letters) for i in xrange(size)])
else:
content = os.urandom(size)
fpath = os.path.join(config.root, fname)
if os.path.isfile(fpath):
file_size = os.stat(fpath).st_size
if file_size == size:
get_logger().warning('fname={} with size={} exists'.format(
fname, size))
return
get_logger().warning('generating fname={} with size={}'.format(
fname, size))
utils.write_content(fpath, content)
def parse_consts(self):
filepath = os.path.join(ENGYN_CONFIG.SRC_DIR, ENGYN_CONFIG.CONSTS_FILE)
# parse
content = utils.parse_content_to_list(filepath)
# locate closing line of struct
idx = utils.locate_element_with_substr(content, '};')
func_name = ENGYN_CONFIG.CONSTS_APP_PREFIX + self.app_stem.upper()
# add app number
content.insert(idx - 1, " {} = {},".format(func_name, self.app_id))
# consolidate
content = '\n'.join(map(str, content))
# write back
status = utils.write_content(content, filepath)
if (status != 0):
self.err = status
return status
def _corrupt(fname, index, size):
get_logger().warning("corrupting disk {}".format(index))
error_fpath = r6.get_real_name(index, fname)
error_content = os.urandom(size)
utils.write_content(error_fpath, error_content)
def _corrupt(fname, index, size):
get_logger().warning("corrupting disk {}".format(index))
error_fpath = r6.get_real_name(index, fname)
error_content = os.urandom(size)
utils.write_content(error_fpath, error_content)
def __write_impl(self, fpath, write_array):
"""
single file write
"""
content = self._1darray_to_str(write_array)
utils.write_content(fpath, content)
def __write_impl(self, fpath, write_array):
"""
single file write
"""
content = self._1darray_to_str(write_array)
utils.write_content(fpath, content)