def recover_disk(base_disk, base_mem, overlay_mem, overlay_disk, recover_path,
chunk_size):
recover_fd = open(recover_path, "wb")
# get delta list from file and recover it to origin
delta_stream = open(overlay_disk, "r")
recovered_memory = Recovered_delta(base_disk, base_mem, chunk_size, \
parent=base_disk, overlay_memory=overlay_mem)
for delta_item in DeltaList.from_stream(delta_stream):
recovered_memory.recover_item(delta_item)
delta_list = recovered_memory.delta_list
# overlay map
chunk_list = []
# sort delta list using offset
delta_list.sort(key=itemgetter('offset'))
for delta_item in delta_list:
if len(delta_item.data) != chunk_size:
raise DiskError("recovered size is not same as page size")
chunk_list.append("%ld:1" % (delta_item.offset / chunk_size))
recover_fd.seek(delta_item.offset)
recover_fd.write(delta_item.data)
last_write_offset = delta_item.offset + len(delta_item.data)
# fill zero to the end of the modified file
if last_write_offset:
diff_offset = os.path.getsize(base_disk) - last_write_offset
if diff_offset > 0:
recover_fd.seek(diff_offset - 1, os.SEEK_CUR)
recover_fd.write('0')
recover_fd.close()
# overlay chunk format: chunk_1:1,chunk_2:1,...
return ','.join(chunk_list)
def recover_disk(base_disk, base_mem, overlay_mem, overlay_disk, recover_path, chunk_size):
recover_fd = open(recover_path, "wb")
# get delta list from file and recover it to origin
delta_stream = open(overlay_disk, "r")
recovered_memory = Recovered_delta(base_disk, base_mem, chunk_size, \
parent=base_disk, overlay_memory=overlay_mem)
for delta_item in DeltaList.from_stream(delta_stream):
recovered_memory.recover_item(delta_item)
delta_list = recovered_memory.delta_list
# overlay map
chunk_list = []
# sort delta list using offset
delta_list.sort(key=itemgetter('offset'))
for delta_item in delta_list:
if len(delta_item.data) != chunk_size:
raise DiskError("recovered size is not same as page size")
chunk_list.append("%ld:1" % (delta_item.offset/chunk_size))
recover_fd.seek(delta_item.offset)
recover_fd.write(delta_item.data)
last_write_offset = delta_item.offset + len(delta_item.data)
# fill zero to the end of the modified file
if last_write_offset:
diff_offset = os.path.getsize(base_disk) - last_write_offset
if diff_offset > 0:
recover_fd.seek(diff_offset-1, os.SEEK_CUR)
recover_fd.write('0')
recover_fd.close()
# overlay chunk format: chunk_1:1,chunk_2:1,...
return ','.join(chunk_list)
def create_memory_overlay(raw_meta, raw_mem, modified_mem, out_delta, print_out=sys.stdout):
# get memory delta
# raw_meta: meta data path of raw memory, e.g. hash_list+header+footer
# raw_mem: raw memory path
# modified_mem: modified memory path
# out_delta: output path of final delta
# Create Base Memory from meta file
base = Memory.import_from_metafile(raw_meta, raw_mem)
# 1.get modified page
print_out.write("[Debug] 1.get modified page list\n")
header_delta, footer_delta, original_delta_list = base.get_modified(modified_mem)
delta_list = []
for item in original_delta_list:
delta_item = DeltaItem(item.offset, item.offset_len,
hash_value=item.hash_value,
ref_id=item.ref_id,
data_len=item.data_len,
data=item.data)
delta_list.append(delta_item)
# 2.find shared with base memory
print_out.write("[Debug] 2.get delta from base Memory\n")
base.get_delta(delta_list, ref_id=DeltaItem.REF_BASE_MEM)
# 3.find shared within self
print_out.write("[Debug] 3.get delta from itself\n")
DeltaList.get_self_delta(delta_list)
DeltaList.statistics(delta_list, print_out)
DeltaList.tofile_with_footer(header_delta, footer_delta, delta_list, out_delta)
def recover_memory(base_path, delta_path, raw_meta, raw_mem, out_path):
# Recover modified memory snapshot
# base_path: base memory snapshot, delta pages will be applied over it
# delta_path: memory overlay
# raw_meta: meta(header/footer/hash list) information of the raw memory
# raw_mem: raw memory of the base memory snapshot
# out_path: path to recovered modified memory snapshot
# Create Base Memory from meta file
base = Memory.import_from_metafile(raw_meta, raw_mem)
header_delta, footer_delta, delta_list = DeltaList.fromfile_with_footer(delta_path)
header = tool.merge_data(base.header_data, header_delta, 1024*1024)
footer = tool.merge_data(base.footer_data, footer_delta, 1024*1024*10)
_recover_modified_list(delta_list, raw_mem)
_recover_memory(base_path, delta_list, header, footer, out_path)
def recover_memory(base_path, delta_path, raw_meta, raw_mem, out_path):
# Recover modified memory snapshot
# base_path: base memory snapshot, delta pages will be applied over it
# delta_path: memory overlay
# raw_meta: meta(header/footer/hash list) information of the raw memory
# raw_mem: raw memory of the base memory snapshot
# out_path: path to recovered modified memory snapshot
# Create Base Memory from meta file
base = Memory.import_from_metafile(raw_meta, raw_mem)
header_delta, footer_delta, delta_list = DeltaList.fromfile_with_footer(
delta_path)
header = tool.merge_data(base.header_data, header_delta, 1024 * 1024)
footer = tool.merge_data(base.footer_data, footer_delta, 1024 * 1024 * 10)
_recover_modified_list(delta_list, raw_mem)
_recover_memory(base_path, delta_list, header, footer, out_path)
def create_memory_overlay(raw_meta,
raw_mem,
modified_mem,
out_delta,
print_out=sys.stdout):
# get memory delta
# raw_meta: meta data path of raw memory, e.g. hash_list+header+footer
# raw_mem: raw memory path
# modified_mem: modified memory path
# out_delta: output path of final delta
# Create Base Memory from meta file
base = Memory.import_from_metafile(raw_meta, raw_mem)
# 1.get modified page
print_out.write("[Debug] 1.get modified page list\n")
header_delta, footer_delta, original_delta_list = base.get_modified(
modified_mem)
delta_list = []
for item in original_delta_list:
delta_item = DeltaItem(item.offset,
item.offset_len,
hash_value=item.hash_value,
ref_id=item.ref_id,
data_len=item.data_len,
data=item.data)
delta_list.append(delta_item)
# 2.find shared with base memory
print_out.write("[Debug] 2.get delta from base Memory\n")
base.get_delta(delta_list, ref_id=DeltaItem.REF_BASE_MEM)
# 3.find shared within self
print_out.write("[Debug] 3.get delta from itself\n")
DeltaList.get_self_delta(delta_list)
DeltaList.statistics(delta_list, print_out)
DeltaList.tofile_with_footer(header_delta, footer_delta, delta_list,
out_delta)
raise MemoryError("footer data is different")
print "[SUCCESS] meta file information is matched with original"
elif command == "delta":
if (not settings.mig_file) or (not settings.base_file):
sys.stderr.write("Error, Cannot find modified memory file. See help\n")
sys.exit(1)
raw_path = settings.base_file
meta_path = settings.base_file + EXT_META
modi_mem_path = settings.mig_file
out_path = settings.mig_file + ".delta"
#delta_list = create_memory_overlay(modi_mem_path, raw_path, modi_mem_path, out_path, print_out=sys.stdout)
mem_deltalist= create_memory_deltalist(modi_mem_path,
basemem_meta=meta_path, basemem_path=raw_path,
print_out=sys.stdout)
DeltaList.statistics(mem_deltalist, print_out=sys.stdout)
DeltaList.tofile(mem_deltalist, modi_mem_path + ".delta")
elif command == "recover":
if (not settings.base_file) or (not settings.delta_file):
sys.stderr.write("Error, Cannot find base/delta file. See help\n")
sys.exit(1)
base_mem = settings.base_file
overlay_mem = settings.delta_file
base_memmeta = settings.base_file + EXT_META
out_path = base_mem + ".recover"
memory_overlay_map = recover_memory(None, base_mem, overlay_mem, \
base_memmeta, out_path, verify_with_original="./tmp/modi")
raise MemoryError("footer data is different")
LOG.info("meta file information is matched with original")
elif command == "delta":
if (not settings.mig_file) or (not settings.base_file):
sys.stderr.write("Error, Cannot find modified memory file. See help\n")
sys.exit(1)
raw_path = settings.base_file
meta_path = settings.base_file + EXT_META
modi_mem_path = settings.mig_file
out_path = settings.mig_file + ".delta"
#delta_list = create_memory_overlay(modi_mem_path, raw_path, \
# modi_mem_path, out_path)
mem_deltalist= create_memory_deltalist(modi_mem_path,
basemem_meta=meta_path, basemem_path=raw_path)
DeltaList.statistics(mem_deltalist)
DeltaList.tofile(mem_deltalist, modi_mem_path + ".delta")
elif command == "recover":
if (not settings.base_file) or (not settings.delta_file):
sys.stderr.write("Error, Cannot find base/delta file. See help\n")
sys.exit(1)
base_mem = settings.base_file
overlay_mem = settings.delta_file
base_memmeta = settings.base_file + EXT_META
out_path = base_mem + ".recover"
memory_overlay_map = recover_memory(None, base_mem, overlay_mem, \
base_memmeta, out_path, verify_with_original="./tmp/modi")
for item in original_delta_list:
delta_item = DeltaItem(item.offset,
item.offset_len,
hash_value=item.hash_value,
ref_id=item.ref_id,
data_len=item.data_len,
data=item.data)
delta_list.append(delta_item)
# 2.find shared with base memory
print "[Debug] get delta from base Memory"
base.get_delta(delta_list, ref_id=DeltaItem.REF_BASE_MEM)
# 3.find shared within self
print "[Debug] get delta from itself"
DeltaList.get_self_delta(delta_list)
DeltaList.statistics(delta_list)
DeltaList.tofile_with_footer(header_delta, footer_delta, delta_list,
out_path)
elif command == "recover":
if (not settings.base_file) or (not settings.delta_file):
sys.stderr.write("Error, Cannot find base/delta file. See help\n")
sys.exit(1)
base_path = settings.base_file
delta_path = settings.delta_file
raw_path = settings.base_file + EXT_RAW
meta_path = settings.base_file + EXT_META
# Create Base Memory from meta file
raise MemoryError("footer data is different")
LOG.info("meta file information is matched with original")
elif command == "delta":
if (not settings.mig_file) or (not settings.base_file):
sys.stderr.write("Error, Cannot find modified memory file. See help\n")
sys.exit(1)
raw_path = settings.base_file
meta_path = settings.base_file + EXT_META
modi_mem_path = settings.mig_file
out_path = settings.mig_file + ".delta"
#delta_list = create_memory_overlay(modi_mem_path, raw_path, \
# modi_mem_path, out_path)
mem_deltalist= create_memory_deltalist(modi_mem_path,
basemem_meta=meta_path, basemem_path=raw_path)
DeltaList.statistics(mem_deltalist)
DeltaList.tofile(mem_deltalist, modi_mem_path + ".delta")
elif command == "recover":
if (not settings.base_file) or (not settings.delta_file):
sys.stderr.write("Error, Cannot find base/delta file. See help\n")
sys.exit(1)
base_mem = settings.base_file
overlay_mem = settings.delta_file
base_memmeta = settings.base_file + EXT_META
out_path = base_mem + ".recover"
memory_overlay_map = recover_memory(None, base_mem, overlay_mem, \
base_memmeta, out_path, verify_with_original="./tmp/modi")
delta_list = []
for item in original_delta_list:
delta_item = DeltaItem(item.offset, item.offset_len,
hash_value=item.hash_value,
ref_id=item.ref_id,
data_len=item.data_len,
data=item.data)
delta_list.append(delta_item)
# 2.find shared with base memory
print "[Debug] get delta from base Memory"
base.get_delta(delta_list, ref_id=DeltaItem.REF_BASE_MEM)
# 3.find shared within self
print "[Debug] get delta from itself"
DeltaList.get_self_delta(delta_list)
DeltaList.statistics(delta_list)
DeltaList.tofile_with_footer(header_delta, footer_delta, delta_list, out_path)
elif command == "recover":
if (not settings.base_file) or (not settings.delta_file):
sys.stderr.write("Error, Cannot find base/delta file. See help\n")
sys.exit(1)
base_path = settings.base_file
delta_path = settings.delta_file
raw_path = settings.base_file + EXT_RAW
meta_path = settings.base_file + EXT_META
# Create Base Memory from meta file
base = Memory.import_from_metafile(meta_path, raw_path)