def recover_memory(base_path, delta_path, raw_meta, raw_mem, out_path):
# Recover modified memory snapshot
# base_path: base memory snapshot, delta pages will be applied over it
# delta_path: memory overlay
# raw_meta: meta(header/footer/hash list) information of the raw memory
# raw_mem: raw memory of the base memory snapshot
# out_path: path to recovered modified memory snapshot
# Create Base Memory from meta file
base = Memory.import_from_metafile(raw_meta, raw_mem)
header_delta, footer_delta, delta_list = DeltaList.fromfile_with_footer(delta_path)
header = tool.merge_data(base.header_data, header_delta, 1024*1024)
footer = tool.merge_data(base.footer_data, footer_delta, 1024*1024*10)
_recover_modified_list(delta_list, raw_mem)
_recover_memory(base_path, delta_list, header, footer, out_path)
def recover_memory(base_path, delta_path, raw_meta, raw_mem, out_path):
# Recover modified memory snapshot
# base_path: base memory snapshot, delta pages will be applied over it
# delta_path: memory overlay
# raw_meta: meta(header/footer/hash list) information of the raw memory
# raw_mem: raw memory of the base memory snapshot
# out_path: path to recovered modified memory snapshot
# Create Base Memory from meta file
base = Memory.import_from_metafile(raw_meta, raw_mem)
header_delta, footer_delta, delta_list = DeltaList.fromfile_with_footer(
delta_path)
header = tool.merge_data(base.header_data, header_delta, 1024 * 1024)
footer = tool.merge_data(base.footer_data, footer_delta, 1024 * 1024 * 10)
_recover_modified_list(delta_list, raw_mem)
_recover_memory(base_path, delta_list, header, footer, out_path)
def recover_item(self, delta_item):
if type(delta_item) != DeltaItem:
raise MemoryError("Need list of DeltaItem")
#LOG.debug("recovering %ld/%ld" % (index, len(delta_list)))
if (delta_item.ref_id == DeltaItem.REF_RAW):
recover_data = delta_item.data
elif (delta_item.ref_id == DeltaItem.REF_ZEROS):
recover_data = self.zero_data
elif (delta_item.ref_id == DeltaItem.REF_BASE_MEM):
offset = delta_item.data
recover_data = self.raw_mem[offset:offset + self.chunk_size]
elif (delta_item.ref_id == DeltaItem.REF_BASE_DISK):
offset = delta_item.data
recover_data = self.raw_disk[offset:offset + self.chunk_size]
elif delta_item.ref_id == DeltaItem.REF_SELF:
ref_index = delta_item.data
self_ref_delta_item = self.recovered_delta_dict.get(
ref_index, None)
if self_ref_delta_item == None:
msg = "Cannot find self reference: type(%ld), offset(%ld), index(%ld), ref_index(%ld)" % \
(delta_item.delta_type, delta_item.offset, delta_item.index, ref_index)
raise MemoryError(msg)
recover_data = self_ref_delta_item.data
elif delta_item.ref_id == DeltaItem.REF_XDELTA:
patch_data = delta_item.data
patch_original_size = delta_item.offset_len
if delta_item.delta_type == DeltaItem.DELTA_MEMORY:
base_data = self.raw_mem[delta_item.offset:delta_item.offset +
patch_original_size]
elif delta_item.delta_type == DeltaItem.DELTA_DISK:
base_data = self.raw_disk[delta_item.offset:delta_item.offset +
patch_original_size]
else:
raise DeltaError("Delta type should be either disk or memory")
recover_data = tool.merge_data(base_data, patch_data,
len(base_data) * 5)
else:
raise MemoryError("Cannot recover: invalid referce id %d" %
delta_item.ref_id)
if len(recover_data) != delta_item.offset_len:
msg = "Error, Recovered Size Error: %d, %d, ref_id: %s, data_len: %ld, offset: %ld, offset_len: %ld" % \
(delta_item.delta_type, len(recover_data), delta_item.ref_id, \
delta_item.data_len, delta_item.offset, delta_item.offset_len)
raise MemoryError(msg)
# recover
delta_item.ref_id = DeltaItem.REF_RAW
delta_item.data = recover_data
return delta_item
def recover_item(self, delta_item):
if type(delta_item) != DeltaItem:
raise MemoryError("Need list of DeltaItem")
#LOG.debug("recovering %ld/%ld" % (index, len(delta_list)))
if (delta_item.ref_id == DeltaItem.REF_RAW):
recover_data = delta_item.data
elif (delta_item.ref_id == DeltaItem.REF_ZEROS):
recover_data = self.zero_data
elif (delta_item.ref_id == DeltaItem.REF_BASE_MEM):
offset = delta_item.data
recover_data = self.raw_mem[offset:offset+self.chunk_size]
elif (delta_item.ref_id == DeltaItem.REF_BASE_DISK):
offset = delta_item.data
recover_data = self.raw_disk[offset:offset+self.chunk_size]
elif delta_item.ref_id == DeltaItem.REF_SELF:
ref_index = delta_item.data
self_ref_delta_item = self.recovered_delta_dict.get(ref_index, None)
if self_ref_delta_item == None:
msg = "Cannot find self reference: type(%ld), offset(%ld), index(%ld), ref_index(%ld)" % \
(delta_item.delta_type, delta_item.offset, delta_item.index, ref_index)
raise MemoryError(msg)
recover_data = self_ref_delta_item.data
elif delta_item.ref_id == DeltaItem.REF_XDELTA:
patch_data = delta_item.data
patch_original_size = delta_item.offset_len
if delta_item.delta_type == DeltaItem.DELTA_MEMORY:
base_data = self.raw_mem[delta_item.offset:delta_item.offset+patch_original_size]
elif delta_item.delta_type == DeltaItem.DELTA_DISK:
base_data = self.raw_disk[delta_item.offset:delta_item.offset+patch_original_size]
else:
raise DeltaError("Delta type should be either disk or memory")
recover_data = tool.merge_data(base_data, patch_data, len(base_data)*5)
else:
raise MemoryError("Cannot recover: invalid referce id %d" % delta_item.ref_id)
if len(recover_data) != delta_item.offset_len:
msg = "Error, Recovered Size Error: %d, %d, ref_id: %s, data_len: %ld, offset: %ld, offset_len: %ld" % \
(delta_item.delta_type, len(recover_data), delta_item.ref_id, \
delta_item.data_len, delta_item.offset, delta_item.offset_len)
raise MemoryError(msg)
# recover
delta_item.ref_id = DeltaItem.REF_RAW
delta_item.data = recover_data
return delta_item
def _recover_modified_list(delta_list, raw_path):
raw_file = open(raw_path, "rb")
raw_mmap = mmap.mmap(raw_file.fileno(), 0, prot=mmap.PROT_READ)
delta_list.sort(key=itemgetter('offset'))
for index, delta_item in enumerate(delta_list):
#print "processing %d/%d, ref_id: %d, offset: %ld" % (index, len(delta_list), delta_item.ref_id, delta_item.offset)
if delta_item.ref_id == DeltaItem.REF_RAW:
continue
elif delta_item.ref_id == DeltaItem.REF_BASE_MEM:
offset = delta_item.data
recover_data = raw_mmap[offset:offset + Memory.RAM_PAGE_SIZE]
elif delta_item.ref_id == DeltaItem.REF_SELF:
ref_offset = delta_item.data
index = 0
while index < len(delta_list):
#print "self referencing : %ld == %ld" % (delta_list[index].offset, ref_offset)
if delta_list[index].offset == ref_offset:
recover_data = delta_list[index].data
break
index += 1
if index >= len(delta_list):
raise MemoryError("Cannot find self reference")
elif delta_item.ref_id == DeltaItem.REF_XDELTA:
patch_data = delta_item.data
base_data = raw_mmap[delta_item.offset:delta_item.offset +
Memory.RAM_PAGE_SIZE]
recover_data = tool.merge_data(base_data, patch_data,
len(base_data) * 2)
else:
raise MemoryError("Cannot recover: invalid referce id %d" %
delta_item.ref_id)
if len(recover_data) != Memory.RAM_PAGE_SIZE:
msg = "Recovered Size Error: %d, ref_id: %d, %ld, %ld" % \
(len(recover_data), delta_item.ref_id, delta_item.data_len, delta_item.data)
raise MemoryError(msg)
delta_item.ref_id = DeltaItem.REF_RAW
delta_item.data = recover_data
raw_file.close()
def _recover_modified_list(delta_list, raw_path):
raw_file = open(raw_path, "rb")
raw_mmap = mmap.mmap(raw_file.fileno(), 0, prot=mmap.PROT_READ)
delta_list.sort(key=itemgetter('offset'))
for index, delta_item in enumerate(delta_list):
#print "processing %d/%d, ref_id: %d, offset: %ld" % (index, len(delta_list), delta_item.ref_id, delta_item.offset)
if delta_item.ref_id == DeltaItem.REF_RAW:
continue
elif delta_item.ref_id == DeltaItem.REF_BASE_MEM:
offset = delta_item.data
recover_data = raw_mmap[offset:offset+Memory.RAM_PAGE_SIZE]
elif delta_item.ref_id == DeltaItem.REF_SELF:
ref_offset = delta_item.data
index = 0
while index < len(delta_list):
#print "self referencing : %ld == %ld" % (delta_list[index].offset, ref_offset)
if delta_list[index].offset == ref_offset:
recover_data = delta_list[index].data
break
index += 1
if index >= len(delta_list):
raise MemoryError("Cannot find self reference")
elif delta_item.ref_id == DeltaItem.REF_XDELTA:
patch_data = delta_item.data
base_data = raw_mmap[delta_item.offset:delta_item.offset+Memory.RAM_PAGE_SIZE]
recover_data = tool.merge_data(base_data, patch_data, len(base_data)*2)
else:
raise MemoryError("Cannot recover: invalid referce id %d" % delta_item.ref_id)
if len(recover_data) != Memory.RAM_PAGE_SIZE:
msg = "Recovered Size Error: %d, ref_id: %d, %ld, %ld" % \
(len(recover_data), delta_item.ref_id, delta_item.data_len, delta_item.data)
raise MemoryError(msg)
delta_item.ref_id = DeltaItem.REF_RAW
delta_item.data = recover_data
raw_file.close()
def recover_item(self, delta_item, delta_counter, delta_times):
if type(delta_item) != DeltaItem:
raise StreamSynthesisError("Need list of DeltaItem")
delta_counter[delta_item.ref_id] += 1
start_time = time.time()
if (delta_item.ref_id == DeltaItem.REF_RAW):
recover_data = delta_item.data
elif (delta_item.ref_id == DeltaItem.REF_ZEROS):
recover_data = self.zero_data
elif (delta_item.ref_id == DeltaItem.REF_BASE_MEM):
offset = delta_item.data
recover_data = self.raw_mem[offset:offset + self.chunk_size]
elif (delta_item.ref_id == DeltaItem.REF_BASE_DISK):
offset = delta_item.data
recover_data = self.raw_disk[offset:offset + self.chunk_size]
elif delta_item.ref_id == DeltaItem.REF_SELF:
ref_index = delta_item.data
self_ref_delta_item = self.recovered_delta_dict.get(
ref_index, None)
if self_ref_delta_item is None:
#msg = "Cannot find self reference: type(%ld), offset(%ld), index(%ld), ref_index(%ld)" % \
# (delta_item.delta_type, delta_item.offset, delta_item.index, ref_index)
#raise StreamSynthesisError(msg)
return None
recover_data = self_ref_delta_item.data
elif delta_item.ref_id == DeltaItem.REF_SELF_HASH:
ref_hashvalue = delta_item.data
self_ref_delta_item = self.recovered_hash_dict.get(
ref_hashvalue, None)
if self_ref_delta_item == None:
return None
recover_data = self_ref_delta_item.data
delta_item.hash_value = ref_hashvalue
elif delta_item.ref_id == DeltaItem.REF_XDELTA:
patch_data = delta_item.data
patch_original_size = delta_item.offset_len
if delta_item.delta_type == DeltaItem.DELTA_MEMORY or\
delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE:
base_data = self.raw_mem[delta_item.offset:delta_item.offset +
patch_original_size]
elif delta_item.delta_type == DeltaItem.DELTA_DISK or\
delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE:
base_data = self.raw_disk[delta_item.offset:delta_item.offset +
patch_original_size]
else:
raise StreamSynthesisError(
"Delta type should be either disk or memory")
recover_data = tool.merge_data(base_data, patch_data,
len(base_data) * 5)
elif delta_item.ref_id == DeltaItem.REF_BSDIFF:
patch_data = delta_item.data
patch_original_size = delta_item.offset_len
if delta_item.delta_type == DeltaItem.DELTA_MEMORY or\
delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE:
base_data = self.raw_mem[delta_item.offset:delta_item.offset +
patch_original_size]
elif delta_item.delta_type == DeltaItem.DELTA_DISK or\
delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE:
base_data = self.raw_disk[delta_item.offset:delta_item.offset +
patch_original_size]
else:
raise StreamSynthesisError(
"Delta type should be either disk or memory")
recover_data = tool.merge_data_bsdiff(base_data, patch_data)
elif delta_item.ref_id == DeltaItem.REF_XOR:
patch_data = delta_item.data
patch_original_size = delta_item.offset_len
if delta_item.delta_type == DeltaItem.DELTA_MEMORY or\
delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE:
base_data = self.raw_mem[delta_item.offset:delta_item.offset +
patch_original_size]
elif delta_item.delta_type == DeltaItem.DELTA_DISK or\
delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE:
base_data = self.raw_disk[delta_item.offset:delta_item.offset +
patch_original_size]
else:
raise StreamSynthesisError(
"Delta type should be either disk or memory")
recover_data = tool.cython_xor(base_data, patch_data)
else:
raise StreamSynthesisError(
"Cannot recover: invalid referce id %d" % delta_item.ref_id)
if len(recover_data) != delta_item.offset_len:
msg = "Error, Recovered Size Error: %d, %d, ref_id: %s, data_len: %ld, offset: %ld, offset_len: %ld" % \
(delta_item.delta_type, len(recover_data), delta_item.ref_id, \
delta_item.data_len, delta_item.offset, delta_item.offset_len)
print msg
raise StreamSynthesisError(msg)
delta_times[delta_item.ref_id] += (time.time() - start_time)
ref_id = delta_item.ref_id
# recover
delta_item.ref_id = DeltaItem.REF_RAW
delta_item.data = recover_data
if delta_item.delta_type == DeltaItem.DELTA_MEMORY or delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE:
self.analysis_queue.put("M,R(%d),%d" %
(ref_id, delta_item.offset / 4096))
elif delta_item.delta_type == DeltaItem.DELTA_DISK or delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE:
self.analysis_queue.put("D,R(%d),%d" %
(ref_id, delta_item.offset / 4096))
if delta_item.hash_value is None or len(delta_item.hash_value) == 0:
delta_counter['sha'] += 1
start_time = time.time()
delta_item.hash_value = sha256(recover_data).digest()
delta_times['sha'] += (time.time() - start_time)
return delta_item
def recover_item(self, delta_item):
if type(delta_item) != DeltaItem:
raise StreamSynthesisError("Need list of DeltaItem")
if (delta_item.ref_id == DeltaItem.REF_RAW):
recover_data = delta_item.data
elif (delta_item.ref_id == DeltaItem.REF_ZEROS):
recover_data = self.zero_data
elif (delta_item.ref_id == DeltaItem.REF_BASE_MEM):
offset = delta_item.data
recover_data = self.raw_mem[offset:offset+self.chunk_size]
elif (delta_item.ref_id == DeltaItem.REF_BASE_DISK):
offset = delta_item.data
recover_data = self.raw_disk[offset:offset+self.chunk_size]
elif delta_item.ref_id == DeltaItem.REF_SELF:
ref_index = delta_item.data
self_ref_delta_item = self.recovered_delta_dict.get(ref_index, None)
if self_ref_delta_item == None:
#msg = "Cannot find self reference: type(%ld), offset(%ld), index(%ld), ref_index(%ld)" % \
# (delta_item.delta_type, delta_item.offset, delta_item.index, ref_index)
#raise StreamSynthesisError(msg)
return None
recover_data = self_ref_delta_item.data
elif delta_item.ref_id == DeltaItem.REF_SELF_HASH:
ref_hashvalue = delta_item.data
self_ref_delta_item = self.recovered_hash_dict.get(ref_hashvalue, None)
if self_ref_delta_item == None:
return None
recover_data = self_ref_delta_item.data
delta_item.hash_value = ref_hashvalue
elif delta_item.ref_id == DeltaItem.REF_XDELTA:
patch_data = delta_item.data
patch_original_size = delta_item.offset_len
if delta_item.delta_type == DeltaItem.DELTA_MEMORY or\
delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE:
base_data = self.raw_mem[delta_item.offset:delta_item.offset+patch_original_size]
elif delta_item.delta_type == DeltaItem.DELTA_DISK or\
delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE:
base_data = self.raw_disk[delta_item.offset:delta_item.offset+patch_original_size]
else:
raise StreamSynthesisError("Delta type should be either disk or memory")
recover_data = tool.merge_data(base_data, patch_data, len(base_data)*5)
elif delta_item.ref_id == DeltaItem.REF_BSDIFF:
patch_data = delta_item.data
patch_original_size = delta_item.offset_len
if delta_item.delta_type == DeltaItem.DELTA_MEMORY or\
delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE:
base_data = self.raw_mem[delta_item.offset:delta_item.offset+patch_original_size]
elif delta_item.delta_type == DeltaItem.DELTA_DISK or\
delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE:
base_data = self.raw_disk[delta_item.offset:delta_item.offset+patch_original_size]
else:
raise DeltaError("Delta type should be either disk or memory")
recover_data = tool.merge_data_bsdiff(base_data, patch_data)
elif delta_item.ref_id == DeltaItem.REF_XOR:
patch_data = delta_item.data
patch_original_size = delta_item.offset_len
if delta_item.delta_type == DeltaItem.DELTA_MEMORY or\
delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE:
base_data = self.raw_mem[delta_item.offset:delta_item.offset+patch_original_size]
elif delta_item.delta_type == DeltaItem.DELTA_DISK or\
delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE:
base_data = self.raw_disk[delta_item.offset:delta_item.offset+patch_original_size]
else:
raise DeltaError("Delta type should be either disk or memory")
recover_data = tool.cython_xor(base_data, patch_data)
else:
raise StreamSynthesisError("Cannot recover: invalid referce id %d" % delta_item.ref_id)
if len(recover_data) != delta_item.offset_len:
msg = "Error, Recovered Size Error: %d, %d, ref_id: %s, data_len: %ld, offset: %ld, offset_len: %ld" % \
(delta_item.delta_type, len(recover_data), delta_item.ref_id, \
delta_item.data_len, delta_item.offset, delta_item.offset_len)
print msg
raise StreamSynthesisError(msg)
# recover
delta_item.ref_id = DeltaItem.REF_RAW
delta_item.data = recover_data
if delta_item.hash_value == None or len(delta_item.hash_value) == 0:
delta_item.hash_value = sha256(recover_data).digest()
return delta_item
base.get_delta(delta_list, ref_id=DeltaItem.REF_BASE_MEM)
# 3.find shared within self
print "[Debug] get delta from itself"
DeltaList.get_self_delta(delta_list)
DeltaList.statistics(delta_list)
DeltaList.tofile_with_footer(header_delta, footer_delta, delta_list,
out_path)
elif command == "recover":
if (not settings.base_file) or (not settings.delta_file):
sys.stderr.write("Error, Cannot find base/delta file. See help\n")
sys.exit(1)
base_path = settings.base_file
delta_path = settings.delta_file
raw_path = settings.base_file + EXT_RAW
meta_path = settings.base_file + EXT_META
# Create Base Memory from meta file
base = Memory.import_from_metafile(meta_path, raw_path)
header_delta, footer_delta, delta_list = DeltaList.fromfile_with_footer(
delta_path)
header = tool.merge_data(base.header_data, header_delta, 1024 * 1024)
footer = tool.merge_data(base.footer_data, footer_delta,
1024 * 1024 * 10)
_recover_modified_list(delta_list, raw_path)
out_path = base_path + ".recover"
_recover_memory(delta_list, header, footer, out_path)
# 2.find shared with base memory
print "[Debug] get delta from base Memory"
base.get_delta(delta_list, ref_id=DeltaItem.REF_BASE_MEM)
# 3.find shared within self
print "[Debug] get delta from itself"
DeltaList.get_self_delta(delta_list)
DeltaList.statistics(delta_list)
DeltaList.tofile_with_footer(header_delta, footer_delta, delta_list, out_path)
elif command == "recover":
if (not settings.base_file) or (not settings.delta_file):
sys.stderr.write("Error, Cannot find base/delta file. See help\n")
sys.exit(1)
base_path = settings.base_file
delta_path = settings.delta_file
raw_path = settings.base_file + EXT_RAW
meta_path = settings.base_file + EXT_META
# Create Base Memory from meta file
base = Memory.import_from_metafile(meta_path, raw_path)
header_delta, footer_delta, delta_list = DeltaList.fromfile_with_footer(delta_path)
header = tool.merge_data(base.header_data, header_delta, 1024*1024)
footer = tool.merge_data(base.footer_data, footer_delta, 1024*1024*10)
_recover_modified_list(delta_list, raw_path)
out_path = base_path+".recover"
_recover_memory(delta_list, header, footer, out_path)
