def recover_memory(base_path, delta_path, raw_meta, raw_mem, out_path): # Recover modified memory snapshot # base_path: base memory snapshot, delta pages will be applied over it # delta_path: memory overlay # raw_meta: meta(header/footer/hash list) information of the raw memory # raw_mem: raw memory of the base memory snapshot # out_path: path to recovered modified memory snapshot # Create Base Memory from meta file base = Memory.import_from_metafile(raw_meta, raw_mem) header_delta, footer_delta, delta_list = DeltaList.fromfile_with_footer(delta_path) header = tool.merge_data(base.header_data, header_delta, 1024*1024) footer = tool.merge_data(base.footer_data, footer_delta, 1024*1024*10) _recover_modified_list(delta_list, raw_mem) _recover_memory(base_path, delta_list, header, footer, out_path)
def recover_memory(base_path, delta_path, raw_meta, raw_mem, out_path): # Recover modified memory snapshot # base_path: base memory snapshot, delta pages will be applied over it # delta_path: memory overlay # raw_meta: meta(header/footer/hash list) information of the raw memory # raw_mem: raw memory of the base memory snapshot # out_path: path to recovered modified memory snapshot # Create Base Memory from meta file base = Memory.import_from_metafile(raw_meta, raw_mem) header_delta, footer_delta, delta_list = DeltaList.fromfile_with_footer( delta_path) header = tool.merge_data(base.header_data, header_delta, 1024 * 1024) footer = tool.merge_data(base.footer_data, footer_delta, 1024 * 1024 * 10) _recover_modified_list(delta_list, raw_mem) _recover_memory(base_path, delta_list, header, footer, out_path)
def recover_item(self, delta_item): if type(delta_item) != DeltaItem: raise MemoryError("Need list of DeltaItem") #LOG.debug("recovering %ld/%ld" % (index, len(delta_list))) if (delta_item.ref_id == DeltaItem.REF_RAW): recover_data = delta_item.data elif (delta_item.ref_id == DeltaItem.REF_ZEROS): recover_data = self.zero_data elif (delta_item.ref_id == DeltaItem.REF_BASE_MEM): offset = delta_item.data recover_data = self.raw_mem[offset:offset + self.chunk_size] elif (delta_item.ref_id == DeltaItem.REF_BASE_DISK): offset = delta_item.data recover_data = self.raw_disk[offset:offset + self.chunk_size] elif delta_item.ref_id == DeltaItem.REF_SELF: ref_index = delta_item.data self_ref_delta_item = self.recovered_delta_dict.get( ref_index, None) if self_ref_delta_item == None: msg = "Cannot find self reference: type(%ld), offset(%ld), index(%ld), ref_index(%ld)" % \ (delta_item.delta_type, delta_item.offset, delta_item.index, ref_index) raise MemoryError(msg) recover_data = self_ref_delta_item.data elif delta_item.ref_id == DeltaItem.REF_XDELTA: patch_data = delta_item.data patch_original_size = delta_item.offset_len if delta_item.delta_type == DeltaItem.DELTA_MEMORY: base_data = self.raw_mem[delta_item.offset:delta_item.offset + patch_original_size] elif delta_item.delta_type == DeltaItem.DELTA_DISK: base_data = self.raw_disk[delta_item.offset:delta_item.offset + patch_original_size] else: raise DeltaError("Delta type should be either disk or memory") recover_data = tool.merge_data(base_data, patch_data, len(base_data) * 5) else: raise MemoryError("Cannot recover: invalid referce id %d" % delta_item.ref_id) if len(recover_data) != delta_item.offset_len: msg = "Error, Recovered Size Error: %d, %d, ref_id: %s, data_len: %ld, offset: %ld, offset_len: %ld" % \ (delta_item.delta_type, len(recover_data), delta_item.ref_id, \ delta_item.data_len, delta_item.offset, delta_item.offset_len) raise MemoryError(msg) # recover delta_item.ref_id = DeltaItem.REF_RAW delta_item.data = recover_data return delta_item
def recover_item(self, delta_item): if type(delta_item) != DeltaItem: raise MemoryError("Need list of DeltaItem") #LOG.debug("recovering %ld/%ld" % (index, len(delta_list))) if (delta_item.ref_id == DeltaItem.REF_RAW): recover_data = delta_item.data elif (delta_item.ref_id == DeltaItem.REF_ZEROS): recover_data = self.zero_data elif (delta_item.ref_id == DeltaItem.REF_BASE_MEM): offset = delta_item.data recover_data = self.raw_mem[offset:offset+self.chunk_size] elif (delta_item.ref_id == DeltaItem.REF_BASE_DISK): offset = delta_item.data recover_data = self.raw_disk[offset:offset+self.chunk_size] elif delta_item.ref_id == DeltaItem.REF_SELF: ref_index = delta_item.data self_ref_delta_item = self.recovered_delta_dict.get(ref_index, None) if self_ref_delta_item == None: msg = "Cannot find self reference: type(%ld), offset(%ld), index(%ld), ref_index(%ld)" % \ (delta_item.delta_type, delta_item.offset, delta_item.index, ref_index) raise MemoryError(msg) recover_data = self_ref_delta_item.data elif delta_item.ref_id == DeltaItem.REF_XDELTA: patch_data = delta_item.data patch_original_size = delta_item.offset_len if delta_item.delta_type == DeltaItem.DELTA_MEMORY: base_data = self.raw_mem[delta_item.offset:delta_item.offset+patch_original_size] elif delta_item.delta_type == DeltaItem.DELTA_DISK: base_data = self.raw_disk[delta_item.offset:delta_item.offset+patch_original_size] else: raise DeltaError("Delta type should be either disk or memory") recover_data = tool.merge_data(base_data, patch_data, len(base_data)*5) else: raise MemoryError("Cannot recover: invalid referce id %d" % delta_item.ref_id) if len(recover_data) != delta_item.offset_len: msg = "Error, Recovered Size Error: %d, %d, ref_id: %s, data_len: %ld, offset: %ld, offset_len: %ld" % \ (delta_item.delta_type, len(recover_data), delta_item.ref_id, \ delta_item.data_len, delta_item.offset, delta_item.offset_len) raise MemoryError(msg) # recover delta_item.ref_id = DeltaItem.REF_RAW delta_item.data = recover_data return delta_item
def _recover_modified_list(delta_list, raw_path): raw_file = open(raw_path, "rb") raw_mmap = mmap.mmap(raw_file.fileno(), 0, prot=mmap.PROT_READ) delta_list.sort(key=itemgetter('offset')) for index, delta_item in enumerate(delta_list): #print "processing %d/%d, ref_id: %d, offset: %ld" % (index, len(delta_list), delta_item.ref_id, delta_item.offset) if delta_item.ref_id == DeltaItem.REF_RAW: continue elif delta_item.ref_id == DeltaItem.REF_BASE_MEM: offset = delta_item.data recover_data = raw_mmap[offset:offset + Memory.RAM_PAGE_SIZE] elif delta_item.ref_id == DeltaItem.REF_SELF: ref_offset = delta_item.data index = 0 while index < len(delta_list): #print "self referencing : %ld == %ld" % (delta_list[index].offset, ref_offset) if delta_list[index].offset == ref_offset: recover_data = delta_list[index].data break index += 1 if index >= len(delta_list): raise MemoryError("Cannot find self reference") elif delta_item.ref_id == DeltaItem.REF_XDELTA: patch_data = delta_item.data base_data = raw_mmap[delta_item.offset:delta_item.offset + Memory.RAM_PAGE_SIZE] recover_data = tool.merge_data(base_data, patch_data, len(base_data) * 2) else: raise MemoryError("Cannot recover: invalid referce id %d" % delta_item.ref_id) if len(recover_data) != Memory.RAM_PAGE_SIZE: msg = "Recovered Size Error: %d, ref_id: %d, %ld, %ld" % \ (len(recover_data), delta_item.ref_id, delta_item.data_len, delta_item.data) raise MemoryError(msg) delta_item.ref_id = DeltaItem.REF_RAW delta_item.data = recover_data raw_file.close()
def _recover_modified_list(delta_list, raw_path): raw_file = open(raw_path, "rb") raw_mmap = mmap.mmap(raw_file.fileno(), 0, prot=mmap.PROT_READ) delta_list.sort(key=itemgetter('offset')) for index, delta_item in enumerate(delta_list): #print "processing %d/%d, ref_id: %d, offset: %ld" % (index, len(delta_list), delta_item.ref_id, delta_item.offset) if delta_item.ref_id == DeltaItem.REF_RAW: continue elif delta_item.ref_id == DeltaItem.REF_BASE_MEM: offset = delta_item.data recover_data = raw_mmap[offset:offset+Memory.RAM_PAGE_SIZE] elif delta_item.ref_id == DeltaItem.REF_SELF: ref_offset = delta_item.data index = 0 while index < len(delta_list): #print "self referencing : %ld == %ld" % (delta_list[index].offset, ref_offset) if delta_list[index].offset == ref_offset: recover_data = delta_list[index].data break index += 1 if index >= len(delta_list): raise MemoryError("Cannot find self reference") elif delta_item.ref_id == DeltaItem.REF_XDELTA: patch_data = delta_item.data base_data = raw_mmap[delta_item.offset:delta_item.offset+Memory.RAM_PAGE_SIZE] recover_data = tool.merge_data(base_data, patch_data, len(base_data)*2) else: raise MemoryError("Cannot recover: invalid referce id %d" % delta_item.ref_id) if len(recover_data) != Memory.RAM_PAGE_SIZE: msg = "Recovered Size Error: %d, ref_id: %d, %ld, %ld" % \ (len(recover_data), delta_item.ref_id, delta_item.data_len, delta_item.data) raise MemoryError(msg) delta_item.ref_id = DeltaItem.REF_RAW delta_item.data = recover_data raw_file.close()
def recover_item(self, delta_item, delta_counter, delta_times): if type(delta_item) != DeltaItem: raise StreamSynthesisError("Need list of DeltaItem") delta_counter[delta_item.ref_id] += 1 start_time = time.time() if (delta_item.ref_id == DeltaItem.REF_RAW): recover_data = delta_item.data elif (delta_item.ref_id == DeltaItem.REF_ZEROS): recover_data = self.zero_data elif (delta_item.ref_id == DeltaItem.REF_BASE_MEM): offset = delta_item.data recover_data = self.raw_mem[offset:offset + self.chunk_size] elif (delta_item.ref_id == DeltaItem.REF_BASE_DISK): offset = delta_item.data recover_data = self.raw_disk[offset:offset + self.chunk_size] elif delta_item.ref_id == DeltaItem.REF_SELF: ref_index = delta_item.data self_ref_delta_item = self.recovered_delta_dict.get( ref_index, None) if self_ref_delta_item is None: #msg = "Cannot find self reference: type(%ld), offset(%ld), index(%ld), ref_index(%ld)" % \ # (delta_item.delta_type, delta_item.offset, delta_item.index, ref_index) #raise StreamSynthesisError(msg) return None recover_data = self_ref_delta_item.data elif delta_item.ref_id == DeltaItem.REF_SELF_HASH: ref_hashvalue = delta_item.data self_ref_delta_item = self.recovered_hash_dict.get( ref_hashvalue, None) if self_ref_delta_item == None: return None recover_data = self_ref_delta_item.data delta_item.hash_value = ref_hashvalue elif delta_item.ref_id == DeltaItem.REF_XDELTA: patch_data = delta_item.data patch_original_size = delta_item.offset_len if delta_item.delta_type == DeltaItem.DELTA_MEMORY or\ delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE: base_data = self.raw_mem[delta_item.offset:delta_item.offset + patch_original_size] elif delta_item.delta_type == DeltaItem.DELTA_DISK or\ delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE: base_data = self.raw_disk[delta_item.offset:delta_item.offset + patch_original_size] else: raise StreamSynthesisError( "Delta type should be either disk or memory") recover_data = tool.merge_data(base_data, patch_data, len(base_data) * 5) elif delta_item.ref_id == DeltaItem.REF_BSDIFF: patch_data = delta_item.data patch_original_size = delta_item.offset_len if delta_item.delta_type == DeltaItem.DELTA_MEMORY or\ delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE: base_data = self.raw_mem[delta_item.offset:delta_item.offset + patch_original_size] elif delta_item.delta_type == DeltaItem.DELTA_DISK or\ delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE: base_data = self.raw_disk[delta_item.offset:delta_item.offset + patch_original_size] else: raise StreamSynthesisError( "Delta type should be either disk or memory") recover_data = tool.merge_data_bsdiff(base_data, patch_data) elif delta_item.ref_id == DeltaItem.REF_XOR: patch_data = delta_item.data patch_original_size = delta_item.offset_len if delta_item.delta_type == DeltaItem.DELTA_MEMORY or\ delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE: base_data = self.raw_mem[delta_item.offset:delta_item.offset + patch_original_size] elif delta_item.delta_type == DeltaItem.DELTA_DISK or\ delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE: base_data = self.raw_disk[delta_item.offset:delta_item.offset + patch_original_size] else: raise StreamSynthesisError( "Delta type should be either disk or memory") recover_data = tool.cython_xor(base_data, patch_data) else: raise StreamSynthesisError( "Cannot recover: invalid referce id %d" % delta_item.ref_id) if len(recover_data) != delta_item.offset_len: msg = "Error, Recovered Size Error: %d, %d, ref_id: %s, data_len: %ld, offset: %ld, offset_len: %ld" % \ (delta_item.delta_type, len(recover_data), delta_item.ref_id, \ delta_item.data_len, delta_item.offset, delta_item.offset_len) print msg raise StreamSynthesisError(msg) delta_times[delta_item.ref_id] += (time.time() - start_time) ref_id = delta_item.ref_id # recover delta_item.ref_id = DeltaItem.REF_RAW delta_item.data = recover_data if delta_item.delta_type == DeltaItem.DELTA_MEMORY or delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE: self.analysis_queue.put("M,R(%d),%d" % (ref_id, delta_item.offset / 4096)) elif delta_item.delta_type == DeltaItem.DELTA_DISK or delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE: self.analysis_queue.put("D,R(%d),%d" % (ref_id, delta_item.offset / 4096)) if delta_item.hash_value is None or len(delta_item.hash_value) == 0: delta_counter['sha'] += 1 start_time = time.time() delta_item.hash_value = sha256(recover_data).digest() delta_times['sha'] += (time.time() - start_time) return delta_item
def recover_item(self, delta_item): if type(delta_item) != DeltaItem: raise StreamSynthesisError("Need list of DeltaItem") if (delta_item.ref_id == DeltaItem.REF_RAW): recover_data = delta_item.data elif (delta_item.ref_id == DeltaItem.REF_ZEROS): recover_data = self.zero_data elif (delta_item.ref_id == DeltaItem.REF_BASE_MEM): offset = delta_item.data recover_data = self.raw_mem[offset:offset+self.chunk_size] elif (delta_item.ref_id == DeltaItem.REF_BASE_DISK): offset = delta_item.data recover_data = self.raw_disk[offset:offset+self.chunk_size] elif delta_item.ref_id == DeltaItem.REF_SELF: ref_index = delta_item.data self_ref_delta_item = self.recovered_delta_dict.get(ref_index, None) if self_ref_delta_item == None: #msg = "Cannot find self reference: type(%ld), offset(%ld), index(%ld), ref_index(%ld)" % \ # (delta_item.delta_type, delta_item.offset, delta_item.index, ref_index) #raise StreamSynthesisError(msg) return None recover_data = self_ref_delta_item.data elif delta_item.ref_id == DeltaItem.REF_SELF_HASH: ref_hashvalue = delta_item.data self_ref_delta_item = self.recovered_hash_dict.get(ref_hashvalue, None) if self_ref_delta_item == None: return None recover_data = self_ref_delta_item.data delta_item.hash_value = ref_hashvalue elif delta_item.ref_id == DeltaItem.REF_XDELTA: patch_data = delta_item.data patch_original_size = delta_item.offset_len if delta_item.delta_type == DeltaItem.DELTA_MEMORY or\ delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE: base_data = self.raw_mem[delta_item.offset:delta_item.offset+patch_original_size] elif delta_item.delta_type == DeltaItem.DELTA_DISK or\ delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE: base_data = self.raw_disk[delta_item.offset:delta_item.offset+patch_original_size] else: raise StreamSynthesisError("Delta type should be either disk or memory") recover_data = tool.merge_data(base_data, patch_data, len(base_data)*5) elif delta_item.ref_id == DeltaItem.REF_BSDIFF: patch_data = delta_item.data patch_original_size = delta_item.offset_len if delta_item.delta_type == DeltaItem.DELTA_MEMORY or\ delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE: base_data = self.raw_mem[delta_item.offset:delta_item.offset+patch_original_size] elif delta_item.delta_type == DeltaItem.DELTA_DISK or\ delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE: base_data = self.raw_disk[delta_item.offset:delta_item.offset+patch_original_size] else: raise DeltaError("Delta type should be either disk or memory") recover_data = tool.merge_data_bsdiff(base_data, patch_data) elif delta_item.ref_id == DeltaItem.REF_XOR: patch_data = delta_item.data patch_original_size = delta_item.offset_len if delta_item.delta_type == DeltaItem.DELTA_MEMORY or\ delta_item.delta_type == DeltaItem.DELTA_MEMORY_LIVE: base_data = self.raw_mem[delta_item.offset:delta_item.offset+patch_original_size] elif delta_item.delta_type == DeltaItem.DELTA_DISK or\ delta_item.delta_type == DeltaItem.DELTA_DISK_LIVE: base_data = self.raw_disk[delta_item.offset:delta_item.offset+patch_original_size] else: raise DeltaError("Delta type should be either disk or memory") recover_data = tool.cython_xor(base_data, patch_data) else: raise StreamSynthesisError("Cannot recover: invalid referce id %d" % delta_item.ref_id) if len(recover_data) != delta_item.offset_len: msg = "Error, Recovered Size Error: %d, %d, ref_id: %s, data_len: %ld, offset: %ld, offset_len: %ld" % \ (delta_item.delta_type, len(recover_data), delta_item.ref_id, \ delta_item.data_len, delta_item.offset, delta_item.offset_len) print msg raise StreamSynthesisError(msg) # recover delta_item.ref_id = DeltaItem.REF_RAW delta_item.data = recover_data if delta_item.hash_value == None or len(delta_item.hash_value) == 0: delta_item.hash_value = sha256(recover_data).digest() return delta_item
base.get_delta(delta_list, ref_id=DeltaItem.REF_BASE_MEM) # 3.find shared within self print "[Debug] get delta from itself" DeltaList.get_self_delta(delta_list) DeltaList.statistics(delta_list) DeltaList.tofile_with_footer(header_delta, footer_delta, delta_list, out_path) elif command == "recover": if (not settings.base_file) or (not settings.delta_file): sys.stderr.write("Error, Cannot find base/delta file. See help\n") sys.exit(1) base_path = settings.base_file delta_path = settings.delta_file raw_path = settings.base_file + EXT_RAW meta_path = settings.base_file + EXT_META # Create Base Memory from meta file base = Memory.import_from_metafile(meta_path, raw_path) header_delta, footer_delta, delta_list = DeltaList.fromfile_with_footer( delta_path) header = tool.merge_data(base.header_data, header_delta, 1024 * 1024) footer = tool.merge_data(base.footer_data, footer_delta, 1024 * 1024 * 10) _recover_modified_list(delta_list, raw_path) out_path = base_path + ".recover" _recover_memory(delta_list, header, footer, out_path)
# 2.find shared with base memory print "[Debug] get delta from base Memory" base.get_delta(delta_list, ref_id=DeltaItem.REF_BASE_MEM) # 3.find shared within self print "[Debug] get delta from itself" DeltaList.get_self_delta(delta_list) DeltaList.statistics(delta_list) DeltaList.tofile_with_footer(header_delta, footer_delta, delta_list, out_path) elif command == "recover": if (not settings.base_file) or (not settings.delta_file): sys.stderr.write("Error, Cannot find base/delta file. See help\n") sys.exit(1) base_path = settings.base_file delta_path = settings.delta_file raw_path = settings.base_file + EXT_RAW meta_path = settings.base_file + EXT_META # Create Base Memory from meta file base = Memory.import_from_metafile(meta_path, raw_path) header_delta, footer_delta, delta_list = DeltaList.fromfile_with_footer(delta_path) header = tool.merge_data(base.header_data, header_delta, 1024*1024) footer = tool.merge_data(base.footer_data, footer_delta, 1024*1024*10) _recover_modified_list(delta_list, raw_path) out_path = base_path+".recover" _recover_memory(delta_list, header, footer, out_path)