def gen_test_data(): server_file = os.path.join(config.server_folder,'test.txt') client_file = os.path.join(config.client_folder,'test.txt') data = '' len = 24 for i in range(0,len): data += 'A' FileUtil.write_file_data(server_file,data) data = '' for i in range(0,len): if i == 14: data += 'B' else: data += 'A' FileUtil.write_file_data(client_file,data)
def TEST_gen_server_blocks_map(self): blocks_data = [] content = FileUtil.get_file_content(self.file.file_path) content_len = len(content) for i in range(0,content_len,config.block_size): if i != content_len and i + config.block_size > content_len: rest_content = content[i:] rest_len = config.block_size - (content_len - i) for j in range(0,rest_len): rest_content+='\0' blocks_data.append(rest_content) else: blocks_data.append(content[i:config.block_size + i]) block_data_len = len(blocks_data) for i in range(0,block_data_len): guid = uuid.uuid4() block_name = os.path.join(config.block_storage,str(guid)) FileUtil.write_file_data(block_name,blocks_data[i]) self.file.blocks.append((block_name,adler32(blocks_data[i]),FileUtil.compute_data_sha256(blocks_data[i])))
def gen_data_block(self,data_adler32_pair): blocks_info = [] total_data = '' for data,adler32_pair in data_adler32_pair: adler32_value,sha256_value = adler32_pair file_name = self._search_for_block(adler32_value,sha256_value) if file_name == None: if data != '': total_data += data break else: if data != '': total_data += data block_data = FileUtil.get_file_content(file_name) total_data += block_data total_data_len = len(total_data) for i in range(0,total_data_len,config.block_size): if i != total_data_len and i + config.block_size > total_data_len: rest_data = total_data[i:] #print "=================================" #print i,total_data_len #print "=================================" replenish_len = config.block_size - len(rest_data) for j in range(0,replenish_len): rest_data += '\0' rest_file_name = os.path.join(config.block_storage,str(uuid.uuid4())) FileUtil.write_file_data(rest_file_name,rest_data) rest_adler32_value = adler32(rest_data) rest_sha256 = FileUtil.compute_data_sha256(rest_data) self._update_global_block(rest_file_name,rest_adler32_value,rest_sha256) blocks_info.append((rest_adler32_value,rest_sha256)) else: block_data = total_data[i:i + config.block_size] file_name = os.path.join(config.block_storage,str(uuid.uuid4())) FileUtil.write_file_data(file_name,block_data) adler32_value = adler32(block_data) sha256 = FileUtil.compute_data_sha256(block_data) self._update_global_block(file_name,adler32_value,sha256) blocks_info.append((adler32_value,sha256)) return blocks_info
if __name__ == "__main__": gen_test_data() server_file_path = os.path.join(config.server_folder,'test.txt') client_file_path = os.path.join(config.client_folder,'test.txt') server_file = FileEntity(server_file_path) client_file = FileEntity(client_file_path) server_handler = IncrementalUploadHandler(server_file) server_handler.TEST_gen_server_blocks_map() adler32_pair = server_handler.compute_check_sum() #for i in range(0,len(adler32_pair) % 10): # print adler32_pair[i] client_handler = IncrementalUploadHandler(client_file,adler32_pair) server_handler.server_adler32_pair = adler32_pair diffs = client_handler.diff() log = '' for i in range(0,len(diffs)): log += str(diffs[i]) FileUtil.write_file_data(r'D:\test\log.log',log) new_server_file_info = server_handler.gen_data_block(diffs) #for i in range(0,len(new_server_file_info)): # print new_server_file_info[i]