def diffPackage(self, oldMD5, newMD5, prefix): diffZipName = oldMD5 + '-' + newMD5 + '.zip' diffZipPath = MEDIA_ROOT + diffZipName oldZipName = oldMD5 + '.zip' oldZipPath = MEDIA_ROOT + oldZipName newZipName = newMD5 + '.zip' newZipPath = MEDIA_ROOT + newZipName #TODO:客户端增量包有问题,暂时全部返回全量包 return (False, prefix + newZipName) #先判断diff文件存不存在,存在说明之前生成过这个增量包,直接返回 exsit = os.path.isfile(diffZipPath) if exsit: return (True, prefix + diffZipName) #判断老的js包存不存在,不存在返回最新全量包的地址 exsit = os.path.isfile(oldZipPath) if not exsit: return (False, prefix + newZipName) exsit = os.path.isfile(newZipPath) if not exsit: return (False, None) bsdiff4.file_diff(oldZipPath, newZipPath, diffZipPath) return (True, prefix + diffZipName)
def _make_patch(patch_info): # Does with the name implies. Used with multiprocessing patch = Patch(patch_info) patch_name = patch_info['patch_name'] dst_path = patch_info['dst'] patch_number = patch_info['patch_num'] src_path = patch_info['src'] patch_name += '-' + str(patch_number) # Updating with full name - number included patch.patch_name = patch_name if not os.path.exists(src_path): log.warning('Src file does not exist to create patch') else: log.debug('Patch source path:{}'.format(src_path)) log.debug('Patch destination path: {}'.format(dst_path)) if patch.ready is True: log.info("Creating patch... " "{}".format(os.path.basename(patch_name))) bsdiff4.file_diff(src_path, patch.dst_path, patch.patch_name) base_name = os.path.basename(patch_name) log.info('Done creating patch... {}'.format(base_name)) else: log.error('Missing patch attr') return patch
def save_blocklist(self, stats, mlbf, id_, previous_id=None): out_file = os.path.join(settings.TMP_PATH, 'mlbf', id_, 'filter') meta_file = os.path.join(settings.TMP_PATH, 'mlbf', id_, 'filter.meta') os.makedirs(os.path.dirname(out_file), exist_ok=True) with default_storage.open(out_file, 'wb') as mlbf_file: log.info("Writing to file {}".format(out_file)) mlbf.tofile(mlbf_file) stats['mlbf_filesize'] = os.stat(out_file).st_size with default_storage.open(meta_file, 'wb') as mlbf_meta_file: log.info("Writing to meta file {}".format(meta_file)) mlbf.saveDiffMeta(mlbf_meta_file) stats['mlbf_metafilesize'] = os.stat(meta_file).st_size if previous_id: diff_base_file = os.path.join(settings.TMP_PATH, 'mlbf', str(previous_id), 'filter') patch_file = os.path.join(settings.TMP_PATH, 'mlbf', id_, 'filter.patch') log.info( "Generating patch file {patch} from {base} to {out}".format( patch=patch_file, base=diff_base_file, out=out_file)) bsdiff4.file_diff(diff_base_file, out_file, patch_file) stats['mlbf_diffsize'] = os.stat(patch_file).st_size
def passed(self, input_name, file_name): self.log("Test ok") self.log("Creating log patch in {0}".format(self.dir_name)) bsdiff4.file_diff( input_name, file_name, os.path.join(self.dir_name, file_name + self.diff_ext)) os.remove(file_name)
def test_inplace(self): a = 1000 * to_bytes('ABCDE') b = 1000 * to_bytes('XYZ') self.write_data('src', a + random_bytes(100) + b) self.write_data('dst', a + random_bytes(100) + b) file_diff(self.path('src'), self.path('dst'), self.path('patch')) file_patch_inplace(self.path('src'), self.path('patch')) self.assert_same_file_content('src', 'dst')
def test_inplace(self): a = 1000 * b'ABCDE' b = 1000 * b'XYZ' self.write_data('src', a + os.urandom(100) + b) self.write_data('dst', a + os.urandom(100) + b) file_diff(self.path('src'), self.path('dst'), self.path('patch')) file_patch_inplace(self.path('src'), self.path('patch')) self.assert_same_file_content('src', 'dst')
def generate_patch(self, file_from, file_to, save_name): mkdir_p(os.path.join(self.storage_dir, "patches")) from_path = os.path.join(self.storage_dir, file_from) to_path = os.path.join(self.storage_dir, file_to) patch_path = os.path.join(self.storage_dir, "patches", save_name) bsdiff4.file_diff(from_path, to_path, patch_path) assert os.path.exists(patch_path), "Patch file from {] to {} => {} not generated!".format(from_path, to_path, patch_path) print("Patch file from {} to {} => {} generated!".format(from_path, to_path, patch_path)) return patch_path
def round_trip(self): # write file 'patch' file_diff(self.path('src'), self.path('dst'), self.path('patch')) # write file 'dst2' file_patch(self.path('src'), self.path('dst2'), self.path('patch')) # compare files 'dst' and 'dst2' self.assert_same_file_content('dst', 'dst2') # patch 'src' in place file_patch(self.path('src'), self.path('src'), self.path('patch')) self.assert_same_file_content('src', 'dst')
def make_patch(patch): log.debug('Patch source path: %s', patch.src) log.debug('Patch destination path: %s', patch.dst) log.info('Creating patch... %s', patch.basename) bsdiff4.file_diff(patch.src, patch.dst, patch.patch_name) log.info('Done creating patch... %s', patch.basename) return patch
def get_patch( package_name, pre_version_code, version_code, new_file_path): # generate patch old_file_path = get_apk_path(package_name, pre_version_code) patch_file_path = get_patch_path(package_name, pre_version_code, version_code) bsdiff4.file_diff(old_file_path, new_file_path, patch_file_path) return patch_file_path
def generate_patch(version, old_jar, patch, latest_jar): """ Generates a diff between latest_jar and old_jar in patch """ bsdiff4.file_diff(latest_jar, old_jar, patch) result = {"name": version} with open(old_jar, "rb") as fp: result["md5"] = md5.new(fp.read()).hexdigest() return result
def _process(objects, current, in_lock, temp_dir): in_total, out_total = 0, 0 while True: position = current.get_and_add() if position >= len(objects): return in_total, out_total # print "Starting %s in %s" % (position, current_thread()) hash, object_file = objects[position] window_copy = objects[max(0, position - 10):position] o_dir = temp_dir.child(hash) o_dir.mkdirs() try: # First, generate diffs and gzipped diffs for h, f in window_copy: p = o_dir.child(h) # Generate diff bsdiff4.file_diff(f.path, object_file.path, p.path) # Generate gzipped diff # g = o_dir.child(h + ".gz") # with open(p.path, "rb") as in_patch: # with gzip.GzipFile(filename=g.path, mode="wb") as out_patch: # shutil.copyfileobj(in_patch, out_patch) # Then gzip the original with object_file.open("rb") as in_file: with gzip.GzipFile(filename=o_dir.child("original.gz").path, mode="wb") as out_file: shutil.copyfileobj(in_file, out_file) # And compare the files. # Gather the files to compare to_compare = [] for h, f in window_copy: p = o_dir.child(h) to_compare.append((p.size, DIFF, p)) # g = o_dir.child(h + ".gz") # to_compare.append((g.size, DIFF_GZIP, g)) to_compare.append((object_file.size, LITERAL, object_file)) object_file_g = o_dir.child("original.gz") to_compare.append((object_file_g.size, LITERAL_GZIP, object_file_g)) # Compare them min_size, min_type, min_file = min(to_compare) # Print a message about how much we compressed original_size = object_file.size fraction_of_original = float(min_size) / float(original_size) amount_saved = 1.0 - fraction_of_original # print "For object %s:" % hash # print "original_size, min_size, min_type, min_file: %s, %s, %s, %s" % ( # original_size, min_size, hex(ord(min_type)), min_file) # print "Compressed to %s%% smaller (larger numbers are better)" % ( # int(amount_saved * 100)) in_total += original_size out_total += min_size print "Compressed %s" % position # print "Space savings so far: %s%%" % int(100*(1.0 - (float(out_total) / float(in_total)))) finally: o_dir.delete_folder(True)
def diff_file(src_path, dst_path, patch_path): """ patch_path = dst_path - src_path >>> diff_dir2zip('/Users/cosyman/Downloads/bs2/car','/Users/cosyman/Downloads/bs2/car 2', ... '/Users/cosyman/Downloads/bs2/car.p.zip') ... None :param file1: :param file2: :param target: :return: """ bsdiff4.file_diff(src_path, dst_path, patch_path)
def generatePatch(originfilepath, targetfilepath, patchesfilepath): # start to diff patch = bsdiff4.file_diff(originfilepath, targetfilepath, patchesfilepath) # create info json patchsJsonFilePath = patchesfilepath + '.json' patchsJsonFile = open(patchsJsonFilePath, 'w') patchFileMd5 = hashlib.md5(open(patchesfilepath, 'r').read()).hexdigest() originFileMd5 = hashlib.md5(open(originfilepath, 'r').read()).hexdigest() targetFileMd5 = hashlib.md5(open(targetfilepath, 'r').read()).hexdigest() outputJson = json.dumps({ 'originFileMd5': originFileMd5, 'targetFileMd5': targetFileMd5, 'patchFileMd5': patchFileMd5 }) # write info patchsJsonFile.seek(0) patchsJsonFile.write(outputJson) patchsJsonFile.close() print 'patch file is ', patchesfilepath
import sys from bsdiff4 import file_diff, file_patch print(sys.argv[1] + "," + sys.argv[2] + "," + sys.argv[3]) if sys.argv[1] == "diff": file_diff(sys.argv[2], sys.argv[3], sys.argv[4]) else: file_patch(sys.argv[2], sys.argv[3], sys.argv[4])
def passed(self, input_name, file_name): self.log("Test ok") self.log("Creating log patch in {0}".format(self.dir_name)) bsdiff4.file_diff(input_name, file_name, os.path.join(self.dir_name, file_name + self.diff_ext)) os.remove(file_name)
# I have modified the game .exe by directly changing assembly instructions in OllyDbg # this script just turned those modifications into a small portable patch file and it isn't used anywhere during # the execution of reternal.py import bsdiff4 print("REternal Daughter patching utility") print("==================================") print() print("Making upscale patch...") # patch 1: this hooks the execution and updates resolution in memory bsdiff4.file_diff("Eternal Daughter.exe", "fixres1.exe", "patch1") print("Done!") print() print("Making pseudo hi-res patch...") # patch 2: this changes the resolution only when the window is drawn; # as a result, the game isn't upscaled but you also don't get artifacts on smaller levels bsdiff4.file_diff("Eternal Daughter.exe", "fixres2.exe", "patch2") print("Done!") print() print("Making screen borders patch...") # patch 3: like patch 2 but adds screen borders; unfortunately, they blink in a really annoying way; # requires compiled bmploader.dll and a 640x480 file 'img.bmp' that contains borders (white = transparent) bsdiff4.file_diff("Eternal Daughter.exe", "fixres3.exe", "patch3") print("Done!") print() print("Making windowed patch...")
import bsdiff4 import shutil from pathlib import Path for path in Path('./nonroot/Arch').rglob('*.apk'): bsdiff4.file_diff( str(path).replace('nonroot', 'root'), path, './patches/Arch/' + path.name.replace('.apk', '.bsdiff')) for path in Path('./nonroot/Language').rglob('*.apk'): bsdiff4.file_diff( str(path).replace('nonroot', 'root'), path, './patches/Language/' + path.name.replace('.apk', '.bsdiff')) for path in Path('./nonroot/Theme').rglob('*.apk'): bsdiff4.file_diff( 'base.apk', path, './patches/Theme/' + path.name.replace('.apk', '.bsdiff')) for path in Path('./root/Theme').rglob('*.apk'): bsdiff4.file_diff('base.apk', path, './patches/root/' + path.name.replace('.apk', '.bsdiff')) for path in Path('./patches').rglob('*.bsdiff'): shutil.copy(path, './patches/' + str(path).replace('/', '-')[8:])
print("\t" + file) if originalHash != fixedHash: if file not in existingManifest[platform][ branch] or originalHash != existingManifest[platform][ branch][file][ "original"] or fixedHash != existingManifest[ platform][branch][file]["fixed"]: fileTimeStart = process_time() os.makedirs(os.path.dirname( os.path.join(patchTargetPathRoot, platform, branch, file)), exist_ok=True) bsdiff4.file_diff( os.path.join(originalPathRoot, platform, file), os.path.join(fixedPathRoot, platform, file), os.path.join(patchTargetPathRoot, platform, branch, file + ".bsdiff")) print("\t\tTook " + str(process_time() - fileTimeStart) + " second(s)") else: print("\t\tSkipped: Up to date") manifest[platform][branch][file] = existingManifest[ platform][branch][file] filesToSkip[platform][branch].append(file) else: print("\t\tSkipped: Original matches Fixed hash") filesToSkip[platform][branch].append(file) print("\nGenerating New Manifest...")
def _compare_file(self, relative_path: str, filename: str) -> DiffItem: self.logger.debug("_compare_file for `%s` in `%s`", filename, relative_path) result_diff = DiffItem(iotype='file', name=filename, base_crc='', target_crc='') # type: DiffItem basepath = os.path.join(self._basepath, relative_path, filename) targetpath = os.path.join(self._targetpath, relative_path, filename) deltapath = os.path.join(self._deltapath, relative_path, filename) if not os.path.exists(basepath): shutil.copy2(targetpath, deltapath) result_diff.action = 'add' result_diff.target_crc = crc32_from_file(targetpath) elif not os.path.exists(targetpath): result_diff.action = 'remove' result_diff.base_crc = crc32_from_file(basepath) elif filecmp.cmp(basepath, targetpath): result_diff.action = 'unchanged' result_diff.base_crc = result_diff.target_crc = crc32_from_file( targetpath) else: if zipfile.is_zipfile(basepath): result_diff.action = 'zipdelta' result_diff.base_crc = result_diff.target_crc = "#ZIPFILE" working_directory = os.getcwd() temp = tempfile.TemporaryDirectory(suffix='_dir', prefix='bireus_') os.chdir(temp.name) temp_basepath = self._basepath temp_targetpath = self._targetpath temp_deltapath = self._deltapath os.makedirs(temp_basepath) os.makedirs(temp_targetpath) os.makedirs(temp_deltapath) with zipfile.ZipFile(os.path.join(working_directory, basepath)) as baseZip: baseZip.extractall(temp_basepath) with zipfile.ZipFile( os.path.join(working_directory, targetpath)) as targetZip: targetZip.extractall(temp_targetpath) self.logger.debug("zipdelta required for `%s`", filename) zip_diff = CompareTask(self.name, self.base, self.target).generate_diff(False) shutil.copytree( temp_deltapath, os.path.join(working_directory, self._deltapath, filename)) result_diff.items.extend(zip_diff.items) os.chdir(working_directory) else: result_diff.action = 'bsdiff' bsdiff4.file_diff(basepath, targetpath, deltapath) result_diff.target_crc = crc32_from_file(targetpath) result_diff.base_crc = crc32_from_file(basepath) return result_diff
if __name__ == "__main__": for filename in os.listdir(dir_new): old_path = os.path.join(dir_current, filename) new_path = os.path.join(dir_new, filename) download_path = os.path.join(dir_download, f"{filename}.gz") if os.path.isfile(old_path): patch_path = os.path.join(dir_patch, filename) os.makedirs(patch_path, exist_ok=True) old_hash = hash(old_path) if old_hash != hash(new_path): if os.path.isfile(old_path): print(f"making patch of {filename}") bsdiff4.file_diff(old_path, new_path, os.path.join(patch_path, old_hash)) print("finished writing patch") shutil.copyfile(new_path, old_path) with open(new_path, 'rb') as f_in: with gzip.open(download_path, 'wb') as f_out: shutil.copyfileobj(f_in, f_out) os.remove(new_path) files = {} for filename in os.listdir(dir_current): files[filename] = hash(os.path.join(dir_current, filename))
def compute_delta(a_folder, a_files, b_folder, b_files): print('Going from %d files %d files' % (len(a_files), len(b_files))) # First let's fill up these categories files_new = [] files_removed = [] files_changed = [] files_unchanged = [] files_renamed = [] # First, let's remove the .so files since we're going to treat # them special. a_files_so = [tup for tup in a_files if tup.endswith('.so')] b_files_so = [tup for tup in b_files if tup.endswith('.so')] # What files appear in B but not in A? for elt in b_files: if elt not in a_files: if not elt.endswith('.so'): files_new.append(elt) # What files appear in A but not in B? for elt in a_files: if elt not in b_files: files_removed.append(elt) # What files have changed contents but not name/path? for elt in b_files: if elt in a_files: if not filecmp.cmp(a_folder+'/'+elt, b_folder+'/'+elt): files_changed.append(elt) else: files_unchanged.append(elt) ''' Special-case the .so files. We ultimately want to diff .so files that are built from the same source even if the names have slightly changed. ''' print('Find matches for .so files') for elt in b_files_so: if elt in a_files_so: # this file is the same name in both! files_changed.append(elt) else: # This one is new, but we can probably bsdiff it with one # of the other .so's since it was likely renamed. a_best_choice_diff = find_best_diff(b_folder+'/'+elt, a_folder, a_files_so) # this will be a 'rename' record in the TOC files_renamed.append([elt, a_best_choice_diff]) print('%d .so files to diff' % (len(b_files_so) + len(a_files_so))) print('%d new files' % len(files_new)) print('%d removed files' % len(files_removed)) print('%d files changed' % len(files_changed)) print('%d files unchanged' % len(files_unchanged)) print('%d files files_renamed' % len(files_renamed)) ''' Ok, now let's write out the patch. The patch is two major sections: - TOC.txt, which contains the instructions of what files to add, remove, and patch - f0, f1, ... the files themselves Legend: -<filename> // This file should be REMOVED +<id>|<dst_filename> // This file should be ADDED c<id>|<filename> // This file shouldb be patched, same name r<id>|<src_filename>|<dst_filename> // this file should be patched to src_filename and named dst_filename ''' # temp dir where we're assembling the patch ensure_dir_exists(g_output_dir) unique_fileid = 0 toc = open(g_output_dir+'/TOC.txt','w') # TODO write SHA1 of result toc.write('sha1 97ff22e8da32324bd1c79fd7b3da8a5b0c5f6dd1\n') for elt in files_removed: toc.write('-%s\n' % elt) for elt in files_new: # write an entry for the file toc.write('+%d|%s\n' % (unique_fileid, elt)) # copy the file contents itself into the folder. shutil.copy(b_folder+'/'+elt, '%s/f%d' % (g_output_dir, unique_fileid)) unique_fileid = unique_fileid + 1 print("writing diff'ed changed files...") for elt in files_changed: toc.write('c%d|%s\n' % (unique_fileid, elt)) bsdiff4.file_diff(b_folder+'/'+elt, a_folder+'/'+elt, '%s/f%d' % (g_output_dir, unique_fileid)) unique_fileid = unique_fileid + 1 for elt in files_renamed: # these files are diffed against a src file with a different name src_file = a_folder+'/'+elt[1] dst_file = b_folder+'/'+elt[0] toc.write('C%d|%s|%s\n' % (unique_fileid, elt[1], elt[0])) bsdiff4.file_diff(src_file, dst_file, '%s/f%d' % (g_output_dir, unique_fileid)) unique_fileid = unique_fileid + 1 toc.close()
def _get_bsdiff(q, old_file, new_file, patch_file): bsdiff4.file_diff(old_file.path, new_file.path, patch_file) q.put(True)
def setUp(self): # pylint: disable=too-many-statements # pylint: disable=too-many-locals self.initialWorkingDir = os.getcwd() userDataDir = appdirs.user_data_dir(APP_NAME, COMPANY_NAME) if os.path.exists(userDataDir): shutil.rmtree(userDataDir) os.makedirs(userDataDir) versionsUserDataFilePath = os.path.join(userDataDir, 'versions.gz') userDataUpdateDir = os.path.join(userDataDir, "update") os.mkdir(userDataUpdateDir) system = get_system() self.currentFilename = \ VERSIONS['updates'][APP_NAME][CURRENT_VERSION_PYU_FORMAT]\ [system]['filename'] currentFilePath = os.path.join(userDataUpdateDir, self.currentFilename) with open(currentFilePath, "wb") as currentFile: currentFile.write("%s" % CURRENT_VERSION) currentFile.seek(FILE_SIZE - 1) currentFile.write("\0") fileHash = get_package_hashes(currentFilePath) VERSIONS['updates'][APP_NAME][CURRENT_VERSION_PYU_FORMAT]\ [system]['file_hash'] = fileHash tempFile = tempfile.NamedTemporaryFile() self.fileServerDir = tempFile.name tempFile.close() os.mkdir(self.fileServerDir) os.chdir(self.fileServerDir) self.updateFilename = \ VERSIONS['updates'][APP_NAME][UPDATE_VERSION_PYU_FORMAT]\ [system]['filename'] with open(self.updateFilename, "wb") as updateFile: updateFile.write("%s" % UPDATE_VERSION) updateFile.seek(FILE_SIZE - 1) updateFile.write("\0") os.chdir(self.fileServerDir) fileHash = get_package_hashes(self.updateFilename) VERSIONS['updates'][APP_NAME][UPDATE_VERSION_PYU_FORMAT]\ [system]['file_hash'] = fileHash self.patchFilename = \ VERSIONS['updates'][APP_NAME][UPDATE_VERSION_PYU_FORMAT]\ [system]['patch_name'] bsdiff4.file_diff(currentFilePath, self.updateFilename, self.patchFilename) os.chdir(self.fileServerDir) fileHash = get_package_hashes(self.patchFilename) VERSIONS['updates'][APP_NAME][UPDATE_VERSION_PYU_FORMAT]\ [system]['patch_hash'] = fileHash os.chdir(self.initialWorkingDir) privateKey = ed25519.SigningKey(PRIVATE_KEY.encode('utf-8'), encoding='base64') signature = privateKey.sign(six.b(json.dumps(VERSIONS, sort_keys=True)), encoding='base64').decode() VERSIONS['signature'] = signature keysFilePath = os.path.join(self.fileServerDir, 'keys.gz') with gzip.open(keysFilePath, 'wb') as keysFile: keysFile.write(json.dumps(KEYS, sort_keys=True)) versionsFilePath = os.path.join(self.fileServerDir, 'versions.gz') with gzip.open(versionsFilePath, 'wb') as versionsFile: versionsFile.write(json.dumps(VERSIONS, sort_keys=True)) with gzip.open(versionsUserDataFilePath, 'wb') as versionsFile: versionsFile.write(json.dumps(VERSIONS, sort_keys=True)) tempFile = tempfile.NamedTemporaryFile() self.tempDir = tempFile.name tempFile.close() settings.CONFIG_DATA_FOLDER = os.path.join(self.tempDir, '.pyupdater') settings.USER_DATA_FOLDER = os.path.join(self.tempDir, 'pyu-data') os.mkdir(self.tempDir) os.mkdir(settings.USER_DATA_FOLDER) os.mkdir(settings.CONFIG_DATA_FOLDER) # The way we set the App name below avoids having to # create .pyupdater/config.pyu: settings.GENERIC_APP_NAME = APP_NAME settings.GENERIC_COMPANY_NAME = COMPANY_NAME os.environ['PYUPDATER_FILESERVER_DIR'] = self.fileServerDir os.environ['WXUPDATEDEMO_TESTING'] = 'True' os.environ['WXUPDATEDEMO_TESTING_FROZEN'] = 'True' os.environ['WXUPDATEDEMO_TESTING_APP_NAME'] = APP_NAME os.environ['WXUPDATEDEMO_TESTING_COMPANY_NAME'] = COMPANY_NAME os.environ['WXUPDATEDEMO_TESTING_APP_VERSION'] = CURRENT_VERSION os.environ['WXUPDATEDEMO_TESTING_PUBLIC_KEY'] = PUBLIC_KEY
def measure_two_filediffs(src, dst): k_patch_filename = 'temp.patch' bsdiff4.file_diff(src, dst, k_patch_filename) result_size = os.path.getsize(k_patch_filename) os.remove(k_patch_filename) return result_size