def generate_ifdefined(all_patches): """Update autogenerated ifdefined patches, which can be used to selectively disable features at compile time.""" enabled_patches = dict([(i, patch) for i, patch in all_patches.iteritems() if not patch.disabled]) for i, patch in enabled_patches.iteritems(): if patch.ifdefined is None: continue filename = os.path.join(patch.directory, config.path_IfDefined) with open(filename, "wb") as fp: fp.write("From: Wine Staging Team <*****@*****.**>\n") fp.write("Subject: Autogenerated #ifdef patch for %s.\n" % patch.name) fp.write("\n") depends = resolve_dependencies(enabled_patches, i) for f in patch.modified_files: # Reconstruct the state after applying the dependencies original = get_wine_file(f) for _, (_, p) in select_patches(enabled_patches, depends, f).iteritems(): original = patchutils.apply_patch(original, p, fuzz=0) # Now apply the main patch p = extract_patch(patch, f)[1] patched = patchutils.apply_patch(original, p, fuzz=0) # Now get the diff between both diff = patchutils.generate_ifdef_patch(original, patched, ifdef=patch.ifdefined) if diff is not None: fp.write("diff --git a/%s b/%s\n" % (f, f)) fp.write("--- a/%s\n" % f) fp.write("+++ b/%s\n" % f) while True: buf = diff.read(16384) if buf == "": break fp.write(buf) diff.close() # Close the file fp.close() # Add changes to git subprocess.call(["git", "add", filename]) # Add the autogenerated file as a last patch patch.files.append(os.path.basename(filename)) for p in patchutils.read_patch(filename): assert p.modified_file in patch.modified_files patch.patches.append(p)
def test_apply(current): set_apply = [(i, all_patches[i]) for i in current] set_skip = [(i, all_patches[i]) for i in indices if i not in current] # Check if there is any patch2 which depends directly or indirectly on patch1. # If this is the case we found an impossible situation, we can be skipped in this test. for i, patch1 in set_apply: for j, patch2 in set_skip: if causal_time_smaller(patch2.verify_time, patch1.verify_time): return True # we can skip this test try: original = original_content for i, _ in set_apply: original = patchutils.apply_patch(original, selected_patches[i][1], fuzz=0) except patchutils.PatchApplyError: return False return True # everything is fine
def generate_ifdefined(all_patches, skip_checks=False): """Update autogenerated ifdefined patches, which can be used to selectively disable features at compile time.""" for i, patch in all_patches.iteritems(): if patch.ifdefined is None: continue if patch.disabled: continue filename = os.path.join(patch.directory, config.path_IfDefined) headers = { 'author': "Wine Staging Team", 'email': "*****@*****.**", 'subject': "Autogenerated #ifdef patch for %s." % patch.name } if skip_checks: patch.files = [os.path.basename(filename)] continue with open(filename, "wb") as fp: fp.write("From: %s <%s>\n" % (headers['author'], headers['email'])) fp.write("Subject: %s\n" % headers['subject']) fp.write("\n") fp.write("Based on patches by:\n") for author, email in sorted(set([(p.patch_author, p.patch_email) for p in patch.patches])): fp.write(" %s <%s>\n" % (author, email)) fp.write("\n") depends = resolve_dependencies(all_patches, i) for f in sorted(patch.modified_files): # Reconstruct the state after applying the dependencies original = get_wine_file(f) selected_patches = select_patches(all_patches, depends, f) failed = [] try: for j in depends: failed.append(j) original = patchutils.apply_patch(original, selected_patches[j][1], fuzz=0) except patchutils.PatchApplyError: raise PatchUpdaterError("Changes to file %s don't apply: %s" % (f, ", ".join([all_patches[j].name for j in failed]))) # Now apply the main patch p = extract_patch(patch, f)[1] try: failed.append(i) patched = patchutils.apply_patch(original, p, fuzz=0) except patchutils.PatchApplyError: raise PatchUpdaterError("Changes to file %s don't apply: %s" % (f, ", ".join([all_patches[j].name for j in failed]))) # Now get the diff between both diff = patchutils.generate_ifdef_patch(original, patched, ifdef=patch.ifdefined) if diff is not None: fp.write("diff --git a/%s b/%s\n" % (f, f)) fp.write("--- a/%s\n" % f) fp.write("+++ b/%s\n" % f) while True: buf = diff.read(16384) if buf == "": break fp.write(buf) diff.close() # Close the file fp.close() # Add changes to git subprocess.call(["git", "add", filename]) # Add the autogenerated file as a last patch patch.files = [os.path.basename(filename)] for p in patch.patches: p.filename = None p.modified_file = None for p in patchutils.read_patch(filename): assert p.modified_file in patch.modified_files p.patch_author = None patch.patches.append(p)
def generate_ifdefined(all_patches, skip_checks=False): """Update autogenerated ifdefined patches, which can be used to selectively disable features at compile time.""" enabled_patches = dict([(i, patch) for i, patch in all_patches.iteritems() if not patch.disabled]) for i, patch in enabled_patches.iteritems(): if patch.ifdefined is None: continue filename = os.path.join(patch.directory, config.path_IfDefined) headers = { 'author': "Wine Staging Team", 'email': "*****@*****.**", 'subject': "Autogenerated #ifdef patch for %s." % patch.name } if skip_checks: patch.files = [os.path.basename(filename)] continue with open(filename, "wb") as fp: fp.write("From: %s <%s>\n" % (headers['author'], headers['email'])) fp.write("Subject: %s\n" % headers['subject']) fp.write("\n") fp.write("Based on patches by:\n") for author, email in sorted(set([(p.patch_author, p.patch_email) for p in patch.patches])): fp.write(" %s <%s>\n" % (author, email)) fp.write("\n") depends = resolve_dependencies(enabled_patches, i) for f in sorted(patch.modified_files): # Reconstruct the state after applying the dependencies original = get_wine_file(f) selected_patches = select_patches(enabled_patches, depends, f) failed = [] try: for j in depends: failed.append(j) original = patchutils.apply_patch(original, selected_patches[j][1], fuzz=0) except patchutils.PatchApplyError: raise PatchUpdaterError("Changes to file %s don't apply: %s" % (f, ", ".join([all_patches[j].name for j in failed]))) # Now apply the main patch p = extract_patch(patch, f)[1] try: failed.append(i) patched = patchutils.apply_patch(original, p, fuzz=0) except patchutils.PatchApplyError: raise PatchUpdaterError("Changes to file %s don't apply: %s" % (f, ", ".join([all_patches[j].name for j in failed]))) # Now get the diff between both diff = patchutils.generate_ifdef_patch(original, patched, ifdef=patch.ifdefined) if diff is not None: fp.write("diff --git a/%s b/%s\n" % (f, f)) fp.write("--- a/%s\n" % f) fp.write("+++ b/%s\n" % f) while True: buf = diff.read(16384) if buf == "": break fp.write(buf) diff.close() # Close the file fp.close() # Add changes to git subprocess.call(["git", "add", filename]) # Add the autogenerated file as a last patch patch.files = [os.path.basename(filename)] for p in patch.patches: p.filename = None p.modified_file = None for p in patchutils.read_patch(filename): assert p.modified_file in patch.modified_files p.patch_author = None patch.patches.append(p)
def generate_script(all_patches): """Resolve dependencies, and afterwards check if everything applies properly.""" depends = sorted([i for i, patch in all_patches.iteritems() if not patch.disabled]) resolved = resolve_dependencies(all_patches, depends=depends) max_patches = max(resolved) + 1 # Generate timestamps based on dependencies, still required for binary patches # Find out which files are modified by multiple patches modified_files = {} for i, patch in [(i, all_patches[i]) for i in resolved]: patch.verify_time = [0]*max_patches patch.verify_time[i] += 1 for j in patch.depends: patch.verify_time = causal_time_combine(patch.verify_time, all_patches[j].verify_time) for f in patch.modified_files: if f not in modified_files: modified_files[f] = [] modified_files[f].append(i) # Check dependencies pool = multiprocessing.pool.ThreadPool(processes=4) try: # Checking all dependencies takes a very long time, so to improve development speed, # run a first quick check with all patches enabled. with progressbar.ProgressBar(desc="pre-check ...", total=len(modified_files)) as progress: for k, (filename, indices) in enumerate(modified_files.iteritems()): # If one of patches is a binary patch, then we cannot / won't verify it - require dependencies in this case if contains_binary_patch(all_patches, indices, filename): if not causal_time_relation_any(all_patches, indices): raise PatchUpdaterError("Because of binary patch modifying file %s the following patches need explicit dependencies: %s" % (filename, ", ".join([all_patches[i].name for i in indices]))) continue original = get_wine_file(filename) selected_patches = select_patches(all_patches, indices, filename) set_apply = [(i, all_patches[i]) for i in indices] try: for i, patch in set_apply: original = patchutils.apply_patch(original, selected_patches[i][1], fuzz=0) except patchutils.PatchApplyError: progress.finish("<failed to apply>") raise PatchUpdaterError("Changes to file %s don't apply: %s" % (filename, ", ".join([all_patches[i].name for i in indices]))) progress.update(k) # More detailed checks, required to make sure that dependencies are set correctly for filename, indices in modified_files.iteritems(): if contains_binary_patch(all_patches, indices, filename): continue original_content = get_wine_file(filename) selected_patches = select_patches(all_patches, indices, filename) # Show a progress bar while applying the patches - this task might take some time chunk_size = 20 with progressbar.ProgressBar(desc=filename, total=2 ** len(indices) / chunk_size) as progress: def test_apply(bitstring): set_apply = [(i, all_patches[i]) for u, i in zip(bitstring, indices) if u] set_skip = [(i, all_patches[i]) for u, i in zip(bitstring, indices) if not u] # Check if there is any patch2 which depends directly or indirectly on patch1. # If this is the case we found an impossible situation, we can be skipped in this test. for i, patch1 in set_apply: for j, patch2 in set_skip: if causal_time_smaller(patch2.verify_time, patch1.verify_time): return True # we can skip this test try: original = original_content for i, patch in set_apply: original = patchutils.apply_patch(original, selected_patches[i][1], fuzz=0) except patchutils.PatchApplyError: return False return True # everything is fine def test_apply_seq(bitstrings): for bitstring in bitstrings: if not test_apply(bitstring): return False return True it = _split_seq(itertools.product([0,1], repeat=len(indices)), chunk_size) for k, res in enumerate(pool.imap_unordered(test_apply_seq, it)): if not res: progress.finish("<failed to apply>") raise PatchUpdaterError("Changes to file %s don't apply: %s" % (filename, ", ".join([all_patches[i].name for i in indices]))) progress.update(k) finally: pool.close() # Generate code for helper functions lines = [] lines.append("# Enable or disable all patchsets\n") lines.append("patch_enable_all ()\n") lines.append("{\n") for i, patch in sorted([(i, all_patches[i]) for i in resolved], key=lambda x:x[1].name): patch.variable = "enable_%s" % patch.name.replace("-","_").replace(".","_") lines.append("\t%s=\"$1\"\n" % patch.variable) lines.append("}\n") lines.append("\n") lines.append("# Enable or disable a specific patchset\n") lines.append("patch_enable ()\n") lines.append("{\n") lines.append("\tcase \"$1\" in\n") for i, patch in sorted([(i, all_patches[i]) for i in resolved], key=lambda x:x[1].name): lines.append("\t\t%s)\n" % patch.name) lines.append("\t\t\t%s=\"$2\"\n" % patch.variable) lines.append("\t\t\t;;\n") lines.append("\t\t*)\n") lines.append("\t\t\treturn 1\n") lines.append("\t\t\t;;\n") lines.append("\tesac\n") lines.append("\treturn 0\n") lines.append("}\n") lines_helpers = lines # Generate code for dependency resolver lines = [] for i, patch in [(i, all_patches[i]) for i in reversed(resolved)]: if len(patch.depends): lines.append("if test \"$%s\" -eq 1; then\n" % patch.variable) for j in sorted(patch.depends): lines.append("\tif test \"$%s\" -gt 1; then\n" % all_patches[j].variable) lines.append("\t\tabort \"Patchset %s disabled, but %s depends on that.\"\n" % (all_patches[j].name, patch.name)) lines.append("\tfi\n") for j in sorted(patch.depends): lines.append("\t%s=1\n" % all_patches[j].variable) lines.append("fi\n\n") lines_resolver = lines # Generate code for applying all patchsets lines = [] for i, patch in [(i, all_patches[i]) for i in resolved]: lines.append("# Patchset %s\n" % patch.name) lines.append("# |\n") # List all bugs fixed by this patchset if any([bugid is not None for bugid, bugname in patch.fixes]): lines.append("# | This patchset fixes the following Wine bugs:\n") for bugid, bugname in patch.fixes: if bugid is not None: lines.append("# | *\t%s\n" % "\n# | \t".join(textwrap.wrap("[#%d] %s" % (bugid, bugname), 120))) lines.append("# |\n") # List all modified files lines.append("# | Modified files:\n") lines.append("# | *\t%s\n" % "\n# | \t".join(textwrap.wrap(", ".join(sorted(patch.modified_files)), 120))) lines.append("# |\n") lines.append("if test \"$%s\" -eq 1; then\n" % patch.variable) for f in patch.files: lines.append("\tpatch_apply %s\n" % os.path.join(patch.name, f)) if len(patch.patches): lines.append("\t(\n") for p in _unique(patch.patches, key=lambda p: (p.patch_author, p.patch_subject, p.patch_revision)): lines.append("\t\techo '+ { \"%s\", \"%s\", %d },';\n" % (_escape(p.patch_author), _escape(p.patch_subject), p.patch_revision)) lines.append("\t) >> \"$patchlist\"\n") lines.append("fi\n\n") lines_apply = lines with open(config.path_template_script) as template_fp: template = template_fp.read() with open(config.path_script, "w") as fp: fp.write(template.format(patch_helpers="".join(lines_helpers).rstrip("\n"), patch_resolver="".join(lines_resolver).rstrip("\n"), patch_apply="".join(lines_apply).rstrip("\n"))) # Add changes to git subprocess.call(["git", "add", config.path_script])
def verify_patch_order(all_patches, indices, filename): """Checks if the dependencies are defined correctly by applying the patches on a (temporary) copy from the git tree.""" global cached_patch_result # If one of patches is a binary patch, then we cannot / won't verify it - require dependencies in this case if contains_binary_patch(all_patches, indices, filename): if not causal_time_relation_any(all_patches, indices): raise PatchUpdaterError( "Because of binary patch modifying file %s the following patches need explicit dependencies: %s" % (filename, ", ".join([all_patches[i].name for i in indices]))) return # Get at least the checksum of the original file original_content_hash, original_content = get_wine_file(filename) # Check for possible ways to apply the patch failed_to_apply = False last_result_hash = None for patches in causal_time_permutations(all_patches, indices, filename): # Calculate unique hash based on the original content and the order in which the patches are applied m = hashlib.sha256() m.update(original_content_hash) for patch in patches: m.update(patch.hash()) unique_hash = m.digest() # Fast path -> we know that it applies properly if cached_patch_result.has_key(unique_hash): result_hash = cached_patch_result[unique_hash] assert result_hash is not None else: # Now really get the file, if we don't have it yet if original_content is None: original_content_hash, original_content = get_wine_file( filename, force=True) # Apply the patches (without fuzz) try: content = patchutils.apply_patch(original_content, patches, fuzz=0) except patchutils.PatchApplyError: # Remember that we failed to apply the patches, but continue, if there is still a chance # that it applies in a different order (to give a better error message). failed_to_apply = True if last_result_hash is None: continue break # Get hash of resulting file and add to cache result_hash = hashlib.sha256(content).digest() cached_patch_result[unique_hash] = result_hash # No known hash yet, remember the result. If we failed applying before, we can stop now. if last_result_hash is None: last_result_hash = result_hash if failed_to_apply: break # Applied successful, but result has a different hash - also treat as failure. elif last_result_hash != result_hash: failed_to_apply = True break # If something failed, then show the appropriate error message. if failed_to_apply and last_result_hash is None: raise PatchUpdaterError( "Changes to file %s don't apply on git source tree: %s" % (filename, ", ".join([all_patches[i].name for i in indices]))) elif failed_to_apply: raise PatchUpdaterError( "Depending on the order some changes to file %s don't apply / lead to different results: %s" % (filename, ", ".join([all_patches[i].name for i in indices]))) else: assert len(last_result_hash) == 32
def generate_script(all_patches): """Resolve dependencies, and afterwards check if everything applies properly.""" depends = sorted( [i for i, patch in all_patches.iteritems() if not patch.disabled]) resolved = resolve_dependencies(all_patches, depends=depends) max_patches = max(resolved) + 1 # Generate timestamps based on dependencies, still required for binary patches # Find out which files are modified by multiple patches modified_files = {} for i, patch in [(i, all_patches[i]) for i in resolved]: patch.verify_time = [0] * max_patches patch.verify_time[i] += 1 for j in patch.depends: patch.verify_time = causal_time_combine(patch.verify_time, all_patches[j].verify_time) for f in patch.modified_files: if f not in modified_files: modified_files[f] = [] modified_files[f].append(i) # Check dependencies for filename, indices in modified_files.iteritems(): # If one of patches is a binary patch, then we cannot / won't verify it - require dependencies in this case if contains_binary_patch(all_patches, indices, filename): if not causal_time_relation_any(all_patches, indices): raise PatchUpdaterError( "Because of binary patch modifying file %s the following patches need explicit dependencies: %s" % (filename, ", ".join( [all_patches[i].name for i in indices]))) continue original_content = get_wine_file(filename) selected_patches = select_patches(all_patches, indices, filename) # Show a progress bar while applying the patches - this task might take some time with progressbar.ProgressBar(desc=filename, total=2**len(indices)) as progress: for k, bitstring in enumerate( itertools.product([0, 1], repeat=len(indices))): progress.update(k) set_apply = [(i, all_patches[i]) for u, i in zip(bitstring, indices) if u] set_skip = [(i, all_patches[i]) for u, i in zip(bitstring, indices) if not u] test_apply = True # Check if there is any patch2 which depends directly or indirectly on patch1. # If this is the case we found an impossible situation, we can be skipped in this test. for i, patch1 in set_apply: for j, patch2 in set_skip: if causal_time_smaller(patch2.verify_time, patch1.verify_time): test_apply = False break if not test_apply: break if test_apply: try: original = original_content for i, patch in set_apply: original = patchutils.apply_patch( original, selected_patches[i][1], fuzz=0) except patchutils.PatchApplyError: progress.finish("<failed to apply>") raise PatchUpdaterError( "Changes to file %s don't apply: %s" % (filename, ", ".join( [all_patches[i].name for i in indices]))) # Generate code for helper functions lines = [] lines.append("# Enable or disable all patchsets\n") lines.append("patch_enable_all ()\n") lines.append("{\n") for i, patch in sorted([(i, all_patches[i]) for i in resolved], key=lambda x: x[1].name): patch.variable = "enable_%s" % patch.name.replace("-", "_") lines.append("\t%s=\"$1\"\n" % patch.variable) lines.append("}\n") lines.append("\n") lines.append("# Enable or disable a specific patchset\n") lines.append("patch_enable ()\n") lines.append("{\n") lines.append("\tcase \"$1\" in\n") for i, patch in sorted([(i, all_patches[i]) for i in resolved], key=lambda x: x[1].name): lines.append("\t\t%s)\n" % patch.name) lines.append("\t\t\t%s=\"$2\"\n" % patch.variable) lines.append("\t\t\t;;\n") lines.append("\t\t*)\n") lines.append("\t\t\treturn 1\n") lines.append("\t\t\t;;\n") lines.append("\tesac\n") lines.append("\treturn 0\n") lines.append("}\n") lines_helpers = lines # Generate code for dependency resolver lines = [] for i, patch in [(i, all_patches[i]) for i in reversed(resolved)]: if len(patch.depends): lines.append("if test \"$%s\" -eq 1; then\n" % patch.variable) for j in sorted(patch.depends): lines.append("\tif test \"$%s\" -gt 1; then\n" % all_patches[j].variable) lines.append( "\t\tabort \"Patchset %s disabled, but %s depends on that.\"\n" % (all_patches[j].name, patch.name)) lines.append("\tfi\n") for j in sorted(patch.depends): lines.append("\t%s=1\n" % all_patches[j].variable) lines.append("fi\n\n") lines_resolver = lines # Generate code for applying all patchsets lines = [] for i, patch in [(i, all_patches[i]) for i in resolved]: lines.append("# Patchset %s\n" % patch.name) lines.append("# |\n") # List all bugs fixed by this patchset if any([bugid is not None for bugid, bugname in patch.fixes]): lines.append("# | This patchset fixes the following Wine bugs:\n") for bugid, bugname in patch.fixes: if bugid is not None: lines.append("# | *\t%s\n" % "\n# | \t".join( textwrap.wrap("[#%d] %s" % (bugid, bugname), 120))) lines.append("# |\n") # List all modified files lines.append("# | Modified files:\n") lines.append("# | *\t%s\n" % "\n# | \t".join( textwrap.wrap(", ".join(sorted(patch.modified_files)), 120))) lines.append("# |\n") lines.append("if test \"$%s\" -eq 1; then\n" % patch.variable) for f in patch.files: lines.append("\tpatch_apply %s\n" % os.path.join(patch.name, f)) if len(patch.patches): lines.append("\t(\n") for p in _unique( patch.patches, key=lambda p: (p.patch_author, p.patch_subject, p.patch_revision)): lines.append("\t\techo '+ { \"%s\", \"%s\", %d },';\n" % (_escape(p.patch_author), _escape( p.patch_subject), p.patch_revision)) lines.append("\t) >> \"$patchlist\"\n") lines.append("fi\n\n") lines_apply = lines with open(config.path_template_script) as template_fp: template = template_fp.read() with open(config.path_script, "w") as fp: fp.write( template.format( patch_helpers="".join(lines_helpers).rstrip("\n"), patch_resolver="".join(lines_resolver).rstrip("\n"), patch_apply="".join(lines_apply).rstrip("\n"))) # Add changes to git subprocess.call(["git", "add", config.path_script])