def script_runner(local_tree: WorkingTree, script: str, commit_pending: Optional[bool] = None) -> str: """Run a script in a tree and commit the result. This ignores newly added files. :param local_tree: Local tree to run script in :param script: Script to run :param commit_pending: Whether to commit pending changes (True, False or None: only commit if there were no commits by the script) :return: Description as reported by script """ last_revision = local_tree.last_revision() p = subprocess.Popen(script, cwd=local_tree.basedir, stdout=subprocess.PIPE, shell=True) (description_encoded, err) = p.communicate(b"") if p.returncode != 0: raise errors.BzrCommandError("Script %s failed with error code %d" % (script, p.returncode)) new_revision = local_tree.last_revision() description = description_encoded.decode() if last_revision == new_revision and commit_pending is None: # Automatically commit pending changes if the script did not # touch the branch. commit_pending = True if commit_pending: try: new_revision = local_tree.commit(description, allow_pointless=False) except PointlessCommit: pass if new_revision == last_revision: raise ScriptMadeNoChanges() return description
def script_runner( # noqa: C901 local_tree: WorkingTree, script: Union[str, List[str]], commit_pending: Optional[bool] = None, resume_metadata=None, subpath: str = '', update_changelog: Optional[bool] = None, extra_env: Optional[Dict[str, str]] = None, committer: Optional[str] = None ) -> CommandResult: # noqa: C901 """Run a script in a tree and commit the result. This ignores newly added files. Args: local_tree: Local tree to run script in script: Script to run commit_pending: Whether to commit pending changes (True, False or None: only commit if there were no commits by the script) """ if control_files_in_root(local_tree, subpath): debian_path = subpath else: debian_path = os.path.join(subpath, "debian") if update_changelog is None: dch_guess = guess_update_changelog(local_tree, debian_path) if dch_guess: if isinstance(dch_guess, tuple): # lintian-brush < 1.22 update_changelog, explanation = dch_guess else: update_changelog = dch_guess.update_changelog explanation = dch_guess.explanation logging.info('%s', explanation) else: # Assume yes. update_changelog = True cl_path = os.path.join(debian_path, 'changelog') try: with open(local_tree.abspath(cl_path), 'r') as f: cl = Changelog(f) source_name = cl[0].package except FileNotFoundError: source_name = None env = dict(os.environ) if extra_env: env.update(extra_env) env['SVP_API'] = '1' if source_name: env['DEB_SOURCE'] = source_name if update_changelog: env['DEB_UPDATE_CHANGELOG'] = 'update' else: env['DEB_UPDATE_CHANGELOG'] = 'leave' last_revision = local_tree.last_revision() orig_tags = local_tree.branch.tags.get_tag_dict() with tempfile.TemporaryDirectory() as td: env['SVP_RESULT'] = os.path.join(td, 'result.json') if resume_metadata: env['SVP_RESUME'] = os.path.join(td, 'resume-metadata.json') with open(env['SVP_RESUME'], 'w') as f: json.dump(resume_metadata, f) p = subprocess.Popen( script, cwd=local_tree.abspath(subpath), stdout=subprocess.PIPE, shell=isinstance(script, str), env=env) (description_encoded, err) = p.communicate(b"") try: with open(env['SVP_RESULT'], 'r') as f: try: result_json = json.load(f) except json.decoder.JSONDecodeError as e: raise ResultFileFormatError(e) except FileNotFoundError: result_json = None if p.returncode != 0: if result_json is not None: raise DetailedFailure.from_json(source_name, result_json) raise ScriptFailed(script, p.returncode) # If the changelog didn't exist earlier, then hopefully it was created # now. if source_name is None: try: with open(local_tree.abspath(cl_path), 'r') as f: cl = Changelog(f) source_name = cl[0].package except FileNotFoundError: raise MissingChangelog(cl_path) if result_json is not None: result = CommandResult.from_json(source_name, result_json) else: result = CommandResult(source=source_name) if not result.description: result.description = description_encoded.decode().replace("\r", "") new_revision = local_tree.last_revision() if result.tags is None: result.tags = [] for name, revid in local_tree.branch.tags.get_tag_dict().items(): if orig_tags.get(name) != revid: result.tags.append((name, revid)) if last_revision == new_revision and commit_pending is None: # Automatically commit pending changes if the script did not # touch the branch. commit_pending = True if commit_pending: if update_changelog and result.description and local_tree.has_changes(): add_changelog_entry( local_tree, os.path.join(debian_path, 'changelog'), [result.description]) local_tree.smart_add([local_tree.abspath(subpath)]) try: new_revision = local_tree.commit( result.description, allow_pointless=False, committer=committer) except PointlessCommit: pass if new_revision == last_revision: raise ScriptMadeNoChanges() result.old_revision = last_revision result.new_revision = new_revision return result
def run_lintian_fixer( # noqa: C901 local_tree: WorkingTree, fixer: Fixer, committer: Optional[str] = None, update_changelog: Union[bool, Callable[[], bool]] = True, compat_release: Optional[str] = None, minimum_certainty: Optional[str] = None, trust_package: bool = False, allow_reformatting: bool = False, dirty_tracker=None, subpath: str = "", net_access: bool = True, opinionated: Optional[bool] = None, diligence: int = 0, timestamp: Optional[datetime] = None, basis_tree: Optional[Tree] = None, changes_by: str = "lintian-brush", ): """Run a lintian fixer on a tree. Args: local_tree: WorkingTree object basis_tree: Tree fixer: Fixer object to apply committer: Optional committer (name and email) update_changelog: Whether to add a new entry to the changelog compat_release: Minimum release that the package should be usable on (e.g. 'stable' or 'unstable') minimum_certainty: How certain the fixer should be about its changes. trust_package: Whether to run code from the package if necessary allow_reformatting: Whether to allow reformatting of changed files dirty_tracker: Optional object that can be used to tell if the tree has been changed. subpath: Path in tree to operate on net_access: Whether to allow accessing external services opinionated: Whether to be opinionated diligence: Level of diligence Returns: tuple with set of FixerResult, summary of the changes """ if basis_tree is None: basis_tree = local_tree.basis_tree() changelog_path = os.path.join(subpath, "debian/changelog") try: with local_tree.get_file(changelog_path) as f: cl = Changelog(f, max_blocks=1) except NoSuchFile: raise NotDebianPackage(local_tree, subpath) package = cl.package if cl.distributions == "UNRELEASED": current_version = cl.version else: current_version = cl.version increment_version(current_version) if compat_release is None: compat_release = "sid" if minimum_certainty is None: minimum_certainty = DEFAULT_MINIMUM_CERTAINTY logger.debug('Running fixer %r', fixer) try: result = fixer.run( local_tree.abspath(subpath), package=package, current_version=current_version, compat_release=compat_release, minimum_certainty=minimum_certainty, trust_package=trust_package, allow_reformatting=allow_reformatting, net_access=net_access, opinionated=opinionated, diligence=diligence, ) except BaseException: reset_tree(local_tree, basis_tree, subpath, dirty_tracker=dirty_tracker) raise if not certainty_sufficient(result.certainty, minimum_certainty): reset_tree(local_tree, basis_tree, subpath, dirty_tracker=dirty_tracker) raise NotCertainEnough( fixer, result.certainty, minimum_certainty, overridden_lintian_issues=result.overridden_lintian_issues) specific_files: Optional[List[str]] if dirty_tracker: relpaths = dirty_tracker.relpaths() # Sort paths so that directories get added before the files they # contain (on VCSes where it matters) local_tree.add( [ p for p in sorted(relpaths) if local_tree.has_filename(p) and not local_tree.is_ignored(p) ] ) specific_files = [p for p in relpaths if local_tree.is_versioned(p)] if not specific_files: raise NoChanges( fixer, "Script didn't make any changes", result.overridden_lintian_issues) else: local_tree.smart_add([local_tree.abspath(subpath)]) specific_files = [subpath] if subpath else None if local_tree.supports_setting_file_ids(): RenameMap.guess_renames(basis_tree, local_tree, dry_run=False) changes = list( local_tree.iter_changes( basis_tree, specific_files=specific_files, want_unversioned=False, require_versioned=True, ) ) if len(local_tree.get_parent_ids()) <= 1 and not changes: raise NoChanges( fixer, "Script didn't make any changes", result.overridden_lintian_issues) if not result.description: raise DescriptionMissing(fixer) lines = result.description.splitlines() summary = lines[0] details = list(itertools.takewhile(lambda line: line, lines[1:])) # If there are upstream changes in a non-native package, perhaps # export them to debian/patches if has_non_debian_changes(changes, subpath) and current_version.debian_revision: try: patch_name, specific_files = _upstream_changes_to_patch( local_tree, basis_tree, dirty_tracker, subpath, result.patch_name or fixer.name, result.description, timestamp=timestamp, ) except BaseException: reset_tree(local_tree, basis_tree, subpath, dirty_tracker=dirty_tracker) raise summary = "Add patch %s: %s" % (patch_name, summary) if only_changes_last_changelog_block( local_tree, basis_tree, changelog_path, changes ): # If the script only changed the last entry in the changelog, # don't update the changelog update_changelog = False if callable(update_changelog): update_changelog = update_changelog() if update_changelog: from .changelog import add_changelog_entry add_changelog_entry(local_tree, changelog_path, [summary] + details) if specific_files: specific_files.append(changelog_path) description = result.description + "\n" description += "\n" description += "Changes-By: %s\n" % changes_by for tag in result.fixed_lintian_tags: description += "Fixes: lintian: %s\n" % tag description += "See-also: https://lintian.debian.org/tags/%s.html\n" % tag if committer is None: committer = get_committer(local_tree) revid = local_tree.commit( description, allow_pointless=False, reporter=NullCommitReporter(), committer=committer, specific_files=specific_files, ) result.revision_id = revid # TODO(jelmer): Support running sbuild & verify lintian warning is gone? return result, summary
def script_runner( # noqa: C901 local_tree: WorkingTree, script: Union[str, List[str]], commit_pending: Optional[bool] = None, resume_metadata=None, subpath: str = '', committer: Optional[str] = None, extra_env: Optional[Dict[str, str]] = None, ) -> CommandResult: # noqa: C901 """Run a script in a tree and commit the result. This ignores newly added files. Args: local_tree: Local tree to run script in script: Script to run commit_pending: Whether to commit pending changes (True, False or None: only commit if there were no commits by the script) """ env = dict(os.environ) if extra_env: env.update(extra_env) env['SVP_API'] = '1' last_revision = local_tree.last_revision() orig_tags = local_tree.branch.tags.get_tag_dict() with tempfile.TemporaryDirectory() as td: env['SVP_RESULT'] = os.path.join(td, 'result.json') if resume_metadata: env['SVP_RESUME'] = os.path.join(td, 'resume-metadata.json') with open(env['SVP_RESUME'], 'w') as f: json.dump(resume_metadata, f) p = subprocess.Popen(script, cwd=local_tree.abspath(subpath), stdout=subprocess.PIPE, shell=isinstance(script, str), env=env) (description_encoded, err) = p.communicate(b"") try: with open(env['SVP_RESULT'], 'r') as f: try: result_json = json.load(f) except json.decoder.JSONDecodeError as e: raise ResultFileFormatError(e) except FileNotFoundError: result_json = None if p.returncode != 0: if result_json is not None: raise DetailedFailure.from_json(result_json) raise ScriptFailed(script, p.returncode) if result_json is not None: result = CommandResult.from_json(result_json) else: result = CommandResult() if not result.description: result.description = description_encoded.decode() new_revision = local_tree.last_revision() if result.tags is None: result.tags = [] for name, revid in local_tree.branch.tags.get_tag_dict().items(): if orig_tags.get(name) != revid: result.tags.append((name, revid)) if last_revision == new_revision and commit_pending is None: # Automatically commit pending changes if the script did not # touch the branch. commit_pending = True if commit_pending: local_tree.smart_add([local_tree.abspath(subpath)]) try: new_revision = local_tree.commit(result.description, allow_pointless=False, committer=committer) except PointlessCommit: pass if new_revision == last_revision: raise ScriptMadeNoChanges() result.old_revision = last_revision result.new_revision = local_tree.last_revision() return result
def scrub_obsolete(wt: WorkingTree, subpath: str, compat_release: str, upgrade_release: str, update_changelog=None, allow_reformatting: bool = False) -> ScrubObsoleteResult: """Scrub obsolete entries. """ if control_files_in_root(wt, subpath): debian_path = subpath else: debian_path = os.path.join(subpath, 'debian') result = _scrub_obsolete(wt, debian_path, compat_release, upgrade_release, allow_reformatting) if not result: return result specific_files = list(result.specific_files) summary = result.itemized() changelog_path = os.path.join(debian_path, "changelog") if update_changelog is None: from .detect_gbp_dch import guess_update_changelog from debian.changelog import Changelog with wt.get_file(changelog_path) as f: cl = Changelog(f, max_blocks=1) dch_guess = guess_update_changelog(wt, debian_path, cl) if dch_guess: update_changelog = dch_guess.update_changelog _note_changelog_policy(update_changelog, dch_guess.explanation) result.changelog_behaviour = dch_guess else: # Assume we should update changelog update_changelog = True result.changelog_behaviour = None if update_changelog: lines = [] for release, entries in summary.items(): lines.append("Remove constraints unnecessary since %s:" % release) lines.extend(["+ " + line for line in entries]) add_changelog_entry(wt, changelog_path, lines) specific_files.append(changelog_path) lines = [] for release, entries in summary.items(): lines.extend(["Remove constraints unnecessary since %s" % release, ""]) lines.extend(["* " + line for line in entries]) lines.extend(["", "Changes-By: deb-scrub-obsolete"]) committer = get_committer(wt) try: wt.commit( specific_files=specific_files, message="\n".join(lines), allow_pointless=False, reporter=NullCommitReporter(), committer=committer, ) except PointlessCommit: pass return result