def main(): # noqa: C901 import argparse import breezy # noqa: E402 import logging breezy.initialize() import breezy.git # noqa: E402 import breezy.bzr # noqa: E402 from breezy.workingtree import WorkingTree from breezy.workspace import ( check_clean_tree, WorkspaceDirty, ) from . import ( get_committer, version_string, ) from .config import Config parser = argparse.ArgumentParser(prog="deb-update-watch") parser.add_argument( "--directory", metavar="DIRECTORY", help="directory to run in", type=str, default=".", ) parser.add_argument( "--no-update-changelog", action="store_false", default=None, dest="update_changelog", help="do not update the changelog", ) parser.add_argument( "--update-changelog", action="store_true", dest="update_changelog", help="force updating of the changelog", default=None, ) parser.add_argument( "--allow-reformatting", default=None, action="store_true", help=argparse.SUPPRESS, ) parser.add_argument("--version", action="version", version="%(prog)s " + version_string) parser.add_argument( "--identity", help="Print user identity that would be used when committing", action="store_true", default=False, ) parser.add_argument("--debug", help="Describe all considerd changes.", action="store_true") args = parser.parse_args() wt, subpath = WorkingTree.open_containing(args.directory) if args.identity: logging.info('%s', get_committer(wt)) return 0 try: check_clean_tree(wt, wt.basis_tree(), subpath) except WorkspaceDirty: logging.info("%s: Please commit pending changes first.", wt.basedir) return 1 if args.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO, format='%(message)s') update_changelog = args.update_changelog allow_reformatting = args.allow_reformatting try: cfg = Config.from_workingtree(wt, subpath) except FileNotFoundError: pass else: if update_changelog is None: update_changelog = cfg.update_changelog() if allow_reformatting is None: allow_reformatting = cfg.allow_reformatting() if allow_reformatting is None: allow_reformatting = False try: with WatchEditor() as updater: fix_watch_issues(updater) except FileNotFoundError: # TODO(jelmer): Reuse logic from ../fixers/debian-watch-file-is-missing.py pass if os.environ.get("SVP_API") == "1": with open(os.environ["SVP_RESULT"], "w") as f: json.dump({"description": "Update watch file.", "context": {}}, f) return 0
def main(): # noqa: C901 import argparse import breezy # noqa: E402 breezy.initialize() import breezy.git # noqa: E402 import breezy.bzr # noqa: E402 parser = argparse.ArgumentParser(prog="deb-transition-apply") parser.add_argument( "--directory", metavar="DIRECTORY", help="directory to run in", type=str, default=".", ) parser.add_argument( "--no-update-changelog", action="store_false", default=None, dest="update_changelog", help="do not update the changelog", ) parser.add_argument( "--update-changelog", action="store_true", dest="update_changelog", help="force updating of the changelog", default=None, ) parser.add_argument( "--allow-reformatting", default=None, action="store_true", help=argparse.SUPPRESS, ) parser.add_argument("--version", action="version", version="%(prog)s " + version_string) parser.add_argument( "--identity", help="Print user identity that would be used when committing", action="store_true", default=False, ) parser.add_argument("--debug", help="Describe all considered changes.", action="store_true") parser.add_argument("benfile", help="Benfile to read transition from.", type=str) args = parser.parse_args() with open(args.benfile, 'r') as f: ben = parse_ben(f) wt, subpath = WorkingTree.open_containing(args.directory) if args.identity: logging.info('%s', get_committer(wt)) return 0 try: check_clean_tree(wt, wt.basis_tree(), subpath) except WorkspaceDirty: logging.info("%s: Please commit pending changes first.", wt.basedir) return 1 if args.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO, format='%(message)s') update_changelog = args.update_changelog allow_reformatting = args.allow_reformatting try: cfg = Config.from_workingtree(wt, subpath) except FileNotFoundError: pass else: if update_changelog is None: update_changelog = cfg.update_changelog() if allow_reformatting is None: allow_reformatting = cfg.allow_reformatting() if allow_reformatting is None: allow_reformatting = False if control_files_in_root(wt, subpath): debian_path = subpath else: debian_path = os.path.join(subpath, 'debian') try: result = apply_transition(wt, debian_path, ben, update_changelog=args.update_changelog, allow_reformatting=allow_reformatting) except PackageNotAffected: report_okay("nothing-to-do", "Package not affected by transition") return 0 except PackageAlreadyGood: report_okay("nothing-to-do", "Transition already applied to package") return 0 except PackageNotBad: report_okay("nothing-to-do", "Package not bad") return 0 except FormattingUnpreservable as e: report_fatal( "formatting-unpreservable", "unable to preserve formatting while editing %s" % e.path, ) return 1 except GeneratedFile as e: report_fatal("generated-file", "unable to edit generated file: %r" % e) return 1 except NotDebianPackage: report_fatal('not-debian-package', 'Not a Debian package.') return 1 except ChangeConflict as e: report_fatal('change-conflict', 'Generated file changes conflict: %s' % e) return 1 if not result: report_okay("nothing-to-do", "no changes from transition") return 0 changelog_path = os.path.join(debian_path, "changelog") if update_changelog is None: from .detect_gbp_dch import guess_update_changelog from debian.changelog import Changelog with wt.get_file(changelog_path) as f: cl = Changelog(f, max_blocks=1) dch_guess = guess_update_changelog(wt, debian_path, cl) if dch_guess: update_changelog = dch_guess[0] _note_changelog_policy(update_changelog, dch_guess[1]) else: # Assume we should update changelog update_changelog = True if update_changelog: summary = 'Apply transition %s.' % ben['title'] if result.bugno: summary += ' Closes: #%d' % result.bugno add_changelog_entry(wt, changelog_path, [summary]) if os.environ.get("SVP_API") == "1": with open(os.environ["SVP_RESULT"], "w") as f: json.dump( { "description": "Apply transition.", "value": result.value(), "context": ben }, f) logging.info("Applied transition %s", ben['title']) return 0
def main(argv: List[str]) -> Optional[int]: # noqa: C901 import argparse parser = argparse.ArgumentParser() parser.add_argument( "--command", help="Path to script to run.", type=str) parser.add_argument( "--diff", action="store_true", help="Show diff of generated changes." ) parser.add_argument( "--no-update-changelog", action="store_false", default=None, dest="update_changelog", help="do not update the changelog", ) parser.add_argument( "--update-changelog", action="store_true", dest="update_changelog", help="force updating of the changelog", default=None, ) parser.add_argument( "--commit-pending", help="Commit pending changes after script.", choices=["yes", "no", "auto"], default=None, type=str, ) parser.add_argument( "--build-verify", help="Build package to verify it.", dest="build_verify", action="store_true", ) parser.add_argument( "--builder", default=DEFAULT_BUILDER, type=str, help="Build command to use when verifying build.", ) parser.add_argument( "--build-target-dir", type=str, help=( "Store built Debian files in specified directory " "(with --build-verify)" ), ) parser.add_argument( "--install", "-i", action="store_true", help="Install built package (implies --build-verify)") parser.add_argument( "--dump-context", action="store_true", help="Report context on success") parser.add_argument( "--recipe", type=str, help="Recipe to use.") args = parser.parse_args(argv) if args.recipe: from ..recipe import Recipe recipe = Recipe.from_path(args.recipe) else: recipe = None if args.commit_pending: commit_pending = {"auto": None, "yes": True, "no": False}[args.commit_pending] elif recipe: commit_pending = recipe.commit_pending else: commit_pending = None if args.command: command = args.command elif recipe and recipe.command: command = recipe.command else: logging.exception('No command or recipe specified.') return 1 local_tree, subpath = WorkingTree.open_containing('.') check_clean_tree(local_tree) try: try: result = script_runner( local_tree, script=command, commit_pending=commit_pending, subpath=subpath, update_changelog=args.update_changelog) except MissingChangelog as e: logging.error('No debian changelog file (%s) present', e.args[0]) return False except ScriptMadeNoChanges: logging.info('Script made no changes') return False if result.description: logging.info('Succeeded: %s', result.description) if args.build_verify or args.install: try: build(local_tree, subpath, builder=args.builder, result_dir=args.build_target_dir) except BuildFailedError: logging.error("%s: build failed", result.source) return False except MissingUpstreamTarball: logging.error("%s: unable to find upstream source", result.source) return False except Exception: reset_tree(local_tree, subpath) raise if args.install: install_built_package(local_tree, subpath, args.build_target_dir) if args.diff: from breezy.diff import show_diff_trees old_tree = local_tree.revision_tree(result.old_revision) new_tree = local_tree.revision_tree(result.new_revision) show_diff_trees( old_tree, new_tree, sys.stdout.buffer, old_label='old/', new_label='new/') if args.dump_context: json.dump(result.context, sys.stdout, indent=5) return 0
def run_lintian_fixers( # noqa: C901 local_tree: WorkingTree, fixers: List[Fixer], update_changelog: bool = True, verbose: bool = False, committer: Optional[str] = None, compat_release: Optional[str] = None, minimum_certainty: Optional[str] = None, trust_package: bool = False, allow_reformatting: bool = False, use_inotify: Optional[bool] = None, subpath: str = "", net_access: bool = True, opinionated: Optional[bool] = None, diligence: int = 0, ): """Run a set of lintian fixers on a tree. Args: local_tree: WorkingTree object fixers: A set of Fixer objects update_changelog: Whether to add an entry to the changelog verbose: Whether to be verbose committer: Optional committer (name and email) compat_release: Minimum release that the package should be usable on (e.g. 'sid' or 'stretch') minimum_certainty: How certain the fixer should be about its changes. trust_package: Whether to run code from the package if necessary allow_reformatting: Whether to allow reformatting of changed files use_inotify: Use inotify to watch changes (significantly improves performance). Defaults to None (automatic) subpath: Subpath in the tree in which the package lives net_access: Whether to allow network access opinionated: Whether to be opinionated diligence: Level of diligence Returns: Tuple with two lists: 1. list of tuples with (lintian-tag, certainty, description) of fixers that ran 2. dictionary mapping fixer names for fixers that failed to run to the error that occurred """ basis_tree = local_tree.basis_tree() check_clean_tree(local_tree, basis_tree=basis_tree, subpath=subpath) fixers = list(fixers) dirty_tracker = get_dirty_tracker( local_tree, subpath=subpath, use_inotify=use_inotify ) # If we don't know whether to update the changelog, then find out *once* if update_changelog is None: changelog_behaviour = None def update_changelog(): nonlocal update_changelog, changelog_behaviour changelog_behaviour = determine_update_changelog( local_tree, os.path.join(subpath, "debian")) update_changelog = changelog_behaviour.update_changelog return update_changelog else: changelog_behaviour = None ret = ManyResult() with trange(len(fixers), leave=False, disable=None) as t: for i, fixer in enumerate(fixers): t.set_description("Running fixer %s" % fixer) t.update() start = time.time() if dirty_tracker: dirty_tracker.mark_clean() try: result, summary = run_lintian_fixer( local_tree, fixer, update_changelog=update_changelog, committer=committer, compat_release=compat_release, minimum_certainty=minimum_certainty, trust_package=trust_package, allow_reformatting=allow_reformatting, dirty_tracker=dirty_tracker, subpath=subpath, net_access=net_access, opinionated=opinionated, diligence=diligence, basis_tree=basis_tree, ) except FormattingUnpreservable as e: ret.formatting_unpreservable[fixer.name] = e.path if verbose: logging.info( "Fixer %r was unable to preserve formatting of %s.", fixer.name, e.path, ) except FixerFailed as e: ret.failed_fixers[fixer.name] = e if verbose: logging.info("Fixer %r failed to run.", fixer.name) sys.stderr.write(str(e)) except MemoryError as e: ret.failed_fixers[fixer.name] = e if verbose: logging.info("Run out of memory while running fixer %r.", fixer.name) except NotCertainEnough as e: if verbose: logging.info( "Fixer %r made changes but not high enough " "certainty (was %r, needed %r). (took: %.2fs)", fixer.name, e.certainty, e.minimum_certainty, time.time() - start, ) except FailedPatchManipulation as e: if verbose: logging.info("Unable to manipulate upstream patches: %s", e.args[2]) ret.failed_fixers[fixer.name] = e except NoChanges as e: if verbose: logging.info( "Fixer %r made no changes. (took: %.2fs)", fixer.name, time.time() - start, ) ret.overridden_lintian_issues.extend( e.overridden_lintian_issues) else: if verbose: logging.info( "Fixer %r made changes. (took %.2fs)", fixer.name, time.time() - start, ) ret.success.append((result, summary)) basis_tree = local_tree.basis_tree() if changelog_behaviour: ret.changelog_behaviour = changelog_behaviour return ret
def update_offical_vcs(wt, subpath, repo_url=None, committer=None, force=False, create=False): # TODO(jelmer): Allow creation of the repository as well check_clean_tree(wt, wt.basis_tree(), subpath) debcargo_path = os.path.join(subpath, 'debian/debcargo.toml') control_path = os.path.join(subpath, 'debian/control') if wt.has_filename(debcargo_path): from debmutate.debcargo import DebcargoControlShimEditor editor = DebcargoControlShimEditor.from_debian_dir( wt.abspath(os.path.join(subpath, 'debian'))) elif wt.has_filename(control_path): control_path = wt.abspath(control_path) editor = ControlEditor(control_path) else: raise FileNotFoundError(control_path) with editor: try: vcs_type, url = source_package_vcs(editor.source) except KeyError: pass else: if not force: raise VcsAlreadySpecified(vcs_type, url) maintainer_email = parseaddr(editor.source['Maintainer'])[1] source = editor.source['Source'] if repo_url is None: repo_url = guess_repository_url(source, maintainer_email) if repo_url is None: raise NoVcsLocation() logging.info('Using repository URL: %s', repo_url) # TODO(jelmer): Detect vcs type in a better way if hasattr(wt.branch.repository, '_git'): vcs_type = 'Git' else: vcs_type = 'Bzr' update_control_for_vcs_url(editor.source, vcs_type, repo_url) if committer is None: committer = get_committer(wt) try: wt.commit( message='Set Vcs headers.', allow_pointless=False, reporter=NullCommitReporter(), committer=committer, ) except PointlessCommit: if not force: # This can't happen raise if create: from breezy.forge import create_project try: forge, project = create_project(repo_url) except AlreadyControlDirError: logging.info('%s already exists', repo_url) except UnsupportedForge: logging.error('Unable to find a way to create %s', repo_url) else: logging.info('Created %s', repo_url) return repo_url
def main(argv: List[str]) -> Optional[int]: # noqa: C901 import argparse parser = argparse.ArgumentParser() parser.add_argument("command", help="Path to script to run.", type=str, nargs='?') parser.add_argument("--diff", action="store_true", help="Show diff of generated changes.") parser.add_argument( "--commit-pending", help="Commit pending changes after script.", choices=["yes", "no", "auto"], default=None, type=str, ) parser.add_argument("--verify-command", type=str, help="Command to run to verify changes.") parser.add_argument("--recipe", type=str, help="Recipe to use.") args = parser.parse_args(argv) if args.recipe: from .recipe import Recipe recipe = Recipe.from_path(args.recipe) else: recipe = None if args.commit_pending: commit_pending = { "auto": None, "yes": True, "no": False }[args.commit_pending] elif recipe: commit_pending = recipe.commit_pending else: commit_pending = None if args.command: command = args.command elif recipe.command: command = recipe.command else: logging.exception('No command specified.') return 1 local_tree, subpath = WorkingTree.open_containing('.') check_clean_tree(local_tree) try: result = script_runner(local_tree, script=command, commit_pending=commit_pending, subpath=subpath) if result.description: logging.info('Succeeded: %s', result.description) if args.verify_command: try: subprocess.check_call(args.verify_command, shell=True, cwd=local_tree.abspath(subpath)) except subprocess.CalledProcessError: logging.exception("Verify command failed.") return 1 except Exception: reset_tree(local_tree, subpath) raise if args.diff: from breezy.diff import show_diff_trees old_tree = local_tree.revision_tree(result.old_revision) new_tree = local_tree.revision_tree(result.new_revision) show_diff_trees(old_tree, new_tree, sys.stdout.buffer, old_label='old/', new_label='new/') return 0
def main(argv=None): # noqa: C901 import argparse from breezy.workingtree import WorkingTree import breezy # noqa: E402 breezy.initialize() import breezy.git # noqa: E402 import breezy.bzr # noqa: E402 from .config import Config parser = argparse.ArgumentParser(prog="multi-arch-fixer") parser.add_argument( "--directory", metavar="DIRECTORY", help="directory to run in", type=str, default=".", ) parser.add_argument( "--disable-inotify", action="store_true", default=False, help=argparse.SUPPRESS ) parser.add_argument( "--identity", help="Print user identity that would be used when committing", action="store_true", default=False, ) # Hide the minimum-certainty option for the moment. parser.add_argument( "--minimum-certainty", type=str, choices=SUPPORTED_CERTAINTIES, default=None, help=argparse.SUPPRESS, ) parser.add_argument( "--no-update-changelog", action="store_false", default=None, dest="update_changelog", help="do not update the changelog", ) parser.add_argument( "--update-changelog", action="store_true", dest="update_changelog", help="force updating of the changelog", default=None, ) parser.add_argument( "--version", action="version", version="%(prog)s " + version_string ) parser.add_argument( "--allow-reformatting", default=None, action="store_true", help=argparse.SUPPRESS, ) args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format='%(message)s') minimum_certainty = args.minimum_certainty wt, subpath = WorkingTree.open_containing(args.directory) if args.identity: logging.info('%s', get_committer(wt)) return 0 update_changelog = args.update_changelog allow_reformatting = args.allow_reformatting try: cfg = Config.from_workingtree(wt, subpath) except FileNotFoundError: pass else: if minimum_certainty is None: minimum_certainty = cfg.minimum_certainty() if allow_reformatting is None: allow_reformatting = cfg.allow_reformatting() if update_changelog is None: update_changelog = cfg.update_changelog() use_inotify = ((False if args.disable_inotify else None),) try: check_clean_tree(wt, wt.basis_tree(), subpath) except WorkspaceDirty: logging.info("%s: Please commit pending changes first.", wt.basedir) return 1 dirty_tracker = get_dirty_tracker(wt, subpath, use_inotify) if dirty_tracker: dirty_tracker.mark_clean() with cache_download_multiarch_hints() as f: hints = multiarch_hints_by_binary(parse_multiarch_hints(f)) if control_files_in_root(wt, subpath): report_fatal( "control-files-in-root", "control files live in root rather than debian/ " "(LarstIQ mode)", ) return 1 if is_debcargo_package(wt, subpath): report_okay("nothing-to-do", "Package uses debcargo") return 0 if not control_file_present(wt, subpath): report_fatal("missing-control-file", "Unable to find debian/control") return 1 try: result, summary = run_lintian_fixer( wt, MultiArchHintFixer(hints), update_changelog=update_changelog, minimum_certainty=minimum_certainty, dirty_tracker=dirty_tracker, subpath=subpath, allow_reformatting=allow_reformatting, net_access=True, changes_by="apply-multiarch-hints", ) except NoChanges: report_okay("nothing-to-do", "no hints to apply") return 0 except FormattingUnpreservable as e: report_fatal( "formatting-unpreservable", "unable to preserve formatting while editing %s" % e.path, ) return 1 except GeneratedFile as e: report_fatal( "generated-file", "unable to edit generated file: %r" % e) return 1 except NotDebianPackage: logging.info("%s: Not a debian package.", wt.basedir) return 1 else: applied_hints = [] hint_names = [] for (binary, hint, description, certainty) in result.changes: hint_names.append(hint["link"].split("#")[-1]) entry = dict(hint.items()) hint_names.append(entry["link"].split("#")[-1]) entry["action"] = description entry["certainty"] = certainty applied_hints.append(entry) logging.info("%s: %s" % (binary["Package"], description)) if os.environ.get('SVP_API') == '1': with open(os.environ['SVP_RESULT'], 'w') as f: json.dump({ 'description': "Applied multi-arch hints.", 'value': calculate_value(hint_names), 'commit-message': 'Apply multi-arch hints', 'context': { 'applied-hints': applied_hints, }}, f)
def main(): # noqa: C901 import argparse import breezy # noqa: E402 breezy.initialize() import breezy.git # noqa: E402 import breezy.bzr # noqa: E402 from breezy.workspace import ( check_clean_tree, WorkspaceDirty, ) from . import ( version_string, ) from .config import Config parser = argparse.ArgumentParser(prog="deb-scrub-obsolete") parser.add_argument( "--directory", metavar="DIRECTORY", help="directory to run in", type=str, default=".", ) parser.add_argument( "--upgrade-release", metavar="UPGRADE-RELEASE", help="Release to allow upgrading from.", default="oldstable", ) parser.add_argument('--compat-release', metavar='COMPAT-RELEASE', help="Release to allow building on.", default=None) parser.add_argument( "--no-update-changelog", action="store_false", default=None, dest="update_changelog", help="do not update the changelog", ) parser.add_argument( "--update-changelog", action="store_true", dest="update_changelog", help="force updating of the changelog", default=None, ) parser.add_argument( "--allow-reformatting", default=None, action="store_true", help=argparse.SUPPRESS, ) parser.add_argument("--version", action="version", version="%(prog)s " + version_string) parser.add_argument( "--identity", help="Print user identity that would be used when committing", action="store_true", default=False, ) parser.add_argument("--debug", help="Describe all considered changes.", action="store_true") args = parser.parse_args() wt, subpath = WorkingTree.open_containing(args.directory) if args.identity: logging.info('%s', get_committer(wt)) return 0 try: check_clean_tree(wt, wt.basis_tree(), subpath) except WorkspaceDirty: logging.info("%s: Please commit pending changes first.", wt.basedir) return 1 import distro_info debian_info = distro_info.DebianDistroInfo() upgrade_release = debian_info.codename(args.upgrade_release) if args.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO, format='%(message)s') update_changelog = args.update_changelog allow_reformatting = args.allow_reformatting if args.compat_release: compat_release = debian_info.codename(args.compat_release) else: compat_release = None try: cfg = Config.from_workingtree(wt, subpath) except FileNotFoundError: pass else: if update_changelog is None: update_changelog = cfg.update_changelog() if allow_reformatting is None: allow_reformatting = cfg.allow_reformatting() if compat_release is None: compat_release = cfg.compat_release() if compat_release is None: compat_release = debian_info.codename('oldstable') if upgrade_release != compat_release: logging.info( "Removing run time constraints unnecessary since %s" " and build time constraints unnecessary since %s", upgrade_release, compat_release) else: logging.info( "Removing run time and build time constraints unnecessary " "since %s", compat_release) if allow_reformatting is None: allow_reformatting = False if is_debcargo_package(wt, subpath): report_fatal("nothing-to-do", "Package uses debcargo") return 1 elif not control_file_present(wt, subpath): report_fatal("missing-control-file", "Unable to find debian/control") return 1 try: result = scrub_obsolete(wt, subpath, compat_release, upgrade_release, update_changelog=args.update_changelog, allow_reformatting=allow_reformatting) except FormattingUnpreservable as e: report_fatal( "formatting-unpreservable", "unable to preserve formatting while editing %s" % e.path, ) return 1 except GeneratedFile as e: report_fatal("generated-file", "unable to edit generated file: %r" % e) return 1 except NotDebianPackage: report_fatal('not-debian-package', 'Not a Debian package.') return 1 except ChangeConflict as e: report_fatal('change-conflict', 'Generated file changes conflict: %s' % e) return 1 except UddTimeout: report_fatal('udd-timeout', 'Timeout communicating with UDD') return 1 if not result: report_okay("nothing-to-do", "no obsolete constraints") return 0 debian_context = {} if result.changelog_behaviour: debian_context['changelog'] = result.changelog_behaviour.json() if os.environ.get("SVP_API") == "1": with open(os.environ["SVP_RESULT"], "w") as f: json.dump( { "description": "Remove constraints unnecessary since %s." % upgrade_release, "value": result.value(), "debian": debian_context, "context": { "specific_files": result.specific_files, "maintscript_removed": result.maintscript_removed, "control_removed": result.control_removed, } }, f) logging.info("Scrub obsolete settings.") for release, lines in result.itemized().items(): for line in lines: logging.info("* %s", line) return 0