def get_debian_source_series(self): title = self.bug_task.bug.title.lower().split() if "experimental" in title: series = "experimental" elif "testing" in title: series = distro_info.DebianDistroInfo().testing() else: series = distro_info.DebianDistroInfo().devel() return series
def test_debian_backports_sloppy(self): try: import distro_info except ImportError: return c = self.__config c.vendor = 'debian' c.suite = 'stable-backports-sloppy' debian = c.get_vendor('debian') self.assertIs(c.vendor, debian) debian_info = distro_info.DebianDistroInfo() sloppy = c.get_suite(debian, 'stable-backports-sloppy') backports = c.get_suite(debian, 'stable-backports') stable = c.get_suite(debian, 'stable') self.assertIs(c.suite, sloppy) self.assertEqual(str(sloppy), debian_info.stable() + '-backports-sloppy') self.assertEqual(sloppy.hierarchy[0], sloppy) self.assertEqual(str(sloppy.hierarchy[1]), str(backports)) self.assertEqual(str(sloppy.hierarchy[2]), str(stable)) self.assertEqual(sloppy.sbuild_resolver, ['--build-dep-resolver=aptitude']) self.assertEqual(c.get_mirrors().lookup_suite(sloppy), 'http://192.168.122.1:3142/debian') self.assertEqual(sloppy.archive, 'debian') self.assertEqual(c.sbuild_resolver, ['--build-dep-resolver=aptitude'])
def test_debian_stable_security(self): c = self.__config c.vendor = 'debian' c.suite = 'stable-security' try: import distro_info except ImportError: return debian = c.get_vendor('debian') self.assertIs(c.vendor, debian) debian_info = distro_info.DebianDistroInfo() security = c.get_suite(debian, 'stable-security') stable = c.get_suite(debian, 'stable') self.assertEqual(security.apt_suite, '{}/updates'.format(debian_info.stable())) self.assertEqual(c.get_mirrors().lookup_suite(security), 'http://192.168.122.1:3142/security.debian.org') self.assertEqual(security.archive, 'security.debian.org') self.assertEqual(security.hierarchy[0], security) self.assertEqual(str(security.hierarchy[1]), str(stable)) with self.assertRaises(AttributeError): c.archive
def resolve_release_codename(name: str, date=None) -> str: def oldest_name(fn): return max(fn("object", date), key=lambda r: r.created).name if '/' in name: distro, name = name.split('/', 1) else: distro = None if distro in ('debian', None): debian = distro_info.DebianDistroInfo() if name == 'lts': return oldest_name(debian.lts_supported) if name == 'elts': return oldest_name(debian.elts_supported) if debian.codename(name): return debian.codename(name) if debian.valid(name): return name if distro in ('ubuntu', None): ubuntu = distro_info.UbuntuDistroInfo() if name == 'esm': return oldest_name(ubuntu.supported_esm) if ubuntu.valid(name): return name return None
def setUp(self): self.cobj = conf.Config('notimportant', {}) debdist = distro_info.DebianDistroInfo() self.stable = debdist.stable() self.unstable = debdist.devel() self.oldstable = debdist.old() self.testing = debdist.testing() self.experimental = 'experimental'
def previous_release(release): import distro_info debian = distro_info.DebianDistroInfo() if release in (debian.devel(), debian.testing(), 'experimental'): return debian.stable() releases = debian.get_all() try: return releases[releases.index(release) - 1] except ValueError: pass # TODO(jelmer): Ubuntu? return None
def prepare_apt(self): distroinfo = distro_info.DebianDistroInfo() if distroinfo.testing() == self.codename: self.suite = "testing" elif self.codename == "sid": self.suite = "unstable" else: self.suite = "stable" if not self.codename or not self.mirror: raise cliapp.AppException( "Misconfiguration: no codename or mirror set") state_dir = tempfile.mkdtemp() os.mkdir(os.path.join(state_dir, 'lists')) self.dirlist.append(state_dir) cache_dir = tempfile.mkdtemp() os.makedirs(os.path.join(cache_dir, 'archives', 'partial')) self.dirlist.append(cache_dir) etc_dir = tempfile.mkdtemp() os.mkdir(os.path.join(etc_dir, 'apt.conf.d')) os.mkdir(os.path.join(etc_dir, 'preferences.d')) os.mkdir(os.path.join(etc_dir, 'trusted.gpg.d')) copy_files('/etc/apt/trusted.gpg.d', os.path.join(etc_dir, 'trusted.gpg.d')) with open(os.path.join(etc_dir, 'sources.list'), 'w') as sources: sources.write( 'deb %s %s %s\n' % (self.mirror, self.codename, ' '.join(self.components))) self.dirlist.append(etc_dir) updates = { 'APT::Architecture': str(self.architecture), 'APT::Architectures': str(self.architecture), 'Dir::State': state_dir, 'Dir::Cache': cache_dir, 'Dir::Etc': etc_dir, } for key, value in updates.items(): try: apt_pkg.config[key] = value # pylint: disable=no-member except TypeError as exc: print(key, value, exc) continue self.cache = apt.cache.Cache() try: self.cache.update() self.cache.open() except apt.cache.FetchFailedException as exc: raise cliapp.AppException('Unable to update cache: %s' % exc) if not os.path.exists(self.destdir): raise cliapp.AppException( 'Destination directory %s does not exist' % self.destdir)
def test_unknown_vendor(self): c = self.__config c.vendor = 'steamos' c.suite = 'brewmaster' steamos = c.get_vendor('steamos') debian = c.get_vendor('debian') brewmaster = c.get_suite(steamos, 'brewmaster') self.assertEqual(str(steamos), 'steamos') self.assertEqual(steamos.components, {'main'}) self.assertEqual(list(brewmaster.hierarchy), [brewmaster]) with self.assertRaises(AttributeError): steamos.archive self.assertEqual(c.components, {'main'}) self.assertEqual(c.vendor, steamos) self.assertIs(c.worker_vendor, debian) self.assertIs(c.pbuilder_worker_vendor, debian) self.assertIs(c.sbuild_worker_vendor, debian) self.assertIs(c.vmdebootstrap_worker_vendor, debian) with self.assertRaises(AttributeError): c.archive self.assertEqual(c.autopkgtest, [ 'schroot', 'qemu:autopkgtest.qcow2', 'qemu:autopkgtest-merged-usr.qcow2', ]) self.assertIsNone(c.get_suite(steamos, 'xyzzy', create=False)) self.assertIsNotNone(c.get_suite(steamos, 'xyzzy')) self.assertIs(c.get_suite(steamos, 'xyzzy'), c.get_suite(steamos, 'xyzzy')) self.assertEqual(c.get_mirrors().lookup_suite(brewmaster), 'http://localhost/steamos') self.assertEqual(brewmaster.archive, 'steamos') try: import distro_info except ImportError: return debian_info = distro_info.DebianDistroInfo() self.assertIs(c.worker_suite, c.get_suite(debian, debian_info.stable()))
def test_distro_info(self): debian = self.__config._get_vendor('debian') ubuntu = self.__config._get_vendor('ubuntu') try: import distro_info except ImportError: return debian_info = distro_info.DebianDistroInfo() ubuntu_info = distro_info.UbuntuDistroInfo() try: ubuntu_devel = ubuntu_info.devel() except distro_info.DistroDataOutdated: ubuntu_devel = ubuntu_info.stable() self.assertEqual(str(ubuntu.get_suite('devel')), ubuntu_devel) self.assertEqual(str(debian.get_suite('unstable')), 'sid') self.assertEqual(str(debian.get_suite('testing')), debian_info.testing()) self.assertEqual(str(debian.get_suite('oldstable')), debian_info.old()) self.assertEqual(str(debian.get_suite('rc-buggy')), 'experimental') stable = debian.get_suite('stable') self.assertEqual(str(stable), debian_info.stable()) self.assertEqual(stable.sbuild_resolver, []) backports = debian.get_suite('stable-backports') self.assertEqual(str(backports), debian_info.stable() + '-backports') self.assertEqual(backports.sbuild_resolver, ['--build-dep-resolver=aptitude']) self.assertEqual(backports.apt_suite, debian_info.stable() + '-backports') self.assertEqual(backports.mirror, 'http://192.168.122.1:3142/debian') self.assertEqual(backports.hierarchy[0], backports) self.assertEqual(str(backports.hierarchy[1]), str(stable)) security = debian.get_suite('stable-security') self.assertEqual(security.apt_suite, '{}/updates'.format(debian_info.stable())) self.assertEqual(security.mirror, 'http://192.168.122.1:3142/security.debian.org') self.assertEqual(security.hierarchy[0], security) self.assertEqual(str(security.hierarchy[1]), str(stable))
def _get_distmap(self): debdist = distro_info.DebianDistroInfo() # start with e.g. "sid" -> "unstable" distmap = collections.defaultdict(lambda: "unknown", [ (debdist.old(), "oldstable"), (debdist.devel(), "unstable"), (debdist.stable(), "stable"), (debdist.testing(), "testing"), ("experimental", "experimental"), ("rc", "experimental"), ]) # add mappings for e.g. "oldstable" -> "oldstable" distmap.update(dict([(val, val) for key, val in distmap.iteritems()])) # map e.g. "Debian6" -> "oldstable" where debdist.old(result="fullname") # currently returns 'Debian 6.0 "Squeeze"' dkey = lambda x: "Debian" + re.split('[ \.]', x(result="fullname"))[1] dfuncs = [debdist.old, debdist.stable, debdist.testing] distmap.update(dict([(dkey(x), distmap[x()]) for x in dfuncs])) return distmap
def test_defaults(self): self.__config = Config(config_layers=({}, ), current_directory='/') c = self.__config self.assertGreaterEqual(c.parallel, 1) self.assertIs(type(c.parallel), int) debian = c.get_vendor('debian') ubuntu = c.get_vendor('ubuntu') self.assertEqual(str(c.vendor), 'debian') self.assertEqual(str(c.worker_vendor), 'debian') self.assertEqual(str(c.vmdebootstrap_worker_vendor), 'debian') self.assertEqual(str(c.sbuild_worker_vendor), 'debian') self.assertIs(c.vendor, debian) self.assertIs(c.worker_vendor, debian) self.assertIs(c.sbuild_worker_vendor, debian) self.assertIs(c.vmdebootstrap_worker_vendor, debian) with self.assertRaises(AttributeError): c.archive self.assertIs(c.sbuild_indep_together, False) self.assertIs(c.sbuild_source_together, False) self.assertEqual(c.output_parent, '..') self.assertEqual(c.qemu_image_size, '10G') self.assertIsNone(c.sbuild_buildables) self.assertEqual(c.sbuild_resolver, []) self.assertEqual(c.debootstrap_script, None) self.assertEqual(c.apt_key, '/usr/share/keyrings/debian-archive-keyring.gpg') self.assertEqual(c.dpkg_source_tar_ignore, []) self.assertIsNone(c.dpkg_source_diff_ignore) self.assertEqual(c.dpkg_source_extend_diff_ignore, []) if ARCHITECTURE is not None: self.assertEqual(c.architecture, ARCHITECTURE) if 0: # FIXME: these raise, because suite is undefined, # but then the error is trapped and __getattr__ is called # instead self.assertEqual( c.qemu_image, '{}/vectis/{}/debian/sid/autopkgtest.qcow2'.format( XDG_CACHE_HOME, ARCHITECTURE)) self.assertEqual( c.write_qemu_image, '{}/vectis/{}/debian/sid/autopkgtest.qcow2'.format( XDG_CACHE_HOME, ARCHITECTURE)) self.assertEqual(c.worker_architecture, ARCHITECTURE) self.assertEqual(c.worker, ['qemu', c.worker_qemu_image]) self.assertEqual( c.worker_qemu_image, '{}/vectis/{}/debian/sid/autopkgtest.qcow2'.format( XDG_CACHE_HOME, ARCHITECTURE)) self.assertEqual(c.sbuild_worker, ['qemu', c.sbuild_worker_qemu_image]) self.assertEqual( c.sbuild_worker_qemu_image, '{}/vectis/{}/debian/sid/autopkgtest.qcow2'.format( XDG_CACHE_HOME, ARCHITECTURE)) self.assertEqual(c.autopkgtest, ['lxc', 'qemu']) self.assertEqual(c.suite, None) with self.assertRaises(AttributeError): c.apt_suite try: import distro_info except ImportError: pass else: self.assertEqual( c.worker_suite, c.get_suite(c.vendor, distro_info.DebianDistroInfo().stable())) self.assertEqual(c.default_worker_suite, distro_info.DebianDistroInfo().stable()) stable = c.get_suite(c.vendor, 'stable') self.assertEqual(c.sbuild_worker_suite, stable) # #860433, #877592 self.assertEqual(str(c.piuparts_worker_suite), 'stretch-backports') self.assertEqual(c.default_suite, 'sid') self.assertEqual(c.components, {'main'}) self.assertEqual(c.extra_components, {'contrib', 'non-free'}) self.assertEqual(c.all_components, {'main', 'contrib', 'non-free'}) self.assertEqual(c.storage, '{}/vectis'.format(XDG_CACHE_HOME))
def make_changes( self, local_tree, subpath, update_changelog, reporter, committer, base_proposal=None, ): from lintian_brush.scrub_obsolete import scrub_obsolete import distro_info debian_info = distro_info.DebianDistroInfo() if self.compat_release: compat_release = debian_info.codename(self.compat_release) else: compat_release = None upgrade_release = debian_info.codename(self.upgrade_release) base_revid = local_tree.last_revision() allow_reformatting = self.allow_reformatting try: cfg = Config.from_workingtree(local_tree, subpath) except FileNotFoundError: pass else: if allow_reformatting is None: allow_reformatting = cfg.allow_reformatting() if update_changelog is None: update_changelog = cfg.update_changelog() if compat_release is None: compat_release = cfg.compat_release() if compat_release is None: compat_release = debian_info.stable() if is_debcargo_package(local_tree, subpath): raise ChangerError("nothing-to-do", "Package uses debcargo") elif not control_file_present(local_tree, subpath): raise ChangerError("missing-control-file", "Unable to find debian/control") try: result = scrub_obsolete( local_tree, subpath, compat_release, upgrade_release, update_changelog=update_changelog, ) except FormattingUnpreservable as e: raise ChangerError( "formatting-unpreservable", "unable to preserve formatting while editing %s" % e.path, ) except GeneratedFile as e: raise ChangerError("generated-file", "unable to edit generated file: %r" % e) except NotDebianPackage: raise ChangerError("not-debian-package", "Not a Debian package") if not result: raise ChangerError("nothing-to-do", "no obsolete constraints") branches = [("main", None, base_revid, local_tree.last_revision())] tags = [] return ChangerResult( description="Scrub obsolete settings.", mutator=result, branches=branches, tags=tags, value=calculate_value(result), sufficient_for_proposal=True, proposed_commit_message="Scrub obsolete settings.", )
def test_debian(self): c = self.__config c.vendor = 'debian' c.suite = 'sid' debian = c.get_vendor('debian') self.assertIs(c.vendor, debian) sid = c.get_suite(debian, 'sid') self.assertIs(c.suite, sid) # Properties of the vendor itself self.assertEqual(str(debian), 'debian') self.assertEqual(debian.default_suite, 'sid') self.assertIs(c.get_suite(debian, 'unstable'), sid) self.assertEqual(debian.components, {'main'}) self.assertEqual(debian.extra_components, {'contrib', 'non-free'}) self.assertEqual(debian.all_components, {'main', 'contrib', 'non-free'}) self.assertIsNone(c.get_suite(debian, 'xenial', create=False)) # Properties of the suite itswelf self.assertEqual(sid.apt_key, '/usr/share/keyrings/debian-archive-keyring.gpg') self.assertEqual(sid.archive, 'debian') self.assertEqual(c.get_mirrors().lookup_suite(sid), 'http://192.168.122.1:3142/debian') self.assertIs(sid.base, None) self.assertEqual(sid.components, {'main'}) self.assertEqual(sid.extra_components, {'contrib', 'non-free'}) self.assertEqual(sid.all_components, {'main', 'contrib', 'non-free'}) self.assertEqual(sid.apt_suite, 'sid') self.assertEqual(sid.sbuild_resolver, []) # Properties of the Config determined by the suite being Debian sid self.assertEqual(c.autopkgtest, ['lxc', 'qemu']) self.assertIs(c.worker_vendor, debian) self.assertIs(c.lxc_worker_vendor, debian) self.assertIs(c.piuparts_worker_vendor, debian) self.assertIs(c.sbuild_worker_vendor, debian) self.assertIs(c.vmdebootstrap_worker_vendor, debian) with self.assertRaises(AttributeError): c.archive self.assertEqual(c.qemu_image_size, '10G') self.assertGreaterEqual(c.parallel, 1) self.assertIs(c.sbuild_indep_together, False) self.assertIs(c.sbuild_source_together, False) self.assertEqual(c.sbuild_resolver, []) self.assertEqual(c.apt_key, '/usr/share/keyrings/debian-archive-keyring.gpg') self.assertIsNone(c.dpkg_source_diff_ignore) self.assertEqual(c.dpkg_source_tar_ignore, []) self.assertEqual(c.dpkg_source_extend_diff_ignore, []) self.assertEqual(c.output_parent, '..') self.assertEqual(c.architecture, 'mips') self.assertEqual(c.worker_architecture, 'mips') self.assertEqual(c.lxc_worker_architecture, 'mips') self.assertEqual(c.piuparts_worker_architecture, 'mips') self.assertEqual(c.sbuild_worker_architecture, 'mips') with self.assertRaises(AttributeError): c.apt_suite # Below this point relies on knowledge of distro_info try: import distro_info except ImportError: return debian_info = distro_info.DebianDistroInfo() self.assertEqual(debian.default_worker_suite, debian_info.stable()) self.assertIs(c.lxc_worker_suite, c.get_suite(debian, debian_info.stable())) # #860433, #877592 self.assertIs(c.piuparts_worker_suite, c.get_suite(debian, 'stretch-backports')) self.assertIs(c.sbuild_worker_suite, c.get_suite(debian, debian_info.stable())) self.assertIs(c.vmdebootstrap_worker_suite, c.get_suite(debian, debian_info.stable())) self.assertIs(c.worker_suite, c.get_suite(debian, debian_info.stable())) self.assertEqual(str(c.get_suite(debian, 'unstable')), 'sid') self.assertEqual(str(c.get_suite(debian, 'testing')), debian_info.testing()) self.assertEqual(str(c.get_suite(debian, 'oldstable')), debian_info.old()) self.assertEqual(str(c.get_suite(debian, 'rc-buggy')), 'experimental') stable = c.get_suite(debian, 'stable') self.assertEqual(str(stable), debian_info.stable())
def make_changes( # noqa: C901 self, local_tree, subpath, update_changelog, reporter, committer, base_proposal=None, ): base_revid = local_tree.last_revision() reporter.report_metadata( "versions", { "lintian-brush": lintian_brush_version_string, "silver-platter": silver_platter.version_string, "breezy": breezy.version_string, }, ) import distro_info debian_info = distro_info.DebianDistroInfo() compat_release = self.compat_release allow_reformatting = self.allow_reformatting minimum_certainty = None try: cfg = Config.from_workingtree(local_tree, subpath) except FileNotFoundError: pass else: compat_release = cfg.compat_release() if compat_release: compat_release = debian_info.codename( compat_release, default=compat_release ) allow_reformatting = cfg.allow_reformatting() minimum_certainty = cfg.minimum_certainty() if compat_release is None: compat_release = debian_info.stable() if allow_reformatting is None: allow_reformatting = False if minimum_certainty is None: minimum_certainty = DEFAULT_MINIMUM_CERTAINTY with local_tree.lock_write(): if control_files_in_root(local_tree, subpath): raise ChangerError( "control-files-in-root", "control files live in root rather than debian/ " "(LarstIQ mode)", ) try: overall_result = run_lintian_fixers( local_tree, self.fixers, committer=committer, update_changelog=update_changelog, compat_release=compat_release, allow_reformatting=allow_reformatting, minimum_certainty=minimum_certainty, subpath=subpath, diligence=self.diligence, opinionated=self.opinionated, trust_package=self.trust_package, ) except NotDebianPackage: raise ChangerError("not-debian-package", "Not a Debian package") except ChangelogCreateError as e: raise ChangerError( "changelog-create-error", "Error creating changelog entry: %s" % e ) applied = [] base_applied = reporter.get_base_metadata("applied", []) if base_applied: applied.extend(base_applied) for result, summary in overall_result.success: applied.append( { "summary": summary, "description": result.description, "fixed_lintian_tags": result.fixed_lintian_tags, "revision_id": result.revision_id.decode("utf-8"), "certainty": result.certainty, } ) reporter.report_metadata("applied", applied) if overall_result.failed_fixers: for fixer_name, failure in overall_result.failed_fixers.items(): logging.info("Fixer %r failed to run:", fixer_name) sys.stderr.write(str(failure)) reporter.report_metadata( "failed", {name: str(e) for (name, e) in overall_result.failed_fixers.items()}, ) if not overall_result.success: raise ChangerError("nothing-to-do", "no fixers to apply") fixed_lintian_tags = set() for result, summary in overall_result.success: fixed_lintian_tags.update(result.fixed_lintian_tags) add_on_only = not has_nontrivial_changes( overall_result.success, self.propose_addon_only ) if not reporter.get_base_metadata("add_on_only", False): add_on_only = False if not add_on_only: if overall_result.success: logging.info("only add-on fixers found") sufficient_for_proposal = False reporter.report_metadata("add_on_only", True) else: sufficient_for_proposal = True reporter.report_metadata("add_on_only", False) branches = [("main", None, base_revid, local_tree.last_revision())] return ChangerResult( description="Applied fixes for %r" % fixed_lintian_tags, mutator=overall_result.success, branches=branches, tags=[], value=calculate_value(fixed_lintian_tags), sufficient_for_proposal=sufficient_for_proposal, proposed_commit_message=update_proposal_commit_message( base_proposal, overall_result.success ), )
def main(argv=None): # noqa: C901 parser = argparse.ArgumentParser(prog="lintian-brush") fixer_group = parser.add_argument_group("fixer selection") fixer_group.add_argument("fixers", metavar="FIXER", nargs="*", help="specific fixer to run") fixer_group.add_argument( "--fixers-dir", type=str, help="path to fixer scripts. [%(default)s]", default=find_fixers_dir(), ) fixer_group.add_argument( "--exclude", metavar="EXCLUDE", type=str, action="append", help="Exclude a fixer.", ) fixer_group.add_argument( "--modern", help=("Use features/compatibility levels that are not available in " "stable. (makes backporting harder)"), action="store_true", default=False, ) fixer_group.add_argument("--compat-release", type=str, help=argparse.SUPPRESS) # Hide the minimum-certainty option for the moment. fixer_group.add_argument( "--minimum-certainty", type=str, choices=SUPPORTED_CERTAINTIES, default=None, help=argparse.SUPPRESS, ) fixer_group.add_argument("--opinionated", action="store_true", help=argparse.SUPPRESS) fixer_group.add_argument( "--diligent", action="count", default=0, dest="diligence", help=argparse.SUPPRESS, ) fixer_group.add_argument("--uncertain", action="store_true", help="Include changes with lower certainty.") fixer_group.add_argument("--yolo", action="store_true", help=argparse.SUPPRESS) fixer_group.add_argument("--force-subprocess", action="store_true", default=False, help=argparse.SUPPRESS) package_group = parser.add_argument_group("package preferences") package_group.add_argument( "--allow-reformatting", default=None, action="store_true", help="Allow file reformatting and stripping of comments.") package_group.add_argument( "--no-update-changelog", action="store_false", default=None, dest="update_changelog", help="do not update the changelog", ) package_group.add_argument( "--update-changelog", action="store_true", dest="update_changelog", help="force updating of the changelog", default=None, ) package_group.add_argument("--trust", action="store_true", help=argparse.SUPPRESS) output_group = parser.add_argument_group("output") output_group.add_argument("--verbose", help="be verbose", action="store_true", default=('SVP_API' in os.environ)) output_group.add_argument("--diff", help="Print resulting diff afterwards.", action="store_true") output_group.add_argument("--version", action="version", version="%(prog)s " + version_string) output_group.add_argument("--list-fixers", action="store_true", help="list available fixers") output_group.add_argument( "--list-tags", action="store_true", help="list lintian tags for which fixers are available", ) output_group.add_argument( "--dry-run", help=("Do not make any changes to the current repository. " "Note: currently creates a temporary clone of the repository."), action="store_true", ) output_group.add_argument( "--identity", help="Print user identity that would be used when committing", action="store_true", default=False, ) parser.add_argument( "-d", "--directory", metavar="DIRECTORY", help="directory to run in", type=str, default=".", ) parser.add_argument( "--disable-net-access", help="Do not probe external services.", action="store_true", default=False, ) parser.add_argument("--disable-inotify", action="store_true", default=False, help=argparse.SUPPRESS) args = parser.parse_args(argv) logging.basicConfig(level=logging.INFO, format='%(message)s') if args.list_fixers and args.list_tags: parser.print_usage() return 1 fixers = available_lintian_fixers(args.fixers_dir, force_subprocess=args.force_subprocess) if args.list_fixers: for script in sorted([fixer.name for fixer in fixers]): print(script) elif args.list_tags: tags = set() for fixer in fixers: tags.update(fixer.lintian_tags) for tag in sorted(tags): print(tag) else: try: if args.dry_run: branch, subpath = Branch.open_containing(args.directory) td = tempfile.mkdtemp() atexit.register(shutil.rmtree, td) # TODO(jelmer): Make a slimmer copy to_dir = branch.controldir.sprout( td, None, create_tree_if_local=True, source_branch=branch, stacked=branch._format.supports_stacking(), ) wt = to_dir.open_workingtree() else: wt, subpath = WorkingTree.open_containing(args.directory) except NotBranchError: logging.error( "No version control directory found (e.g. a .git directory).") return 1 except DependencyNotPresent as e: logging.error( "Unable to open tree at %s: missing package %s", args.directory, e.library, ) return 1 if args.identity: print("Committer identity: %s" % get_committer(wt)) print("Changelog identity: %s <%s>" % get_maintainer()) return 0 since_revid = wt.last_revision() if args.fixers: try: fixers = select_fixers(fixers, args.fixers, args.exclude) except KeyError as e: logging.error("Unknown fixer specified: %s", e.args[0]) return 1 debian_info = distro_info.DebianDistroInfo() if args.modern: if args.compat_release: logging.error( "--compat-release and --modern are incompatible.") return 1 compat_release = debian_info.devel() else: compat_release = args.compat_release minimum_certainty = args.minimum_certainty allow_reformatting = args.allow_reformatting update_changelog = args.update_changelog try: cfg = Config.from_workingtree(wt, subpath) except FileNotFoundError: pass else: if minimum_certainty is None: minimum_certainty = cfg.minimum_certainty() if compat_release is None: compat_release = cfg.compat_release() if allow_reformatting is None: allow_reformatting = cfg.allow_reformatting() if update_changelog is None: update_changelog = cfg.update_changelog() if minimum_certainty is None: if args.uncertain or args.yolo: minimum_certainty = "possible" else: minimum_certainty = DEFAULT_MINIMUM_CERTAINTY if compat_release is None: compat_release = debian_info.stable() if allow_reformatting is None: allow_reformatting = False with wt.lock_write(): if control_files_in_root(wt, subpath): report_fatal( "control-files-in-root", "control files live in root rather than debian/ " "(LarstIQ mode)", ) try: overall_result = run_lintian_fixers( wt, fixers, update_changelog=update_changelog, compat_release=compat_release, verbose=args.verbose, minimum_certainty=minimum_certainty, trust_package=args.trust, allow_reformatting=allow_reformatting, use_inotify=(False if args.disable_inotify else None), subpath=subpath, net_access=not args.disable_net_access, opinionated=args.opinionated, diligence=args.diligence, ) except NotDebianPackage: report_fatal("not-debian-package", "Not a Debian package") return 1 except WorkspaceDirty: logging.error("%s: Please commit pending changes first.", wt.basedir) if args.verbose: from breezy.status import show_tree_status show_tree_status(wt) return 1 except ChangelogCreateError as e: report_fatal("changelog-create-error", "Error creating changelog entry: %s" % e) return 1 except DescriptionMissing as e: report_fatal( "fixer-description-missing", "Fixer %s made changes but did not provide description." % e.fixer) return 1 if overall_result.overridden_lintian_issues: if len(overall_result.overridden_lintian_issues) == 1: logging.info("%d change skipped because of lintian overrides.", len(overall_result.overridden_lintian_issues)) else: logging.info( "%d changes skipped because of lintian overrides.", len(overall_result.overridden_lintian_issues)) if overall_result.success: all_tags = set() for result, summary in overall_result.success: all_tags.update(result.fixed_lintian_tags) if all_tags: logging.info("Lintian tags fixed: %r", all_tags) else: logging.info("Some changes were made, " "but there are no affected lintian tags.") min_certainty = overall_result.minimum_success_certainty() if min_certainty != "certain": logging.info( "Some changes were made with lower certainty (%s); " "please double check the changes.", min_certainty, ) else: report_fatal("nothing-to-do", "No changes made.") return 0 if overall_result.failed_fixers and not args.verbose: logging.info( "Some fixer scripts failed to run: %r. " "Run with --verbose for details.", set(overall_result.failed_fixers.keys()), ) if overall_result.formatting_unpreservable and not args.verbose: logging.info( "Some fixer scripts were unable to preserve formatting: %r. " "Run with --allow-reformatting to reformat %r.", set(overall_result.formatting_unpreservable), set(overall_result.formatting_unpreservable.values()), ) if args.diff: from breezy.diff import show_diff_trees show_diff_trees(wt.branch.repository.revision_tree(since_revid), wt, sys.stdout.buffer) if os.environ.get('SVP_API') == '1': applied = [] if 'SVP_RESUME' in os.environ: with open(os.environ['SVP_RESUME'], 'r') as f: base = json.load(f) applied.extend(base['applied']) all_fixed_lintian_tags = set() for result, summary in overall_result.success: applied.append({ "summary": summary, "description": result.description, "fixed_lintian_tags": result.fixed_lintian_tags, "revision_id": result.revision_id.decode("utf-8"), "certainty": result.certainty, }) all_fixed_lintian_tags.update(result.fixed_lintian_tags) failed = { name: str(e) for (name, e) in overall_result.failed_fixers.items() } debian_context = {} if overall_result.changelog_behaviour: debian_context[ 'changelog'] = overall_result.changelog_behaviour.json() with open(os.environ['SVP_RESULT'], 'w') as f: json.dump( { 'value': calculate_value(all_fixed_lintian_tags), 'debian': debian_context, 'context': { 'applied': applied, 'failed': failed, "versions": { "lintian-brush": lintian_brush_version_string, "breezy": breezy.version_string, } } }, f)
def make_changes( # noqa: C901 self, local_tree, subpath, update_changelog, reporter, committer, base_proposal=None): base_revid = local_tree.last_revision() upstream_base_revid = NULL_REVISION reporter.report_metadata( "versions", { "lintian-brush": lintian_brush_version_string, "silver-platter": silver_platter.version_string, "breezy": breezy.version_string, }, ) import distro_info debian_info = distro_info.DebianDistroInfo() compat_release = self.compat_release try: cfg = Config.from_workingtree(local_tree, subpath) except FileNotFoundError: pass else: compat_release = cfg.compat_release() if compat_release: compat_release = debian_info.codename(compat_release, default=compat_release) if compat_release is None: compat_release = debian_info.stable() # For now... upstream_branch = local_tree.branch upstream_subpath = subpath with local_tree.lock_write(): try: result = debianize( local_tree, subpath=subpath, upstream_branch=upstream_branch, upstream_subpath=upstream_subpath, compat_release=self.compat_release, schroot=self.schroot, diligence=self.diligence, trust=self.trust, verbose=self.verbose, force_new_directory=self.force_new_directory, create_dist=getattr(self, 'create_dist', None)) except OSError as e: if e.errno == errno.ENOSPC: raise ChangerError('no-space-on-device', str(e)) else: raise except DebianDirectoryExists: raise ChangerError( 'debian-directory-exists', "A debian/ directory already exists in the upstream project." ) except SourcePackageNameInvalid as e: raise ChangerError( 'invalid-source-package-name', "Generated source package name %r is not valid." % e.source) except NoBuildToolsFound: raise ChangerError( 'no-build-tools', "Unable to find any build systems in upstream sources.") except NoUpstreamReleases: raise ChangerError( 'no-upstream-releases', 'The upstream project does not appear to have made any releases.' ) except DistCommandFailed as e: raise ChangerError("dist-command-failed", str(e), e) except DetailedFailure as e: raise ChangerError('dist-%s' % e.error.kind, str(e.error)) except UnidentifiedError as e: if e.secondary: raise ChangerError('dist-command-failed', str(e.secondary.line)) else: raise ChangerError('dist-command-failed', e.lines[-1]) except DistCreationFailed as e: if e.inner: raise ChangerError('dist-%s' % e.inner.kind, e.msg) else: raise ChangerError('dist-command-failed', e.msg) # TODO(jelmer): Pristine tar branch? branches = [ ("main", None, base_revid, local_tree.last_revision()), ] if result.upstream_branch_name: branches.append(( "upstream", result.upstream_branch_name, upstream_base_revid, local_tree.controldir.open_branch( result.upstream_branch_name).last_revision(), )) tags = [(("upstream", str(result.upstream_version), component), tag, local_tree.branch.tags.lookup_tag(tag)) for (component, tag) in result.tag_names.items()] reporter.report_metadata("wnpp_bugs", result.wnpp_bugs) return ChangerResult( description="Debianized package.", mutator=None, branches=branches, tags=tags, value=None, sufficient_for_proposal=True, )
def main(): # noqa: C901 import argparse import breezy # noqa: E402 breezy.initialize() import breezy.git # noqa: E402 import breezy.bzr # noqa: E402 from breezy.workspace import ( check_clean_tree, WorkspaceDirty, ) from . import ( version_string, ) from .config import Config parser = argparse.ArgumentParser(prog="deb-scrub-obsolete") parser.add_argument( "--directory", metavar="DIRECTORY", help="directory to run in", type=str, default=".", ) parser.add_argument( "--upgrade-release", metavar="UPGRADE-RELEASE", help="Release to allow upgrading from.", default="oldstable", ) parser.add_argument('--compat-release', metavar='COMPAT-RELEASE', help="Release to allow building on.", default=None) parser.add_argument( "--no-update-changelog", action="store_false", default=None, dest="update_changelog", help="do not update the changelog", ) parser.add_argument( "--update-changelog", action="store_true", dest="update_changelog", help="force updating of the changelog", default=None, ) parser.add_argument( "--allow-reformatting", default=None, action="store_true", help=argparse.SUPPRESS, ) parser.add_argument("--version", action="version", version="%(prog)s " + version_string) parser.add_argument( "--identity", help="Print user identity that would be used when committing", action="store_true", default=False, ) parser.add_argument("--debug", help="Describe all considered changes.", action="store_true") args = parser.parse_args() wt, subpath = WorkingTree.open_containing(args.directory) if args.identity: logging.info('%s', get_committer(wt)) return 0 try: check_clean_tree(wt, wt.basis_tree(), subpath) except WorkspaceDirty: logging.info("%s: Please commit pending changes first.", wt.basedir) return 1 import distro_info debian_info = distro_info.DebianDistroInfo() upgrade_release = debian_info.codename(args.upgrade_release) if args.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO, format='%(message)s') update_changelog = args.update_changelog allow_reformatting = args.allow_reformatting if args.compat_release: compat_release = debian_info.codename(args.compat_release) else: compat_release = None try: cfg = Config.from_workingtree(wt, subpath) except FileNotFoundError: pass else: if update_changelog is None: update_changelog = cfg.update_changelog() if allow_reformatting is None: allow_reformatting = cfg.allow_reformatting() if compat_release is None: compat_release = cfg.compat_release() if compat_release is None: compat_release = debian_info.codename('oldstable') if upgrade_release != compat_release: logging.info( "Removing run time constraints unnecessary since %s" " and build time constraints unnecessary since %s", upgrade_release, compat_release) else: logging.info( "Removing run time and build time constraints unnecessary " "since %s", compat_release) if allow_reformatting is None: allow_reformatting = False if is_debcargo_package(wt, subpath): report_fatal("nothing-to-do", "Package uses debcargo") return 1 elif not control_file_present(wt, subpath): report_fatal("missing-control-file", "Unable to find debian/control") return 1 try: result = scrub_obsolete(wt, subpath, compat_release, upgrade_release, update_changelog=args.update_changelog, allow_reformatting=allow_reformatting) except FormattingUnpreservable as e: report_fatal( "formatting-unpreservable", "unable to preserve formatting while editing %s" % e.path, ) return 1 except GeneratedFile as e: report_fatal("generated-file", "unable to edit generated file: %r" % e) return 1 except NotDebianPackage: report_fatal('not-debian-package', 'Not a Debian package.') return 1 except ChangeConflict as e: report_fatal('change-conflict', 'Generated file changes conflict: %s' % e) return 1 except UddTimeout: report_fatal('udd-timeout', 'Timeout communicating with UDD') return 1 if not result: report_okay("nothing-to-do", "no obsolete constraints") return 0 debian_context = {} if result.changelog_behaviour: debian_context['changelog'] = result.changelog_behaviour.json() if os.environ.get("SVP_API") == "1": with open(os.environ["SVP_RESULT"], "w") as f: json.dump( { "description": "Remove constraints unnecessary since %s." % upgrade_release, "value": result.value(), "debian": debian_context, "context": { "specific_files": result.specific_files, "maintscript_removed": result.maintscript_removed, "control_removed": result.control_removed, } }, f) logging.info("Scrub obsolete settings.") for release, lines in result.itemized().items(): for line in lines: logging.info("* %s", line) return 0
def main(args): import distro_info import socket import subprocess import silver_platter # noqa: F401 from . import ( propose_or_push, BuildFailedError, MissingUpstreamTarball, NoSuchPackage, ) from breezy import ( errors, ) from breezy.trace import note from breezy.plugins.propose.propose import ( NoSuchProject, UnsupportedHoster, ) possible_transports = [] possible_hosters = [] fixer_scripts = {} for fixer in available_lintian_fixers(): for tag in fixer.lintian_tags: fixer_scripts[tag] = fixer available_fixers = set(fixer_scripts) if args.fixers: available_fixers = available_fixers.intersection(set(args.fixers)) debian_info = distro_info.DebianDistroInfo() for pkg in args.packages: if args.pre_check: def pre_check(local_tree): try: subprocess.check_call(args.pre_check, shell=True, cwd=local_tree.basedir) except subprocess.CalledProcessError: note('%r: pre-check failed, skipping', pkg) return False return True else: pre_check = None if args.post_check: def post_check(local_tree, since_revid): try: subprocess.check_call(args.post_check, shell=True, cwd=local_tree.basedir, env={'SINCE_REVID': since_revid}) except subprocess.CalledProcessError: note('%r: post-check failed, skipping', pkg) return False return True else: post_check = None note('Processing: %s', pkg) try: main_branch = open_packaging_branch( pkg, possible_transports=possible_transports) except NoSuchPackage: note('%s: no such package', pkg) except socket.error: note('%s: ignoring, socket error', pkg) except errors.NotBranchError as e: note('%s: Branch does not exist: %s', pkg, e) except errors.UnsupportedProtocol: note('%s: Branch available over unsupported protocol', pkg) except errors.ConnectionError as e: note('%s: %s', pkg, e) except errors.PermissionDenied as e: note('%s: %s', pkg, e) except errors.InvalidHttpResponse as e: note('%s: %s', pkg, e) except errors.TransportError as e: note('%s: %s', pkg, e) else: # If it's unknown which fixers are relevant, just try all of them. if args.fixers: fixers = args.fixers else: fixers = available_fixers branch_changer = LintianFixer( pkg, fixers=[fixer_scripts[fixer] for fixer in fixers], update_changelog=args.update_changelog, compat_release=debian_info.stable(), build_verify=args.build_verify, pre_check=pre_check, post_check=post_check, propose_addon_only=args.propose_addon_only, committer=args.committer) try: result = propose_or_push( main_branch, "lintian-fixes", branch_changer, args.mode, possible_transports=possible_transports, possible_hosters=possible_hosters, refresh=args.refresh, dry_run=args.dry_run) except UnsupportedHoster: note('%s: Hoster unsupported', pkg) continue except NoSuchProject as e: note('%s: project %s was not found', pkg, e.project) continue except BuildFailedError: note('%s: build failed', pkg) continue except MissingUpstreamTarball: note('%s: unable to find upstream source', pkg) continue except errors.PermissionDenied as e: note('%s: %s', pkg, e) continue except PostCheckFailed as e: note('%s: %s', pkg, e) continue else: if result.merge_proposal: tags = set() for brush_result, unused_summary in branch_changer.applied: tags.update(brush_result.fixed_lintian_tags) if result.is_new: note('%s: Proposed fixes %r: %s', pkg, tags, result.merge_proposal.url) elif tags: note('%s: Updated proposal %s with fixes %r', pkg, result.merge_proposal.url, tags) else: note('%s: No new fixes for proposal %s', pkg, result.merge_proposal.url) if args.diff: result.show_base_diff(sys.stdout.buffer)
def __init__(self, config_layers=(), current_directory=None): super(Config, self).__init__() self._suites = WeakValueDictionary() self._vendors = {} self._overrides = {} self._relevant_directory = None d = yaml.safe_load( open(os.path.join(os.path.dirname(__file__), 'defaults.yaml'))) # Some things can have better defaults that can't be hard-coded d['defaults']['parallel'] = str(os.cpu_count()) try: d['defaults']['architecture'] = subprocess.check_output( ['dpkg', '--print-architecture'], universal_newlines=True).strip() except subprocess.CalledProcessError: pass d['vendors']['debian']['default_suite'] = 'sid' try: import distro_info except ImportError: d['vendors']['debian']['default_worker_suite'] = 'sid' else: debian = distro_info.DebianDistroInfo() ubuntu = distro_info.UbuntuDistroInfo() d['vendors']['debian']['default_worker_suite'] = debian.stable() d['vendors']['debian']['suites']['stable'] = { 'alias_for': debian.stable(), } d['vendors']['debian']['suites']['testing'] = { 'alias_for': debian.testing(), } d['vendors']['debian']['suites']['oldstable'] = { 'alias_for': debian.old(), } # According to autopkgtest-buildvm-ubuntu-cloud, just after # an Ubuntu release there is briefly no development version # at all. try: ubuntu_devel = ubuntu.devel() except distro_info.DistroDataOutdated: ubuntu_devel = ubuntu.stable() d['vendors']['ubuntu']['default_suite'] = ubuntu_devel d['vendors']['ubuntu']['default_worker_suite'] = (ubuntu.lts() + '-backports') d['vendors']['ubuntu']['suites']['devel'] = { 'alias_for': ubuntu_devel, } for suite in debian.all: d['vendors']['debian']['suites'].setdefault(suite, {}) for suite in ubuntu.all: d['vendors']['ubuntu']['suites'].setdefault(suite, {}) self._raw = [] self._raw.append(d) if config_layers: self._raw[:0] = list(config_layers) else: config_dirs = XDG_CONFIG_DIRS.split(':') config_dirs = list(reversed(config_dirs)) config_dirs.append(XDG_CONFIG_HOME) for p in config_dirs: conffile = os.path.join(p, 'vectis', 'vectis.yaml') try: reader = open(conffile) except FileNotFoundError: continue with reader: raw = yaml.safe_load(reader) if not isinstance(raw, dict): raise ConfigError( 'Reading {!r} did not yield a dict'.format( conffile)) self._raw.insert(0, raw) if current_directory is None: current_directory = os.getcwd() self._relevant_directory = None while self._relevant_directory is None: for r in self._raw: if current_directory in r.get('directories', {}): self._relevant_directory = current_directory break else: parent, _ = os.path.split(current_directory) # Guard against infinite recursion. If current_directory == '/' # we would already have found directories./ in the hard-coded # defaults, and broken out of the loop assert len(parent) < len(current_directory) current_directory = parent continue assert self._relevant_directory is not None self._path_based = Directory(self._relevant_directory, self._raw)