def get_debian_source_series(self):
title = self.bug_task.bug.title.lower().split()
if "experimental" in title:
series = "experimental"
elif "testing" in title:
series = distro_info.DebianDistroInfo().testing()
else:
series = distro_info.DebianDistroInfo().devel()
return series
def test_debian_backports_sloppy(self):
try:
import distro_info
except ImportError:
return
c = self.__config
c.vendor = 'debian'
c.suite = 'stable-backports-sloppy'
debian = c.get_vendor('debian')
self.assertIs(c.vendor, debian)
debian_info = distro_info.DebianDistroInfo()
sloppy = c.get_suite(debian, 'stable-backports-sloppy')
backports = c.get_suite(debian, 'stable-backports')
stable = c.get_suite(debian, 'stable')
self.assertIs(c.suite, sloppy)
self.assertEqual(str(sloppy),
debian_info.stable() + '-backports-sloppy')
self.assertEqual(sloppy.hierarchy[0], sloppy)
self.assertEqual(str(sloppy.hierarchy[1]), str(backports))
self.assertEqual(str(sloppy.hierarchy[2]), str(stable))
self.assertEqual(sloppy.sbuild_resolver,
['--build-dep-resolver=aptitude'])
self.assertEqual(c.get_mirrors().lookup_suite(sloppy),
'http://192.168.122.1:3142/debian')
self.assertEqual(sloppy.archive, 'debian')
self.assertEqual(c.sbuild_resolver, ['--build-dep-resolver=aptitude'])
def test_debian_stable_security(self):
c = self.__config
c.vendor = 'debian'
c.suite = 'stable-security'
try:
import distro_info
except ImportError:
return
debian = c.get_vendor('debian')
self.assertIs(c.vendor, debian)
debian_info = distro_info.DebianDistroInfo()
security = c.get_suite(debian, 'stable-security')
stable = c.get_suite(debian, 'stable')
self.assertEqual(security.apt_suite,
'{}/updates'.format(debian_info.stable()))
self.assertEqual(c.get_mirrors().lookup_suite(security),
'http://192.168.122.1:3142/security.debian.org')
self.assertEqual(security.archive, 'security.debian.org')
self.assertEqual(security.hierarchy[0], security)
self.assertEqual(str(security.hierarchy[1]), str(stable))
with self.assertRaises(AttributeError):
c.archive
def resolve_release_codename(name: str, date=None) -> str:
def oldest_name(fn):
return max(fn("object", date), key=lambda r: r.created).name
if '/' in name:
distro, name = name.split('/', 1)
else:
distro = None
if distro in ('debian', None):
debian = distro_info.DebianDistroInfo()
if name == 'lts':
return oldest_name(debian.lts_supported)
if name == 'elts':
return oldest_name(debian.elts_supported)
if debian.codename(name):
return debian.codename(name)
if debian.valid(name):
return name
if distro in ('ubuntu', None):
ubuntu = distro_info.UbuntuDistroInfo()
if name == 'esm':
return oldest_name(ubuntu.supported_esm)
if ubuntu.valid(name):
return name
return None
def setUp(self):
self.cobj = conf.Config('notimportant', {})
debdist = distro_info.DebianDistroInfo()
self.stable = debdist.stable()
self.unstable = debdist.devel()
self.oldstable = debdist.old()
self.testing = debdist.testing()
self.experimental = 'experimental'
def previous_release(release):
import distro_info
debian = distro_info.DebianDistroInfo()
if release in (debian.devel(), debian.testing(), 'experimental'):
return debian.stable()
releases = debian.get_all()
try:
return releases[releases.index(release) - 1]
except ValueError:
pass
# TODO(jelmer): Ubuntu?
return None
def prepare_apt(self):
distroinfo = distro_info.DebianDistroInfo()
if distroinfo.testing() == self.codename:
self.suite = "testing"
elif self.codename == "sid":
self.suite = "unstable"
else:
self.suite = "stable"
if not self.codename or not self.mirror:
raise cliapp.AppException(
"Misconfiguration: no codename or mirror set")
state_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(state_dir, 'lists'))
self.dirlist.append(state_dir)
cache_dir = tempfile.mkdtemp()
os.makedirs(os.path.join(cache_dir, 'archives', 'partial'))
self.dirlist.append(cache_dir)
etc_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(etc_dir, 'apt.conf.d'))
os.mkdir(os.path.join(etc_dir, 'preferences.d'))
os.mkdir(os.path.join(etc_dir, 'trusted.gpg.d'))
copy_files('/etc/apt/trusted.gpg.d',
os.path.join(etc_dir, 'trusted.gpg.d'))
with open(os.path.join(etc_dir, 'sources.list'), 'w') as sources:
sources.write(
'deb %s %s %s\n' %
(self.mirror, self.codename, ' '.join(self.components)))
self.dirlist.append(etc_dir)
updates = {
'APT::Architecture': str(self.architecture),
'APT::Architectures': str(self.architecture),
'Dir::State': state_dir,
'Dir::Cache': cache_dir,
'Dir::Etc': etc_dir,
}
for key, value in updates.items():
try:
apt_pkg.config[key] = value # pylint: disable=no-member
except TypeError as exc:
print(key, value, exc)
continue
self.cache = apt.cache.Cache()
try:
self.cache.update()
self.cache.open()
except apt.cache.FetchFailedException as exc:
raise cliapp.AppException('Unable to update cache: %s' % exc)
if not os.path.exists(self.destdir):
raise cliapp.AppException(
'Destination directory %s does not exist' % self.destdir)
def test_unknown_vendor(self):
c = self.__config
c.vendor = 'steamos'
c.suite = 'brewmaster'
steamos = c.get_vendor('steamos')
debian = c.get_vendor('debian')
brewmaster = c.get_suite(steamos, 'brewmaster')
self.assertEqual(str(steamos), 'steamos')
self.assertEqual(steamos.components, {'main'})
self.assertEqual(list(brewmaster.hierarchy), [brewmaster])
with self.assertRaises(AttributeError):
steamos.archive
self.assertEqual(c.components, {'main'})
self.assertEqual(c.vendor, steamos)
self.assertIs(c.worker_vendor, debian)
self.assertIs(c.pbuilder_worker_vendor, debian)
self.assertIs(c.sbuild_worker_vendor, debian)
self.assertIs(c.vmdebootstrap_worker_vendor, debian)
with self.assertRaises(AttributeError):
c.archive
self.assertEqual(c.autopkgtest, [
'schroot',
'qemu:autopkgtest.qcow2',
'qemu:autopkgtest-merged-usr.qcow2',
])
self.assertIsNone(c.get_suite(steamos, 'xyzzy', create=False))
self.assertIsNotNone(c.get_suite(steamos, 'xyzzy'))
self.assertIs(c.get_suite(steamos, 'xyzzy'),
c.get_suite(steamos, 'xyzzy'))
self.assertEqual(c.get_mirrors().lookup_suite(brewmaster),
'http://localhost/steamos')
self.assertEqual(brewmaster.archive, 'steamos')
try:
import distro_info
except ImportError:
return
debian_info = distro_info.DebianDistroInfo()
self.assertIs(c.worker_suite, c.get_suite(debian,
debian_info.stable()))
def test_distro_info(self):
debian = self.__config._get_vendor('debian')
ubuntu = self.__config._get_vendor('ubuntu')
try:
import distro_info
except ImportError:
return
debian_info = distro_info.DebianDistroInfo()
ubuntu_info = distro_info.UbuntuDistroInfo()
try:
ubuntu_devel = ubuntu_info.devel()
except distro_info.DistroDataOutdated:
ubuntu_devel = ubuntu_info.stable()
self.assertEqual(str(ubuntu.get_suite('devel')), ubuntu_devel)
self.assertEqual(str(debian.get_suite('unstable')), 'sid')
self.assertEqual(str(debian.get_suite('testing')),
debian_info.testing())
self.assertEqual(str(debian.get_suite('oldstable')), debian_info.old())
self.assertEqual(str(debian.get_suite('rc-buggy')), 'experimental')
stable = debian.get_suite('stable')
self.assertEqual(str(stable), debian_info.stable())
self.assertEqual(stable.sbuild_resolver, [])
backports = debian.get_suite('stable-backports')
self.assertEqual(str(backports), debian_info.stable() + '-backports')
self.assertEqual(backports.sbuild_resolver,
['--build-dep-resolver=aptitude'])
self.assertEqual(backports.apt_suite,
debian_info.stable() + '-backports')
self.assertEqual(backports.mirror, 'http://192.168.122.1:3142/debian')
self.assertEqual(backports.hierarchy[0], backports)
self.assertEqual(str(backports.hierarchy[1]), str(stable))
security = debian.get_suite('stable-security')
self.assertEqual(security.apt_suite,
'{}/updates'.format(debian_info.stable()))
self.assertEqual(security.mirror,
'http://192.168.122.1:3142/security.debian.org')
self.assertEqual(security.hierarchy[0], security)
self.assertEqual(str(security.hierarchy[1]), str(stable))
def _get_distmap(self):
debdist = distro_info.DebianDistroInfo()
# start with e.g. "sid" -> "unstable"
distmap = collections.defaultdict(lambda: "unknown", [
(debdist.old(), "oldstable"),
(debdist.devel(), "unstable"),
(debdist.stable(), "stable"),
(debdist.testing(), "testing"),
("experimental", "experimental"),
("rc", "experimental"),
])
# add mappings for e.g. "oldstable" -> "oldstable"
distmap.update(dict([(val, val) for key, val in distmap.iteritems()]))
# map e.g. "Debian6" -> "oldstable" where debdist.old(result="fullname")
# currently returns 'Debian 6.0 "Squeeze"'
dkey = lambda x: "Debian" + re.split('[ \.]', x(result="fullname"))[1]
dfuncs = [debdist.old, debdist.stable, debdist.testing]
distmap.update(dict([(dkey(x), distmap[x()]) for x in dfuncs]))
return distmap
def test_defaults(self):
self.__config = Config(config_layers=({}, ), current_directory='/')
c = self.__config
self.assertGreaterEqual(c.parallel, 1)
self.assertIs(type(c.parallel), int)
debian = c.get_vendor('debian')
ubuntu = c.get_vendor('ubuntu')
self.assertEqual(str(c.vendor), 'debian')
self.assertEqual(str(c.worker_vendor), 'debian')
self.assertEqual(str(c.vmdebootstrap_worker_vendor), 'debian')
self.assertEqual(str(c.sbuild_worker_vendor), 'debian')
self.assertIs(c.vendor, debian)
self.assertIs(c.worker_vendor, debian)
self.assertIs(c.sbuild_worker_vendor, debian)
self.assertIs(c.vmdebootstrap_worker_vendor, debian)
with self.assertRaises(AttributeError):
c.archive
self.assertIs(c.sbuild_indep_together, False)
self.assertIs(c.sbuild_source_together, False)
self.assertEqual(c.output_parent, '..')
self.assertEqual(c.qemu_image_size, '10G')
self.assertIsNone(c.sbuild_buildables)
self.assertEqual(c.sbuild_resolver, [])
self.assertEqual(c.debootstrap_script, None)
self.assertEqual(c.apt_key,
'/usr/share/keyrings/debian-archive-keyring.gpg')
self.assertEqual(c.dpkg_source_tar_ignore, [])
self.assertIsNone(c.dpkg_source_diff_ignore)
self.assertEqual(c.dpkg_source_extend_diff_ignore, [])
if ARCHITECTURE is not None:
self.assertEqual(c.architecture, ARCHITECTURE)
if 0:
# FIXME: these raise, because suite is undefined,
# but then the error is trapped and __getattr__ is called
# instead
self.assertEqual(
c.qemu_image,
'{}/vectis/{}/debian/sid/autopkgtest.qcow2'.format(
XDG_CACHE_HOME, ARCHITECTURE))
self.assertEqual(
c.write_qemu_image,
'{}/vectis/{}/debian/sid/autopkgtest.qcow2'.format(
XDG_CACHE_HOME, ARCHITECTURE))
self.assertEqual(c.worker_architecture, ARCHITECTURE)
self.assertEqual(c.worker, ['qemu', c.worker_qemu_image])
self.assertEqual(
c.worker_qemu_image,
'{}/vectis/{}/debian/sid/autopkgtest.qcow2'.format(
XDG_CACHE_HOME, ARCHITECTURE))
self.assertEqual(c.sbuild_worker,
['qemu', c.sbuild_worker_qemu_image])
self.assertEqual(
c.sbuild_worker_qemu_image,
'{}/vectis/{}/debian/sid/autopkgtest.qcow2'.format(
XDG_CACHE_HOME, ARCHITECTURE))
self.assertEqual(c.autopkgtest, ['lxc', 'qemu'])
self.assertEqual(c.suite, None)
with self.assertRaises(AttributeError):
c.apt_suite
try:
import distro_info
except ImportError:
pass
else:
self.assertEqual(
c.worker_suite,
c.get_suite(c.vendor,
distro_info.DebianDistroInfo().stable()))
self.assertEqual(c.default_worker_suite,
distro_info.DebianDistroInfo().stable())
stable = c.get_suite(c.vendor, 'stable')
self.assertEqual(c.sbuild_worker_suite, stable)
# #860433, #877592
self.assertEqual(str(c.piuparts_worker_suite), 'stretch-backports')
self.assertEqual(c.default_suite, 'sid')
self.assertEqual(c.components, {'main'})
self.assertEqual(c.extra_components, {'contrib', 'non-free'})
self.assertEqual(c.all_components, {'main', 'contrib', 'non-free'})
self.assertEqual(c.storage, '{}/vectis'.format(XDG_CACHE_HOME))
def make_changes(
self,
local_tree,
subpath,
update_changelog,
reporter,
committer,
base_proposal=None,
):
from lintian_brush.scrub_obsolete import scrub_obsolete
import distro_info
debian_info = distro_info.DebianDistroInfo()
if self.compat_release:
compat_release = debian_info.codename(self.compat_release)
else:
compat_release = None
upgrade_release = debian_info.codename(self.upgrade_release)
base_revid = local_tree.last_revision()
allow_reformatting = self.allow_reformatting
try:
cfg = Config.from_workingtree(local_tree, subpath)
except FileNotFoundError:
pass
else:
if allow_reformatting is None:
allow_reformatting = cfg.allow_reformatting()
if update_changelog is None:
update_changelog = cfg.update_changelog()
if compat_release is None:
compat_release = cfg.compat_release()
if compat_release is None:
compat_release = debian_info.stable()
if is_debcargo_package(local_tree, subpath):
raise ChangerError("nothing-to-do", "Package uses debcargo")
elif not control_file_present(local_tree, subpath):
raise ChangerError("missing-control-file",
"Unable to find debian/control")
try:
result = scrub_obsolete(
local_tree,
subpath,
compat_release,
upgrade_release,
update_changelog=update_changelog,
)
except FormattingUnpreservable as e:
raise ChangerError(
"formatting-unpreservable",
"unable to preserve formatting while editing %s" % e.path,
)
except GeneratedFile as e:
raise ChangerError("generated-file",
"unable to edit generated file: %r" % e)
except NotDebianPackage:
raise ChangerError("not-debian-package", "Not a Debian package")
if not result:
raise ChangerError("nothing-to-do", "no obsolete constraints")
branches = [("main", None, base_revid, local_tree.last_revision())]
tags = []
return ChangerResult(
description="Scrub obsolete settings.",
mutator=result,
branches=branches,
tags=tags,
value=calculate_value(result),
sufficient_for_proposal=True,
proposed_commit_message="Scrub obsolete settings.",
)
def test_debian(self):
c = self.__config
c.vendor = 'debian'
c.suite = 'sid'
debian = c.get_vendor('debian')
self.assertIs(c.vendor, debian)
sid = c.get_suite(debian, 'sid')
self.assertIs(c.suite, sid)
# Properties of the vendor itself
self.assertEqual(str(debian), 'debian')
self.assertEqual(debian.default_suite, 'sid')
self.assertIs(c.get_suite(debian, 'unstable'), sid)
self.assertEqual(debian.components, {'main'})
self.assertEqual(debian.extra_components, {'contrib', 'non-free'})
self.assertEqual(debian.all_components,
{'main', 'contrib', 'non-free'})
self.assertIsNone(c.get_suite(debian, 'xenial', create=False))
# Properties of the suite itswelf
self.assertEqual(sid.apt_key,
'/usr/share/keyrings/debian-archive-keyring.gpg')
self.assertEqual(sid.archive, 'debian')
self.assertEqual(c.get_mirrors().lookup_suite(sid),
'http://192.168.122.1:3142/debian')
self.assertIs(sid.base, None)
self.assertEqual(sid.components, {'main'})
self.assertEqual(sid.extra_components, {'contrib', 'non-free'})
self.assertEqual(sid.all_components, {'main', 'contrib', 'non-free'})
self.assertEqual(sid.apt_suite, 'sid')
self.assertEqual(sid.sbuild_resolver, [])
# Properties of the Config determined by the suite being Debian sid
self.assertEqual(c.autopkgtest, ['lxc', 'qemu'])
self.assertIs(c.worker_vendor, debian)
self.assertIs(c.lxc_worker_vendor, debian)
self.assertIs(c.piuparts_worker_vendor, debian)
self.assertIs(c.sbuild_worker_vendor, debian)
self.assertIs(c.vmdebootstrap_worker_vendor, debian)
with self.assertRaises(AttributeError):
c.archive
self.assertEqual(c.qemu_image_size, '10G')
self.assertGreaterEqual(c.parallel, 1)
self.assertIs(c.sbuild_indep_together, False)
self.assertIs(c.sbuild_source_together, False)
self.assertEqual(c.sbuild_resolver, [])
self.assertEqual(c.apt_key,
'/usr/share/keyrings/debian-archive-keyring.gpg')
self.assertIsNone(c.dpkg_source_diff_ignore)
self.assertEqual(c.dpkg_source_tar_ignore, [])
self.assertEqual(c.dpkg_source_extend_diff_ignore, [])
self.assertEqual(c.output_parent, '..')
self.assertEqual(c.architecture, 'mips')
self.assertEqual(c.worker_architecture, 'mips')
self.assertEqual(c.lxc_worker_architecture, 'mips')
self.assertEqual(c.piuparts_worker_architecture, 'mips')
self.assertEqual(c.sbuild_worker_architecture, 'mips')
with self.assertRaises(AttributeError):
c.apt_suite
# Below this point relies on knowledge of distro_info
try:
import distro_info
except ImportError:
return
debian_info = distro_info.DebianDistroInfo()
self.assertEqual(debian.default_worker_suite, debian_info.stable())
self.assertIs(c.lxc_worker_suite,
c.get_suite(debian, debian_info.stable()))
# #860433, #877592
self.assertIs(c.piuparts_worker_suite,
c.get_suite(debian, 'stretch-backports'))
self.assertIs(c.sbuild_worker_suite,
c.get_suite(debian, debian_info.stable()))
self.assertIs(c.vmdebootstrap_worker_suite,
c.get_suite(debian, debian_info.stable()))
self.assertIs(c.worker_suite, c.get_suite(debian,
debian_info.stable()))
self.assertEqual(str(c.get_suite(debian, 'unstable')), 'sid')
self.assertEqual(str(c.get_suite(debian, 'testing')),
debian_info.testing())
self.assertEqual(str(c.get_suite(debian, 'oldstable')),
debian_info.old())
self.assertEqual(str(c.get_suite(debian, 'rc-buggy')), 'experimental')
stable = c.get_suite(debian, 'stable')
self.assertEqual(str(stable), debian_info.stable())
def make_changes( # noqa: C901
self,
local_tree,
subpath,
update_changelog,
reporter,
committer,
base_proposal=None,
):
base_revid = local_tree.last_revision()
reporter.report_metadata(
"versions",
{
"lintian-brush": lintian_brush_version_string,
"silver-platter": silver_platter.version_string,
"breezy": breezy.version_string,
},
)
import distro_info
debian_info = distro_info.DebianDistroInfo()
compat_release = self.compat_release
allow_reformatting = self.allow_reformatting
minimum_certainty = None
try:
cfg = Config.from_workingtree(local_tree, subpath)
except FileNotFoundError:
pass
else:
compat_release = cfg.compat_release()
if compat_release:
compat_release = debian_info.codename(
compat_release, default=compat_release
)
allow_reformatting = cfg.allow_reformatting()
minimum_certainty = cfg.minimum_certainty()
if compat_release is None:
compat_release = debian_info.stable()
if allow_reformatting is None:
allow_reformatting = False
if minimum_certainty is None:
minimum_certainty = DEFAULT_MINIMUM_CERTAINTY
with local_tree.lock_write():
if control_files_in_root(local_tree, subpath):
raise ChangerError(
"control-files-in-root",
"control files live in root rather than debian/ " "(LarstIQ mode)",
)
try:
overall_result = run_lintian_fixers(
local_tree,
self.fixers,
committer=committer,
update_changelog=update_changelog,
compat_release=compat_release,
allow_reformatting=allow_reformatting,
minimum_certainty=minimum_certainty,
subpath=subpath,
diligence=self.diligence,
opinionated=self.opinionated,
trust_package=self.trust_package,
)
except NotDebianPackage:
raise ChangerError("not-debian-package", "Not a Debian package")
except ChangelogCreateError as e:
raise ChangerError(
"changelog-create-error", "Error creating changelog entry: %s" % e
)
applied = []
base_applied = reporter.get_base_metadata("applied", [])
if base_applied:
applied.extend(base_applied)
for result, summary in overall_result.success:
applied.append(
{
"summary": summary,
"description": result.description,
"fixed_lintian_tags": result.fixed_lintian_tags,
"revision_id": result.revision_id.decode("utf-8"),
"certainty": result.certainty,
}
)
reporter.report_metadata("applied", applied)
if overall_result.failed_fixers:
for fixer_name, failure in overall_result.failed_fixers.items():
logging.info("Fixer %r failed to run:", fixer_name)
sys.stderr.write(str(failure))
reporter.report_metadata(
"failed",
{name: str(e) for (name, e) in overall_result.failed_fixers.items()},
)
if not overall_result.success:
raise ChangerError("nothing-to-do", "no fixers to apply")
fixed_lintian_tags = set()
for result, summary in overall_result.success:
fixed_lintian_tags.update(result.fixed_lintian_tags)
add_on_only = not has_nontrivial_changes(
overall_result.success, self.propose_addon_only
)
if not reporter.get_base_metadata("add_on_only", False):
add_on_only = False
if not add_on_only:
if overall_result.success:
logging.info("only add-on fixers found")
sufficient_for_proposal = False
reporter.report_metadata("add_on_only", True)
else:
sufficient_for_proposal = True
reporter.report_metadata("add_on_only", False)
branches = [("main", None, base_revid, local_tree.last_revision())]
return ChangerResult(
description="Applied fixes for %r" % fixed_lintian_tags,
mutator=overall_result.success,
branches=branches,
tags=[],
value=calculate_value(fixed_lintian_tags),
sufficient_for_proposal=sufficient_for_proposal,
proposed_commit_message=update_proposal_commit_message(
base_proposal, overall_result.success
),
)
def main(argv=None): # noqa: C901
parser = argparse.ArgumentParser(prog="lintian-brush")
fixer_group = parser.add_argument_group("fixer selection")
fixer_group.add_argument("fixers",
metavar="FIXER",
nargs="*",
help="specific fixer to run")
fixer_group.add_argument(
"--fixers-dir",
type=str,
help="path to fixer scripts. [%(default)s]",
default=find_fixers_dir(),
)
fixer_group.add_argument(
"--exclude",
metavar="EXCLUDE",
type=str,
action="append",
help="Exclude a fixer.",
)
fixer_group.add_argument(
"--modern",
help=("Use features/compatibility levels that are not available in "
"stable. (makes backporting harder)"),
action="store_true",
default=False,
)
fixer_group.add_argument("--compat-release",
type=str,
help=argparse.SUPPRESS)
# Hide the minimum-certainty option for the moment.
fixer_group.add_argument(
"--minimum-certainty",
type=str,
choices=SUPPORTED_CERTAINTIES,
default=None,
help=argparse.SUPPRESS,
)
fixer_group.add_argument("--opinionated",
action="store_true",
help=argparse.SUPPRESS)
fixer_group.add_argument(
"--diligent",
action="count",
default=0,
dest="diligence",
help=argparse.SUPPRESS,
)
fixer_group.add_argument("--uncertain",
action="store_true",
help="Include changes with lower certainty.")
fixer_group.add_argument("--yolo",
action="store_true",
help=argparse.SUPPRESS)
fixer_group.add_argument("--force-subprocess",
action="store_true",
default=False,
help=argparse.SUPPRESS)
package_group = parser.add_argument_group("package preferences")
package_group.add_argument(
"--allow-reformatting",
default=None,
action="store_true",
help="Allow file reformatting and stripping of comments.")
package_group.add_argument(
"--no-update-changelog",
action="store_false",
default=None,
dest="update_changelog",
help="do not update the changelog",
)
package_group.add_argument(
"--update-changelog",
action="store_true",
dest="update_changelog",
help="force updating of the changelog",
default=None,
)
package_group.add_argument("--trust",
action="store_true",
help=argparse.SUPPRESS)
output_group = parser.add_argument_group("output")
output_group.add_argument("--verbose",
help="be verbose",
action="store_true",
default=('SVP_API' in os.environ))
output_group.add_argument("--diff",
help="Print resulting diff afterwards.",
action="store_true")
output_group.add_argument("--version",
action="version",
version="%(prog)s " + version_string)
output_group.add_argument("--list-fixers",
action="store_true",
help="list available fixers")
output_group.add_argument(
"--list-tags",
action="store_true",
help="list lintian tags for which fixers are available",
)
output_group.add_argument(
"--dry-run",
help=("Do not make any changes to the current repository. "
"Note: currently creates a temporary clone of the repository."),
action="store_true",
)
output_group.add_argument(
"--identity",
help="Print user identity that would be used when committing",
action="store_true",
default=False,
)
parser.add_argument(
"-d",
"--directory",
metavar="DIRECTORY",
help="directory to run in",
type=str,
default=".",
)
parser.add_argument(
"--disable-net-access",
help="Do not probe external services.",
action="store_true",
default=False,
)
parser.add_argument("--disable-inotify",
action="store_true",
default=False,
help=argparse.SUPPRESS)
args = parser.parse_args(argv)
logging.basicConfig(level=logging.INFO, format='%(message)s')
if args.list_fixers and args.list_tags:
parser.print_usage()
return 1
fixers = available_lintian_fixers(args.fixers_dir,
force_subprocess=args.force_subprocess)
if args.list_fixers:
for script in sorted([fixer.name for fixer in fixers]):
print(script)
elif args.list_tags:
tags = set()
for fixer in fixers:
tags.update(fixer.lintian_tags)
for tag in sorted(tags):
print(tag)
else:
try:
if args.dry_run:
branch, subpath = Branch.open_containing(args.directory)
td = tempfile.mkdtemp()
atexit.register(shutil.rmtree, td)
# TODO(jelmer): Make a slimmer copy
to_dir = branch.controldir.sprout(
td,
None,
create_tree_if_local=True,
source_branch=branch,
stacked=branch._format.supports_stacking(),
)
wt = to_dir.open_workingtree()
else:
wt, subpath = WorkingTree.open_containing(args.directory)
except NotBranchError:
logging.error(
"No version control directory found (e.g. a .git directory).")
return 1
except DependencyNotPresent as e:
logging.error(
"Unable to open tree at %s: missing package %s",
args.directory,
e.library,
)
return 1
if args.identity:
print("Committer identity: %s" % get_committer(wt))
print("Changelog identity: %s <%s>" % get_maintainer())
return 0
since_revid = wt.last_revision()
if args.fixers:
try:
fixers = select_fixers(fixers, args.fixers, args.exclude)
except KeyError as e:
logging.error("Unknown fixer specified: %s", e.args[0])
return 1
debian_info = distro_info.DebianDistroInfo()
if args.modern:
if args.compat_release:
logging.error(
"--compat-release and --modern are incompatible.")
return 1
compat_release = debian_info.devel()
else:
compat_release = args.compat_release
minimum_certainty = args.minimum_certainty
allow_reformatting = args.allow_reformatting
update_changelog = args.update_changelog
try:
cfg = Config.from_workingtree(wt, subpath)
except FileNotFoundError:
pass
else:
if minimum_certainty is None:
minimum_certainty = cfg.minimum_certainty()
if compat_release is None:
compat_release = cfg.compat_release()
if allow_reformatting is None:
allow_reformatting = cfg.allow_reformatting()
if update_changelog is None:
update_changelog = cfg.update_changelog()
if minimum_certainty is None:
if args.uncertain or args.yolo:
minimum_certainty = "possible"
else:
minimum_certainty = DEFAULT_MINIMUM_CERTAINTY
if compat_release is None:
compat_release = debian_info.stable()
if allow_reformatting is None:
allow_reformatting = False
with wt.lock_write():
if control_files_in_root(wt, subpath):
report_fatal(
"control-files-in-root",
"control files live in root rather than debian/ "
"(LarstIQ mode)",
)
try:
overall_result = run_lintian_fixers(
wt,
fixers,
update_changelog=update_changelog,
compat_release=compat_release,
verbose=args.verbose,
minimum_certainty=minimum_certainty,
trust_package=args.trust,
allow_reformatting=allow_reformatting,
use_inotify=(False if args.disable_inotify else None),
subpath=subpath,
net_access=not args.disable_net_access,
opinionated=args.opinionated,
diligence=args.diligence,
)
except NotDebianPackage:
report_fatal("not-debian-package", "Not a Debian package")
return 1
except WorkspaceDirty:
logging.error("%s: Please commit pending changes first.",
wt.basedir)
if args.verbose:
from breezy.status import show_tree_status
show_tree_status(wt)
return 1
except ChangelogCreateError as e:
report_fatal("changelog-create-error",
"Error creating changelog entry: %s" % e)
return 1
except DescriptionMissing as e:
report_fatal(
"fixer-description-missing",
"Fixer %s made changes but did not provide description." %
e.fixer)
return 1
if overall_result.overridden_lintian_issues:
if len(overall_result.overridden_lintian_issues) == 1:
logging.info("%d change skipped because of lintian overrides.",
len(overall_result.overridden_lintian_issues))
else:
logging.info(
"%d changes skipped because of lintian overrides.",
len(overall_result.overridden_lintian_issues))
if overall_result.success:
all_tags = set()
for result, summary in overall_result.success:
all_tags.update(result.fixed_lintian_tags)
if all_tags:
logging.info("Lintian tags fixed: %r", all_tags)
else:
logging.info("Some changes were made, "
"but there are no affected lintian tags.")
min_certainty = overall_result.minimum_success_certainty()
if min_certainty != "certain":
logging.info(
"Some changes were made with lower certainty (%s); "
"please double check the changes.",
min_certainty,
)
else:
report_fatal("nothing-to-do", "No changes made.")
return 0
if overall_result.failed_fixers and not args.verbose:
logging.info(
"Some fixer scripts failed to run: %r. "
"Run with --verbose for details.",
set(overall_result.failed_fixers.keys()),
)
if overall_result.formatting_unpreservable and not args.verbose:
logging.info(
"Some fixer scripts were unable to preserve formatting: %r. "
"Run with --allow-reformatting to reformat %r.",
set(overall_result.formatting_unpreservable),
set(overall_result.formatting_unpreservable.values()),
)
if args.diff:
from breezy.diff import show_diff_trees
show_diff_trees(wt.branch.repository.revision_tree(since_revid),
wt, sys.stdout.buffer)
if os.environ.get('SVP_API') == '1':
applied = []
if 'SVP_RESUME' in os.environ:
with open(os.environ['SVP_RESUME'], 'r') as f:
base = json.load(f)
applied.extend(base['applied'])
all_fixed_lintian_tags = set()
for result, summary in overall_result.success:
applied.append({
"summary":
summary,
"description":
result.description,
"fixed_lintian_tags":
result.fixed_lintian_tags,
"revision_id":
result.revision_id.decode("utf-8"),
"certainty":
result.certainty,
})
all_fixed_lintian_tags.update(result.fixed_lintian_tags)
failed = {
name: str(e)
for (name, e) in overall_result.failed_fixers.items()
}
debian_context = {}
if overall_result.changelog_behaviour:
debian_context[
'changelog'] = overall_result.changelog_behaviour.json()
with open(os.environ['SVP_RESULT'], 'w') as f:
json.dump(
{
'value': calculate_value(all_fixed_lintian_tags),
'debian': debian_context,
'context': {
'applied': applied,
'failed': failed,
"versions": {
"lintian-brush": lintian_brush_version_string,
"breezy": breezy.version_string,
}
}
}, f)
def make_changes( # noqa: C901
self,
local_tree,
subpath,
update_changelog,
reporter,
committer,
base_proposal=None):
base_revid = local_tree.last_revision()
upstream_base_revid = NULL_REVISION
reporter.report_metadata(
"versions",
{
"lintian-brush": lintian_brush_version_string,
"silver-platter": silver_platter.version_string,
"breezy": breezy.version_string,
},
)
import distro_info
debian_info = distro_info.DebianDistroInfo()
compat_release = self.compat_release
try:
cfg = Config.from_workingtree(local_tree, subpath)
except FileNotFoundError:
pass
else:
compat_release = cfg.compat_release()
if compat_release:
compat_release = debian_info.codename(compat_release,
default=compat_release)
if compat_release is None:
compat_release = debian_info.stable()
# For now...
upstream_branch = local_tree.branch
upstream_subpath = subpath
with local_tree.lock_write():
try:
result = debianize(
local_tree,
subpath=subpath,
upstream_branch=upstream_branch,
upstream_subpath=upstream_subpath,
compat_release=self.compat_release,
schroot=self.schroot,
diligence=self.diligence,
trust=self.trust,
verbose=self.verbose,
force_new_directory=self.force_new_directory,
create_dist=getattr(self, 'create_dist', None))
except OSError as e:
if e.errno == errno.ENOSPC:
raise ChangerError('no-space-on-device', str(e))
else:
raise
except DebianDirectoryExists:
raise ChangerError(
'debian-directory-exists',
"A debian/ directory already exists in the upstream project."
)
except SourcePackageNameInvalid as e:
raise ChangerError(
'invalid-source-package-name',
"Generated source package name %r is not valid." %
e.source)
except NoBuildToolsFound:
raise ChangerError(
'no-build-tools',
"Unable to find any build systems in upstream sources.")
except NoUpstreamReleases:
raise ChangerError(
'no-upstream-releases',
'The upstream project does not appear to have made any releases.'
)
except DistCommandFailed as e:
raise ChangerError("dist-command-failed", str(e), e)
except DetailedFailure as e:
raise ChangerError('dist-%s' % e.error.kind, str(e.error))
except UnidentifiedError as e:
if e.secondary:
raise ChangerError('dist-command-failed',
str(e.secondary.line))
else:
raise ChangerError('dist-command-failed', e.lines[-1])
except DistCreationFailed as e:
if e.inner:
raise ChangerError('dist-%s' % e.inner.kind, e.msg)
else:
raise ChangerError('dist-command-failed', e.msg)
# TODO(jelmer): Pristine tar branch?
branches = [
("main", None, base_revid, local_tree.last_revision()),
]
if result.upstream_branch_name:
branches.append((
"upstream",
result.upstream_branch_name,
upstream_base_revid,
local_tree.controldir.open_branch(
result.upstream_branch_name).last_revision(),
))
tags = [(("upstream", str(result.upstream_version), component), tag,
local_tree.branch.tags.lookup_tag(tag))
for (component, tag) in result.tag_names.items()]
reporter.report_metadata("wnpp_bugs", result.wnpp_bugs)
return ChangerResult(
description="Debianized package.",
mutator=None,
branches=branches,
tags=tags,
value=None,
sufficient_for_proposal=True,
)
def main(): # noqa: C901
import argparse
import breezy # noqa: E402
breezy.initialize()
import breezy.git # noqa: E402
import breezy.bzr # noqa: E402
from breezy.workspace import (
check_clean_tree,
WorkspaceDirty,
)
from . import (
version_string, )
from .config import Config
parser = argparse.ArgumentParser(prog="deb-scrub-obsolete")
parser.add_argument(
"--directory",
metavar="DIRECTORY",
help="directory to run in",
type=str,
default=".",
)
parser.add_argument(
"--upgrade-release",
metavar="UPGRADE-RELEASE",
help="Release to allow upgrading from.",
default="oldstable",
)
parser.add_argument('--compat-release',
metavar='COMPAT-RELEASE',
help="Release to allow building on.",
default=None)
parser.add_argument(
"--no-update-changelog",
action="store_false",
default=None,
dest="update_changelog",
help="do not update the changelog",
)
parser.add_argument(
"--update-changelog",
action="store_true",
dest="update_changelog",
help="force updating of the changelog",
default=None,
)
parser.add_argument(
"--allow-reformatting",
default=None,
action="store_true",
help=argparse.SUPPRESS,
)
parser.add_argument("--version",
action="version",
version="%(prog)s " + version_string)
parser.add_argument(
"--identity",
help="Print user identity that would be used when committing",
action="store_true",
default=False,
)
parser.add_argument("--debug",
help="Describe all considered changes.",
action="store_true")
args = parser.parse_args()
wt, subpath = WorkingTree.open_containing(args.directory)
if args.identity:
logging.info('%s', get_committer(wt))
return 0
try:
check_clean_tree(wt, wt.basis_tree(), subpath)
except WorkspaceDirty:
logging.info("%s: Please commit pending changes first.", wt.basedir)
return 1
import distro_info
debian_info = distro_info.DebianDistroInfo()
upgrade_release = debian_info.codename(args.upgrade_release)
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO, format='%(message)s')
update_changelog = args.update_changelog
allow_reformatting = args.allow_reformatting
if args.compat_release:
compat_release = debian_info.codename(args.compat_release)
else:
compat_release = None
try:
cfg = Config.from_workingtree(wt, subpath)
except FileNotFoundError:
pass
else:
if update_changelog is None:
update_changelog = cfg.update_changelog()
if allow_reformatting is None:
allow_reformatting = cfg.allow_reformatting()
if compat_release is None:
compat_release = cfg.compat_release()
if compat_release is None:
compat_release = debian_info.codename('oldstable')
if upgrade_release != compat_release:
logging.info(
"Removing run time constraints unnecessary since %s"
" and build time constraints unnecessary since %s",
upgrade_release, compat_release)
else:
logging.info(
"Removing run time and build time constraints unnecessary "
"since %s", compat_release)
if allow_reformatting is None:
allow_reformatting = False
if is_debcargo_package(wt, subpath):
report_fatal("nothing-to-do", "Package uses debcargo")
return 1
elif not control_file_present(wt, subpath):
report_fatal("missing-control-file", "Unable to find debian/control")
return 1
try:
result = scrub_obsolete(wt,
subpath,
compat_release,
upgrade_release,
update_changelog=args.update_changelog,
allow_reformatting=allow_reformatting)
except FormattingUnpreservable as e:
report_fatal(
"formatting-unpreservable",
"unable to preserve formatting while editing %s" % e.path,
)
return 1
except GeneratedFile as e:
report_fatal("generated-file", "unable to edit generated file: %r" % e)
return 1
except NotDebianPackage:
report_fatal('not-debian-package', 'Not a Debian package.')
return 1
except ChangeConflict as e:
report_fatal('change-conflict',
'Generated file changes conflict: %s' % e)
return 1
except UddTimeout:
report_fatal('udd-timeout', 'Timeout communicating with UDD')
return 1
if not result:
report_okay("nothing-to-do", "no obsolete constraints")
return 0
debian_context = {}
if result.changelog_behaviour:
debian_context['changelog'] = result.changelog_behaviour.json()
if os.environ.get("SVP_API") == "1":
with open(os.environ["SVP_RESULT"], "w") as f:
json.dump(
{
"description": "Remove constraints unnecessary since %s." %
upgrade_release,
"value": result.value(),
"debian": debian_context,
"context": {
"specific_files": result.specific_files,
"maintscript_removed": result.maintscript_removed,
"control_removed": result.control_removed,
}
}, f)
logging.info("Scrub obsolete settings.")
for release, lines in result.itemized().items():
for line in lines:
logging.info("* %s", line)
return 0
def main(args):
import distro_info
import socket
import subprocess
import silver_platter # noqa: F401
from . import (
propose_or_push,
BuildFailedError,
MissingUpstreamTarball,
NoSuchPackage,
)
from breezy import (
errors, )
from breezy.trace import note
from breezy.plugins.propose.propose import (
NoSuchProject,
UnsupportedHoster,
)
possible_transports = []
possible_hosters = []
fixer_scripts = {}
for fixer in available_lintian_fixers():
for tag in fixer.lintian_tags:
fixer_scripts[tag] = fixer
available_fixers = set(fixer_scripts)
if args.fixers:
available_fixers = available_fixers.intersection(set(args.fixers))
debian_info = distro_info.DebianDistroInfo()
for pkg in args.packages:
if args.pre_check:
def pre_check(local_tree):
try:
subprocess.check_call(args.pre_check,
shell=True,
cwd=local_tree.basedir)
except subprocess.CalledProcessError:
note('%r: pre-check failed, skipping', pkg)
return False
return True
else:
pre_check = None
if args.post_check:
def post_check(local_tree, since_revid):
try:
subprocess.check_call(args.post_check,
shell=True,
cwd=local_tree.basedir,
env={'SINCE_REVID': since_revid})
except subprocess.CalledProcessError:
note('%r: post-check failed, skipping', pkg)
return False
return True
else:
post_check = None
note('Processing: %s', pkg)
try:
main_branch = open_packaging_branch(
pkg, possible_transports=possible_transports)
except NoSuchPackage:
note('%s: no such package', pkg)
except socket.error:
note('%s: ignoring, socket error', pkg)
except errors.NotBranchError as e:
note('%s: Branch does not exist: %s', pkg, e)
except errors.UnsupportedProtocol:
note('%s: Branch available over unsupported protocol', pkg)
except errors.ConnectionError as e:
note('%s: %s', pkg, e)
except errors.PermissionDenied as e:
note('%s: %s', pkg, e)
except errors.InvalidHttpResponse as e:
note('%s: %s', pkg, e)
except errors.TransportError as e:
note('%s: %s', pkg, e)
else:
# If it's unknown which fixers are relevant, just try all of them.
if args.fixers:
fixers = args.fixers
else:
fixers = available_fixers
branch_changer = LintianFixer(
pkg,
fixers=[fixer_scripts[fixer] for fixer in fixers],
update_changelog=args.update_changelog,
compat_release=debian_info.stable(),
build_verify=args.build_verify,
pre_check=pre_check,
post_check=post_check,
propose_addon_only=args.propose_addon_only,
committer=args.committer)
try:
result = propose_or_push(
main_branch,
"lintian-fixes",
branch_changer,
args.mode,
possible_transports=possible_transports,
possible_hosters=possible_hosters,
refresh=args.refresh,
dry_run=args.dry_run)
except UnsupportedHoster:
note('%s: Hoster unsupported', pkg)
continue
except NoSuchProject as e:
note('%s: project %s was not found', pkg, e.project)
continue
except BuildFailedError:
note('%s: build failed', pkg)
continue
except MissingUpstreamTarball:
note('%s: unable to find upstream source', pkg)
continue
except errors.PermissionDenied as e:
note('%s: %s', pkg, e)
continue
except PostCheckFailed as e:
note('%s: %s', pkg, e)
continue
else:
if result.merge_proposal:
tags = set()
for brush_result, unused_summary in branch_changer.applied:
tags.update(brush_result.fixed_lintian_tags)
if result.is_new:
note('%s: Proposed fixes %r: %s', pkg, tags,
result.merge_proposal.url)
elif tags:
note('%s: Updated proposal %s with fixes %r', pkg,
result.merge_proposal.url, tags)
else:
note('%s: No new fixes for proposal %s', pkg,
result.merge_proposal.url)
if args.diff:
result.show_base_diff(sys.stdout.buffer)
def __init__(self, config_layers=(), current_directory=None):
super(Config, self).__init__()
self._suites = WeakValueDictionary()
self._vendors = {}
self._overrides = {}
self._relevant_directory = None
d = yaml.safe_load(
open(os.path.join(os.path.dirname(__file__), 'defaults.yaml')))
# Some things can have better defaults that can't be hard-coded
d['defaults']['parallel'] = str(os.cpu_count())
try:
d['defaults']['architecture'] = subprocess.check_output(
['dpkg', '--print-architecture'],
universal_newlines=True).strip()
except subprocess.CalledProcessError:
pass
d['vendors']['debian']['default_suite'] = 'sid'
try:
import distro_info
except ImportError:
d['vendors']['debian']['default_worker_suite'] = 'sid'
else:
debian = distro_info.DebianDistroInfo()
ubuntu = distro_info.UbuntuDistroInfo()
d['vendors']['debian']['default_worker_suite'] = debian.stable()
d['vendors']['debian']['suites']['stable'] = {
'alias_for': debian.stable(),
}
d['vendors']['debian']['suites']['testing'] = {
'alias_for': debian.testing(),
}
d['vendors']['debian']['suites']['oldstable'] = {
'alias_for': debian.old(),
}
# According to autopkgtest-buildvm-ubuntu-cloud, just after
# an Ubuntu release there is briefly no development version
# at all.
try:
ubuntu_devel = ubuntu.devel()
except distro_info.DistroDataOutdated:
ubuntu_devel = ubuntu.stable()
d['vendors']['ubuntu']['default_suite'] = ubuntu_devel
d['vendors']['ubuntu']['default_worker_suite'] = (ubuntu.lts() +
'-backports')
d['vendors']['ubuntu']['suites']['devel'] = {
'alias_for': ubuntu_devel,
}
for suite in debian.all:
d['vendors']['debian']['suites'].setdefault(suite, {})
for suite in ubuntu.all:
d['vendors']['ubuntu']['suites'].setdefault(suite, {})
self._raw = []
self._raw.append(d)
if config_layers:
self._raw[:0] = list(config_layers)
else:
config_dirs = XDG_CONFIG_DIRS.split(':')
config_dirs = list(reversed(config_dirs))
config_dirs.append(XDG_CONFIG_HOME)
for p in config_dirs:
conffile = os.path.join(p, 'vectis', 'vectis.yaml')
try:
reader = open(conffile)
except FileNotFoundError:
continue
with reader:
raw = yaml.safe_load(reader)
if not isinstance(raw, dict):
raise ConfigError(
'Reading {!r} did not yield a dict'.format(
conffile))
self._raw.insert(0, raw)
if current_directory is None:
current_directory = os.getcwd()
self._relevant_directory = None
while self._relevant_directory is None:
for r in self._raw:
if current_directory in r.get('directories', {}):
self._relevant_directory = current_directory
break
else:
parent, _ = os.path.split(current_directory)
# Guard against infinite recursion. If current_directory == '/'
# we would already have found directories./ in the hard-coded
# defaults, and broken out of the loop
assert len(parent) < len(current_directory)
current_directory = parent
continue
assert self._relevant_directory is not None
self._path_based = Directory(self._relevant_directory, self._raw)
