def _depend_checks(ebuild, pkg, portdb, qatracker, repo_metadata):
'''Checks the ebuild dependencies for errors
@param pkg: Package in which we check (object).
@param ebuild: Ebuild which we check (object).
@param portdb: portdb instance
@param qatracker: QATracker instance
@param repo_metadata: dictionary of various repository items.
@returns: (unknown_pkgs, badlicsyntax)
'''
unknown_pkgs = set()
inherited_java_eclass = "java-pkg-2" in ebuild.inherited or \
"java-pkg-opt-2" in ebuild.inherited,
inherited_wxwidgets_eclass = "wxwidgets" in ebuild.inherited
# operator_tokens = set(["||", "(", ")"])
badsyntax = []
for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
mydepstr = ebuild.metadata[mytype]
buildtime = mytype in Package._buildtime_keys
runtime = mytype in Package._runtime_keys
token_class = None
if mytype.endswith("DEPEND"):
token_class = portage.dep.Atom
try:
atoms = portage.dep.use_reduce(
mydepstr, matchall=1, flat=True,
is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
except portage.exception.InvalidDependString as e:
atoms = None
badsyntax.append((mytype, str(e)))
if atoms and mytype.endswith("DEPEND"):
if runtime and \
"test?" in mydepstr.split():
qatracker.add_error(
mytype + '.suspect',
"%s: 'test?' USE conditional in %s" %
(ebuild.relative_path, mytype))
for atom in atoms:
if atom == "||":
continue
is_blocker = atom.blocker
# Skip dependency.unknown for blockers, so that we
# don't encourage people to remove necessary blockers,
# as discussed in bug 382407. We use atom.without_use
# due to bug 525376.
if not is_blocker and \
not portdb.xmatch("match-all", atom.without_use) and \
not atom.cp.startswith("virtual/"):
unknown_pkgs.add((mytype, atom.unevaluated_atom))
if pkg.category != "virtual":
if not is_blocker and \
atom.cp in suspect_virtual:
qatracker.add_error(
'virtual.suspect', ebuild.relative_path +
": %s: consider using '%s' instead of '%s'" %
(mytype, suspect_virtual[atom.cp], atom))
if not is_blocker and \
atom.cp.startswith("perl-core/"):
qatracker.add_error('dependency.perlcore',
ebuild.relative_path +
": %s: please use '%s' instead of '%s'" %
(mytype,
atom.replace("perl-core/","virtual/perl-"),
atom))
if buildtime and \
not is_blocker and \
not inherited_java_eclass and \
atom.cp == "virtual/jdk":
qatracker.add_error(
'java.eclassesnotused', ebuild.relative_path)
elif buildtime and \
not is_blocker and \
not inherited_wxwidgets_eclass and \
atom.cp == "x11-libs/wxGTK":
qatracker.add_error(
'wxwidgets.eclassnotused',
"%s: %ss on x11-libs/wxGTK without inheriting"
" wxwidgets.eclass" % (ebuild.relative_path, mytype))
elif runtime:
if not is_blocker and \
atom.cp in suspect_rdepend:
qatracker.add_error(
mytype + '.suspect',
ebuild.relative_path + ": '%s'" % atom)
if atom.operator == "~" and \
portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
qacat = 'dependency.badtilde'
qatracker.add_error(
qacat, "%s: %s uses the ~ operator"
" with a non-zero revision: '%s'" %
(ebuild.relative_path, mytype, atom))
check_missingslot(atom, mytype, ebuild.eapi, portdb, qatracker,
ebuild.relative_path, ebuild.metadata)
if runtime:
check_slotop(mydepstr, pkg.iuse.is_valid_flag,
badsyntax, mytype, qatracker, ebuild.relative_path)
baddepsyntax = False
dedup = collections.defaultdict(set)
for m, b in badsyntax:
if b in dedup[m]:
continue
dedup[m].add(b)
if m.endswith("DEPEND"):
baddepsyntax = True
qacat = "dependency.syntax"
else:
qacat = m + ".syntax"
qatracker.add_error(
qacat, "%s: %s: %s" % (ebuild.relative_path, m, b))
# Parse the LICENSE variable, remove USE conditions and flatten it.
licenses = portage.dep.use_reduce(
ebuild.metadata["LICENSE"], matchall=1, flat=True)
# Check each entry to ensure that it exists in ${PORTDIR}/licenses/.
for lic in licenses:
# Need to check for "||" manually as no portage
# function will remove it without removing values.
if lic not in repo_metadata['liclist'] and lic != "||":
qatracker.add_error("LICENSE.invalid",
"%s: %s" % (ebuild.relative_path, lic))
elif lic in repo_metadata['lic_deprecated']:
qatracker.add_error("LICENSE.deprecated",
"%s: %s" % (ebuild.relative_path, lic))
return unknown_pkgs, baddepsyntax
def _scan_ebuilds(self, ebuildlist, xpkg, catdir, pkgdir):
# detect unused local USE-descriptions
used_useflags = set()
for y_ebuild in ebuildlist:
ebuild = Ebuild(
self.repo_settings, self.repolevel, pkgdir, catdir, self.vcs_settings,
xpkg, y_ebuild)
if self.check['changelog'] and not self.changelog_modified \
and ebuild.ebuild_path in self.changed.new_ebuilds:
self.qatracker.add_error('changelog.ebuildadded', ebuild.relative_path)
if ebuild.untracked(self.check['ebuild_notadded'], y_ebuild, self.eadded):
# ebuild not added to vcs
self.qatracker.add_error(
"ebuild.notadded", xpkg + "/" + y_ebuild + ".ebuild")
if bad_split_check(xpkg, y_ebuild, pkgdir, self.qatracker):
continue
pkg = self.pkgs[y_ebuild]
if pkg_invalid(pkg, self.qatracker, ebuild):
self.allvalid = False
continue
myaux = pkg._metadata
eapi = myaux["EAPI"]
inherited = pkg.inherited
live_ebuild = self.live_eclasses.intersection(inherited)
self.eapicheck.check(pkg, ebuild)
for k, v in myaux.items():
if not isinstance(v, basestring):
continue
m = NON_ASCII_RE.search(v)
if m is not None:
self.qatracker.add_error(
"variable.invalidchar",
"%s: %s variable contains non-ASCII "
"character at position %s" %
(ebuild.relative_path, k, m.start() + 1))
if not self.fetchcheck.src_uri_error:
self.thirdparty.check(myaux, ebuild.relative_path)
if myaux.get("PROVIDE"):
self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
for pos, missing_var in enumerate(missingvars):
if not myaux.get(missing_var):
if catdir == "virtual" and \
missing_var in ("HOMEPAGE", "LICENSE"):
continue
if live_ebuild and missing_var == "KEYWORDS":
continue
myqakey = missingvars[pos] + ".missing"
self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")
if catdir == "virtual":
for var in ("HOMEPAGE", "LICENSE"):
if myaux.get(var):
myqakey = var + ".virtual"
self.qatracker.add_error(myqakey, ebuild.relative_path)
self.descriptioncheck.check(pkg, ebuild)
keywords = myaux["KEYWORDS"].split()
ebuild_archs = set(
kw.lstrip("~") for kw in keywords if not kw.startswith("-"))
self.keywordcheck.check(
pkg, xpkg, ebuild, y_ebuild, keywords, ebuild_archs, self.changed,
live_ebuild, self.repo_metadata['kwlist'], self.profiles)
if live_ebuild and self.repo_settings.repo_config.name == "gentoo":
self.liveeclasscheck.check(
pkg, xpkg, ebuild, y_ebuild, keywords, self.repo_metadata['pmaskdict'])
if self.options.ignore_arches:
arches = [[
self.repo_settings.repoman_settings["ARCH"], self.repo_settings.repoman_settings["ARCH"],
self.repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split()]]
else:
arches = set()
for keyword in keywords:
if keyword[0] == "-":
continue
elif keyword[0] == "~":
arch = keyword[1:]
if arch == "*":
for expanded_arch in self.profiles:
if expanded_arch == "**":
continue
arches.add(
(keyword, expanded_arch, (
expanded_arch, "~" + expanded_arch)))
else:
arches.add((keyword, arch, (arch, keyword)))
else:
# For ebuilds with stable keywords, check if the
# dependencies are satisfiable for unstable
# configurations, since use.stable.mask is not
# applied for unstable configurations (see bug
# 563546).
if keyword == "*":
for expanded_arch in self.profiles:
if expanded_arch == "**":
continue
arches.add(
(keyword, expanded_arch, (expanded_arch,)))
arches.add(
(keyword, expanded_arch,
(expanded_arch, "~" + expanded_arch)))
else:
arches.add((keyword, keyword, (keyword,)))
arches.add((keyword, keyword,
(keyword, "~" + keyword)))
if not arches:
# Use an empty profile for checking dependencies of
# packages that have empty KEYWORDS.
arches.add(('**', '**', ('**',)))
unknown_pkgs = set()
baddepsyntax = False
badlicsyntax = False
badprovsyntax = False
# catpkg = catdir + "/" + y_ebuild
inherited_java_eclass = "java-pkg-2" in inherited or \
"java-pkg-opt-2" in inherited
inherited_wxwidgets_eclass = "wxwidgets" in inherited
# operator_tokens = set(["||", "(", ")"])
type_list, badsyntax = [], []
for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
mydepstr = myaux[mytype]
buildtime = mytype in Package._buildtime_keys
runtime = mytype in Package._runtime_keys
token_class = None
if mytype.endswith("DEPEND"):
token_class = portage.dep.Atom
try:
atoms = portage.dep.use_reduce(
mydepstr, matchall=1, flat=True,
is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
except portage.exception.InvalidDependString as e:
atoms = None
badsyntax.append(str(e))
if atoms and mytype.endswith("DEPEND"):
if runtime and \
"test?" in mydepstr.split():
self.qatracker.add_error(
mytype + '.suspect',
"%s: 'test?' USE conditional in %s" %
(ebuild.relative_path, mytype))
for atom in atoms:
if atom == "||":
continue
is_blocker = atom.blocker
# Skip dependency.unknown for blockers, so that we
# don't encourage people to remove necessary blockers,
# as discussed in bug 382407. We use atom.without_use
# due to bug 525376.
if not is_blocker and \
not self.portdb.xmatch("match-all", atom.without_use) and \
not atom.cp.startswith("virtual/"):
unknown_pkgs.add((mytype, atom.unevaluated_atom))
if catdir != "virtual":
if not is_blocker and \
atom.cp in suspect_virtual:
self.qatracker.add_error(
'virtual.suspect', ebuild.relative_path +
": %s: consider using '%s' instead of '%s'" %
(mytype, suspect_virtual[atom.cp], atom))
if not is_blocker and \
atom.cp.startswith("perl-core/"):
self.qatracker.add_error('dependency.perlcore',
ebuild.relative_path +
": %s: please use '%s' instead of '%s'" %
(mytype,
atom.replace("perl-core/","virtual/perl-"),
atom))
if buildtime and \
not is_blocker and \
not inherited_java_eclass and \
atom.cp == "virtual/jdk":
self.qatracker.add_error(
'java.eclassesnotused', ebuild.relative_path)
elif buildtime and \
not is_blocker and \
not inherited_wxwidgets_eclass and \
atom.cp == "x11-libs/wxGTK":
self.qatracker.add_error(
'wxwidgets.eclassnotused',
"%s: %ss on x11-libs/wxGTK without inheriting"
" wxwidgets.eclass" % (ebuild.relative_path, mytype))
elif runtime:
if not is_blocker and \
atom.cp in suspect_rdepend:
self.qatracker.add_error(
mytype + '.suspect',
ebuild.relative_path + ": '%s'" % atom)
if atom.operator == "~" and \
portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
qacat = 'dependency.badtilde'
self.qatracker.add_error(
qacat, "%s: %s uses the ~ operator"
" with a non-zero revision: '%s'" %
(ebuild.relative_path, mytype, atom))
check_missingslot(atom, mytype, eapi, self.portdb, self.qatracker,
ebuild.relative_path, myaux)
type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
for m, b in zip(type_list, badsyntax):
if m.endswith("DEPEND"):
qacat = "dependency.syntax"
else:
qacat = m + ".syntax"
self.qatracker.add_error(
qacat, "%s: %s: %s" % (ebuild.relative_path, m, b))
badlicsyntax = len([z for z in type_list if z == "LICENSE"])
badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
badlicsyntax = badlicsyntax > 0
badprovsyntax = badprovsyntax > 0
self.use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild, self.muselist)
ebuild_used_useflags = self.use_flag_checks.getUsedUseFlags()
used_useflags = used_useflags.union(ebuild_used_useflags)
self.rubyeclasscheck.check(pkg, ebuild)
# license checks
if not badlicsyntax:
self.licensecheck.check(pkg, xpkg, ebuild, y_ebuild)
self.restrictcheck.check(pkg, xpkg, ebuild, y_ebuild)
# Syntax Checks
if not self.vcs_settings.vcs_preserves_mtime:
if ebuild.ebuild_path not in self.changed.new_ebuilds and \
ebuild.ebuild_path not in self.changed.ebuilds:
pkg.mtime = None
try:
# All ebuilds should have utf_8 encoding.
f = io.open(
_unicode_encode(
ebuild.full_path, encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'])
try:
for check_name, e in run_checks(f, pkg):
self.qatracker.add_error(
check_name, ebuild.relative_path + ': %s' % e)
finally:
f.close()
except UnicodeDecodeError:
# A file.UTF8 failure will have already been recorded above.
pass
if self.options.force:
# The dep_check() calls are the most expensive QA test. If --force
# is enabled, there's no point in wasting time on these since the
# user is intent on forcing the commit anyway.
continue
relevant_profiles = []
for keyword, arch, groups in arches:
if arch not in self.profiles:
# A missing profile will create an error further down
# during the KEYWORDS verification.
continue
if self.include_arches is not None:
if arch not in self.include_arches:
continue
relevant_profiles.extend(
(keyword, groups, prof) for prof in self.profiles[arch])
relevant_profiles.sort(key=sort_key)
for keyword, groups, prof in relevant_profiles:
is_stable_profile = prof.status == "stable"
is_dev_profile = prof.status == "dev" and \
self.options.include_dev
is_exp_profile = prof.status == "exp" and \
self.options.include_exp_profiles == 'y'
if not (is_stable_profile or is_dev_profile or is_exp_profile):
continue
dep_settings = self.caches['arch'].get(prof.sub_path)
if dep_settings is None:
dep_settings = portage.config(
config_profile_path=prof.abs_path,
config_incrementals=self.repoman_incrementals,
config_root=self.config_root,
local_config=False,
_unmatched_removal=self.options.unmatched_removal,
env=self.env, repositories=self.repo_settings.repoman_settings.repositories)
dep_settings.categories = self.repo_settings.repoman_settings.categories
if self.options.without_mask:
dep_settings._mask_manager_obj = \
copy.deepcopy(dep_settings._mask_manager)
dep_settings._mask_manager._pmaskdict.clear()
self.caches['arch'][prof.sub_path] = dep_settings
xmatch_cache_key = (prof.sub_path, tuple(groups))
xcache = self.caches['arch_xmatch'].get(xmatch_cache_key)
if xcache is None:
self.portdb.melt()
self.portdb.freeze()
xcache = self.portdb.xcache
xcache.update(self.caches['shared_xmatch'])
self.caches['arch_xmatch'][xmatch_cache_key] = xcache
self.repo_settings.trees[self.repo_settings.root]["porttree"].settings = dep_settings
self.portdb.settings = dep_settings
self.portdb.xcache = xcache
dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
# just in case, prevent config.reset() from nuking these.
dep_settings.backup_changes("ACCEPT_KEYWORDS")
# This attribute is used in dbapi._match_use() to apply
# use.stable.{mask,force} settings based on the stable
# status of the parent package. This is required in order
# for USE deps of unstable packages to be resolved correctly,
# since otherwise use.stable.{mask,force} settings of
# dependencies may conflict (see bug #456342).
dep_settings._parent_stable = dep_settings._isStable(pkg)
# Handle package.use*.{force,mask) calculation, for use
# in dep_check.
dep_settings.useforce = dep_settings._use_manager.getUseForce(
pkg, stable=dep_settings._parent_stable)
dep_settings.usemask = dep_settings._use_manager.getUseMask(
pkg, stable=dep_settings._parent_stable)
if not baddepsyntax:
ismasked = not ebuild_archs or \
pkg.cpv not in self.portdb.xmatch("match-visible",
Atom("%s::%s" % (pkg.cp, self.repo_settings.repo_config.name)))
if ismasked:
if not self.have['pmasked']:
self.have['pmasked'] = bool(dep_settings._getMaskAtom(
pkg.cpv, pkg._metadata))
if self.options.ignore_masked:
continue
# we are testing deps for a masked package; give it some lee-way
suffix = "masked"
matchmode = "minimum-all-ignore-profile"
else:
suffix = ""
matchmode = "minimum-visible"
if not self.have['dev_keywords']:
self.have['dev_keywords'] = \
bool(self.dev_keywords.intersection(keywords))
if prof.status == "dev":
suffix = suffix + "indev"
for mytype in Package._dep_keys:
mykey = "dependency.bad" + suffix
myvalue = myaux[mytype]
if not myvalue:
continue
success, atoms = portage.dep_check(
myvalue, self.portdb, dep_settings,
use="all", mode=matchmode, trees=self.repo_settings.trees)
if success:
if atoms:
# Don't bother with dependency.unknown for
# cases in which *DEPEND.bad is triggered.
for atom in atoms:
# dep_check returns all blockers and they
# aren't counted for *DEPEND.bad, so we
# ignore them here.
if not atom.blocker:
unknown_pkgs.discard(
(mytype, atom.unevaluated_atom))
if not prof.sub_path:
# old-style virtuals currently aren't
# resolvable with empty profile, since
# 'virtuals' mappings are unavailable
# (it would be expensive to search
# for PROVIDE in all ebuilds)
atoms = [
atom for atom in atoms if not (
atom.cp.startswith('virtual/')
and not self.portdb.cp_list(atom.cp))]
# we have some unsolvable deps
# remove ! deps, which always show up as unsatisfiable
atoms = [
str(atom.unevaluated_atom)
for atom in atoms if not atom.blocker]
# if we emptied out our list, continue:
if not atoms:
continue
if self.options.output_style in ['column']:
self.qatracker.add_error(mykey,
"%s: %s: %s(%s) %s"
% (ebuild.relative_path, mytype, keyword,
prof, repr(atoms)))
else:
self.qatracker.add_error(mykey,
"%s: %s: %s(%s)\n%s"
% (ebuild.relative_path, mytype, keyword,
prof, pformat(atoms, indent=6)))
else:
if self.options.output_style in ['column']:
self.qatracker.add_error(mykey,
"%s: %s: %s(%s) %s"
% (ebuild.relative_path, mytype, keyword,
prof, repr(atoms)))
else:
self.qatracker.add_error(mykey,
"%s: %s: %s(%s)\n%s"
% (ebuild.relative_path, mytype, keyword,
prof, pformat(atoms, indent=6)))
if not baddepsyntax and unknown_pkgs:
type_map = {}
for mytype, atom in unknown_pkgs:
type_map.setdefault(mytype, set()).add(atom)
for mytype, atoms in type_map.items():
self.qatracker.add_error(
"dependency.unknown", "%s: %s: %s"
% (ebuild.relative_path, mytype, ", ".join(sorted(atoms))))
# check if there are unused local USE-descriptions in metadata.xml
# (unless there are any invalids, to avoid noise)
if self.allvalid:
for myflag in self.muselist.difference(used_useflags):
self.qatracker.add_error(
"metadata.warning",
"%s/metadata.xml: unused local USE-description: '%s'"
% (xpkg, myflag))
def _depend_checks(ebuild, pkg, portdb, qatracker, repo_metadata, qadata):
'''Checks the ebuild dependencies for errors
@param pkg: Package in which we check (object).
@param ebuild: Ebuild which we check (object).
@param portdb: portdb instance
@param qatracker: QATracker instance
@param repo_metadata: dictionary of various repository items.
@returns: (unknown_pkgs, badlicsyntax)
'''
unknown_pkgs = set()
inherited_java_eclass = "java-pkg-2" in ebuild.inherited or \
"java-pkg-opt-2" in ebuild.inherited,
inherited_wxwidgets_eclass = "wxwidgets" in ebuild.inherited
# operator_tokens = set(["||", "(", ")"])
badsyntax = []
for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES"):
mydepstr = ebuild.metadata[mytype]
buildtime = mytype in Package._buildtime_keys
runtime = mytype in Package._runtime_keys
token_class = None
if mytype.endswith("DEPEND"):
token_class = portage.dep.Atom
try:
atoms = portage.dep.use_reduce(
mydepstr,
matchall=1,
flat=True,
is_valid_flag=pkg.iuse.is_valid_flag,
token_class=token_class)
except portage.exception.InvalidDependString as e:
atoms = None
badsyntax.append((mytype, str(e)))
if atoms and mytype.endswith("DEPEND"):
if runtime and \
"test?" in mydepstr.split():
qatracker.add_error(
mytype + '.suspect', "%s: 'test?' USE conditional in %s" %
(ebuild.relative_path, mytype))
for atom in atoms:
if atom == "||":
continue
is_blocker = atom.blocker
# Skip dependency.unknown for blockers, so that we
# don't encourage people to remove necessary blockers,
# as discussed in bug 382407. We use atom.without_use
# due to bug 525376.
if not is_blocker and \
not portdb.xmatch("match-all", atom.without_use) and \
not atom.cp.startswith("virtual/"):
unknown_pkgs.add((mytype, atom.unevaluated_atom))
if pkg.category != "virtual":
if not is_blocker and \
atom.cp in qadata.suspect_virtual:
qatracker.add_error(
'virtual.suspect', ebuild.relative_path +
": %s: consider using '%s' instead of '%s'" %
(mytype, qadata.suspect_virtual[atom.cp], atom))
if not is_blocker and \
atom.cp.startswith("perl-core/"):
qatracker.add_error(
'dependency.perlcore', ebuild.relative_path +
": %s: please use '%s' instead of '%s'" %
(mytype, atom.replace("perl-core/",
"virtual/perl-"), atom))
if buildtime and \
not is_blocker and \
not inherited_java_eclass and \
atom.cp == "virtual/jdk":
qatracker.add_error('java.eclassesnotused',
ebuild.relative_path)
elif buildtime and \
not is_blocker and \
not inherited_wxwidgets_eclass and \
atom.cp == "x11-libs/wxGTK":
qatracker.add_error(
'wxwidgets.eclassnotused',
"%s: %ss on x11-libs/wxGTK without inheriting"
" wxwidgets.eclass" % (ebuild.relative_path, mytype))
elif runtime:
if not is_blocker and \
atom.cp in qadata.suspect_rdepend:
qatracker.add_error(
mytype + '.suspect',
ebuild.relative_path + ": '%s'" % atom)
if atom.operator == "~" and \
portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
qacat = 'dependency.badtilde'
qatracker.add_error(
qacat, "%s: %s uses the ~ operator"
" with a non-zero revision: '%s'" %
(ebuild.relative_path, mytype, atom))
# plain =foo-1.2.3 without revision or *
if atom.operator == "=" and '-r' not in atom.version:
qacat = 'dependency.equalsversion'
qatracker.add_error(
qacat, "%s: %s uses the = operator with"
" no revision: '%s'; if any revision is"
" acceptable, use '~' instead; if only -r0"
" then please append '-r0' to the dep" %
(ebuild.relative_path, mytype, atom))
check_missingslot(atom, mytype, ebuild.eapi, portdb, qatracker,
ebuild.relative_path, ebuild.metadata)
if runtime:
check_slotop(mydepstr, pkg.iuse.is_valid_flag, badsyntax, mytype,
qatracker, ebuild.relative_path)
baddepsyntax = False
dedup = collections.defaultdict(set)
for m, b in badsyntax:
if b in dedup[m]:
continue
dedup[m].add(b)
if m.endswith("DEPEND"):
baddepsyntax = True
qacat = "dependency.syntax"
else:
qacat = m + ".syntax"
qatracker.add_error(qacat, "%s: %s: %s" % (ebuild.relative_path, m, b))
# Parse the LICENSE variable, remove USE conditions and flatten it.
licenses = portage.dep.use_reduce(ebuild.metadata["LICENSE"],
matchall=1,
flat=True)
# Check each entry to ensure that it exists in ${PORTDIR}/licenses/.
for lic in licenses:
# Need to check for "||" manually as no portage
# function will remove it without removing values.
if lic not in repo_metadata['liclist'] and lic != "||":
qatracker.add_error("LICENSE.invalid",
"%s: %s" % (ebuild.relative_path, lic))
elif lic in repo_metadata['lic_deprecated']:
qatracker.add_error("LICENSE.deprecated",
"%s: %s" % (ebuild.relative_path, lic))
return unknown_pkgs, baddepsyntax
def _scan_ebuilds(self, ebuildlist, xpkg, catdir, pkgdir):
# detect unused local USE-descriptions
used_useflags = set()
for y_ebuild in ebuildlist:
ebuild = Ebuild(
self.repo_settings, self.repolevel, pkgdir, catdir, self.vcs_settings,
xpkg, y_ebuild)
if self.check['changelog'] and not self.changelog_modified \
and ebuild.ebuild_path in self.changed.new_ebuilds:
self.qatracker.add_error('changelog.ebuildadded', ebuild.relative_path)
if ebuild.untracked(self.check['ebuild_notadded'], y_ebuild, self.eadded):
# ebuild not added to vcs
self.qatracker.add_error(
"ebuild.notadded", xpkg + "/" + y_ebuild + ".ebuild")
if bad_split_check(xpkg, y_ebuild, pkgdir, self.qatracker):
continue
pkg = self.pkgs[y_ebuild]
if pkg_invalid(pkg, self.qatracker, ebuild):
self.allvalid = False
continue
myaux = pkg._metadata
eapi = myaux["EAPI"]
inherited = pkg.inherited
live_ebuild = self.live_eclasses.intersection(inherited)
self.eapicheck.check(pkg, ebuild)
for k, v in myaux.items():
if not isinstance(v, basestring):
continue
m = NON_ASCII_RE.search(v)
if m is not None:
self.qatracker.add_error(
"variable.invalidchar",
"%s: %s variable contains non-ASCII "
"character at position %s" %
(ebuild.relative_path, k, m.start() + 1))
if not self.fetchcheck.src_uri_error:
self.thirdparty.check(myaux, ebuild.relative_path)
if myaux.get("PROVIDE"):
self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path)
for pos, missing_var in enumerate(missingvars):
if not myaux.get(missing_var):
if catdir == "virtual" and \
missing_var in ("HOMEPAGE", "LICENSE"):
continue
if live_ebuild and missing_var == "KEYWORDS":
continue
myqakey = missingvars[pos] + ".missing"
self.qatracker.add_error(myqakey, xpkg + "/" + y_ebuild + ".ebuild")
if catdir == "virtual":
for var in ("HOMEPAGE", "LICENSE"):
if myaux.get(var):
myqakey = var + ".virtual"
self.qatracker.add_error(myqakey, ebuild.relative_path)
self.descriptioncheck.check(pkg, ebuild)
keywords = myaux["KEYWORDS"].split()
ebuild_archs = set(
kw.lstrip("~") for kw in keywords if not kw.startswith("-"))
self.keywordcheck.check(
pkg, xpkg, ebuild, y_ebuild, keywords, ebuild_archs, self.changed,
live_ebuild, self.repo_metadata['kwlist'], self.profiles)
if live_ebuild and self.repo_settings.repo_config.name == "gentoo":
self.liveeclasscheck.check(
pkg, xpkg, ebuild, y_ebuild, keywords, self.repo_metadata['pmaskdict'])
if self.options.ignore_arches:
arches = [[
self.repo_settings.repoman_settings["ARCH"], self.repo_settings.repoman_settings["ARCH"],
self.repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split()]]
else:
arches = set()
for keyword in keywords:
if keyword[0] == "-":
continue
elif keyword[0] == "~":
arch = keyword[1:]
if arch == "*":
for expanded_arch in self.profiles:
if expanded_arch == "**":
continue
arches.add(
(keyword, expanded_arch, (
expanded_arch, "~" + expanded_arch)))
else:
arches.add((keyword, arch, (arch, keyword)))
else:
# For ebuilds with stable keywords, check if the
# dependencies are satisfiable for unstable
# configurations, since use.stable.mask is not
# applied for unstable configurations (see bug
# 563546).
if keyword == "*":
for expanded_arch in self.profiles:
if expanded_arch == "**":
continue
arches.add(
(keyword, expanded_arch, (expanded_arch,)))
arches.add(
(keyword, expanded_arch,
(expanded_arch, "~" + expanded_arch)))
else:
arches.add((keyword, keyword, (keyword,)))
arches.add((keyword, keyword,
(keyword, "~" + keyword)))
if not arches:
# Use an empty profile for checking dependencies of
# packages that have empty KEYWORDS.
arches.add(('**', '**', ('**',)))
unknown_pkgs = set()
baddepsyntax = False
badlicsyntax = False
badprovsyntax = False
# catpkg = catdir + "/" + y_ebuild
inherited_java_eclass = "java-pkg-2" in inherited or \
"java-pkg-opt-2" in inherited
inherited_wxwidgets_eclass = "wxwidgets" in inherited
# operator_tokens = set(["||", "(", ")"])
type_list, badsyntax = [], []
for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"):
mydepstr = myaux[mytype]
buildtime = mytype in Package._buildtime_keys
runtime = mytype in Package._runtime_keys
token_class = None
if mytype.endswith("DEPEND"):
token_class = portage.dep.Atom
try:
atoms = portage.dep.use_reduce(
mydepstr, matchall=1, flat=True,
is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class)
except portage.exception.InvalidDependString as e:
atoms = None
badsyntax.append(str(e))
if atoms and mytype.endswith("DEPEND"):
if runtime and \
"test?" in mydepstr.split():
self.qatracker.add_error(
mytype + '.suspect',
"%s: 'test?' USE conditional in %s" %
(ebuild.relative_path, mytype))
for atom in atoms:
if atom == "||":
continue
is_blocker = atom.blocker
# Skip dependency.unknown for blockers, so that we
# don't encourage people to remove necessary blockers,
# as discussed in bug 382407. We use atom.without_use
# due to bug 525376.
if not is_blocker and \
not self.portdb.xmatch("match-all", atom.without_use) and \
not atom.cp.startswith("virtual/"):
unknown_pkgs.add((mytype, atom.unevaluated_atom))
if catdir != "virtual":
if not is_blocker and \
atom.cp in suspect_virtual:
self.qatracker.add_error(
'virtual.suspect', ebuild.relative_path +
": %s: consider using '%s' instead of '%s'" %
(mytype, suspect_virtual[atom.cp], atom))
if not is_blocker and \
atom.cp.startswith("perl-core/"):
self.qatracker.add_error('dependency.perlcore',
ebuild.relative_path +
": %s: please use '%s' instead of '%s'" %
(mytype,
atom.replace("perl-core/","virtual/perl-"),
atom))
if buildtime and \
not is_blocker and \
not inherited_java_eclass and \
atom.cp == "virtual/jdk":
self.qatracker.add_error(
'java.eclassesnotused', ebuild.relative_path)
elif buildtime and \
not is_blocker and \
not inherited_wxwidgets_eclass and \
atom.cp == "x11-libs/wxGTK":
self.qatracker.add_error(
'wxwidgets.eclassnotused',
"%s: %ss on x11-libs/wxGTK without inheriting"
" wxwidgets.eclass" % (ebuild.relative_path, mytype))
elif runtime:
if not is_blocker and \
atom.cp in suspect_rdepend:
self.qatracker.add_error(
mytype + '.suspect',
ebuild.relative_path + ": '%s'" % atom)
if atom.operator == "~" and \
portage.versions.catpkgsplit(atom.cpv)[3] != "r0":
qacat = 'dependency.badtilde'
self.qatracker.add_error(
qacat, "%s: %s uses the ~ operator"
" with a non-zero revision: '%s'" %
(ebuild.relative_path, mytype, atom))
check_missingslot(atom, mytype, eapi, self.portdb, self.qatracker,
ebuild.relative_path, myaux)
type_list.extend([mytype] * (len(badsyntax) - len(type_list)))
for m, b in zip(type_list, badsyntax):
if m.endswith("DEPEND"):
qacat = "dependency.syntax"
else:
qacat = m + ".syntax"
self.qatracker.add_error(
qacat, "%s: %s: %s" % (ebuild.relative_path, m, b))
badlicsyntax = len([z for z in type_list if z == "LICENSE"])
badprovsyntax = len([z for z in type_list if z == "PROVIDE"])
baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax
badlicsyntax = badlicsyntax > 0
badprovsyntax = badprovsyntax > 0
self.use_flag_checks.check(pkg, xpkg, ebuild, y_ebuild, self.muselist)
ebuild_used_useflags = self.use_flag_checks.getUsedUseFlags()
used_useflags = used_useflags.union(ebuild_used_useflags)
self.rubyeclasscheck.check(pkg, ebuild)
# license checks
if not badlicsyntax:
self.licensecheck.check(pkg, xpkg, ebuild, y_ebuild)
self.restrictcheck.check(pkg, xpkg, ebuild, y_ebuild)
# Syntax Checks
if not self.vcs_settings.vcs_preserves_mtime:
if ebuild.ebuild_path not in self.changed.new_ebuilds and \
ebuild.ebuild_path not in self.changed.ebuilds:
pkg.mtime = None
try:
# All ebuilds should have utf_8 encoding.
f = io.open(
_unicode_encode(
ebuild.full_path, encoding=_encodings['fs'], errors='strict'),
mode='r', encoding=_encodings['repo.content'])
try:
for check_name, e in run_checks(f, pkg):
self.qatracker.add_error(
check_name, ebuild.relative_path + ': %s' % e)
finally:
f.close()
except UnicodeDecodeError:
# A file.UTF8 failure will have already been recorded above.
pass
if self.options.force:
# The dep_check() calls are the most expensive QA test. If --force
# is enabled, there's no point in wasting time on these since the
# user is intent on forcing the commit anyway.
continue
relevant_profiles = []
for keyword, arch, groups in arches:
if arch not in self.profiles:
# A missing profile will create an error further down
# during the KEYWORDS verification.
continue
if self.include_arches is not None:
if arch not in self.include_arches:
continue
relevant_profiles.extend(
(keyword, groups, prof) for prof in self.profiles[arch])
relevant_profiles.sort(key=sort_key)
for keyword, groups, prof in relevant_profiles:
is_stable_profile = prof.status == "stable"
is_dev_profile = prof.status == "dev" and \
self.options.include_dev
is_exp_profile = prof.status == "exp" and \
self.options.include_exp_profiles == 'y'
if not (is_stable_profile or is_dev_profile or is_exp_profile):
continue
dep_settings = self.caches['arch'].get(prof.sub_path)
if dep_settings is None:
dep_settings = portage.config(
config_profile_path=prof.abs_path,
config_incrementals=self.repoman_incrementals,
config_root=self.config_root,
local_config=False,
_unmatched_removal=self.options.unmatched_removal,
env=self.env, repositories=self.repo_settings.repoman_settings.repositories)
dep_settings.categories = self.repo_settings.repoman_settings.categories
if self.options.without_mask:
dep_settings._mask_manager_obj = \
copy.deepcopy(dep_settings._mask_manager)
dep_settings._mask_manager._pmaskdict.clear()
self.caches['arch'][prof.sub_path] = dep_settings
xmatch_cache_key = (prof.sub_path, tuple(groups))
xcache = self.caches['arch_xmatch'].get(xmatch_cache_key)
if xcache is None:
self.portdb.melt()
self.portdb.freeze()
xcache = self.portdb.xcache
xcache.update(self.caches['shared_xmatch'])
self.caches['arch_xmatch'][xmatch_cache_key] = xcache
self.repo_settings.trees[self.repo_settings.root]["porttree"].settings = dep_settings
self.portdb.settings = dep_settings
self.portdb.xcache = xcache
dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups)
# just in case, prevent config.reset() from nuking these.
dep_settings.backup_changes("ACCEPT_KEYWORDS")
# This attribute is used in dbapi._match_use() to apply
# use.stable.{mask,force} settings based on the stable
# status of the parent package. This is required in order
# for USE deps of unstable packages to be resolved correctly,
# since otherwise use.stable.{mask,force} settings of
# dependencies may conflict (see bug #456342).
dep_settings._parent_stable = dep_settings._isStable(pkg)
# Handle package.use*.{force,mask) calculation, for use
# in dep_check.
dep_settings.useforce = dep_settings._use_manager.getUseForce(
pkg, stable=dep_settings._parent_stable)
dep_settings.usemask = dep_settings._use_manager.getUseMask(
pkg, stable=dep_settings._parent_stable)
if not baddepsyntax:
ismasked = not ebuild_archs or \
pkg.cpv not in self.portdb.xmatch("match-visible",
Atom("%s::%s" % (pkg.cp, self.repo_settings.repo_config.name)))
if ismasked:
if not self.have['pmasked']:
self.have['pmasked'] = bool(dep_settings._getMaskAtom(
pkg.cpv, pkg._metadata))
if self.options.ignore_masked:
continue
# we are testing deps for a masked package; give it some lee-way
suffix = "masked"
matchmode = "minimum-all-ignore-profile"
else:
suffix = ""
matchmode = "minimum-visible"
if not self.have['dev_keywords']:
self.have['dev_keywords'] = \
bool(self.dev_keywords.intersection(keywords))
if prof.status == "dev":
suffix = suffix + "indev"
for mytype in Package._dep_keys:
mykey = "dependency.bad" + suffix
myvalue = myaux[mytype]
if not myvalue:
continue
success, atoms = portage.dep_check(
myvalue, self.portdb, dep_settings,
use="all", mode=matchmode, trees=self.repo_settings.trees)
if success:
if atoms:
# Don't bother with dependency.unknown for
# cases in which *DEPEND.bad is triggered.
for atom in atoms:
# dep_check returns all blockers and they
# aren't counted for *DEPEND.bad, so we
# ignore them here.
if not atom.blocker:
unknown_pkgs.discard(
(mytype, atom.unevaluated_atom))
if not prof.sub_path:
# old-style virtuals currently aren't
# resolvable with empty profile, since
# 'virtuals' mappings are unavailable
# (it would be expensive to search
# for PROVIDE in all ebuilds)
atoms = [
atom for atom in atoms if not (
atom.cp.startswith('virtual/')
and not self.portdb.cp_list(atom.cp))]
# we have some unsolvable deps
# remove ! deps, which always show up as unsatisfiable
atoms = [
str(atom.unevaluated_atom)
for atom in atoms if not atom.blocker]
# if we emptied out our list, continue:
if not atoms:
continue
if self.options.output_style in ['column']:
self.qatracker.add_error(mykey,
"%s: %s: %s(%s) %s"
% (ebuild.relative_path, mytype, keyword,
prof, repr(atoms)))
else:
self.qatracker.add_error(mykey,
"%s: %s: %s(%s)\n%s"
% (ebuild.relative_path, mytype, keyword,
prof, pformat(atoms, indent=6)))
else:
if self.options.output_style in ['column']:
self.qatracker.add_error(mykey,
"%s: %s: %s(%s) %s"
% (ebuild.relative_path, mytype, keyword,
prof, repr(atoms)))
else:
self.qatracker.add_error(mykey,
"%s: %s: %s(%s)\n%s"
% (ebuild.relative_path, mytype, keyword,
prof, pformat(atoms, indent=6)))
if not baddepsyntax and unknown_pkgs:
type_map = {}
for mytype, atom in unknown_pkgs:
type_map.setdefault(mytype, set()).add(atom)
for mytype, atoms in type_map.items():
self.qatracker.add_error(
"dependency.unknown", "%s: %s: %s"
% (ebuild.relative_path, mytype, ", ".join(sorted(atoms))))
# check if there are unused local USE-descriptions in metadata.xml
# (unless there are any invalids, to avoid noise)
if self.allvalid:
for myflag in self.muselist.difference(used_useflags):
self.qatracker.add_error(
"metadata.warning",
"%s/metadata.xml: unused local USE-description: '%s'"
% (xpkg, myflag))
