def assign_packages(broken, logger, settings):
''' Finds and returns packages that owns files placed in broken.
Broken is list of files
'''
assigned = set()
if not broken:
return assigned
pkgset = set(get_installed_cpvs())
# Map all files in CONTENTS database to package names
fname_pkg_dict = {}
for pkg in pkgset:
contents = Package(pkg).parsed_contents()
for fname in contents.keys():
if contents[fname][0] == "obj":
fname_pkg_dict[fname] = str(pkg)
for fname in broken:
realname = os.path.realpath(fname)
if realname in fname_pkg_dict.keys():
pkgname = fname_pkg_dict[realname]
elif fname in fname_pkg_dict.keys():
pkgname = fname_pkg_dict[fname]
else:
pkgname = None
if pkgname and pkgname not in assigned:
assigned.add(pkgname)
if not pkgname:
pkgname = "(none)"
logger.info('\t' + fname + ' -> ' + bold(pkgname))
return assigned
def stabilization_candidates(days_to_candidate=30, logger=None):
"""
Collect stabilization candidates
"""
if logger is None:
logger = FakeLogger()
date_diff = datetime.utcnow() - timedelta(days=days_to_candidate)
date_limit = date_diff.date()
logger.info("Collecting stabilization candidates - date_limit=%s", str(date_limit))
# Set all versions to not be stabilization_candidates
# Version.objects.update(stabilization_candidate=False)
# For every version check if it's unstable.
# If it is then check if can be a stabilization candidate
versions = Version.objects.filter(overlay="gentoo").filter(Q(vtype="release") | Q(vtype="p"))
for version in versions:
pkg = Package(version.cpv())
try:
keywords = pkg.environment("KEYWORDS").split()
except GentoolkitFatalError:
logger.warning("Error while processing %s!", version)
continue
if all([x.startswith("~") for x in keywords]):
version_date = get_version_date(version, date_limit)
if version_date:
logger.info("+ [s] %s @ %s", version, version_date)
# XXX: What should we save? A flag and the date?Just the date?
version.stabilization_candidate = version_date
version.save()
logger.info("Finished collecting stabilization candidates")
def find_best(self, include_keyworded=True, include_masked=True):
"""Returns the "best" version available.
Order of preference:
highest available stable =>
highest available keyworded =>
highest available masked
@rtype: Package object or None
@return: best of up to three options
@raise errors.GentoolkitInvalidAtom: if query is not valid input
"""
best = keyworded = masked = None
try:
best = portage.db[portage.root]["porttree"].dbapi.xmatch(
"bestmatch-visible", self.query)
except portage.exception.InvalidAtom as err:
message = ("query.py: find_best(), bestmatch-visible, " +
"query=%s, InvalidAtom=%s" % (self.query, str(err)))
raise errors.GentoolkitInvalidAtom(message)
# xmatch can return an empty string, so checking for None is not enough
if not best:
if not (include_keyworded or include_masked):
return None
try:
matches = portage.db[portage.root]["porttree"].dbapi.xmatch(
"match-all", self.query)
except portage.exception.InvalidAtom as err:
message = (
"query.py: find_best(), match-all, query=%s, InvalidAtom=%s"
% (self.query, str(err)))
raise errors.GentoolkitInvalidAtom(message)
masked = portage.best(matches)
keywordable = []
for m in matches:
status = portage.getmaskingstatus(m)
if "package.mask" not in status or "profile" not in status:
keywordable.append(m)
if matches:
keyworded = portage.best(keywordable)
else:
return Package(best)
if include_keyworded and keyworded:
return Package(keyworded)
if include_masked and masked:
return Package(masked)
return None
def find_owners(self, query_re, use_match=False, pkgset=None):
"""Find owners and feed data to supplied output function.
@type query_re: _sre.SRE_Pattern
@param query_re: file regex
@type use_match: bool
@param use_match: use re.match or re.search
@type pkgset: iterable or None
@param pkgset: list of packages to look through
"""
# FIXME: Remove when lazyimport supports objects:
from gentoolkit.package import Package
if use_match:
query_fn = query_re.match
else:
query_fn = query_re.search
results = []
found_match = False
for pkg in sorted([Package(x) for x in pkgset]):
files = pkg.parsed_contents()
for cfile in files:
match = query_fn(cfile)
if match:
results.append((pkg, cfile))
if self.printer_fn is not None:
self.printer_fn(pkg, cfile)
if self.early_out:
found_match = True
break
if found_match:
break
return results
def find(self, in_installed=True, include_masked=True):
"""Returns a list of Package objects that matched the query.
@rtype: list
@return: matching Package objects
"""
if not self.query:
return []
try:
if include_masked:
matches = portage.db[portage.root]["porttree"].dbapi.xmatch(
"match-all", self.query)
else:
matches = portage.db[portage.root]["porttree"].dbapi.match(
self.query)
if in_installed:
matches.extend(portage.db[portage.root]["vartree"].dbapi.match(
self.query))
except portage.exception.InvalidAtom as err:
message = "query.py: find(), query=%s, InvalidAtom=%s" % (
self.query, str(err))
raise errors.GentoolkitInvalidAtom(message)
return [Package(x) for x in set(matches)]
def gather_keywords_info(
cpvs=None,
system_keywords=None,
use_portage=False,
# override-able for testing
keywords=portage.settings["ACCEPT_KEYWORDS"],
analyser = None
):
"""Analyze the installed pkgs 'keywords' for frequency of use
@param cpvs: optional list of [cat/pkg-ver,...] to analyze or
defaults to entire installed pkg db
@param system_keywords: list of the system keywords
@param keywords: user defined list of keywords to check and report on
or reports on all relevant keywords found to have been used.
@param _get_kwds: overridable function for testing
@param _get_used: overridable function for testing
@rtype dict. {keyword:{"stable":[cat/pkg-ver,...], "testing":[cat/pkg-ver,...]}
"""
if cpvs is None:
cpvs = portage.db[portage.root]["vartree"].dbapi.cpv_all()
keyword_users = {}
for cpv in cpvs:
if cpv.startswith("virtual"):
continue
if use_portage:
keyword = analyser.get_inst_keyword_cpv(cpv)
else:
pkg = Package(cpv)
keyword = analyser.get_inst_keyword_pkg(pkg)
#print "returned keyword =", cpv, keyword, keyword[0]
key = keyword[0]
if key in ["~", "-"]:
_kwd = keyword[1:]
if _kwd in keyword_users:
if key in ["~"]:
keyword_users[_kwd]["testing"].append(cpv)
elif key in ["-"]:
#print "adding cpv to missing:", cpv
keyword_users[_kwd]["missing"].append(cpv)
else:
if key in ["~"]:
keyword_users[_kwd] = {"stable": [],
"testing": [cpv], "missing": []}
elif key in ["-"]:
keyword_users[_kwd] = {"stable": [],
"testing": [], "missing": [cpv]}
else:
keyword_users[_kwd] = {"stable": [cpv],
"testing": [], "missing": []}
elif keyword in keyword_users:
keyword_users[keyword]["stable"].append(cpv)
else:
keyword_users[keyword] = {
"stable": [cpv],
"testing": [],
"missing": []
}
return keyword_users
def cpv_all_diff_keywords(
cpvs=None,
system_keywords=None,
use_portage=False,
# override-able for testing
keywords=portage.settings["ACCEPT_KEYWORDS"],
analyser=None,
):
"""Analyze the installed pkgs 'keywords' for difference from ACCEPT_KEYWORDS
@param cpvs: optional list of [cat/pkg-ver,...] to analyze or
defaults to entire installed pkg db
@param system_keywords: list of the system keywords
@param keywords: user defined list of keywords to check and report on
or reports on all relevant keywords found to have been used.
@param _get_kwds: overridable function for testing
@param _get_used: overridable function for testing
@rtype dict. {keyword:{"stable":[cat/pkg-ver,...],
"testing":[cat/pkg-ver,...]}
"""
if cpvs is None:
cpvs = portage.db[portage.root]["vartree"].dbapi.cpv_all()
keyword_users = {}
cp_counts = {}
for cpv in cpvs:
if cpv.startswith("virtual"):
continue
if use_portage:
keyword = analyser.get_inst_keyword_cpv(cpv)
else:
pkg = Package(cpv)
keyword = analyser.get_inst_keyword_pkg(pkg)
# print "returned keyword =", cpv, keyword, keyword[0]
key = keyword[0]
if key in ["~", "-"] and keyword not in system_keywords:
atom = Atom("=" + cpv)
if atom.cp not in keyword_users:
keyword_users[atom.cp] = []
if atom.cp not in cp_counts:
cp_counts[atom.cp] = 0
if key in ["~"]:
atom.keyword = keyword
atom.slot = portage.db[portage.root]["vartree"].dbapi.aux_get(
atom.cpv, ["SLOT"]
)[0]
keyword_users[atom.cp].append(atom)
cp_counts[atom.cp] += 1
elif key in ["-"]:
# print "adding cpv to missing:", cpv
atom.keyword = "**"
atom.slot = portage.db[portage.root]["vartree"].dbapi.aux_get(
atom.cpv, ["SLOT"]
)[0]
keyword_users[atom.cp].append(atom)
cp_counts[atom.cp] += 1
return keyword_users, cp_counts
def stabilization_candidates(days_to_candidate=30, logger=None):
"""
Collect stabilization candidates
"""
if logger is None:
logger = FakeLogger()
date_diff = (datetime.utcnow() - timedelta(days=days_to_candidate))
date_limit = date_diff.date()
logger.info("Collecting stabilization candidates - date_limit=%s",
str(date_limit))
# Set all versions to not be stabilization_candidates
#Version.objects.update(stabilization_candidate=False)
# For every version check if it's unstable.
# If it is then check if can be a stabilization candidate
versions = Version.objects.filter(overlay='gentoo').filter(
Q(vtype='release') | Q(vtype='p'))
for version in versions:
pkg = Package(version.cpv())
try:
keywords = pkg.environment("KEYWORDS").split()
except GentoolkitFatalError:
logger.warning("Error while processing %s!", version)
continue
if all([x.startswith("~") for x in keywords]):
version_date = get_version_date(version, date_limit)
if version_date:
logger.info('+ [s] %s @ %s', version, version_date)
# XXX: What should we save? A flag and the date?Just the date?
version.stabilization_candidate = version_date
version.save()
logger.info("Finished collecting stabilization candidates")
def find_installed(self):
"""Return a list of Package objects that matched the search key."""
try:
matches = portage.db[portage.root]["vartree"].dbapi.match(
self.query)
# catch the ambiguous package Exception
except portage.exception.AmbiguousPackageName as err:
matches = []
for pkgkey in err.args[0]:
matches.extend(
portage.db[portage.root]["vartree"].dbapi.match(pkgkey))
except portage.exception.InvalidAtom as err:
raise errors.GentoolkitInvalidAtom(err)
return [Package(x) for x in set(matches)]
def _do_complex_lookup(self, show_progress=True):
"""Find matches for a query which is a regex or includes globbing."""
result = []
if show_progress and not CONFIG["piping"]:
self.print_summary()
try:
cat = CPV(self.query).category
except errors.GentoolkitInvalidCPV:
cat = ''
pre_filter = []
# The "get_" functions can pre-filter against the whole package key,
# but since we allow globbing now, we run into issues like:
# >>> portage.dep.dep_getkey("sys-apps/portage-*")
# 'sys-apps/portage-'
# So the only way to guarantee we don't overrun the key is to
# prefilter by cat only.
if cat:
if self.is_regex:
cat_re = cat
else:
cat_re = fnmatch.translate(cat)
predicate = lambda x: re.match(cat_re, x.split("/", 1)[0])
pre_filter = self.package_finder(predicate=predicate)
# Post-filter
if self.is_regex:
try:
re.compile(self.query)
except re.error:
raise errors.GentoolkitInvalidRegex(self.query)
predicate = lambda x: re.search(self.query, x)
else:
if cat:
query_re = fnmatch.translate(self.query)
else:
query_re = fnmatch.translate("*/%s" % self.query)
predicate = lambda x: re.search(query_re, x)
if pre_filter:
result = [x for x in pre_filter if predicate(x)]
else:
result = self.package_finder(predicate=predicate)
return [Package(x) for x in result]
def format_depend(self, dep, dep_is_displayed):
"""Format a dependency for printing.
@type dep: L{gentoolkit.dependencies.Dependencies}
@param dep: the dependency to display
"""
# Don't print blank lines
if dep_is_displayed and not self.verbose:
return
depth = getattr(dep, 'depth', 0)
indent = " " * depth
mdep = dep.matching_dep
use_conditional = ""
if None != QUERY_OPTS["package_format"]:
pkg = Package(str(dep.cpv))
self.print_formated(pkg)
else:
if mdep.use_conditional:
use_conditional = " & ".join(
pp.useflag(u) for u in mdep.use_conditional.split())
if mdep.operator == '=*':
formatted_dep = '=%s*' % str(mdep.cpv)
else:
formatted_dep = mdep.operator + str(mdep.cpv)
if mdep.slot:
formatted_dep += pp.emph(':') + pp.slot(mdep.slot)
if mdep.sub_slot:
formatted_dep += pp.slot('/') + pp.slot(mdep.sub_slot)
if mdep.use:
useflags = pp.useflag(','.join(mdep.use.tokens))
formatted_dep += (pp.emph('[') + useflags + pp.emph(']'))
if dep_is_displayed:
indent = indent + " " * len(str(dep.cpv))
self.print_fn(indent, '', use_conditional, formatted_dep)
else:
self.print_fn(indent, \
str(dep.cpv), use_conditional, formatted_dep)
def gather_flags_info(
cpvs=None,
system_flags=None,
include_unset=False,
target="USE",
use_portage=False,
# override-able for testing
_get_flags=get_flags,
_get_used=get_installed_use
):
"""Analyze the installed pkgs USE flags for frequency of use
@type cpvs: list
@param cpvs: optional list of [cat/pkg-ver,...] to analyze or
defaults to entire installed pkg db
@type: system_flags: list
@param system_flags: the current default USE flags as defined
by portage.settings["USE"].split()
@type include_unset: bool
@param include_unset: controls the inclusion of unset USE flags in the report.
@type target: string
@param target: the environment variable being analyzed
one of ["USE", "PKGUSE"]
@type _get_flags: function
@param _get_flags: ovride-able for testing,
defaults to gentoolkit.enalyze.lib.get_flags
@param _get_used: ovride-able for testing,
defaults to gentoolkit.enalyze.lib.get_installed_use
@rtype dict. {flag:{"+":[cat/pkg-ver,...], "-":[cat/pkg-ver,...], "unset":[]}
"""
if cpvs is None:
cpvs = portage.db[portage.root]["vartree"].dbapi.cpv_all()
# pass them in to override for tests
flags = FlagAnalyzer(system_flags,
filter_defaults=False,
target=target,
_get_flags=_get_flags,
_get_used=get_installed_use
)
flag_users = {}
for cpv in cpvs:
if cpv.startswith("virtual"):
continue
if use_portage:
plus, minus, unset = flags.analyse_cpv(cpv)
else:
pkg = Package(cpv)
plus, minus, unset = flags.analyse_pkg(pkg)
for flag in plus:
if flag in flag_users:
flag_users[flag]["+"].append(cpv)
else:
flag_users[flag] = {"+": [cpv], "-": []}
for flag in minus:
if flag in flag_users:
flag_users[flag]["-"].append(cpv)
else:
flag_users[flag] = {"+":[], "-": [cpv]}
if include_unset:
for flag in unset:
if flag in flag_users:
if "unset" in flag_users[flag]:
flag_users[flag]["unset"].append(cpv)
else:
flag_users[flag]["unset"] = [cpv]
else:
flag_users[flag] = {"+": [], "-": [], "unset": [cpv]}
return flag_users
def scan_upstream(query, on_progress=None):
"""
Scans the upstream searching new versions for the given query
"""
matches = []
if query.endswith(".ebuild"):
cpv = package_from_ebuild(query)
reload_gentoolkit()
if cpv:
matches = [Package(cpv)]
else:
matches = Query(query).find(
include_masked=True,
in_installed=False,
)
if not matches:
output.ewarn(pp.warn("No package matching '%s'" % pp.pkgquery(query)))
return None
matches = sorted(matches)
pkg = matches.pop()
while '9999' in pkg.version and len(matches):
pkg = matches.pop()
if not pkg:
output.ewarn(
pp.warn("Package '%s' only have a dev version (9999)" %
pp.pkgquery(pkg.cp)))
return None
# useful data only for formatted output
start_time = datetime.now()
output.metadata("datetime", start_time.isoformat(), show=False)
output.metadata("cp", pkg.cp, show=False)
output.metadata("cpv", pkg.cpv, show=False)
if on_progress:
on_progress(increment=10)
if pkg.cp in BLACKLIST_PACKAGES:
output.ewarn(
pp.warn("Package '%s' is blacklisted" % pp.pkgquery(pkg.cp)))
return None
if not CONFIG['quiet']:
if not CONFIG['format']:
pp.uprint(" * %s [%s]" %
(pp.cpv(pkg.cpv), pp.section(pkg.repo_name())))
pp.uprint()
else:
output.metadata("overlay", pp.section(pkg.repo_name()))
ebuild_path = pkg.ebuild_path()
if ebuild_path:
output.metadata("ebuild", pp.path(os.path.normpath(ebuild_path)))
uris, homepage, description = pkg.environment(
('SRC_URI', 'HOMEPAGE', 'DESCRIPTION'))
output.metadata("repository", pkg.repo_name())
output.metadata("homepage", homepage)
output.metadata("description", description)
else:
uris = pkg.environment('SRC_URI')
cpv = pkg.cpv
uris = parse_src_uri(uris)
uris_expanded = [
from_mirror(uri) if 'mirror://' in uri else uri for uri in uris
]
pkg._uris = uris
pkg._uris_expanded = uris_expanded
versions = handlers.scan(pkg, uris, on_progress)
cp, ver, rev = portage.pkgsplit(pkg.cpv)
result = filter_versions(cp, versions)
if on_progress:
on_progress(increment=10)
# output scan time for formatted output
scan_time = (datetime.now() - start_time).total_seconds()
output.metadata("scan_time", scan_time, show=False)
is_current_version_stable = is_version_stable(ver)
if len(result) > 0:
if not (CONFIG['format'] or CONFIG['quiet']):
print("")
for cp, url, version, handler, confidence in result:
if CONFIG["ignore-pre-release"]:
if not is_version_stable(version):
continue
if CONFIG["ignore-pre-release-if-stable"]:
if is_current_version_stable and \
not is_version_stable(version):
continue
if CONFIG['progress']:
print("", file=sys.stderr)
output.result(cp, version, url, handler, confidence)
return result