def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False):
if all:
useflags = None
elif useflags is None:
if mysettings:
useflags = mysettings["USE"].split()
myfiles = self.getFetchMap(mypkg, useflags=useflags)
myebuild = self.findname(mypkg)
if myebuild is None:
raise AssertionError("ebuild not found for '%s'" % mypkg)
pkgdir = os.path.dirname(myebuild)
mf = Manifest(pkgdir, self.settings["DISTDIR"])
mysums = mf.getDigests()
failures = {}
for x in myfiles:
if not mysums or x not in mysums:
ok = False
reason = _("digest missing")
else:
try:
ok, reason = portage.checksum.verify_all(
os.path.join(self.settings["DISTDIR"], x), mysums[x])
except FileNotFound as e:
ok = False
reason = _("File Not Found: '%s'") % (e, )
if not ok:
failures[x] = reason
if failures:
return False
return True
def getfetchsizes(self, mypkg, useflags=None, debug=0):
# returns a filename:size dictionnary of remaining downloads
myebuild = self.findname(mypkg)
if myebuild is None:
raise AssertionError("ebuild not found for '%s'" % mypkg)
pkgdir = os.path.dirname(myebuild)
mf = Manifest(pkgdir, self.settings["DISTDIR"])
checksums = mf.getDigests()
if not checksums:
if debug:
writemsg("[empty/missing/bad digest]: %s\n" % (mypkg, ))
return {}
filesdict = {}
myfiles = self.getFetchMap(mypkg, useflags=useflags)
#XXX: maybe this should be improved: take partial downloads
# into account? check checksums?
for myfile in myfiles:
try:
fetch_size = int(checksums[myfile]["size"])
except (KeyError, ValueError):
if debug:
writemsg(
_("[bad digest]: missing %(file)s for %(pkg)s\n") % {
"file": myfile,
"pkg": mypkg
})
continue
file_path = os.path.join(self.settings["DISTDIR"], myfile)
mystat = None
try:
mystat = os.stat(file_path)
except OSError as e:
pass
if mystat is None:
existing_size = 0
ro_distdirs = self.settings.get("PORTAGE_RO_DISTDIRS")
if ro_distdirs is not None:
for x in shlex_split(ro_distdirs):
try:
mystat = os.stat(os.path.join(x, myfile))
except OSError:
pass
else:
if mystat.st_size == fetch_size:
existing_size = fetch_size
break
else:
existing_size = mystat.st_size
remaining_size = fetch_size - existing_size
if remaining_size > 0:
# Assume the download is resumable.
filesdict[myfile] = remaining_size
elif remaining_size < 0:
# The existing file is too large and therefore corrupt.
filesdict[myfile] = int(checksums[myfile]["size"])
return filesdict
def process(self, args):
# Call ebuild ... digest
try:
portage._doebuild_manifest_exempt_depend += 1
pkgdir = os.path.dirname(args['output'])
fetchlist_dict = portage.FetchlistDict(pkgdir, portage.settings,
portage.portdb)
mf = Manifest(pkgdir,
args['--portage-distfiles'],
fetchlist_dict=fetchlist_dict,
manifest1_compat=False)
mf.create(requiredDistfiles=None,
assumeDistHashesSometimes=True,
assumeDistHashesAlways=True)
mf.write()
finally:
portage._doebuild_manifest_exempt_depend -= 1
def digestParseFile(myfilename, mysettings=None):
"""(filename) -- Parses a given file for entries matching:
<checksumkey> <checksum_hex_string> <filename> <filesize>
Ignores lines that don't start with a valid checksum identifier
and returns a dict with the filenames as keys and {checksumkey:checksum}
as the values.
DEPRECATED: this function is now only a compability wrapper for
portage.manifest.Manifest()."""
warnings.warn("portage.digestParseFile() is deprecated",
DeprecationWarning,
stacklevel=2)
mysplit = myfilename.split(os.sep)
if mysplit[-2] == "files" and mysplit[-1].startswith("digest-"):
pkgdir = os.sep + os.sep.join(mysplit[:-2]).strip(os.sep)
elif mysplit[-1] == "Manifest":
pkgdir = os.sep + os.sep.join(mysplit[:-1]).strip(os.sep)
return Manifest(pkgdir, None).getDigests()
def getfetchsize(pkg):
# from /usr/bin/emerge
try:
myebuild = portage.portdb.findname(pkg)
pkgdir = os.path.dirname(myebuild)
mf = Manifest(pkgdir, portage.settings["DISTDIR"])
if hasattr(portage.portdb, "getFetchMap"):
fetchlist = portage.portdb.getFetchMap(pkg)
else:
fetchlist = portage.portdb.getfetchlist(
pkg, mysettings=portage.settings, all=True)[1]
mysum = mf.getDistfilesSize(fetchlist)
mystr = str(mysum / 1024)
mycount = len(mystr)
while (mycount > 3):
mycount -= 3
mystr = mystr[:mycount] + "," + mystr[mycount:]
mysum = mystr + " kB"
return mysum
except (PortageException, KeyError):
return "[no/bad digest]"
def digestcheck(myfiles, mysettings, strict=False, justmanifest=None):
"""
Verifies checksums. Assumes all files have been downloaded.
@rtype: int
@returns: 1 on success and 0 on failure
"""
if justmanifest is not None:
warnings.warn("The justmanifest parameter of the " + \
"portage.package.ebuild.digestcheck.digestcheck()" + \
" function is now unused.",
DeprecationWarning, stacklevel=2)
justmanifest = None
if mysettings.get("EBUILD_SKIP_MANIFEST") == "1":
return 1
pkgdir = mysettings["O"]
manifest_path = os.path.join(pkgdir, "Manifest")
if not os.path.exists(manifest_path):
writemsg(_("!!! Manifest file not found: '%s'\n") % manifest_path,
noiselevel=-1)
if strict:
return 0
else:
return 1
mf = Manifest(pkgdir, mysettings["DISTDIR"])
manifest_empty = True
for d in mf.fhashdict.values():
if d:
manifest_empty = False
break
if manifest_empty:
writemsg(_("!!! Manifest is empty: '%s'\n") % manifest_path,
noiselevel=-1)
if strict:
return 0
else:
return 1
eout = EOutput()
eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1"
try:
if strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings:
eout.ebegin(_("checking ebuild checksums ;-)"))
mf.checkTypeHashes("EBUILD")
eout.eend(0)
eout.ebegin(_("checking auxfile checksums ;-)"))
mf.checkTypeHashes("AUX")
eout.eend(0)
eout.ebegin(_("checking miscfile checksums ;-)"))
mf.checkTypeHashes("MISC", ignoreMissingFiles=True)
eout.eend(0)
for f in myfiles:
eout.ebegin(_("checking %s ;-)") % f)
ftype = mf.findFile(f)
if ftype is None:
raise KeyError(f)
mf.checkFileHashes(ftype, f)
eout.eend(0)
except KeyError as e:
eout.eend(1)
writemsg(_("\n!!! Missing digest for %s\n") % str(e), noiselevel=-1)
return 0
except FileNotFound as e:
eout.eend(1)
writemsg(
_("\n!!! A file listed in the Manifest could not be found: %s\n") %
str(e),
noiselevel=-1)
return 0
except DigestException as e:
eout.eend(1)
writemsg(_("\n!!! Digest verification failed:\n"), noiselevel=-1)
writemsg("!!! %s\n" % e.value[0], noiselevel=-1)
writemsg(_("!!! Reason: %s\n") % e.value[1], noiselevel=-1)
writemsg(_("!!! Got: %s\n") % e.value[2], noiselevel=-1)
writemsg(_("!!! Expected: %s\n") % e.value[3], noiselevel=-1)
return 0
# Make sure that all of the ebuilds are actually listed in the Manifest.
for f in os.listdir(pkgdir):
pf = None
if f[-7:] == '.ebuild':
pf = f[:-7]
if pf is not None and not mf.hasFile("EBUILD", f):
writemsg(_("!!! A file is not listed in the Manifest: '%s'\n") % \
os.path.join(pkgdir, f), noiselevel=-1)
if strict:
return 0
""" epatch will just grab all the patches out of a directory, so we have to
make sure there aren't any foreign files that it might grab."""
filesdir = os.path.join(pkgdir, "files")
for parent, dirs, files in os.walk(filesdir):
try:
parent = _unicode_decode(parent,
encoding=_encodings['fs'],
errors='strict')
except UnicodeDecodeError:
parent = _unicode_decode(parent,
encoding=_encodings['fs'],
errors='replace')
writemsg(_("!!! Path contains invalid "
"character(s) for encoding '%s': '%s'") \
% (_encodings['fs'], parent), noiselevel=-1)
if strict:
return 0
continue
for d in dirs:
d_bytes = d
try:
d = _unicode_decode(d,
encoding=_encodings['fs'],
errors='strict')
except UnicodeDecodeError:
d = _unicode_decode(d,
encoding=_encodings['fs'],
errors='replace')
writemsg(_("!!! Path contains invalid "
"character(s) for encoding '%s': '%s'") \
% (_encodings['fs'], os.path.join(parent, d)),
noiselevel=-1)
if strict:
return 0
dirs.remove(d_bytes)
continue
if d.startswith(".") or d == "CVS":
dirs.remove(d_bytes)
for f in files:
try:
f = _unicode_decode(f,
encoding=_encodings['fs'],
errors='strict')
except UnicodeDecodeError:
f = _unicode_decode(f,
encoding=_encodings['fs'],
errors='replace')
if f.startswith("."):
continue
f = os.path.join(parent, f)[len(filesdir) + 1:]
writemsg(_("!!! File name contains invalid "
"character(s) for encoding '%s': '%s'") \
% (_encodings['fs'], f), noiselevel=-1)
if strict:
return 0
continue
if f.startswith("."):
continue
f = os.path.join(parent, f)[len(filesdir) + 1:]
file_type = mf.findFile(f)
if file_type != "AUX" and not f.startswith("digest-"):
writemsg(_("!!! A file is not listed in the Manifest: '%s'\n") % \
os.path.join(filesdir, f), noiselevel=-1)
if strict:
return 0
return 1
def digestgen(myarchives=None,
mysettings=None,
overwrite=None,
manifestonly=None,
myportdb=None):
"""
Generates a digest file if missing. Fetches files if necessary.
NOTE: myarchives and mysettings used to be positional arguments,
so their order must be preserved for backward compatibility.
@param mysettings: the ebuild config (mysettings["O"] must correspond
to the ebuild's parent directory)
@type mysettings: config
@param myportdb: a portdbapi instance
@type myportdb: portdbapi
@rtype: int
@returns: 1 on success and 0 on failure
"""
if mysettings is None:
raise TypeError("portage.digestgen(): missing" + \
" required 'mysettings' parameter")
if myportdb is None:
warnings.warn(
"portage.digestgen() called without 'myportdb' parameter",
DeprecationWarning,
stacklevel=2)
myportdb = portage.portdb
if overwrite is not None:
warnings.warn("portage.digestgen() called with " + \
"deprecated 'overwrite' parameter",
DeprecationWarning, stacklevel=2)
if manifestonly is not None:
warnings.warn("portage.digestgen() called with " + \
"deprecated 'manifestonly' parameter",
DeprecationWarning, stacklevel=2)
try:
portage._doebuild_manifest_exempt_depend += 1
distfiles_map = {}
fetchlist_dict = FetchlistDict(mysettings["O"], mysettings, myportdb)
for cpv in fetchlist_dict:
try:
for myfile in fetchlist_dict[cpv]:
distfiles_map.setdefault(myfile, []).append(cpv)
except InvalidDependString as e:
writemsg("!!! %s\n" % str(e), noiselevel=-1)
del e
return 0
mytree = os.path.dirname(os.path.dirname(mysettings["O"]))
manifest1_compat = False
mf = Manifest(mysettings["O"],
mysettings["DISTDIR"],
fetchlist_dict=fetchlist_dict,
manifest1_compat=manifest1_compat)
# Don't require all hashes since that can trigger excessive
# fetches when sufficient digests already exist. To ease transition
# while Manifest 1 is being removed, only require hashes that will
# exist before and after the transition.
required_hash_types = set()
required_hash_types.add("size")
required_hash_types.add(MANIFEST2_REQUIRED_HASH)
dist_hashes = mf.fhashdict.get("DIST", {})
# To avoid accidental regeneration of digests with the incorrect
# files (such as partially downloaded files), trigger the fetch
# code if the file exists and it's size doesn't match the current
# manifest entry. If there really is a legitimate reason for the
# digest to change, `ebuild --force digest` can be used to avoid
# triggering this code (or else the old digests can be manually
# removed from the Manifest).
missing_files = []
for myfile in distfiles_map:
myhashes = dist_hashes.get(myfile)
if not myhashes:
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError:
st = None
if st is None or st.st_size == 0:
missing_files.append(myfile)
continue
size = myhashes.get("size")
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError as e:
if e.errno != errno.ENOENT:
raise
del e
if size == 0:
missing_files.append(myfile)
continue
if required_hash_types.difference(myhashes):
missing_files.append(myfile)
continue
else:
if st.st_size == 0 or size is not None and size != st.st_size:
missing_files.append(myfile)
continue
if missing_files:
mytree = os.path.realpath(
os.path.dirname(os.path.dirname(mysettings["O"])))
fetch_settings = config(clone=mysettings)
debug = mysettings.get("PORTAGE_DEBUG") == "1"
for myfile in missing_files:
uris = set()
for cpv in distfiles_map[myfile]:
myebuild = os.path.join(mysettings["O"],
catsplit(cpv)[1] + ".ebuild")
# for RESTRICT=fetch, mirror, etc...
doebuild_environment(myebuild, "fetch", mysettings["ROOT"],
fetch_settings, debug, 1, myportdb)
uris.update(
myportdb.getFetchMap(cpv, mytree=mytree)[myfile])
fetch_settings["A"] = myfile # for use by pkg_nofetch()
try:
st = os.stat(os.path.join(mysettings["DISTDIR"], myfile))
except OSError:
st = None
if not fetch({myfile: uris}, fetch_settings):
writemsg(_("!!! Fetch failed for %s, can't update "
"Manifest\n") % myfile,
noiselevel=-1)
if myfile in dist_hashes and \
st is not None and st.st_size > 0:
# stat result is obtained before calling fetch(),
# since fetch may rename the existing file if the
# digest does not match.
writemsg(_("!!! If you would like to "
"forcefully replace the existing "
"Manifest entry\n!!! for %s, use "
"the following command:\n") % myfile + \
"!!! " + colorize("INFORM",
"ebuild --force %s manifest" % \
os.path.basename(myebuild)) + "\n",
noiselevel=-1)
return 0
writemsg_stdout(_(">>> Creating Manifest for %s\n") % mysettings["O"])
try:
mf.create(assumeDistHashesSometimes=True,
assumeDistHashesAlways=("assume-digests"
in mysettings.features))
except FileNotFound as e:
writemsg(_("!!! File %s doesn't exist, can't update "
"Manifest\n") % e,
noiselevel=-1)
return 0
except PortagePackageException as e:
writemsg(("!!! %s\n") % (e, ), noiselevel=-1)
return 0
try:
mf.write(sign=False)
except PermissionDenied as e:
writemsg(_("!!! Permission Denied: %s\n") % (e, ), noiselevel=-1)
return 0
if "assume-digests" not in mysettings.features:
distlist = list(mf.fhashdict.get("DIST", {}))
distlist.sort()
auto_assumed = []
for filename in distlist:
if not os.path.exists(
os.path.join(mysettings["DISTDIR"], filename)):
auto_assumed.append(filename)
if auto_assumed:
mytree = os.path.realpath(
os.path.dirname(os.path.dirname(mysettings["O"])))
cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:])
pkgs = myportdb.cp_list(cp, mytree=mytree)
pkgs.sort()
writemsg_stdout(" digest.assumed" +
colorize("WARN",
str(len(auto_assumed)).rjust(18)) +
"\n")
for pkg_key in pkgs:
fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree)
pv = pkg_key.split("/")[1]
for filename in auto_assumed:
if filename in fetchlist:
writemsg_stdout(" %s::%s\n" % (pv, filename))
return 1
finally:
portage._doebuild_manifest_exempt_depend -= 1