def fetch(self, target):
"""
fetch a file
:type target: :obj:`pkgcore.fetch.fetchable` instance
:return: None if fetching failed,
else on disk location of the copied file
"""
if not isinstance(target, fetchable):
raise TypeError(
"target must be fetchable instance/derivative: %s" % target)
kw = {"mode": 0775}
if self.readonly:
kw["mode"] = 0555
if self.userpriv:
kw["gid"] = portage_gid
kw["minimal"] = True
if not ensure_dirs(self.distdir, **kw):
raise errors.distdirPerms(
self.distdir, "if userpriv, uid must be %i, gid must be %i. "
"if not readonly, directory must be 0775, else 0555" %
(portage_uid, portage_gid))
fp = pjoin(self.distdir, target.filename)
filename = os.path.basename(fp)
uri = iter(target.uri)
if self.userpriv and is_userpriv_capable():
extra = {"uid": portage_uid, "gid": portage_gid}
else:
extra = {}
extra["umask"] = 0002
extra["env"] = self.extra_env
attempts = self.attempts
last_exc = None
try:
while attempts >= 0:
try:
c = self._verify(fp, target)
return fp
except errors.MissingDistfile:
command = self.command
last_exc = sys.exc_info()
except errors.FetchFailed as e:
last_exc = sys.exc_info()
if not e.resumable:
try:
os.unlink(fp)
command = self.command
except OSError as oe:
raise_from(errors.UnmodifiableFile(fp, oe))
else:
command = self.resume_command
# yeah, it's funky, but it works.
if attempts > 0:
u = uri.next()
# note we're not even checking the results. the
# verify portion of the loop handles this. iow,
# don't trust their exit code. trust our chksums
# instead.
spawn_bash(command % {"URI": u, "FILE": filename}, **extra)
attempts -= 1
assert last_exc is not None
raise last_exc[0], last_exc[1], last_exc[2]
except StopIteration:
# ran out of uris
raise errors.FetchFailed(fp, "Ran out of urls to fetch from")
def fetch(self, target):
"""Fetch a file.
:type target: :obj:`pkgcore.fetch.fetchable` instance
:return: None if fetching failed,
else on disk location of the copied file
"""
if not isinstance(target, fetchable):
raise TypeError(
f"target must be fetchable instance/derivative: {target}")
kw = {"mode": 0o775}
if self.readonly:
kw["mode"] = 0o555
if self.userpriv:
kw["gid"] = portage_gid
kw["minimal"] = True
if not ensure_dirs(self.distdir, **kw):
raise errors.DistdirPerms(
self.distdir, "if userpriv, uid must be %i, gid must be %i. "
"if not readonly, directory must be 0775, else 0555" %
(portage_uid, portage_gid))
path = pjoin(self.distdir, target.filename)
uris = iter(target.uri)
last_exc = RuntimeError("fetching failed for an unknown reason")
spawn_opts = {'umask': 0o002, 'env': self.extra_env}
if self.userpriv and is_userpriv_capable():
spawn_opts.update({"uid": portage_uid, "gid": portage_gid})
for _attempt in range(self.attempts):
try:
self._verify(path, target)
return path
except errors.MissingDistfile as e:
command = self.command
last_exc = e
except errors.ChksumFailure:
raise
except errors.FetchFailed as e:
last_exc = e
if not e.resumable:
try:
os.unlink(path)
command = self.command
except OSError as e:
raise errors.UnmodifiableFile(path, e) from e
else:
command = self.resume_command
# Note we're not even checking the results, the verify portion of
# the loop handles this. In other words, don't trust the external
# fetcher's exit code, trust our chksums instead.
try:
spawn_bash(
command % {
"URI": next(uris),
"FILE": target.filename
}, **spawn_opts)
except StopIteration:
raise errors.FetchFailed(target.filename,
"ran out of urls to fetch from")
else:
raise last_exc
def _verify(self, file_location, target, all_chksums=True, handlers=None):
"""
Internal function for derivatives.
Digs through chksums, and either returns None, or throws an
errors.FetchFailed exception.
- -2: file doesn't exist.
- -1: if (size chksum is available, and
file is smaller than stated chksum)
- 0: if all chksums match
- 1: if file is too large (if size chksums are available)
or else size is right but a chksum didn't match.
if all_chksums is True, all chksums must be verified; if false, all
a handler can be found for are used.
"""
nondefault_handlers = handlers
if handlers is None:
try:
handlers = get_handlers(target.chksums)
except KeyError:
compatibility.raise_from(
errors.FetchFailed(
file_location,
"Couldn't find a required checksum handler"))
if all_chksums:
missing = set(target.chksums).difference(handlers)
if missing:
raise errors.RequiredChksumDataMissing(target,
*sorted(missing))
if "size" in handlers:
val = handlers["size"](file_location)
if val == -1:
raise errors.MissingDistfile(file_location)
c = cmp(val, target.chksums["size"])
if c:
resumable = (c < 0)
if resumable:
msg = "File is too small."
else:
msg = "File is too big."
raise errors.FetchFailed(file_location,
msg,
resumable=resumable)
elif not os.path.exists(file_location):
raise errors.MissingDistfile(file_location)
elif not os.stat(file_location).st_size:
raise errors.FetchFailed(file_location,
'file is empty',
resumable=False)
chfs = set(target.chksums).intersection(handlers)
chfs.discard("size")
chfs = list(chfs)
if nondefault_handlers:
for x in chfs:
val = handlers[x](file_location)
if val != target.chksums[x]:
raise errors.FetchFailed(
file_location,
"Validation handler %s: expected %s, got %s" %
(x, target.chksums[x], val))
else:
desired_vals = [target.chksums[x] for x in chfs]
calced = get_chksums(file_location, *chfs)
for desired, got, chf in zip(desired_vals, calced, chfs):
if desired != got:
raise errors.FetchFailed(
file_location,
"Validation handler %s: expected %s, got %s" %
(chf, desired, got))
def _verify(self, file_location, target, all_chksums=True, handlers=None):
"""Internal function for derivatives.
Digs through chksums, and either returns None, or throws an
errors.FetchFailed exception.
- -2: file doesn't exist.
- -1: if (size chksum is available, and
file is smaller than stated chksum)
- 0: if all chksums match
- 1: if file is too large (if size chksums are available)
or else size is right but a chksum didn't match.
if all_chksums is True, all chksums must be verified; if false, all
a handler can be found for are used.
"""
nondefault_handlers = handlers
if handlers is None:
try:
handlers = get_handlers(target.chksums)
except MissingChksumHandler as e:
raise errors.MissingChksumHandler(
f'missing required checksum handler: {e}')
if all_chksums:
missing = set(target.chksums).difference(handlers)
if missing:
raise errors.RequiredChksumDataMissing(target,
*sorted(missing))
if "size" in handlers:
val = handlers["size"](file_location)
if val == -1:
raise errors.MissingDistfile(file_location)
if val != target.chksums["size"]:
if val < target.chksums["size"]:
raise errors.FetchFailed(file_location,
'file is too small',
resumable=True)
raise errors.ChksumFailure(file_location,
chksum='size',
expected=target.chksums["size"],
value=val)
elif not os.path.exists(file_location):
raise errors.MissingDistfile(file_location)
elif not os.stat(file_location).st_size:
raise errors.FetchFailed(file_location,
'file is empty',
resumable=False)
chfs = set(target.chksums).intersection(handlers)
chfs.discard("size")
chfs = list(chfs)
if nondefault_handlers:
for x in chfs:
val = handlers[x](file_location)
if val != target.chksums[x]:
raise errors.ChksumFailure(file_location,
chksum=x,
expected=target.chksums[x],
value=val)
else:
desired_vals = [target.chksums[x] for x in chfs]
calced = get_chksums(file_location, *chfs)
for desired, got, chf in zip(desired_vals, calced, chfs):
if desired != got:
raise errors.ChksumFailure(file_location,
chksum=chf,
expected=desired,
value=got)