def test_get_handlers(self):
expected = {"x": 1, "y": 2}
chksum.chksum_types.update(expected)
assert expected == chksum.get_handlers()
assert self._inited_count == 1
assert expected == chksum.get_handlers(None)
assert {"x": 1} == chksum.get_handlers(["x"])
assert expected == chksum.get_handlers(["x", "y"])
assert self._inited_count == 1
def test_get_handlers(self):
expected = {"x":1, "y":2}
chksum.chksum_types.update(expected)
self.assertEqual(expected, chksum.get_handlers())
self.assertEqual(self._inited_count, 1)
self.assertEqual(expected, chksum.get_handlers(None))
self.assertEqual({"x":1}, chksum.get_handlers(["x"]))
self.assertEqual(expected, chksum.get_handlers(["x", "y"]))
self.assertEqual(self._inited_count, 1)
def test_get_handlers(self):
expected = {"x": 1, "y": 2}
chksum.chksum_types.update(expected)
self.assertEqual(expected, chksum.get_handlers())
self.assertEqual(self._inited_count, 1)
self.assertEqual(expected, chksum.get_handlers(None))
self.assertEqual({"x": 1}, chksum.get_handlers(["x"]))
self.assertEqual(expected, chksum.get_handlers(["x", "y"]))
self.assertEqual(self._inited_count, 1)
def _verify(self, file_location, target, all_chksums=True, handlers=None):
"""
Internal function for derivatives.
Digs through chksums, and either returns None, or throws an
errors.FetchFailed exception.
- -2: file doesn't exist.
- -1: if (size chksum is available, and
file is smaller than stated chksum)
- 0: if all chksums match
- 1: if file is too large (if size chksums are available)
or else size is right but a chksum didn't match.
if all_chksums is True, all chksums must be verified; if false, all
a handler can be found for are used.
"""
nondefault_handlers = handlers
if handlers is None:
try:
handlers = get_handlers(target.chksums)
except KeyError, e:
compatibility.raise_from(
errors.FetchFailed(file_location,
"Couldn't find a required checksum handler"))
def iter_scan(path,
offset=None,
follow_symlinks=False,
chksum_types=None,
hidden=True,
backup=True):
"""
Recursively scan a path.
Does not follow symlinks pointing at dirs, just merely yields an
obj representing said symlink
:return: an iterator of :obj:`pkgcore.fs.fs.fsBase` objects.
:param path: str path of what directory to scan in the livefs
:type path: str
:param offset: if not None, prefix to strip from each objects location.
if offset is /tmp, /tmp/blah becomes /blah
:type nonexistent: str or None
"""
chksum_handlers = get_handlers(chksum_types)
stat_func = follow_symlinks and os.stat or os.lstat
if offset is None:
return _internal_iter_scan(path,
chksum_handlers,
stat_func,
hidden=hidden,
backup=backup)
return _internal_offset_iter_scan(path,
chksum_handlers,
offset,
stat_func,
hidden=hidden,
backup=backup)
def _verify(self, file_location, target, all_chksums=True, handlers=None):
"""Internal function for derivatives.
Digs through chksums, and either returns None, or throws an
errors.FetchFailed exception.
- -2: file doesn't exist.
- -1: if (size chksum is available, and
file is smaller than stated chksum)
- 0: if all chksums match
- 1: if file is too large (if size chksums are available)
or else size is right but a chksum didn't match.
if all_chksums is True, all chksums must be verified; if false, all
a handler can be found for are used.
"""
nondefault_handlers = handlers
if handlers is None:
try:
handlers = get_handlers(target.chksums)
except MissingChksumHandler as e:
raise errors.MissingChksumHandler(
f'missing required checksum handler: {e}')
if all_chksums:
missing = set(target.chksums).difference(handlers)
if missing:
raise errors.RequiredChksumDataMissing(target, *sorted(missing))
if "size" in handlers:
val = handlers["size"](file_location)
if val == -1:
raise errors.MissingDistfile(file_location)
if val != target.chksums["size"]:
if val < target.chksums["size"]:
raise errors.FetchFailed(
file_location, 'file is too small', resumable=True)
raise errors.ChksumFailure(
file_location, chksum='size', expected=target.chksums["size"], value=val)
elif not os.path.exists(file_location):
raise errors.MissingDistfile(file_location)
elif not os.stat(file_location).st_size:
raise errors.FetchFailed(
file_location, 'file is empty', resumable=False)
chfs = set(target.chksums).intersection(handlers)
chfs.discard("size")
chfs = list(chfs)
if nondefault_handlers:
for x in chfs:
val = handlers[x](file_location)
if val != target.chksums[x]:
raise errors.ChksumFailure(
file_location, chksum=x, expected=target.chksums[x], value=val)
else:
desired_vals = [target.chksums[x] for x in chfs]
calced = get_chksums(file_location, *chfs)
for desired, got, chf in zip(desired_vals, calced, chfs):
if desired != got:
raise errors.ChksumFailure(
file_location, chksum=chf, expected=desired, value=got)
def __init__(self, location, chksums=None, data=None, **kwds):
"""
:param chksums: dict of checksums, key chksum_type: val hash val.
See :obj:`snakeoil.chksum`.
"""
assert 'data_source' not in kwds
if data is None:
data = local_source(location)
kwds["data"] = data
if chksums is None:
# this can be problematic offhand if the file is modified
# but chksum not triggered
chf_types = kwds.pop("chf_types", None)
if chf_types is None:
chf_types = tuple(get_handlers())
chksums = _LazyChksums(chf_types, self._chksum_callback)
kwds["chksums"] = chksums
fsBase.__init__(self, location, **kwds)
def __init__(self, location, chksums=None, data=None, **kwds):
"""
:param chksums: dict of checksums, key chksum_type: val hash val.
See :obj:`snakeoil.chksum`.
"""
assert 'data_source' not in kwds
if data is None:
data = local_source(location)
kwds["data"] = data
if chksums is None:
# this can be problematic offhand if the file is modified
# but chksum not triggered
chf_types = kwds.pop("chf_types", None)
if chf_types is None:
chf_types = tuple(get_handlers())
chksums = _LazyChksums(chf_types, self._chksum_callback)
kwds["chksums"] = chksums
fsBase.__init__(self, location, **kwds)
def iter_scan(path, offset=None, follow_symlinks=False, chksum_types=None):
"""
Recursively scan a path.
Does not follow symlinks pointing at dirs, just merely yields an
obj representing said symlink
:return: an iterator of :obj:`pkgcore.fs.fs.fsBase` objects.
:param path: str path of what directory to scan in the livefs
:param offset: if not None, prefix to strip from each objects location.
if offset is /tmp, /tmp/blah becomes /blah
"""
chksum_handlers = get_handlers(chksum_types)
stat_func = follow_symlinks and os.stat or os.lstat
if offset is None:
return _internal_iter_scan(path, chksum_handlers, stat_func)
return _internal_offset_iter_scan(path, chksum_handlers, offset, stat_func)
# Copyright: 2006-2009 Brian Harring <*****@*****.**>
# License: GPL2/BSD
import os
from snakeoil import data_source
from snakeoil.chksum import get_handlers
from snakeoil.currying import partial
from snakeoil.test.mixins import TempDirMixin
from pkgcore.fetch import base, fetchable, errors
from pkgcore.test import TestCase
repeating_str = 'asdf'
data = repeating_str * 4000
handlers = get_handlers()
from snakeoil.mappings import LazyValDict
def _callback(chf):
return handlers[chf](data_source.data_source(data))
chksums = LazyValDict(frozenset(handlers.iterkeys()), _callback)
# get a non size based chksum
known_chksum = [x for x in handlers.iterkeys() if x != "size"][0]
class TestFetcher(TempDirMixin, TestCase):
# Copyright: 2006-2009 Brian Harring <*****@*****.**>
# License: GPL2/BSD
from functools import partial
import os
from snakeoil import data_source
from snakeoil.chksum import get_handlers
from snakeoil.test.mixins import TempDirMixin
from pkgcore.fetch import base, fetchable, errors
from pkgcore.test import TestCase
repeating_str = 'asdf'
data = repeating_str * 4000
handlers = get_handlers()
from snakeoil.mappings import LazyValDict
def _callback(chf):
return handlers[chf](data_source.data_source(data))
chksums = LazyValDict(frozenset(handlers.iterkeys()), _callback)
# get a non size based chksum
known_chksum = [x for x in handlers.iterkeys() if x != "size"][0]
class TestFetcher(TempDirMixin, TestCase):
def setUp(self):
TempDirMixin.setUp(self)
self.fp = os.path.join(self.dir, "test")
def _verify(self, file_location, target, all_chksums=True, handlers=None):
"""
Internal function for derivatives.
Digs through chksums, and either returns None, or throws an
errors.FetchFailed exception.
- -2: file doesn't exist.
- -1: if (size chksum is available, and
file is smaller than stated chksum)
- 0: if all chksums match
- 1: if file is too large (if size chksums are available)
or else size is right but a chksum didn't match.
if all_chksums is True, all chksums must be verified; if false, all
a handler can be found for are used.
"""
nondefault_handlers = handlers
if handlers is None:
try:
handlers = get_handlers(target.chksums)
except KeyError:
compatibility.raise_from(errors.FetchFailed(
file_location, "Couldn't find a required checksum handler"))
if all_chksums:
missing = set(target.chksums).difference(handlers)
if missing:
raise errors.RequiredChksumDataMissing(target, *sorted(missing))
if "size" in handlers:
val = handlers["size"](file_location)
if val == -1:
raise errors.MissingDistfile(file_location)
c = cmp(val, target.chksums["size"])
if c:
resumable = (c < 0)
if resumable:
msg = "File is too small."
else:
msg = "File is too big."
raise errors.FetchFailed(
file_location, msg, resumable=resumable)
elif not os.path.exists(file_location):
raise errors.MissingDistfile(file_location)
elif not os.stat(file_location).st_size:
raise errors.FetchFailed(
file_location, 'file is empty', resumable=False)
chfs = set(target.chksums).intersection(handlers)
chfs.discard("size")
chfs = list(chfs)
if nondefault_handlers:
for x in chfs:
val = handlers[x](file_location)
if val != target.chksums[x]:
raise errors.FetchFailed(
file_location,
"Validation handler %s: expected %s, got %s" %
(x, target.chksums[x], val))
else:
desired_vals = [target.chksums[x] for x in chfs]
calced = get_chksums(file_location, *chfs)
for desired, got, chf in zip(desired_vals, calced, chfs):
if desired != got:
raise errors.FetchFailed(
file_location,
"Validation handler %s: expected %s, got %s" %
(chf, desired, got))
def _verify(self, file_location, target, all_chksums=True, handlers=None):
"""
Internal function for derivatives.
Digs through chksums, and either returns None, or throws an
errors.FetchFailed exception.
- -2: file doesn't exist.
- -1: if (size chksum is available, and
file is smaller than stated chksum)
- 0: if all chksums match
- 1: if file is too large (if size chksums are available)
or else size is right but a chksum didn't match.
if all_chksums is True, all chksums must be verified; if false, all
a handler can be found for are used.
"""
nondefault_handlers = handlers
if handlers is None:
try:
handlers = get_handlers(target.chksums)
except KeyError:
compatibility.raise_from(
errors.FetchFailed(
file_location,
"Couldn't find a required checksum handler"))
if all_chksums:
missing = set(target.chksums).difference(handlers)
if missing:
raise errors.RequiredChksumDataMissing(target,
*sorted(missing))
if "size" in handlers:
val = handlers["size"](file_location)
if val == -1:
raise errors.MissingDistfile(file_location)
c = cmp(val, target.chksums["size"])
if c:
resumable = (c < 0)
if resumable:
msg = "File is too small."
else:
msg = "File is too big."
raise errors.FetchFailed(file_location,
msg,
resumable=resumable)
elif not os.path.exists(file_location):
raise errors.MissingDistfile(file_location)
elif not os.stat(file_location).st_size:
raise errors.FetchFailed(file_location,
'file is empty',
resumable=False)
chfs = set(target.chksums).intersection(handlers)
chfs.discard("size")
chfs = list(chfs)
if nondefault_handlers:
for x in chfs:
val = handlers[x](file_location)
if val != target.chksums[x]:
raise errors.FetchFailed(
file_location,
"Validation handler %s: expected %s, got %s" %
(x, target.chksums[x], val))
else:
desired_vals = [target.chksums[x] for x in chfs]
calced = get_chksums(file_location, *chfs)
for desired, got, chf in zip(desired_vals, calced, chfs):
if desired != got:
raise errors.FetchFailed(
file_location,
"Validation handler %s: expected %s, got %s" %
(chf, desired, got))
def _verify(self, file_location, target, all_chksums=True, handlers=None):
"""Internal function for derivatives.
Digs through chksums, and either returns None, or throws an
errors.FetchFailed exception.
- -2: file doesn't exist.
- -1: if (size chksum is available, and
file is smaller than stated chksum)
- 0: if all chksums match
- 1: if file is too large (if size chksums are available)
or else size is right but a chksum didn't match.
if all_chksums is True, all chksums must be verified; if false, all
a handler can be found for are used.
"""
nondefault_handlers = handlers
if handlers is None:
try:
handlers = get_handlers(target.chksums)
except MissingChksumHandler as e:
raise errors.MissingChksumHandler(
f'missing required checksum handler: {e}')
if all_chksums:
missing = set(target.chksums).difference(handlers)
if missing:
raise errors.RequiredChksumDataMissing(target,
*sorted(missing))
if "size" in handlers:
val = handlers["size"](file_location)
if val == -1:
raise errors.MissingDistfile(file_location)
if val != target.chksums["size"]:
if val < target.chksums["size"]:
raise errors.FetchFailed(file_location,
'file is too small',
resumable=True)
raise errors.ChksumFailure(file_location,
chksum='size',
expected=target.chksums["size"],
value=val)
elif not os.path.exists(file_location):
raise errors.MissingDistfile(file_location)
elif not os.stat(file_location).st_size:
raise errors.FetchFailed(file_location,
'file is empty',
resumable=False)
chfs = set(target.chksums).intersection(handlers)
chfs.discard("size")
chfs = list(chfs)
if nondefault_handlers:
for x in chfs:
val = handlers[x](file_location)
if val != target.chksums[x]:
raise errors.ChksumFailure(file_location,
chksum=x,
expected=target.chksums[x],
value=val)
else:
desired_vals = [target.chksums[x] for x in chfs]
calced = get_chksums(file_location, *chfs)
for desired, got, chf in zip(desired_vals, calced, chfs):
if desired != got:
raise errors.ChksumFailure(file_location,
chksum=chf,
expected=desired,
value=got)
