def test_get_handlers(self): expected = {"x": 1, "y": 2} chksum.chksum_types.update(expected) assert expected == chksum.get_handlers() assert self._inited_count == 1 assert expected == chksum.get_handlers(None) assert {"x": 1} == chksum.get_handlers(["x"]) assert expected == chksum.get_handlers(["x", "y"]) assert self._inited_count == 1
def test_get_handlers(self): expected = {"x":1, "y":2} chksum.chksum_types.update(expected) self.assertEqual(expected, chksum.get_handlers()) self.assertEqual(self._inited_count, 1) self.assertEqual(expected, chksum.get_handlers(None)) self.assertEqual({"x":1}, chksum.get_handlers(["x"])) self.assertEqual(expected, chksum.get_handlers(["x", "y"])) self.assertEqual(self._inited_count, 1)
def test_get_handlers(self): expected = {"x": 1, "y": 2} chksum.chksum_types.update(expected) self.assertEqual(expected, chksum.get_handlers()) self.assertEqual(self._inited_count, 1) self.assertEqual(expected, chksum.get_handlers(None)) self.assertEqual({"x": 1}, chksum.get_handlers(["x"])) self.assertEqual(expected, chksum.get_handlers(["x", "y"])) self.assertEqual(self._inited_count, 1)
def _verify(self, file_location, target, all_chksums=True, handlers=None): """ Internal function for derivatives. Digs through chksums, and either returns None, or throws an errors.FetchFailed exception. - -2: file doesn't exist. - -1: if (size chksum is available, and file is smaller than stated chksum) - 0: if all chksums match - 1: if file is too large (if size chksums are available) or else size is right but a chksum didn't match. if all_chksums is True, all chksums must be verified; if false, all a handler can be found for are used. """ nondefault_handlers = handlers if handlers is None: try: handlers = get_handlers(target.chksums) except KeyError, e: compatibility.raise_from( errors.FetchFailed(file_location, "Couldn't find a required checksum handler"))
def iter_scan(path, offset=None, follow_symlinks=False, chksum_types=None, hidden=True, backup=True): """ Recursively scan a path. Does not follow symlinks pointing at dirs, just merely yields an obj representing said symlink :return: an iterator of :obj:`pkgcore.fs.fs.fsBase` objects. :param path: str path of what directory to scan in the livefs :type path: str :param offset: if not None, prefix to strip from each objects location. if offset is /tmp, /tmp/blah becomes /blah :type nonexistent: str or None """ chksum_handlers = get_handlers(chksum_types) stat_func = follow_symlinks and os.stat or os.lstat if offset is None: return _internal_iter_scan(path, chksum_handlers, stat_func, hidden=hidden, backup=backup) return _internal_offset_iter_scan(path, chksum_handlers, offset, stat_func, hidden=hidden, backup=backup)
def _verify(self, file_location, target, all_chksums=True, handlers=None): """Internal function for derivatives. Digs through chksums, and either returns None, or throws an errors.FetchFailed exception. - -2: file doesn't exist. - -1: if (size chksum is available, and file is smaller than stated chksum) - 0: if all chksums match - 1: if file is too large (if size chksums are available) or else size is right but a chksum didn't match. if all_chksums is True, all chksums must be verified; if false, all a handler can be found for are used. """ nondefault_handlers = handlers if handlers is None: try: handlers = get_handlers(target.chksums) except MissingChksumHandler as e: raise errors.MissingChksumHandler( f'missing required checksum handler: {e}') if all_chksums: missing = set(target.chksums).difference(handlers) if missing: raise errors.RequiredChksumDataMissing(target, *sorted(missing)) if "size" in handlers: val = handlers["size"](file_location) if val == -1: raise errors.MissingDistfile(file_location) if val != target.chksums["size"]: if val < target.chksums["size"]: raise errors.FetchFailed( file_location, 'file is too small', resumable=True) raise errors.ChksumFailure( file_location, chksum='size', expected=target.chksums["size"], value=val) elif not os.path.exists(file_location): raise errors.MissingDistfile(file_location) elif not os.stat(file_location).st_size: raise errors.FetchFailed( file_location, 'file is empty', resumable=False) chfs = set(target.chksums).intersection(handlers) chfs.discard("size") chfs = list(chfs) if nondefault_handlers: for x in chfs: val = handlers[x](file_location) if val != target.chksums[x]: raise errors.ChksumFailure( file_location, chksum=x, expected=target.chksums[x], value=val) else: desired_vals = [target.chksums[x] for x in chfs] calced = get_chksums(file_location, *chfs) for desired, got, chf in zip(desired_vals, calced, chfs): if desired != got: raise errors.ChksumFailure( file_location, chksum=chf, expected=desired, value=got)
def __init__(self, location, chksums=None, data=None, **kwds): """ :param chksums: dict of checksums, key chksum_type: val hash val. See :obj:`snakeoil.chksum`. """ assert 'data_source' not in kwds if data is None: data = local_source(location) kwds["data"] = data if chksums is None: # this can be problematic offhand if the file is modified # but chksum not triggered chf_types = kwds.pop("chf_types", None) if chf_types is None: chf_types = tuple(get_handlers()) chksums = _LazyChksums(chf_types, self._chksum_callback) kwds["chksums"] = chksums fsBase.__init__(self, location, **kwds)
def iter_scan(path, offset=None, follow_symlinks=False, chksum_types=None): """ Recursively scan a path. Does not follow symlinks pointing at dirs, just merely yields an obj representing said symlink :return: an iterator of :obj:`pkgcore.fs.fs.fsBase` objects. :param path: str path of what directory to scan in the livefs :param offset: if not None, prefix to strip from each objects location. if offset is /tmp, /tmp/blah becomes /blah """ chksum_handlers = get_handlers(chksum_types) stat_func = follow_symlinks and os.stat or os.lstat if offset is None: return _internal_iter_scan(path, chksum_handlers, stat_func) return _internal_offset_iter_scan(path, chksum_handlers, offset, stat_func)
# Copyright: 2006-2009 Brian Harring <*****@*****.**> # License: GPL2/BSD import os from snakeoil import data_source from snakeoil.chksum import get_handlers from snakeoil.currying import partial from snakeoil.test.mixins import TempDirMixin from pkgcore.fetch import base, fetchable, errors from pkgcore.test import TestCase repeating_str = 'asdf' data = repeating_str * 4000 handlers = get_handlers() from snakeoil.mappings import LazyValDict def _callback(chf): return handlers[chf](data_source.data_source(data)) chksums = LazyValDict(frozenset(handlers.iterkeys()), _callback) # get a non size based chksum known_chksum = [x for x in handlers.iterkeys() if x != "size"][0] class TestFetcher(TempDirMixin, TestCase):
# Copyright: 2006-2009 Brian Harring <*****@*****.**> # License: GPL2/BSD from functools import partial import os from snakeoil import data_source from snakeoil.chksum import get_handlers from snakeoil.test.mixins import TempDirMixin from pkgcore.fetch import base, fetchable, errors from pkgcore.test import TestCase repeating_str = 'asdf' data = repeating_str * 4000 handlers = get_handlers() from snakeoil.mappings import LazyValDict def _callback(chf): return handlers[chf](data_source.data_source(data)) chksums = LazyValDict(frozenset(handlers.iterkeys()), _callback) # get a non size based chksum known_chksum = [x for x in handlers.iterkeys() if x != "size"][0] class TestFetcher(TempDirMixin, TestCase): def setUp(self): TempDirMixin.setUp(self) self.fp = os.path.join(self.dir, "test")
def _verify(self, file_location, target, all_chksums=True, handlers=None): """ Internal function for derivatives. Digs through chksums, and either returns None, or throws an errors.FetchFailed exception. - -2: file doesn't exist. - -1: if (size chksum is available, and file is smaller than stated chksum) - 0: if all chksums match - 1: if file is too large (if size chksums are available) or else size is right but a chksum didn't match. if all_chksums is True, all chksums must be verified; if false, all a handler can be found for are used. """ nondefault_handlers = handlers if handlers is None: try: handlers = get_handlers(target.chksums) except KeyError: compatibility.raise_from(errors.FetchFailed( file_location, "Couldn't find a required checksum handler")) if all_chksums: missing = set(target.chksums).difference(handlers) if missing: raise errors.RequiredChksumDataMissing(target, *sorted(missing)) if "size" in handlers: val = handlers["size"](file_location) if val == -1: raise errors.MissingDistfile(file_location) c = cmp(val, target.chksums["size"]) if c: resumable = (c < 0) if resumable: msg = "File is too small." else: msg = "File is too big." raise errors.FetchFailed( file_location, msg, resumable=resumable) elif not os.path.exists(file_location): raise errors.MissingDistfile(file_location) elif not os.stat(file_location).st_size: raise errors.FetchFailed( file_location, 'file is empty', resumable=False) chfs = set(target.chksums).intersection(handlers) chfs.discard("size") chfs = list(chfs) if nondefault_handlers: for x in chfs: val = handlers[x](file_location) if val != target.chksums[x]: raise errors.FetchFailed( file_location, "Validation handler %s: expected %s, got %s" % (x, target.chksums[x], val)) else: desired_vals = [target.chksums[x] for x in chfs] calced = get_chksums(file_location, *chfs) for desired, got, chf in zip(desired_vals, calced, chfs): if desired != got: raise errors.FetchFailed( file_location, "Validation handler %s: expected %s, got %s" % (chf, desired, got))
def _verify(self, file_location, target, all_chksums=True, handlers=None): """ Internal function for derivatives. Digs through chksums, and either returns None, or throws an errors.FetchFailed exception. - -2: file doesn't exist. - -1: if (size chksum is available, and file is smaller than stated chksum) - 0: if all chksums match - 1: if file is too large (if size chksums are available) or else size is right but a chksum didn't match. if all_chksums is True, all chksums must be verified; if false, all a handler can be found for are used. """ nondefault_handlers = handlers if handlers is None: try: handlers = get_handlers(target.chksums) except KeyError: compatibility.raise_from( errors.FetchFailed( file_location, "Couldn't find a required checksum handler")) if all_chksums: missing = set(target.chksums).difference(handlers) if missing: raise errors.RequiredChksumDataMissing(target, *sorted(missing)) if "size" in handlers: val = handlers["size"](file_location) if val == -1: raise errors.MissingDistfile(file_location) c = cmp(val, target.chksums["size"]) if c: resumable = (c < 0) if resumable: msg = "File is too small." else: msg = "File is too big." raise errors.FetchFailed(file_location, msg, resumable=resumable) elif not os.path.exists(file_location): raise errors.MissingDistfile(file_location) elif not os.stat(file_location).st_size: raise errors.FetchFailed(file_location, 'file is empty', resumable=False) chfs = set(target.chksums).intersection(handlers) chfs.discard("size") chfs = list(chfs) if nondefault_handlers: for x in chfs: val = handlers[x](file_location) if val != target.chksums[x]: raise errors.FetchFailed( file_location, "Validation handler %s: expected %s, got %s" % (x, target.chksums[x], val)) else: desired_vals = [target.chksums[x] for x in chfs] calced = get_chksums(file_location, *chfs) for desired, got, chf in zip(desired_vals, calced, chfs): if desired != got: raise errors.FetchFailed( file_location, "Validation handler %s: expected %s, got %s" % (chf, desired, got))
def _verify(self, file_location, target, all_chksums=True, handlers=None): """Internal function for derivatives. Digs through chksums, and either returns None, or throws an errors.FetchFailed exception. - -2: file doesn't exist. - -1: if (size chksum is available, and file is smaller than stated chksum) - 0: if all chksums match - 1: if file is too large (if size chksums are available) or else size is right but a chksum didn't match. if all_chksums is True, all chksums must be verified; if false, all a handler can be found for are used. """ nondefault_handlers = handlers if handlers is None: try: handlers = get_handlers(target.chksums) except MissingChksumHandler as e: raise errors.MissingChksumHandler( f'missing required checksum handler: {e}') if all_chksums: missing = set(target.chksums).difference(handlers) if missing: raise errors.RequiredChksumDataMissing(target, *sorted(missing)) if "size" in handlers: val = handlers["size"](file_location) if val == -1: raise errors.MissingDistfile(file_location) if val != target.chksums["size"]: if val < target.chksums["size"]: raise errors.FetchFailed(file_location, 'file is too small', resumable=True) raise errors.ChksumFailure(file_location, chksum='size', expected=target.chksums["size"], value=val) elif not os.path.exists(file_location): raise errors.MissingDistfile(file_location) elif not os.stat(file_location).st_size: raise errors.FetchFailed(file_location, 'file is empty', resumable=False) chfs = set(target.chksums).intersection(handlers) chfs.discard("size") chfs = list(chfs) if nondefault_handlers: for x in chfs: val = handlers[x](file_location) if val != target.chksums[x]: raise errors.ChksumFailure(file_location, chksum=x, expected=target.chksums[x], value=val) else: desired_vals = [target.chksums[x] for x in chfs] calced = get_chksums(file_location, *chfs) for desired, got, chf in zip(desired_vals, calced, chfs): if desired != got: raise errors.ChksumFailure(file_location, chksum=chf, expected=desired, value=got)