def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
config = MagicMock()
config.get_settings.return_value = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'redis://localhost',
}
RedisCache.configure(config)
cls.redis = RedisCache.db
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
config = MagicMock()
config.get_settings.return_value = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'redis://localhost',
}
RedisCache.configure(config)
cls.redis = RedisCache.db
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'redis://localhost',
}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs['db']
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'redis://localhost',
}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs['db']
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {"pypi.storage": "tests.DummyStorage", "db.url": "redis://localhost"}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs["db"]
try:
cls.redis.flushdb()
except redis.exceptions.ConnectionError:
msg = "Redis not found on port 6379"
setattr(cls, "setUp", lambda cls: unittest.TestCase.skipTest(cls, msg))
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {"pypi.storage": "tests.DummyStorage", "db.url": "redis://localhost"}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs["db"]
try:
cls.redis.flushdb()
except redis.exceptions.ConnectionError:
msg = "Redis not found on port 6379"
setattr(cls, "setUp", lambda cls: unittest.TestCase.skipTest(cls, msg))
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
redis_host = os.environ.get("REDIS_HOST", "localhost")
redis_port = os.environ.get("REDIS_PORT", "6379")
redis_url = f"redis://{redis_host}:{redis_port}"
settings = {"pypi.storage": "tests.DummyStorage", "db.url": redis_url}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs["db"]
try:
cls.redis.flushdb()
except redis.exceptions.ConnectionError:
msg = f"Redis not found at {redis_url}"
setattr(cls, "setUp",
lambda cls: unittest.TestCase.skipTest(cls, msg))
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'redis://localhost',
}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs['db']
try:
cls.redis.flushdb()
except ConnectionError:
msg = "Redis not found on port 6379"
setattr(
cls,
"setUp",
lambda cls: unittest.TestCase.skipTest(cls, msg),
)
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'redis://localhost',
}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs['db']
try:
cls.redis.flushdb()
except ConnectionError:
msg = "Redis not found on port 6379"
if sys.version_info < (2, 7):
raise unittest.SkipTest(msg)
else:
setattr(
cls,
"setUp",
lambda cls: unittest.TestCase.skipTest(cls, msg),
)
def setUp(self):
super(TestRedisCache, self).setUp()
self.db = RedisCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
class TestRedisCache(unittest.TestCase):
""" Tests for the redis cache """
@classmethod
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {
"pypi.storage": "tests.DummyStorage",
"db.url": "redis://localhost"
}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs["db"]
try:
cls.redis.flushdb()
except redis.exceptions.ConnectionError:
msg = "Redis not found on port 6379"
setattr(cls, "setUp",
lambda cls: unittest.TestCase.skipTest(cls, msg))
def setUp(self):
super(TestRedisCache, self).setUp()
self.db = RedisCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestRedisCache, self).tearDown()
self.redis.flushdb()
def assert_in_redis(self, pkg):
""" Assert that a package exists in redis """
self.assertTrue(self.redis.sismember(self.db.redis_set, pkg.name))
data = self.redis.hgetall(self.db.redis_key(pkg.filename))
dt = pkg.last_modified
lm = calendar.timegm(dt.utctimetuple()) + dt.microsecond / 1000000.0
lm_str = ("%.6f" % lm).rstrip("0").rstrip(".")
pkg_data = {
"name": pkg.name,
"version": pkg.version,
"filename": pkg.filename,
"last_modified": lm_str,
"summary": pkg.summary,
}
pkg_data.update(pkg.data)
self.assertEqual(data, pkg_data)
def test_load(self):
""" Loading from redis deserializes all fields """
kwargs = {"url": "my.url", "expire": 7237}
pkg = make_package(**kwargs)
# Due to some rounding weirdness in old Py3 versions, we need to remove
# the microseconds to avoid a flappy test.
# See: https://bugs.python.org/issue23517
pkg.last_modified = pkg.last_modified.replace(microsecond=0)
self.db.save(pkg)
loaded = self.db.fetch(pkg.filename)
self.assertEqual(loaded.name, pkg.name)
self.assertEqual(loaded.version, pkg.version)
self.assertEqual(loaded.filename, pkg.filename)
self.assertEqual(loaded.last_modified, pkg.last_modified)
self.assertEqual(loaded.summary, pkg.summary)
self.assertEqual(loaded.data, kwargs)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package()
key = self.db.redis_key(pkg.filename)
self.redis[key] = "foobar"
self.db.delete(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package()
key = self.db.redis_key(pkg.filename)
self.redis[key] = "foobar"
self.db.clear(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_clear_leave_distinct(self):
""" clear() doesn't remove package from list of distinct """
p1 = make_package()
p2 = make_package(filename="another-1.2.tar.gz")
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1.filename)
self.db.clear(p1)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 1)
def test_clear_all(self):
""" clear_all() removes all packages from db """
p1 = make_package()
p2 = make_package(version="1.2")
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1)
self.db.clear_all()
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
for pkg in keys:
self.assert_in_redis(pkg)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package()
self.db.save(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch("missing_pkg-1.2.tar.gz")
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version="1.3", filename="mypath3",
factory=SQLPackage),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.all("mypkg")
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version="1.3", filename="mypath3",
factory=SQLPackage),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_delete_package(self):
""" Deleting the last package of a name removes from distinct() """
pkgs = [
make_package(factory=SQLPackage),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
self.db.clear(pkgs[0])
saved_pkgs = self.db.distinct()
self.assertEqual(saved_pkgs, ["mypkg2"])
summaries = self.db.summary()
self.assertEqual(len(summaries), 1)
def test_search_or(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=SQLPackage,
),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
make_package("package", factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
criteria = {"name": ["mypkg"], "summary": ["mypkg"]}
packages = self.db.search(criteria, "or")
self.assertItemsEqual(packages, pkgs[:-1])
def test_search_and(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=SQLPackage,
),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
make_package("package", factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
criteria = {"name": ["my", "pkg"], "summary": ["this", "mypkg"]}
packages = self.db.search(criteria, "and")
self.assertItemsEqual(packages, pkgs[:-1])
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, "allow_overwrite", False):
name, version = "a", "1"
path1 = "old_package_path-1.tar.gz"
self.db.upload(path1, BytesIO(b"test1234"), name, version)
path2 = "new_path-1.whl"
self.db.upload(path2, BytesIO(b"test1234"), name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
def test_summary(self):
""" summary constructs per-package metadata summary """
self.db.upload("pkg1-0.3a2.tar.gz", BytesIO(b"test1234"), "pkg1",
"0.3a2")
self.db.upload("pkg1-1.1.tar.gz", BytesIO(b"test1234"), "pkg1", "1.1")
p1 = self.db.upload("pkg1a2.tar.gz", BytesIO(b"test1234"), "pkg1",
"1.1.1a2", "summary")
p2 = self.db.upload("pkg2.tar.gz", BytesIO(b"test1234"), "pkg2",
"0.1dev2", "summary")
summaries = self.db.summary()
self.assertItemsEqual(
summaries,
[
{
"name": "pkg1",
"summary": "summary",
"last_modified": ANY
},
{
"name": "pkg2",
"summary": "summary",
"last_modified": ANY
},
],
)
# Have to compare the last_modified fuzzily
self.assertEqual(
summaries[0]["last_modified"].utctimetuple(),
p1.last_modified.utctimetuple(),
)
self.assertEqual(
summaries[1]["last_modified"].utctimetuple(),
p2.last_modified.utctimetuple(),
)
def test_check_health_success(self):
""" check_health returns True for good connection """
ok, msg = self.db.check_health()
self.assertTrue(ok)
def test_check_health_fail(self):
""" check_health returns False for bad connection """
dbmock = self.db.db = MagicMock()
def throw(*_, **__):
""" Throw an exception """
raise redis.RedisError("DB exception")
dbmock.echo.side_effect = throw
ok, msg = self.db.check_health()
self.assertFalse(ok)
def test_reload_none_summary(self):
""" reload_from_storage() doesn't break on packages with None summary """
pkg = make_package("mypkg3",
"1.2",
"some/other/path",
summary=None,
factory=SQLPackage)
keys = [pkg]
self.storage.list.return_value = keys
self.db.reload_from_storage()
# The shim will convert None summary to ""
pkg.summary = ""
self.assert_in_redis(pkg)
class TestRedisCache(unittest.TestCase):
""" Tests for the RedisCache """
@classmethod
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {
"pypi.storage": "tests.DummyStorage",
"db.url": "redis://localhost",
"db.graceful_reload": True,
}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs["db"]
try:
cls.redis.flushdb()
except redis.exceptions.ConnectionError:
msg = "Redis not found on port 6379"
setattr(cls, "setUp",
lambda cls: unittest.TestCase.skipTest(cls, msg))
@classmethod
def tearDownClass(cls):
super(TestRedisCache, cls).tearDownClass()
def setUp(self):
super(TestRedisCache, self).setUp()
self.db = RedisCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestRedisCache, self).tearDown()
self.redis.flushdb()
def _save_pkgs(self, *pkgs):
""" Save packages to the db """
pipe = self.redis.pipeline()
for pkg in pkgs:
self.db.save(pkg, pipe)
pipe.execute()
def test_add_missing(self):
""" Add missing packages to cache """
keys = [make_package()]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.db._load_all_packages()
self.assertItemsEqual(all_pkgs, keys)
self.assertEqual(len(self.db.summary()), 1)
def test_remove_extra(self):
""" Remove extra packages from cache """
keys = [make_package(), make_package("mypkg2", "1.3.4")]
self.db.save(keys[0])
self.db.save(keys[1])
self.storage.list.return_value = keys[:1]
self.db.reload_from_storage()
all_pkgs = self.db._load_all_packages()
self.assertItemsEqual(all_pkgs, keys[:1])
# It should have removed the summary as well
self.assertEqual(len(self.db.summary()), 1)
def test_remove_extra_leave_concurrent(self):
""" Removing extra packages will leave packages that were uploaded concurrently """
pkgs = [make_package(), make_package("mypkg2")]
self.db.save(pkgs[0])
self.db.save(pkgs[1])
# Return first pkgs[1], then pkgs[1:] because the second time we list
# we will have "uploaded" pkgs[2]
return_values = [lambda: pkgs[1:2], lambda: pkgs[1:]]
def list_storage(factory):
""" mocked method for listing storage packages """
# The first time we list from storage, concurrently "upload"
# pkgs[2]
if len(return_values) == 2:
pkg = make_package("mypkg3")
pkgs.append(pkg)
self.db.save(pkg)
return return_values.pop(0)()
self.storage.list.side_effect = list_storage
self.db.reload_from_storage()
all_pkgs = self.db._load_all_packages()
self.assertItemsEqual(all_pkgs, pkgs[1:])
self.assertEqual(len(self.db.summary()), 2)
def test_remove_extra_concurrent_deletes(self):
""" Remove packages from cache that were concurrently deleted """
pkgs = [make_package(), make_package("mypkg2")]
self.db.save(pkgs[0])
# Return first pkgs[:], then pkgs[:1] because the second time we list
# we will have "deleted" pkgs[1]
return_values = [pkgs[:], pkgs[:1]]
self.storage.list.side_effect = lambda _: return_values.pop(0)
self.db.reload_from_storage()
all_pkgs = self.db._load_all_packages()
self.assertItemsEqual(all_pkgs, pkgs[:1])
self.assertEqual(len(self.db.summary()), 1)
def test_add_missing_more_recent(self):
""" If we sync a more recent package, update the summary """
pkgs = [
make_package(last_modified=utcnow() - timedelta(hours=1)),
make_package(version="1.5"),
]
self.db.save(pkgs[0])
self.storage.list.return_value = pkgs
self.db.reload_from_storage()
all_pkgs = self.db._load_all_packages()
self.assertItemsEqual(all_pkgs, pkgs)
summaries = self.db.summary()
self.assertEqual(len(summaries), 1)
summary = summaries[0]
self.assertEqual(summary["last_modified"].hour,
pkgs[1].last_modified.hour)
def test_same_package_name_version(self):
""" Storage can have packages with the same name and version (different filename) """
pkgs = [
make_package(filename="mypkg-1.1-win32.whl"),
make_package(filename="mypkg-1.1-macosx.whl"),
make_package(filename="mypkg-1.1-x86_64.whl"),
]
self.storage.list.return_value = pkgs
self.db.reload_from_storage()
all_pkgs = self.db._load_all_packages()
self.assertCountEqual(all_pkgs, pkgs)
summaries = self.db.summary()
self.assertEqual(len(summaries), 1)
def setUp(self):
super(TestRedisCache, self).setUp()
self.db = RedisCache(DummyRequest())
self.storage = self.db.storage = MagicMock(spec=IStorage)
class TestRedisCache(unittest.TestCase):
""" Tests for the redis cache """
@classmethod
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {"pypi.storage": "tests.DummyStorage", "db.url": "redis://localhost"}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs["db"]
try:
cls.redis.flushdb()
except redis.exceptions.ConnectionError:
msg = "Redis not found on port 6379"
setattr(cls, "setUp", lambda cls: unittest.TestCase.skipTest(cls, msg))
def setUp(self):
super(TestRedisCache, self).setUp()
self.db = RedisCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestRedisCache, self).tearDown()
self.redis.flushdb()
def assert_in_redis(self, pkg):
""" Assert that a package exists in redis """
self.assertTrue(self.redis.sismember(self.db.redis_set, pkg.name))
data = self.redis.hgetall(self.db.redis_key(pkg.filename))
dt = pkg.last_modified
lm = calendar.timegm(dt.utctimetuple()) + dt.microsecond / 1000000.0
lm_str = ("%.6f" % lm).rstrip("0").rstrip(".")
pkg_data = {
"name": pkg.name,
"version": pkg.version,
"filename": pkg.filename,
"last_modified": lm_str,
"summary": pkg.summary,
}
pkg_data.update(pkg.data)
self.assertEqual(data, pkg_data)
def test_load(self):
""" Loading from redis deserializes all fields """
kwargs = {"url": "my.url", "expire": 7237}
pkg = make_package(**kwargs)
# Due to some rounding weirdness in old Py3 versions, we need to remove
# the microseconds to avoid a flappy test.
# See: https://bugs.python.org/issue23517
pkg.last_modified = pkg.last_modified.replace(microsecond=0)
self.db.save(pkg)
loaded = self.db.fetch(pkg.filename)
self.assertEqual(loaded.name, pkg.name)
self.assertEqual(loaded.version, pkg.version)
self.assertEqual(loaded.filename, pkg.filename)
self.assertEqual(loaded.last_modified, pkg.last_modified)
self.assertEqual(loaded.summary, pkg.summary)
self.assertEqual(loaded.data, kwargs)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package()
key = self.db.redis_key(pkg.filename)
self.redis[key] = "foobar"
self.db.delete(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package()
key = self.db.redis_key(pkg.filename)
self.redis[key] = "foobar"
self.db.clear(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_clear_leave_distinct(self):
""" clear() doesn't remove package from list of distinct """
p1 = make_package()
p2 = make_package(filename="another-1.2.tar.gz")
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1.filename)
self.db.clear(p1)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 1)
def test_clear_all(self):
""" clear_all() removes all packages from db """
p1 = make_package()
p2 = make_package(version="1.2")
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1)
self.db.clear_all()
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
for pkg in keys:
self.assert_in_redis(pkg)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package()
self.db.save(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch("missing_pkg-1.2.tar.gz")
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version="1.3", filename="mypath3", factory=SQLPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.all("mypkg")
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version="1.3", filename="mypath3", factory=SQLPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_delete_package(self):
""" Deleting the last package of a name removes from distinct() """
pkgs = [
make_package(factory=SQLPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
self.db.clear(pkgs[0])
saved_pkgs = self.db.distinct()
self.assertEqual(saved_pkgs, ["mypkg2"])
summaries = self.db.summary()
self.assertEqual(len(summaries), 1)
def test_search_or(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=SQLPackage,
),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
make_package("package", factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
criteria = {"name": ["mypkg"], "summary": ["mypkg"]}
packages = self.db.search(criteria, "or")
self.assertItemsEqual(packages, pkgs[:-1])
def test_search_and(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=SQLPackage,
),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
make_package("package", factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
criteria = {"name": ["my", "pkg"], "summary": ["this", "mypkg"]}
packages = self.db.search(criteria, "and")
self.assertItemsEqual(packages, pkgs[:-1])
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, "allow_overwrite", False):
name, version = "a", "1"
path1 = "old_package_path-1.tar.gz"
self.db.upload(path1, None, name, version)
path2 = "new_path-1.whl"
self.db.upload(path2, None, name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
def test_summary(self):
""" summary constructs per-package metadata summary """
self.db.upload("pkg1-0.3a2.tar.gz", None, "pkg1", "0.3a2")
self.db.upload("pkg1-1.1.tar.gz", None, "pkg1", "1.1")
p1 = self.db.upload("pkg1a2.tar.gz", None, "pkg1", "1.1.1a2", "summary")
p2 = self.db.upload("pkg2.tar.gz", None, "pkg2", "0.1dev2", "summary")
summaries = self.db.summary()
self.assertItemsEqual(
summaries,
[
{"name": "pkg1", "summary": "summary", "last_modified": ANY},
{"name": "pkg2", "summary": "summary", "last_modified": ANY},
],
)
# Have to compare the last_modified fuzzily
self.assertEqual(
summaries[0]["last_modified"].utctimetuple(),
p1.last_modified.utctimetuple(),
)
self.assertEqual(
summaries[1]["last_modified"].utctimetuple(),
p2.last_modified.utctimetuple(),
)
def test_check_health_success(self):
""" check_health returns True for good connection """
ok, msg = self.db.check_health()
self.assertTrue(ok)
def test_check_health_fail(self):
""" check_health returns False for bad connection """
dbmock = self.db.db = MagicMock()
def throw(*_, **__):
""" Throw an exception """
raise redis.RedisError("DB exception")
dbmock.echo.side_effect = throw
ok, msg = self.db.check_health()
self.assertFalse(ok)
class TestRedisCache(unittest.TestCase):
""" Tests for the redis cache """
@classmethod
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
config = MagicMock()
config.get_settings.return_value = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'redis://localhost',
}
RedisCache.configure(config)
cls.redis = RedisCache.db
def setUp(self):
super(TestRedisCache, self).setUp()
self.db = RedisCache(DummyRequest())
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestRedisCache, self).tearDown()
self.redis.flushdb()
def assert_in_redis(self, pkg):
""" Assert that a package exists in redis """
self.assertTrue(self.redis.sismember(self.db.redis_set, pkg.name))
data = self.redis.hgetall(self.db.redis_key(pkg))
pkg_data = {
'name': pkg.name,
'version': pkg.version,
'path': pkg.path,
'last_modified': pkg.last_modified.strftime('%s.%f'),
}
if pkg.url is not None:
pkg_data['url'] = pkg.url
pkg_data['expire'] = pkg.expire.strftime('%s.%f')
self.assertEqual(data, pkg_data)
def test_load(self):
""" Loading from redis deserializes all fields """
pkg = make_package()
pkg.url = 'my.url'
pkg.expire = datetime.utcnow()
self.db.save(pkg)
loaded = self.db.fetch(pkg.name, pkg.version)
self.assertEqual(loaded.name, pkg.name)
self.assertEqual(loaded.version, pkg.version)
self.assertEqual(loaded.path, pkg.path)
self.assertEqual(loaded.last_modified, pkg.last_modified)
self.assertEqual(loaded.url, pkg.url)
self.assertEqual(loaded.expire, pkg.expire)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package()
key = self.db.redis_key(pkg)
self.redis[key] = 'foobar'
self.db.delete(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg.path)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package()
key = self.db.redis_key(pkg)
self.redis[key] = 'foobar'
self.db.clear(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_clear_leave_distinct(self):
""" clear() doesn't remove package from list of distinct """
p1 = make_package()
p2 = make_package(version='1.2')
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1)
self.db.clear(p1)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 1)
def test_clear_all(self):
""" clear_all() removes all packages from db """
p1 = make_package()
p2 = make_package(version='1.2')
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1)
self.db.clear_all()
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package('mypkg2', '1.3.4', 'my/other/path',
factory=SQLPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
for pkg in keys:
self.assert_in_redis(pkg)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package()
self.db.save(pkg)
saved_pkg = self.db.fetch(pkg.name, pkg.version)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch('missing_pkg', '1.2')
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', path='mypath3', factory=SQLPackage),
make_package('mypkg2', '1.3.4', 'my/other/path',
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.all('mypkg')
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', path='mypath3', factory=SQLPackage),
make_package('mypkg2', '1.3.4', 'my/other/path',
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
class TestRedisCache(unittest.TestCase):
""" Tests for the RedisCache """
@classmethod
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {
"pypi.storage": "tests.DummyStorage",
"db.url": "redis://localhost",
"db.graceful_reload": True,
}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs["db"]
try:
cls.redis.flushdb()
except redis.exceptions.ConnectionError:
msg = "Redis not found on port 6379"
setattr(cls, "setUp", lambda cls: unittest.TestCase.skipTest(cls, msg))
@classmethod
def tearDownClass(cls):
super(TestRedisCache, cls).tearDownClass()
def setUp(self):
super(TestRedisCache, self).setUp()
self.db = RedisCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestRedisCache, self).tearDown()
self.redis.flushdb()
def _save_pkgs(self, *pkgs):
""" Save packages to the db """
pipe = self.redis.pipeline()
for pkg in pkgs:
self.db.save(pkg, pipe)
pipe.execute()
def test_add_missing(self):
""" Add missing packages to cache """
keys = [make_package()]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.db._load_all_packages()
self.assertItemsEqual(all_pkgs, keys)
self.assertEqual(len(self.db.summary()), 1)
def test_remove_extra(self):
""" Remove extra packages from cache """
keys = [make_package(), make_package("mypkg2", "1.3.4")]
self.db.save(keys[0])
self.db.save(keys[1])
self.storage.list.return_value = keys[:1]
self.db.reload_from_storage()
all_pkgs = self.db._load_all_packages()
self.assertItemsEqual(all_pkgs, keys[:1])
# It should have removed the summary as well
self.assertEqual(len(self.db.summary()), 1)
def test_remove_extra_leave_concurrent(self):
""" Removing extra packages will leave packages that were uploaded concurrently """
pkgs = [make_package(), make_package("mypkg2")]
self.db.save(pkgs[0])
self.db.save(pkgs[1])
# Return first pkgs[1], then pkgs[1:] because the second time we list
# we will have "uploaded" pkgs[2]
return_values = [lambda: pkgs[1:2], lambda: pkgs[1:]]
def list_storage(package_class):
""" mocked method for listing storage packages """
# The first time we list from storage, concurrently "upload"
# pkgs[2]
if len(return_values) == 2:
pkg = make_package("mypkg3")
pkgs.append(pkg)
self.db.save(pkg)
return return_values.pop(0)()
self.storage.list.side_effect = list_storage
self.db.reload_from_storage()
all_pkgs = self.db._load_all_packages()
self.assertItemsEqual(all_pkgs, pkgs[1:])
self.assertEqual(len(self.db.summary()), 2)
def test_remove_extra_concurrent_deletes(self):
""" Remove packages from cache that were concurrently deleted """
pkgs = [make_package(), make_package("mypkg2")]
self.db.save(pkgs[0])
# Return first pkgs[:], then pkgs[:1] because the second time we list
# we will have "deleted" pkgs[1]
return_values = [pkgs[:], pkgs[:1]]
self.storage.list.side_effect = lambda _: return_values.pop(0)
self.db.reload_from_storage()
all_pkgs = self.db._load_all_packages()
self.assertItemsEqual(all_pkgs, pkgs[:1])
self.assertEqual(len(self.db.summary()), 1)
def test_add_missing_more_recent(self):
""" If we sync a more recent package, update the summary """
pkgs = [
make_package(last_modified=datetime.utcnow() - timedelta(hours=1)),
make_package(version="1.5"),
]
self.db.save(pkgs[0])
self.storage.list.return_value = pkgs
self.db.reload_from_storage()
all_pkgs = self.db._load_all_packages()
self.assertItemsEqual(all_pkgs, pkgs)
summaries = self.db.summary()
self.assertEqual(len(summaries), 1)
summary = summaries[0]
self.assertEqual(summary["last_modified"].hour, pkgs[1].last_modified.hour)
class TestRedisCache(unittest.TestCase):
""" Tests for the redis cache """
@classmethod
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
config = MagicMock()
config.get_settings.return_value = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'redis://localhost',
}
RedisCache.configure(config)
cls.redis = RedisCache.db
def setUp(self):
super(TestRedisCache, self).setUp()
self.db = RedisCache(DummyRequest())
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestRedisCache, self).tearDown()
self.redis.flushdb()
def assert_in_redis(self, pkg):
""" Assert that a package exists in redis """
self.assertTrue(self.redis.sismember(self.db.redis_set, pkg.name))
data = self.redis.hgetall(self.db.redis_key(pkg))
pkg_data = {
'name': pkg.name,
'version': pkg.version,
'path': pkg.path,
'last_modified': pkg.last_modified.strftime('%s.%f'),
}
if pkg.url is not None:
pkg_data['url'] = pkg.url
pkg_data['expire'] = pkg.expire.strftime('%s.%f')
self.assertEqual(data, pkg_data)
def test_load(self):
""" Loading from redis deserializes all fields """
pkg = make_package()
pkg.url = 'my.url'
pkg.expire = datetime.utcnow()
self.db.save(pkg)
loaded = self.db.fetch(pkg.name, pkg.version)
self.assertEqual(loaded.name, pkg.name)
self.assertEqual(loaded.version, pkg.version)
self.assertEqual(loaded.path, pkg.path)
self.assertEqual(loaded.last_modified, pkg.last_modified)
self.assertEqual(loaded.url, pkg.url)
self.assertEqual(loaded.expire, pkg.expire)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package()
key = self.db.redis_key(pkg)
self.redis[key] = 'foobar'
self.db.delete(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg.path)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package()
key = self.db.redis_key(pkg)
self.redis[key] = 'foobar'
self.db.clear(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_clear_leave_distinct(self):
""" clear() doesn't remove package from list of distinct """
p1 = make_package()
p2 = make_package(version='1.2')
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1)
self.db.clear(p1)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 1)
def test_clear_all(self):
""" clear_all() removes all packages from db """
p1 = make_package()
p2 = make_package(version='1.2')
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1)
self.db.clear_all()
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
for pkg in keys:
self.assert_in_redis(pkg)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package()
self.db.save(pkg)
saved_pkg = self.db.fetch(pkg.name, pkg.version)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch('missing_pkg', '1.2')
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', path='mypath3', factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.all('mypkg')
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', path='mypath3', factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
class TestRedisCache(unittest.TestCase):
""" Tests for the redis cache """
@classmethod
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {"pypi.storage": "tests.DummyStorage", "db.url": "redis://localhost"}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs["db"]
def setUp(self):
super(TestRedisCache, self).setUp()
self.db = RedisCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestRedisCache, self).tearDown()
self.redis.flushdb()
def assert_in_redis(self, pkg):
""" Assert that a package exists in redis """
self.assertTrue(self.redis.sismember(self.db.redis_set, pkg.name))
data = self.redis.hgetall(self.db.redis_key(pkg.filename))
pkg_data = {
"name": pkg.name,
"version": pkg.version,
"filename": pkg.filename,
"last_modified": pkg.last_modified.strftime("%s.%f"),
}
pkg_data.update(pkg.data)
self.assertEqual(data, pkg_data)
def test_load(self):
""" Loading from redis deserializes all fields """
kwargs = {"url": "my.url", "expire": 7237}
pkg = make_package(**kwargs)
self.db.save(pkg)
loaded = self.db.fetch(pkg.filename)
self.assertEqual(loaded.name, pkg.name)
self.assertEqual(loaded.version, pkg.version)
self.assertEqual(loaded.filename, pkg.filename)
self.assertEqual(loaded.last_modified, pkg.last_modified)
self.assertEqual(loaded.data, kwargs)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package()
key = self.db.redis_key(pkg.filename)
self.redis[key] = "foobar"
self.db.delete(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package()
key = self.db.redis_key(pkg.filename)
self.redis[key] = "foobar"
self.db.clear(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_clear_leave_distinct(self):
""" clear() doesn't remove package from list of distinct """
p1 = make_package()
p2 = make_package(filename="another-1.2.tar.gz")
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1.filename)
self.db.clear(p1)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 1)
def test_clear_all(self):
""" clear_all() removes all packages from db """
p1 = make_package()
p2 = make_package(version="1.2")
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1)
self.db.clear_all()
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [make_package(factory=SQLPackage), make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage)]
self.storage.list.return_value = keys
self.db.reload_from_storage()
for pkg in keys:
self.assert_in_redis(pkg)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package()
self.db.save(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch("missing_pkg-1.2.tar.gz")
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version="1.3", filename="mypath3", factory=SQLPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.all("mypkg")
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version="1.3", filename="mypath3", factory=SQLPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, "allow_overwrite", False):
name, version = "a", "1"
path1 = "old_package_path-1.tar.gz"
self.db.upload(path1, None, name, version)
path2 = "new_path-1.whl"
self.db.upload(path2, None, name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
class TestRedisCache(unittest.TestCase):
""" Tests for the redis cache """
@classmethod
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'redis://localhost',
}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs['db']
try:
cls.redis.flushdb()
except ConnectionError:
msg = "Redis not found on port 6379"
if sys.version_info < (2, 7):
raise unittest.SkipTest(msg)
else:
setattr(
cls,
"setUp",
lambda cls: unittest.TestCase.skipTest(cls, msg),
)
def setUp(self):
super(TestRedisCache, self).setUp()
self.db = RedisCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestRedisCache, self).tearDown()
self.redis.flushdb()
def assert_in_redis(self, pkg):
""" Assert that a package exists in redis """
self.assertTrue(self.redis.sismember(self.db.redis_set, pkg.name))
data = self.redis.hgetall(self.db.redis_key(pkg.filename))
pkg_data = {
'name': pkg.name,
'version': pkg.version,
'filename': pkg.filename,
'last_modified': pkg.last_modified.strftime('%s.%f'),
}
pkg_data.update(pkg.data)
self.assertEqual(data, pkg_data)
def test_load(self):
""" Loading from redis deserializes all fields """
kwargs = {
'url': 'my.url',
'expire': 7237,
}
pkg = make_package(**kwargs)
self.db.save(pkg)
loaded = self.db.fetch(pkg.filename)
self.assertEqual(loaded.name, pkg.name)
self.assertEqual(loaded.version, pkg.version)
self.assertEqual(loaded.filename, pkg.filename)
self.assertEqual(loaded.last_modified, pkg.last_modified)
self.assertEqual(loaded.data, kwargs)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package()
key = self.db.redis_key(pkg.filename)
self.redis[key] = 'foobar'
self.db.delete(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package()
key = self.db.redis_key(pkg.filename)
self.redis[key] = 'foobar'
self.db.clear(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_clear_leave_distinct(self):
""" clear() doesn't remove package from list of distinct """
p1 = make_package()
p2 = make_package(filename='another-1.2.tar.gz')
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1.filename)
self.db.clear(p1)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 1)
def test_clear_all(self):
""" clear_all() removes all packages from db """
p1 = make_package()
p2 = make_package(version='1.2')
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1)
self.db.clear_all()
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package('mypkg2', '1.3.4', 'my/other/path',
factory=SQLPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
for pkg in keys:
self.assert_in_redis(pkg)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package()
self.db.save(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch('missing_pkg-1.2.tar.gz')
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', filename='mypath3',
factory=SQLPackage),
make_package('mypkg2', '1.3.4', 'my/other/path',
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.all('mypkg')
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', filename='mypath3',
factory=SQLPackage),
make_package('mypkg2', '1.3.4', 'my/other/path',
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, 'allow_overwrite', False):
name, version = 'a', '1'
path1 = 'old_package_path-1.tar.gz'
self.db.upload(path1, None, name, version)
path2 = 'new_path-1.whl'
self.db.upload(path2, None, name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
class TestRedisCache(unittest.TestCase):
""" Tests for the redis cache """
@classmethod
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'redis://localhost',
}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs['db']
try:
cls.redis.flushdb()
except ConnectionError:
msg = "Redis not found on port 6379"
setattr(
cls,
"setUp",
lambda cls: unittest.TestCase.skipTest(cls, msg),
)
def setUp(self):
super(TestRedisCache, self).setUp()
self.db = RedisCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestRedisCache, self).tearDown()
self.redis.flushdb()
def assert_in_redis(self, pkg):
""" Assert that a package exists in redis """
self.assertTrue(self.redis.sismember(self.db.redis_set, pkg.name))
data = self.redis.hgetall(self.db.redis_key(pkg.filename))
dt = pkg.last_modified
lm = calendar.timegm(dt.utctimetuple()) + dt.microsecond / 1000000.0
lm_str = ("%.6f" % lm).rstrip('0').rstrip('.')
pkg_data = {
'name': pkg.name,
'version': pkg.version,
'filename': pkg.filename,
'last_modified': lm_str,
'summary': pkg.summary,
}
pkg_data.update(pkg.data)
self.assertEqual(data, pkg_data)
def test_load(self):
""" Loading from redis deserializes all fields """
kwargs = {
'url': 'my.url',
'expire': 7237,
}
pkg = make_package(**kwargs)
# Due to some rounding weirdness in old Py3 versions, we need to remove
# the microseconds to avoid a flappy test.
# See: https://bugs.python.org/issue23517
pkg.last_modified = pkg.last_modified.replace(microsecond=0)
self.db.save(pkg)
loaded = self.db.fetch(pkg.filename)
self.assertEqual(loaded.name, pkg.name)
self.assertEqual(loaded.version, pkg.version)
self.assertEqual(loaded.filename, pkg.filename)
self.assertEqual(loaded.last_modified, pkg.last_modified)
self.assertEqual(loaded.summary, pkg.summary)
self.assertEqual(loaded.data, kwargs)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package()
key = self.db.redis_key(pkg.filename)
self.redis[key] = 'foobar'
self.db.delete(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package()
key = self.db.redis_key(pkg.filename)
self.redis[key] = 'foobar'
self.db.clear(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_clear_leave_distinct(self):
""" clear() doesn't remove package from list of distinct """
p1 = make_package()
p2 = make_package(filename='another-1.2.tar.gz')
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1.filename)
self.db.clear(p1)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 1)
def test_clear_all(self):
""" clear_all() removes all packages from db """
p1 = make_package()
p2 = make_package(version='1.2')
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1)
self.db.clear_all()
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
for pkg in keys:
self.assert_in_redis(pkg)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package()
self.db.save(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch('missing_pkg-1.2.tar.gz')
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', filename='mypath3',
factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.all('mypkg')
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', filename='mypath3',
factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_delete_package(self):
""" Deleting the last package of a name removes from distinct() """
pkgs = [
make_package(factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
self.db.clear(pkgs[0])
saved_pkgs = self.db.distinct()
self.assertEqual(saved_pkgs, ['mypkg2'])
summaries = self.db.summary()
self.assertEqual(len(summaries), 1)
def test_search_or(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package('somepackage',
version='1.3',
filename='mypath3',
summary='this is mypkg',
factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
make_package('package', factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
criteria = {'name': ['mypkg'], 'summary': ['mypkg']}
packages = self.db.search(criteria, 'or')
self.assertItemsEqual(packages, pkgs[:-1])
def test_search_and(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package('somepackage',
version='1.3',
filename='mypath3',
summary='this is mypkg',
factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
make_package('package', factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
criteria = {'name': ['my', 'pkg'], 'summary': ['this', 'mypkg']}
packages = self.db.search(criteria, 'and')
self.assertItemsEqual(packages, pkgs[:-1])
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, 'allow_overwrite', False):
name, version = 'a', '1'
path1 = 'old_package_path-1.tar.gz'
self.db.upload(path1, None, name, version)
path2 = 'new_path-1.whl'
self.db.upload(path2, None, name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
def test_summary(self):
""" summary constructs per-package metadata summary """
self.db.upload('pkg1-0.3a2.tar.gz', None, 'pkg1', '0.3a2')
self.db.upload('pkg1-1.1.tar.gz', None, 'pkg1', '1.1')
p1 = self.db.upload('pkg1a2.tar.gz', None, 'pkg1', '1.1.1a2',
'summary')
p2 = self.db.upload('pkg2.tar.gz', None, 'pkg2', '0.1dev2', 'summary')
summaries = self.db.summary()
self.assertItemsEqual(summaries, [
{
'name': 'pkg1',
'summary': 'summary',
'last_modified': ANY,
},
{
'name': 'pkg2',
'summary': 'summary',
'last_modified': ANY,
},
])
# Have to compare the last_modified fuzzily
self.assertEqual(
summaries[0]['last_modified'].utctimetuple(),
p1.last_modified.utctimetuple(),
)
self.assertEqual(
summaries[1]['last_modified'].utctimetuple(),
p2.last_modified.utctimetuple(),
)
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {"pypi.storage": "tests.DummyStorage", "db.url": "redis://localhost"}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs["db"]
class TestRedisCache(unittest.TestCase):
""" Tests for the redis cache """
@classmethod
def setUpClass(cls):
super(TestRedisCache, cls).setUpClass()
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'redis://localhost',
}
cls.kwargs = RedisCache.configure(settings)
cls.redis = cls.kwargs['db']
def setUp(self):
super(TestRedisCache, self).setUp()
self.db = RedisCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestRedisCache, self).tearDown()
self.redis.flushdb()
def assert_in_redis(self, pkg):
""" Assert that a package exists in redis """
self.assertTrue(self.redis.sismember(self.db.redis_set, pkg.name))
data = self.redis.hgetall(self.db.redis_key(pkg.filename))
pkg_data = {
'name': pkg.name,
'version': pkg.version,
'filename': pkg.filename,
'last_modified': pkg.last_modified.strftime('%s.%f'),
}
pkg_data.update(pkg.data)
self.assertEqual(data, pkg_data)
def test_load(self):
""" Loading from redis deserializes all fields """
kwargs = {
'url': 'my.url',
'expire': 7237,
}
pkg = make_package(**kwargs)
self.db.save(pkg)
loaded = self.db.fetch(pkg.filename)
self.assertEqual(loaded.name, pkg.name)
self.assertEqual(loaded.version, pkg.version)
self.assertEqual(loaded.filename, pkg.filename)
self.assertEqual(loaded.last_modified, pkg.last_modified)
self.assertEqual(loaded.data, kwargs)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package()
key = self.db.redis_key(pkg.filename)
self.redis[key] = 'foobar'
self.db.delete(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package()
key = self.db.redis_key(pkg.filename)
self.redis[key] = 'foobar'
self.db.clear(pkg)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_clear_leave_distinct(self):
""" clear() doesn't remove package from list of distinct """
p1 = make_package()
p2 = make_package(filename='another-1.2.tar.gz')
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1.filename)
self.db.clear(p1)
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 1)
def test_clear_all(self):
""" clear_all() removes all packages from db """
p1 = make_package()
p2 = make_package(version='1.2')
self.db.save(p1)
self.db.save(p2)
key = self.db.redis_key(p1)
self.db.clear_all()
val = self.redis.get(key)
self.assertIsNone(val)
count = self.redis.scard(self.db.redis_set)
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
for pkg in keys:
self.assert_in_redis(pkg)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package()
self.db.save(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch('missing_pkg-1.2.tar.gz')
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', filename='mypath3',
factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.all('mypkg')
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', filename='mypath3',
factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
for pkg in pkgs:
self.db.save(pkg)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, 'allow_overwrite', False):
name, version = 'a', '1'
path1 = 'old_package_path-1.tar.gz'
self.db.upload(path1, None, name, version)
path2 = 'new_path-1.whl'
self.db.upload(path2, None, name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
