class TestSQLiteCache(unittest.TestCase):
""" Tests for the SQLAlchemy cache """
DB_URL = "sqlite://"
@classmethod
def setUpClass(cls):
super(TestSQLiteCache, cls).setUpClass()
settings = {"pypi.storage": "tests.DummyStorage", "db.url": cls.DB_URL}
try:
cls.kwargs = SQLCache.configure(settings)
except OperationalError:
raise unittest.SkipTest("Couldn't connect to database")
def setUp(self):
super(TestSQLiteCache, self).setUp()
transaction.begin()
self.request = DummyRequest()
self.request.tm = transaction.manager
self.db = SQLCache(self.request, **self.kwargs)
self.sql = self.db.db
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestSQLiteCache, self).tearDown()
transaction.abort()
self.sql.query(SQLPackage).delete()
transaction.commit()
self.request._process_finished_callbacks()
def test_upload(self):
""" upload() saves package and uploads to storage """
pkg = make_package(factory=SQLPackage)
content = BytesIO(b"test1234")
self.db.upload(pkg.filename, content, pkg.name, pkg.version)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
# If calculate hashes is on, it'll read the data
# and rewrap with BytesIO
self.storage.upload.assert_called_with(pkg, ANY)
def test_upload_overwrite(self):
""" Uploading a preexisting packages overwrites current package """
self.db.allow_overwrite = True
name, filename = "a", "a-1.tar.gz"
self.db.upload(filename, BytesIO(b"old"), name)
self.db.upload(filename, BytesIO(b"new"), name)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 1)
def test_save(self):
""" save() puts object into database """
pkg = make_package(factory=SQLPackage)
self.db.save(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_save_unicode(self):
""" save() can store packages with unicode in the names """
pkg = make_package("mypackage™", factory=SQLPackage)
self.db.save(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
transaction.commit()
self.sql.add(pkg)
self.db.delete(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
transaction.commit()
self.sql.add(pkg)
self.db.delete(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package(
"mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage,
hash_md5="md5",
hash_sha256="sha256",
),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.sql.query(SQLPackage).all()
self.assertItemsEqual(all_pkgs, keys)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch("missing_pkg-1.2.tar.gz")
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version="1.3", filename="mypath3",
factory=SQLPackage),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
]
self.sql.add_all(pkgs)
saved_pkgs = self.db.all("mypkg")
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version="1.3", filename="mypath3",
factory=SQLPackage),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
]
self.sql.add_all(pkgs)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_search_or(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=SQLPackage,
),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
make_package("package", factory=SQLPackage),
]
self.sql.add_all(pkgs)
criteria = {"name": ["mypkg"], "summary": ["mypkg"]}
packages = self.db.search(criteria, "or")
self.assertItemsEqual(packages, pkgs[:-1])
def test_search_and(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=SQLPackage,
),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
make_package("package", factory=SQLPackage),
]
self.sql.add_all(pkgs)
criteria = {"name": ["my", "pkg"], "summary": ["this", "mypkg"]}
packages = self.db.search(criteria, "and")
self.assertItemsEqual(packages, pkgs[:-1])
def test_summary(self):
""" summary constructs per-package metadata summary """
self.db.upload("pkg1-0.3.tar.gz", BytesIO(b"test1234"), "pkg1", "0.3")
self.db.upload("pkg1-1.1.tar.gz", BytesIO(b"test1234"), "pkg1", "1.1")
p1 = self.db.upload("pkg1a2.tar.gz", BytesIO(b"test1234"), "pkg1",
"1.1.1a2")
p2 = self.db.upload("pkg2.tar.gz", BytesIO(b"test1234"), "pkg2",
"0.1dev2")
s1, s2 = self.db.summary() # pylint: disable=E0632
# Order them correctly. assertItemsEqual isn't playing nice in py2.6
if s1["name"] == "pkg2":
s1, s2 = s2, s1
# last_modified may be rounded when stored in MySQL,
# so the best we can do is make sure they're close.
self.assertTrue(
calendar.timegm(s1["last_modified"].timetuple()) -
calendar.timegm(p1.last_modified.timetuple()) <= 1)
self.assertTrue(
calendar.timegm(s2["last_modified"].timetuple()) -
calendar.timegm(p2.last_modified.timetuple()) <= 1)
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, "allow_overwrite", False):
name, version = "a", "1"
path1 = "old_package_path-1.tar.gz"
self.db.upload(path1, BytesIO(b"test1234"), name, version)
path2 = "new_path-1.whl"
self.db.upload(path2, BytesIO(b"test1234"), name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
def test_reload_if_needed(self):
""" Reload the cache if it's empty """
self.db.storage = MagicMock()
self.db.storage.list.return_value = [make_package(factory=SQLPackage)]
self.db.reload_if_needed()
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
def test_check_health_success(self):
""" check_health returns True for good connection """
ok, msg = self.db.check_health()
self.assertTrue(ok)
def test_check_health_fail(self):
""" check_health returns False for bad connection """
dbmock = self.db.db = MagicMock()
def throw(*_, **__):
""" Throw an exception """
raise SQLAlchemyError("DB exception")
dbmock.query.side_effect = throw
ok, msg = self.db.check_health()
self.assertFalse(ok)
class TestSQLiteCache(unittest.TestCase):
""" Tests for the SQLAlchemy cache """
DB_URL = "sqlite://"
@classmethod
def setUpClass(cls):
super(TestSQLiteCache, cls).setUpClass()
settings = {"pypi.storage": "tests.DummyStorage", "db.url": cls.DB_URL}
try:
cls.kwargs = SQLCache.configure(settings)
except OperationalError:
raise unittest.SkipTest("Couldn't connect to database")
def setUp(self):
super(TestSQLiteCache, self).setUp()
transaction.begin()
self.request = DummyRequest()
self.request.tm = transaction.manager
self.db = SQLCache(self.request, **self.kwargs)
self.sql = self.db.db
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestSQLiteCache, self).tearDown()
transaction.abort()
self.sql.query(SQLPackage).delete()
transaction.commit()
self.request._process_finished_callbacks()
def test_upload(self):
""" upload() saves package and uploads to storage """
pkg = make_package(factory=SQLPackage)
self.db.upload(pkg.filename, None, pkg.name, pkg.version)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
self.storage.upload.assert_called_with(pkg, None)
def test_upload_overwrite(self):
""" Uploading a preexisting packages overwrites current package """
self.db.allow_overwrite = True
name, filename = "a", "a-1.tar.gz"
self.db.upload(filename, "old", name)
self.db.upload(filename, "new", name)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 1)
def test_save(self):
""" save() puts object into database """
pkg = make_package(factory=SQLPackage)
self.db.save(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_save_unicode(self):
""" save() can store packages with unicode in the names """
pkg = make_package("mypackage™", factory=SQLPackage)
self.db.save(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
transaction.commit()
self.sql.add(pkg)
self.db.delete(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
transaction.commit()
self.sql.add(pkg)
self.db.delete(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.sql.query(SQLPackage).all()
self.assertItemsEqual(all_pkgs, keys)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch("missing_pkg-1.2.tar.gz")
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version="1.3", filename="mypath3", factory=SQLPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
]
self.sql.add_all(pkgs)
saved_pkgs = self.db.all("mypkg")
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version="1.3", filename="mypath3", factory=SQLPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
]
self.sql.add_all(pkgs)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_search_or(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=SQLPackage,
),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
make_package("package", factory=SQLPackage),
]
self.sql.add_all(pkgs)
criteria = {"name": ["mypkg"], "summary": ["mypkg"]}
packages = self.db.search(criteria, "or")
self.assertItemsEqual(packages, pkgs[:-1])
def test_search_and(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=SQLPackage,
),
make_package("mypkg2", "1.3.4", "my/other/path", factory=SQLPackage),
make_package("package", factory=SQLPackage),
]
self.sql.add_all(pkgs)
criteria = {"name": ["my", "pkg"], "summary": ["this", "mypkg"]}
packages = self.db.search(criteria, "and")
self.assertItemsEqual(packages, pkgs[:-1])
def test_summary(self):
""" summary constructs per-package metadata summary """
self.db.upload("pkg1-0.3.tar.gz", None, "pkg1", "0.3")
self.db.upload("pkg1-1.1.tar.gz", None, "pkg1", "1.1")
p1 = self.db.upload("pkg1a2.tar.gz", None, "pkg1", "1.1.1a2")
p2 = self.db.upload("pkg2.tar.gz", None, "pkg2", "0.1dev2")
s1, s2 = self.db.summary() # pylint: disable=E0632
# Order them correctly. assertItemsEqual isn't playing nice in py2.6
if s1["name"] == "pkg2":
s1, s2 = s2, s1
# last_modified may be rounded when stored in MySQL,
# so the best we can do is make sure they're close.
self.assertTrue(
calendar.timegm(s1["last_modified"].timetuple())
- calendar.timegm(p1.last_modified.timetuple())
<= 1
)
self.assertTrue(
calendar.timegm(s2["last_modified"].timetuple())
- calendar.timegm(p2.last_modified.timetuple())
<= 1
)
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, "allow_overwrite", False):
name, version = "a", "1"
path1 = "old_package_path-1.tar.gz"
self.db.upload(path1, None, name, version)
path2 = "new_path-1.whl"
self.db.upload(path2, None, name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
def test_reload_if_needed(self):
""" Reload the cache if it's empty """
self.db.storage = MagicMock()
self.db.storage.list.return_value = [make_package(factory=SQLPackage)]
self.db.reload_if_needed()
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
def test_check_health_success(self):
""" check_health returns True for good connection """
ok, msg = self.db.check_health()
self.assertTrue(ok)
def test_check_health_fail(self):
""" check_health returns False for bad connection """
dbmock = self.db.db = MagicMock()
def throw(*_, **__):
""" Throw an exception """
raise SQLAlchemyError("DB exception")
dbmock.query.side_effect = throw
ok, msg = self.db.check_health()
self.assertFalse(ok)
