def setUpClass(cls):
super(TestSQLCache, cls).setUpClass()
config = MagicMock()
config.get_settings.return_value = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'sqlite:///:memory:',
}
SQLCache.configure(config)
def setUp(self):
super(TestSQLiteCache, self).setUp()
transaction.begin()
self.request = DummyRequest()
self.request.tm = transaction.manager
self.db = SQLCache(self.request, **self.kwargs)
self.sql = self.db.db
self.storage = self.db.storage = MagicMock(spec=IStorage)
def test_reload_if_needed(self):
""" Reload the cache if it's empty """
with patch.object(SQLCache, 'storage_impl') as storage_impl:
storage_impl().list.return_value = [
make_package(factory=SQLPackage)
]
SQLCache.reload_if_needed()
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
def setUpClass(cls):
super(TestSQLiteCache, cls).setUpClass()
settings = {"pypi.storage": "tests.DummyStorage", "db.url": cls.DB_URL}
try:
cls.kwargs = SQLCache.configure(settings)
except OperationalError:
raise unittest.SkipTest("Couldn't connect to database")
def setUpClass(cls):
super(TestSQLCache, cls).setUpClass()
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'sqlite:///:memory:',
}
cls.kwargs = SQLCache.configure(settings)
def setUpClass(cls):
super(TestSQLiteCache, cls).setUpClass()
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.url': cls.DB_URL,
'db.graceful_reload': True,
}
try:
cls.kwargs = SQLCache.configure(settings)
except OperationalError:
raise unittest.SkipTest("Couldn't connect to database")
def setUpClass(cls):
super(TestSQLiteCache, cls).setUpClass()
db_url = cls.get_db_url()
settings = EnvironSettings(
{
"pypi.storage": "tests.DummyStorage",
"db.url": db_url
}, {})
try:
cls.kwargs = SQLCache.configure(settings)
except OperationalError:
raise unittest.SkipTest(f"Couldn't connect to database {db_url}")
class TestSQLiteCache(unittest.TestCase):
""" Tests for the SQLAlchemy cache """
DB_URL = "sqlite://"
@classmethod
def setUpClass(cls):
super(TestSQLiteCache, cls).setUpClass()
settings = {"pypi.storage": "tests.DummyStorage", "db.url": cls.DB_URL}
try:
cls.kwargs = SQLCache.configure(settings)
except OperationalError:
raise unittest.SkipTest("Couldn't connect to database")
def setUp(self):
super(TestSQLiteCache, self).setUp()
transaction.begin()
self.request = DummyRequest()
self.request.tm = transaction.manager
self.db = SQLCache(self.request, **self.kwargs)
self.sql = self.db.db
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestSQLiteCache, self).tearDown()
transaction.abort()
self.sql.query(SQLPackage).delete()
transaction.commit()
self.request._process_finished_callbacks()
def test_upload(self):
""" upload() saves package and uploads to storage """
pkg = make_package(factory=SQLPackage)
content = BytesIO(b"test1234")
self.db.upload(pkg.filename, content, pkg.name, pkg.version)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
# If calculate hashes is on, it'll read the data
# and rewrap with BytesIO
self.storage.upload.assert_called_with(pkg, ANY)
def test_upload_overwrite(self):
""" Uploading a preexisting packages overwrites current package """
self.db.allow_overwrite = True
name, filename = "a", "a-1.tar.gz"
self.db.upload(filename, BytesIO(b"old"), name)
self.db.upload(filename, BytesIO(b"new"), name)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 1)
def test_save(self):
""" save() puts object into database """
pkg = make_package(factory=SQLPackage)
self.db.save(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_save_unicode(self):
""" save() can store packages with unicode in the names """
pkg = make_package("mypackage™", factory=SQLPackage)
self.db.save(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
transaction.commit()
self.sql.add(pkg)
self.db.delete(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
transaction.commit()
self.sql.add(pkg)
self.db.delete(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package(
"mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage,
hash_md5="md5",
hash_sha256="sha256",
),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.sql.query(SQLPackage).all()
self.assertItemsEqual(all_pkgs, keys)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch("missing_pkg-1.2.tar.gz")
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version="1.3", filename="mypath3",
factory=SQLPackage),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
]
self.sql.add_all(pkgs)
saved_pkgs = self.db.all("mypkg")
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version="1.3", filename="mypath3",
factory=SQLPackage),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
]
self.sql.add_all(pkgs)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_search_or(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=SQLPackage,
),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
make_package("package", factory=SQLPackage),
]
self.sql.add_all(pkgs)
criteria = {"name": ["mypkg"], "summary": ["mypkg"]}
packages = self.db.search(criteria, "or")
self.assertItemsEqual(packages, pkgs[:-1])
def test_search_and(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=SQLPackage,
),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=SQLPackage),
make_package("package", factory=SQLPackage),
]
self.sql.add_all(pkgs)
criteria = {"name": ["my", "pkg"], "summary": ["this", "mypkg"]}
packages = self.db.search(criteria, "and")
self.assertItemsEqual(packages, pkgs[:-1])
def test_summary(self):
""" summary constructs per-package metadata summary """
self.db.upload("pkg1-0.3.tar.gz", BytesIO(b"test1234"), "pkg1", "0.3")
self.db.upload("pkg1-1.1.tar.gz", BytesIO(b"test1234"), "pkg1", "1.1")
p1 = self.db.upload("pkg1a2.tar.gz", BytesIO(b"test1234"), "pkg1",
"1.1.1a2")
p2 = self.db.upload("pkg2.tar.gz", BytesIO(b"test1234"), "pkg2",
"0.1dev2")
s1, s2 = self.db.summary() # pylint: disable=E0632
# Order them correctly. assertItemsEqual isn't playing nice in py2.6
if s1["name"] == "pkg2":
s1, s2 = s2, s1
# last_modified may be rounded when stored in MySQL,
# so the best we can do is make sure they're close.
self.assertTrue(
calendar.timegm(s1["last_modified"].timetuple()) -
calendar.timegm(p1.last_modified.timetuple()) <= 1)
self.assertTrue(
calendar.timegm(s2["last_modified"].timetuple()) -
calendar.timegm(p2.last_modified.timetuple()) <= 1)
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, "allow_overwrite", False):
name, version = "a", "1"
path1 = "old_package_path-1.tar.gz"
self.db.upload(path1, BytesIO(b"test1234"), name, version)
path2 = "new_path-1.whl"
self.db.upload(path2, BytesIO(b"test1234"), name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
def test_reload_if_needed(self):
""" Reload the cache if it's empty """
self.db.storage = MagicMock()
self.db.storage.list.return_value = [make_package(factory=SQLPackage)]
self.db.reload_if_needed()
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
def test_check_health_success(self):
""" check_health returns True for good connection """
ok, msg = self.db.check_health()
self.assertTrue(ok)
def test_check_health_fail(self):
""" check_health returns False for bad connection """
dbmock = self.db.db = MagicMock()
def throw(*_, **__):
""" Throw an exception """
raise SQLAlchemyError("DB exception")
dbmock.query.side_effect = throw
ok, msg = self.db.check_health()
self.assertFalse(ok)
class TestSQLiteCache(unittest.TestCase):
""" Tests for the SQLCache """
DB_URL = "sqlite://"
@classmethod
def setUpClass(cls):
super(TestSQLiteCache, cls).setUpClass()
settings = {
"pypi.storage": "tests.DummyStorage",
"db.url": cls.DB_URL,
"db.graceful_reload": True,
}
try:
cls.kwargs = SQLCache.configure(settings)
except OperationalError:
raise unittest.SkipTest("Couldn't connect to database")
def setUp(self):
super(TestSQLiteCache, self).setUp()
transaction.begin()
self.request = DummyRequest()
self.request.tm = transaction.manager
self.db = SQLCache(self.request, **self.kwargs)
self.sql = self.db.db
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestSQLiteCache, self).tearDown()
transaction.abort()
self.sql.query(SQLPackage).delete()
transaction.commit()
self.request._process_finished_callbacks()
def _make_package(self, *args, **kwargs):
""" Wrapper around make_package """
# Some SQL dbs are rounding the timestamps (looking at you MySQL >:|
# which is a problem if they round UP to the future, as our
# calculations depend on the timestamps being monotonically increasing.
now = utcnow() - timedelta(seconds=1)
kwargs.setdefault("last_modified", now)
kwargs.setdefault("factory", SQLPackage)
return make_package(*args, **kwargs)
def test_add_missing(self):
""" Add missing packages to cache """
keys = [self._make_package()]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.sql.query(SQLPackage).all()
self.assertItemsEqual(all_pkgs, keys)
def test_remove_extra(self):
""" Remove extra packages from cache """
keys = [self._make_package(), self._make_package("mypkg2", "1.3.4")]
self.db.save(keys[0])
self.db.save(keys[1])
self.storage.list.return_value = keys[:1]
self.db.reload_from_storage()
all_pkgs = self.sql.query(SQLPackage).all()
self.assertItemsEqual(all_pkgs, keys[:1])
def test_remove_extra_leave_concurrent(self):
""" Removing extra packages will leave packages that were uploaded concurrently """
pkgs = [self._make_package(), self._make_package("mypkg2")]
self.db.save(pkgs[0])
self.db.save(pkgs[1])
# Return first pkgs[1], then pkgs[1:] because the second time we list
# we will have "uploaded" pkgs[2]
return_values = [lambda: pkgs[1:2], lambda: pkgs[1:]]
def list_storage(factory):
""" mocked method for listing storage packages """
# The first time we list from storage, concurrently "upload"
# pkgs[2]
if len(return_values) == 2:
nowish = utcnow() + timedelta(seconds=1)
pkg = self._make_package("mypkg3", last_modified=nowish)
pkgs.append(pkg)
self.db.save(pkg)
return return_values.pop(0)()
self.storage.list.side_effect = list_storage
self.db.reload_from_storage()
all_pkgs = self.sql.query(SQLPackage).all()
self.assertItemsEqual(all_pkgs, pkgs[1:])
def test_remove_extra_concurrent_deletes(self):
""" Remove packages from cache that were concurrently deleted """
pkgs = [self._make_package(), self._make_package("mypkg2")]
self.db.save(pkgs[0])
# Return first pkgs[:], then pkgs[:1] because the second time we list
# we will have "deleted" pkgs[1]
return_values = [pkgs[:], pkgs[:1]]
self.storage.list.side_effect = lambda _: return_values.pop(0)
self.db.reload_from_storage()
all_pkgs = self.sql.query(SQLPackage).all()
self.assertItemsEqual(all_pkgs, pkgs[:1])
def test_add_missing_more_recent(self):
""" If we sync a more recent package, update the summary """
pkgs = [
self._make_package(last_modified=utcnow() - timedelta(hours=1)),
self._make_package(version="1.5"),
]
self.db.save(pkgs[0])
self.storage.list.return_value = pkgs
self.db.reload_from_storage()
all_pkgs = self.sql.query(SQLPackage).all()
self.assertItemsEqual(all_pkgs, pkgs)
def test_same_package_name_version(self):
""" Storage can have packages with the same name and version (different filename) """
pkgs = [
self._make_package(filename="mypkg-1.1-win32.whl"),
self._make_package(filename="mypkg-1.1-macosx.whl"),
self._make_package(filename="mypkg-1.1-x86_64.whl"),
]
self.storage.list.return_value = pkgs
self.db.reload_from_storage()
all_pkgs = self.sql.query(SQLPackage).all()
self.assertCountEqual(all_pkgs, pkgs)
def setUp(self):
super(TestSQLCache, self).setUp()
self.request = DummyRequest()
self.db = SQLCache(self.request)
self.sql = self.db.db
self.storage = self.db.storage = MagicMock(spec=IStorage)
class TestSQLCache(unittest.TestCase):
""" Tests for the SQLAlchemy cache """
@classmethod
def setUpClass(cls):
super(TestSQLCache, cls).setUpClass()
config = MagicMock()
config.get_settings.return_value = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'sqlite:///:memory:',
}
SQLCache.configure(config)
def setUp(self):
super(TestSQLCache, self).setUp()
self.request = DummyRequest()
self.db = SQLCache(self.request)
self.sql = self.db.db
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestSQLCache, self).tearDown()
transaction.abort()
self.sql.query(SQLPackage).delete()
transaction.commit()
self.request._process_finished_callbacks()
def test_upload(self):
""" upload() saves package and uploads to storage """
pkg = make_package(factory=SQLPackage)
self.storage.upload.return_value = pkg.path
self.db.upload(pkg.name, pkg.version, pkg.path, None)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
self.storage.upload.assert_called_with(pkg.name, pkg.version, pkg.path,
None)
def test_save(self):
""" save() puts object into database """
pkg = make_package(factory=SQLPackage)
self.db.save(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
transaction.commit()
self.sql.add(pkg)
self.db.delete(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg.path)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
transaction.commit()
self.sql.add(pkg)
self.db.delete(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.sql.query(SQLPackage).all()
self.assertItemsEqual(all_pkgs, keys)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
saved_pkg = self.db.fetch(pkg.name, pkg.version)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch('missing_pkg', '1.2')
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', path='mypath3', factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
self.sql.add_all(pkgs)
saved_pkgs = self.db.all('mypkg')
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', path='mypath3', factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
self.sql.add_all(pkgs)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_summary(self):
""" summary constructs per-package metadata summary """
self.storage.upload.side_effect = lambda x, y, z, _: z
self.db.upload('pkg1', '0.3', 'pkg1.tar.gz', None)
self.db.upload('pkg1', '1.1', 'pkg1.tar.gz', None)
p1 = self.db.upload('pkg1', '1.1.1a2', 'pkg1a2.tar.gz', None)
p2 = self.db.upload('pkg2', '0.1dev2', 'pkg2.tar.gz', None)
summaries = self.db.summary()
self.assertItemsEqual(summaries, [
{
'name': 'pkg1',
'stable': '1.1',
'unstable': '1.1.1a2',
'last_modified': p1.last_modified,
},
{
'name': 'pkg2',
'stable': None,
'unstable': '0.1dev2',
'last_modified': p2.last_modified,
},
])
def test_reload_if_needed(self):
""" Reload the cache if it's empty """
with patch.object(SQLCache, 'storage_impl') as storage_impl:
storage_impl().list.return_value = [
make_package(factory=SQLPackage)
]
SQLCache.reload_if_needed()
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
class TestSQLiteCache(unittest.TestCase):
""" Tests for the SQLAlchemy cache """
DB_URL = 'sqlite://'
@classmethod
def setUpClass(cls):
super(TestSQLiteCache, cls).setUpClass()
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.url': cls.DB_URL,
}
try:
cls.kwargs = SQLCache.configure(settings)
except OperationalError:
raise unittest.SkipTest("Couldn't connect to database")
def setUp(self):
super(TestSQLiteCache, self).setUp()
transaction.begin()
self.request = DummyRequest()
self.request.tm = transaction.manager
self.db = SQLCache(self.request, **self.kwargs)
self.sql = self.db.db
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestSQLiteCache, self).tearDown()
transaction.abort()
self.sql.query(SQLPackage).delete()
transaction.commit()
self.request._process_finished_callbacks()
def test_upload(self):
""" upload() saves package and uploads to storage """
pkg = make_package(factory=SQLPackage)
self.db.upload(pkg.filename, None, pkg.name, pkg.version)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
self.storage.upload.assert_called_with(pkg, None)
def test_upload_overwrite(self):
""" Uploading a preexisting packages overwrites current package """
self.db.allow_overwrite = True
name, filename = 'a', 'a-1.tar.gz'
self.db.upload(filename, 'old', name)
self.db.upload(filename, 'new', name)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 1)
def test_save(self):
""" save() puts object into database """
pkg = make_package(factory=SQLPackage)
self.db.save(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_save_unicode(self):
""" save() can store packages with unicode in the names """
pkg = make_package(u'mypackage™', factory=SQLPackage)
self.db.save(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
transaction.commit()
self.sql.add(pkg)
self.db.delete(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
transaction.commit()
self.sql.add(pkg)
self.db.delete(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.sql.query(SQLPackage).all()
self.assertItemsEqual(all_pkgs, keys)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch('missing_pkg-1.2.tar.gz')
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', filename='mypath3',
factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
self.sql.add_all(pkgs)
saved_pkgs = self.db.all('mypkg')
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', filename='mypath3',
factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
self.sql.add_all(pkgs)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_search_or(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package('somepackage',
version='1.3',
filename='mypath3',
summary='this is mypkg',
factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
make_package('package', factory=SQLPackage),
]
self.sql.add_all(pkgs)
criteria = {'name': ['mypkg'], 'summary': ['mypkg']}
packages = self.db.search(criteria, 'or')
self.assertItemsEqual(packages, pkgs[:-1])
def test_search_and(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=SQLPackage),
make_package('somepackage',
version='1.3',
filename='mypath3',
summary='this is mypkg',
factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
make_package('package', factory=SQLPackage),
]
self.sql.add_all(pkgs)
criteria = {'name': ['my', 'pkg'], 'summary': ['this', 'mypkg']}
packages = self.db.search(criteria, 'and')
self.assertItemsEqual(packages, pkgs[:-1])
def test_summary(self):
""" summary constructs per-package metadata summary """
self.db.upload('pkg1-0.3.tar.gz', None, 'pkg1', '0.3')
self.db.upload('pkg1-1.1.tar.gz', None, 'pkg1', '1.1')
p1 = self.db.upload('pkg1a2.tar.gz', None, 'pkg1', '1.1.1a2')
p2 = self.db.upload('pkg2.tar.gz', None, 'pkg2', '0.1dev2')
s1, s2 = self.db.summary() # pylint: disable=E0632
# Order them correctly. assertItemsEqual isn't playing nice in py2.6
if s1['name'] == 'pkg2':
s1, s2 = s2, s1
# last_modified may be rounded when stored in MySQL,
# so the best we can do is make sure they're close.
self.assertTrue(
calendar.timegm(s1['last_modified'].timetuple()) -
calendar.timegm(p1.last_modified.timetuple()) <= 1)
self.assertTrue(
calendar.timegm(s2['last_modified'].timetuple()) -
calendar.timegm(p2.last_modified.timetuple()) <= 1)
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, 'allow_overwrite', False):
name, version = 'a', '1'
path1 = 'old_package_path-1.tar.gz'
self.db.upload(path1, None, name, version)
path2 = 'new_path-1.whl'
self.db.upload(path2, None, name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
def test_reload_if_needed(self):
""" Reload the cache if it's empty """
self.db.storage = MagicMock()
self.db.storage.list.return_value = [make_package(factory=SQLPackage)]
self.db.reload_if_needed()
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
class TestSQLCache(unittest.TestCase):
""" Tests for the SQLAlchemy cache """
@classmethod
def setUpClass(cls):
super(TestSQLCache, cls).setUpClass()
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.url': 'sqlite:///:memory:',
}
cls.kwargs = SQLCache.configure(settings)
def setUp(self):
super(TestSQLCache, self).setUp()
self.request = DummyRequest()
self.db = SQLCache(self.request, **self.kwargs)
self.sql = self.db.db
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestSQLCache, self).tearDown()
transaction.abort()
self.sql.query(SQLPackage).delete()
transaction.commit()
self.request._process_finished_callbacks()
def test_upload(self):
""" upload() saves package and uploads to storage """
pkg = make_package(factory=SQLPackage)
self.db.upload(pkg.filename, None, pkg.name, pkg.version)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
self.storage.upload.assert_called_with(pkg, None)
def test_save(self):
""" save() puts object into database """
pkg = make_package(factory=SQLPackage)
self.db.save(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.sql.query(SQLPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
transaction.commit()
self.sql.add(pkg)
self.db.delete(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
transaction.commit()
self.sql.add(pkg)
self.db.delete(pkg)
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.sql.query(SQLPackage).all()
self.assertItemsEqual(all_pkgs, keys)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package(factory=SQLPackage)
self.sql.add(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch('missing_pkg-1.2.tar.gz')
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', filename='mypath3',
factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
self.sql.add_all(pkgs)
saved_pkgs = self.db.all('mypkg')
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=SQLPackage),
make_package(version='1.3', filename='mypath3',
factory=SQLPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=SQLPackage),
]
self.sql.add_all(pkgs)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_summary(self):
""" summary constructs per-package metadata summary """
self.db.upload('pkg1-0.3.tar.gz', None, 'pkg1', '0.3')
self.db.upload('pkg1-1.1.tar.gz', None, 'pkg1', '1.1')
p1 = self.db.upload('pkg1a2.tar.gz', None, 'pkg1', '1.1.1a2')
p2 = self.db.upload('pkg2.tar.gz', None, 'pkg2', '0.1dev2')
summaries = self.db.summary()
self.assertItemsEqual(summaries, [
{
'name': 'pkg1',
'stable': '1.1',
'unstable': '1.1.1a2',
'last_modified': p1.last_modified,
},
{
'name': 'pkg2',
'stable': None,
'unstable': '0.1dev2',
'last_modified': p2.last_modified,
},
])
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, 'allow_overwrite', False):
name, version = 'a', '1'
path1 = 'old_package_path-1.tar.gz'
self.db.upload(path1, None, name, version)
path2 = 'new_path-1.whl'
self.db.upload(path2, None, name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
def test_reload_if_needed(self):
""" Reload the cache if it's empty """
self.db.storage = MagicMock()
self.db.storage.list.return_value = [make_package(factory=SQLPackage)]
self.db.reload_if_needed()
count = self.sql.query(SQLPackage).count()
self.assertEqual(count, 1)
