def setUpClass(cls):
super(TestDynamoCache, cls).setUpClass()
host = cls.dynamo.host[cls.dynamo.host.index('//') + 2:]
host, port = host.split(':')
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.host': host,
'db.port': port,
'db.namespace': 'test.',
'db.access_key': '',
'db.secret_key': '',
}
cls.kwargs = DynamoCache.configure(settings)
cls.engine = cls.kwargs['engine']
def setUpClass(cls):
super(TestDynamoCache, cls).setUpClass()
host = cls.dynamo.host[cls.dynamo.host.index('//') + 2:]
host, port = host.split(':')
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.host': host,
'db.port': port,
'db.namespace': 'test.',
'db.access_key': '',
'db.secret_key': '',
}
cls.kwargs = DynamoCache.configure(settings)
cls.engine = cls.kwargs['engine']
def setUpClass(cls):
super(TestDynamoCache, cls).setUpClass()
host = cls.dynamo.host[cls.dynamo.host.index("//") + 2 :]
host, port = host.split(":")
settings = {
"pypi.storage": "tests.DummyStorage",
"db.host": host,
"db.port": port,
"db.namespace": "test",
"db.access_key": "",
"db.secret_key": "",
}
cls.kwargs = DynamoCache.configure(settings)
cls.engine = cls.kwargs["engine"]
def setUpClass(cls):
super(TestDynamoCache, cls).setUpClass()
host = cls.dynamo.host[cls.dynamo.host.index("//") + 2:]
host, port = host.split(":")
settings = {
"pypi.storage": "tests.DummyStorage",
"db.region_name": "us-east-1",
"db.host": host,
"db.port": port,
"db.namespace": "test.",
"db.aws_access_key_id": "",
"db.aws_secret_access_key": "",
}
cls.kwargs = DynamoCache.configure(settings)
cls.engine = cls.kwargs["engine"]
def setUpClass(cls):
super(TestDynamoCache, cls).setUpClass()
host = cls.dynamo.host[cls.dynamo.host.index("//") + 2 :]
host, port = host.split(":")
settings = {
"pypi.storage": "tests.DummyStorage",
"db.region_name": "us-east-1",
"db.host": host,
"db.port": port,
"db.namespace": "test.",
"db.aws_access_key_id": "",
"db.aws_secret_access_key": "",
"db.graceful_reload": True,
}
cls.kwargs = DynamoCache.configure(settings)
cls.engine = cls.kwargs["engine"]
def setUpClass(cls):
super(TestDynamoCache, cls).setUpClass()
host = cls.dynamo.host[cls.dynamo.host.index('//') + 2:]
host, port = host.split(':')
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.region_name': 'us-east-1',
'db.host': host,
'db.port': port,
'db.namespace': 'test.',
'db.aws_access_key_id': '',
'db.aws_secret_access_key': '',
'db.graceful_reload': True,
}
cls.kwargs = DynamoCache.configure(settings)
cls.engine = cls.kwargs['engine']
def setUp(self):
super(TestDynamoCache, self).setUp()
self.db = DynamoCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
class TestDynamoCache(unittest.TestCase):
""" Tests for the DynamoCache """
dynamo = None
@classmethod
def setUpClass(cls):
super(TestDynamoCache, cls).setUpClass()
host = cls.dynamo.host[cls.dynamo.host.index("//") + 2:]
host, port = host.split(":")
settings = {
"pypi.storage": "tests.DummyStorage",
"db.region_name": "us-east-1",
"db.host": host,
"db.port": port,
"db.namespace": "test.",
"db.aws_access_key_id": "",
"db.aws_secret_access_key": "",
}
cls.kwargs = DynamoCache.configure(settings)
cls.engine = cls.kwargs["engine"]
@classmethod
def tearDownClass(cls):
super(TestDynamoCache, cls).tearDownClass()
cls.engine.delete_schema()
def setUp(self):
super(TestDynamoCache, self).setUp()
self.db = DynamoCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestDynamoCache, self).tearDown()
for model in (DynamoPackage, PackageSummary):
self.engine.scan(model).delete()
def _save_pkgs(self, *pkgs):
""" Save a DynamoPackage to the db """
for pkg in pkgs:
self.engine.save(pkg)
summary = PackageSummary(pkg)
self.engine.save(summary, overwrite=True)
def test_upload(self):
""" upload() saves package and uploads to storage """
pkg = make_package(factory=DynamoPackage)
self.db.upload(pkg.filename, BytesIO(b"test1234"), pkg.name,
pkg.version)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.engine.scan(DynamoPackage).first()
self.assertEqual(saved_pkg, pkg)
self.storage.upload.assert_called_with(pkg, ANY)
def test_save(self):
""" save() puts object into database """
pkg = make_package(factory=DynamoPackage)
self.db.save(pkg)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.engine.scan(DynamoPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package(factory=DynamoPackage)
self._save_pkgs(pkg)
self.db.delete(pkg)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 0)
count = self.engine.scan(PackageSummary).count()
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package(factory=DynamoPackage)
self._save_pkgs(pkg)
self.db.delete(pkg)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 0)
count = self.engine.scan(PackageSummary).count()
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=DynamoPackage),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=DynamoPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, keys)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package(factory=DynamoPackage)
self._save_pkgs(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch("missing_pkg-1.2.tar.gz")
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=DynamoPackage),
make_package(version="1.3",
filename="mypath3",
factory=DynamoPackage),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
saved_pkgs = self.db.all("mypkg")
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=DynamoPackage),
make_package(version="1.3",
filename="mypath3",
factory=DynamoPackage),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_search_or(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=DynamoPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=DynamoPackage,
),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=DynamoPackage),
make_package("package", factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
criteria = {"name": ["mypkg"], "summary": ["mypkg"]}
packages = self.db.search(criteria, "or")
self.assertItemsEqual(packages, pkgs[:-1])
def test_search_and(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=DynamoPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=DynamoPackage,
),
make_package("mypkg2",
"1.3.4",
"my/other/path",
factory=DynamoPackage),
make_package("package", factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
criteria = {"name": ["my", "pkg"], "summary": ["this", "mypkg"]}
packages = self.db.search(criteria, "and")
self.assertItemsEqual(packages, pkgs[:-1])
def test_summary(self):
""" summary constructs per-package metadata summary """
self.db.upload("pkg1-0.3a2.tar.gz", BytesIO(b"test1234"), "pkg1",
"0.3a2")
self.db.upload("pkg1-1.1.tar.gz", BytesIO(b"test1234"), "pkg1", "1.1")
p1 = self.db.upload("pkg1a2.tar.gz", BytesIO(b"test1234"), "pkg1",
"1.1.1a2", "summary")
p2 = self.db.upload("pkg2.tar.gz", BytesIO(b"test1234"), "pkg2",
"0.1dev2", "summary")
summaries = self.db.summary()
self.assertItemsEqual(
summaries,
[
{
"name": "pkg1",
"summary": "summary",
"last_modified": p1.last_modified.replace(tzinfo=UTC),
},
{
"name": "pkg2",
"summary": "summary",
"last_modified": p2.last_modified.replace(tzinfo=UTC),
},
],
)
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, "allow_overwrite", False):
name, version = "a", "1"
path1 = "old_package_path-1.tar.gz"
self.db.upload(path1, BytesIO(b"test1234"), name, version)
path2 = "new_path-1.whl"
self.db.upload(path2, BytesIO(b"test1234"), name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
def test_clear_all_keep_throughput(self):
""" Calling clear_all will keep same table throughput """
throughput = {}
for model in (DynamoPackage, PackageSummary):
tablename = model.meta_.ddb_tablename(self.engine.namespace)
desc = self.dynamo.describe_table(tablename)
self.dynamo.update_table(desc.name, Throughput(7, 7))
for index in desc.global_indexes:
self.dynamo.update_table(
desc.name, global_indexes={index.name: Throughput(7, 7)})
self.db.clear_all()
for model in (DynamoPackage, PackageSummary):
tablename = model.meta_.ddb_tablename(self.engine.namespace)
desc = self.dynamo.describe_table(tablename)
self.assertEqual(desc.throughput.read, 7)
self.assertEqual(desc.throughput.write, 7)
for index in desc.global_indexes:
self.assertEqual(index.throughput.read, 7)
self.assertEqual(index.throughput.write, 7)
def test_upload_no_summary(self):
""" upload() saves package even when there is no summary """
pkg = make_package(factory=DynamoPackage)
self.db.upload(pkg.filename,
BytesIO(b"test1234"),
pkg.name,
pkg.version,
summary="")
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.engine.scan(DynamoPackage).first()
self.assertEqual(saved_pkg, pkg)
self.storage.upload.assert_called_with(pkg, ANY)
def test_check_health_success(self):
""" check_health returns True for good connection """
ok, msg = self.db.check_health()
self.assertTrue(ok)
def test_check_health_fail(self):
""" check_health returns False for bad connection """
dbmock = self.db.engine = MagicMock()
def throw(*_, **__):
""" Throw an exception """
raise Exception("DB exception")
dbmock.scan.side_effect = throw
ok, msg = self.db.check_health()
self.assertFalse(ok)
class TestDynamoCache(unittest.TestCase):
""" Tests for the DynamoCache """
dynamo = None
@classmethod
def setUpClass(cls):
super(TestDynamoCache, cls).setUpClass()
host = cls.dynamo.host[cls.dynamo.host.index("//") + 2:]
host, port = host.split(":")
settings = {
"pypi.storage": "tests.DummyStorage",
"db.region_name": "us-east-1",
"db.host": host,
"db.port": port,
"db.namespace": "test.",
"db.aws_access_key_id": "",
"db.aws_secret_access_key": "",
"db.graceful_reload": True,
}
cls.kwargs = DynamoCache.configure(settings)
cls.engine = cls.kwargs["engine"]
@classmethod
def tearDownClass(cls):
super(TestDynamoCache, cls).tearDownClass()
cls.engine.delete_schema()
def setUp(self):
super(TestDynamoCache, self).setUp()
self.db = DynamoCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestDynamoCache, self).tearDown()
for model in (DynamoPackage, PackageSummary):
self.engine.scan(model).delete()
def _save_pkgs(self, *pkgs):
""" Save a DynamoPackage to the db """
for pkg in pkgs:
self.engine.save(pkg)
summary = PackageSummary(pkg)
self.engine.save(summary, overwrite=True)
def test_add_missing(self):
""" Add missing packages to cache """
keys = [make_package(factory=DynamoPackage)]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, keys)
all_summaries = self.engine.scan(PackageSummary).all()
self.assertEqual(len(all_summaries), 1)
def test_remove_extra(self):
""" Remove extra packages from cache """
keys = [
make_package(factory=DynamoPackage),
make_package("mypkg2", "1.3.4", factory=DynamoPackage),
]
self.db.save(keys[0])
self.db.save(keys[1])
self.storage.list.return_value = keys[:1]
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, keys[:1])
# It should have removed the summary as well
self.assertEqual(self.engine.scan(PackageSummary).count(), 1)
def test_remove_extra_leave_concurrent(self):
""" Removing extra packages will leave packages that were uploaded concurrently """
pkgs = [
make_package(factory=DynamoPackage),
make_package("mypkg2", factory=DynamoPackage),
]
self.db.save(pkgs[0])
self.db.save(pkgs[1])
# Return first pkgs[1], then pkgs[1:] because the second time we list
# we will have "uploaded" pkgs[2]
return_values = [lambda: pkgs[1:2], lambda: pkgs[1:]]
def list_storage(factory):
""" mocked method for listing storage packages """
# The first time we list from storage, concurrently "upload"
# pkgs[2]
if len(return_values) == 2:
pkg = make_package("mypkg3", factory=DynamoPackage)
pkgs.append(pkg)
self.db.save(pkg)
return return_values.pop(0)()
self.storage.list.side_effect = list_storage
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, pkgs[1:])
self.assertEqual(self.engine.scan(PackageSummary).count(), 2)
def test_remove_extra_concurrent_deletes(self):
""" Remove packages from cache that were concurrently deleted """
pkgs = [
make_package(factory=DynamoPackage),
make_package("mypkg2", factory=DynamoPackage),
]
self.db.save(pkgs[0])
# Return first pkgs[:], then pkgs[:1] because the second time we list
# we will have "deleted" pkgs[1]
return_values = [pkgs[:], pkgs[:1]]
self.storage.list.side_effect = lambda _: return_values.pop(0)
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, pkgs[:1])
self.assertEqual(self.engine.scan(PackageSummary).count(), 1)
def test_add_missing_more_recent(self):
""" If we sync a more recent package, update the summary """
pkgs = [
make_package(
last_modified=utcnow() - timedelta(hours=1),
factory=DynamoPackage,
),
make_package(version="1.5", factory=DynamoPackage),
]
self.db.save(pkgs[0])
self.storage.list.return_value = pkgs
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, pkgs)
summaries = self.db.summary()
self.assertEqual(len(summaries), 1)
summary = summaries[0]
self.assertEqual(summary["last_modified"], pkgs[1].last_modified)
def test_same_package_name_version(self):
""" Storage can have packages with the same name and version (different filename) """
pkgs = [
make_package(filename="mypkg-1.1-win32.whl",
factory=DynamoPackage),
make_package(filename="mypkg-1.1-macosx.whl",
factory=DynamoPackage),
make_package(filename="mypkg-1.1-x86_64.whl",
factory=DynamoPackage),
]
self.storage.list.return_value = pkgs
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertCountEqual(all_pkgs, pkgs)
summaries = self.db.summary()
self.assertEqual(len(summaries), 1)
class TestDynamoCache(unittest.TestCase):
""" Tests for the DynamoCache """
dynamo = None
@classmethod
def setUpClass(cls):
super(TestDynamoCache, cls).setUpClass()
host = cls.dynamo.host[cls.dynamo.host.index('//') + 2:]
host, port = host.split(':')
settings = {
'pypi.storage': 'tests.DummyStorage',
'db.region_name': 'us-east-1',
'db.host': host,
'db.port': port,
'db.namespace': 'test.',
'db.aws_access_key_id': '',
'db.aws_secret_access_key': '',
}
cls.kwargs = DynamoCache.configure(settings)
cls.engine = cls.kwargs['engine']
@classmethod
def tearDownClass(cls):
super(TestDynamoCache, cls).tearDownClass()
cls.engine.delete_schema()
def setUp(self):
super(TestDynamoCache, self).setUp()
self.db = DynamoCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestDynamoCache, self).tearDown()
for model in (DynamoPackage, PackageSummary):
self.engine.scan(model).delete()
def _save_pkgs(self, *pkgs):
""" Save a DynamoPackage to the db """
for pkg in pkgs:
self.engine.save(pkg)
summary = PackageSummary(pkg)
self.engine.save(summary, overwrite=True)
def test_upload(self):
""" upload() saves package and uploads to storage """
pkg = make_package(factory=DynamoPackage)
self.db.upload(pkg.filename, None, pkg.name, pkg.version)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.engine.scan(DynamoPackage).first()
self.assertEqual(saved_pkg, pkg)
self.storage.upload.assert_called_with(pkg, None)
def test_save(self):
""" save() puts object into database """
pkg = make_package(factory=DynamoPackage)
self.db.save(pkg)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.engine.scan(DynamoPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package(factory=DynamoPackage)
self._save_pkgs(pkg)
self.db.delete(pkg)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 0)
count = self.engine.scan(PackageSummary).count()
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package(factory=DynamoPackage)
self._save_pkgs(pkg)
self.db.delete(pkg)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 0)
count = self.engine.scan(PackageSummary).count()
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=DynamoPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=DynamoPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, keys)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package(factory=DynamoPackage)
self._save_pkgs(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch('missing_pkg-1.2.tar.gz')
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=DynamoPackage),
make_package(version='1.3',
filename='mypath3',
factory=DynamoPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
saved_pkgs = self.db.all('mypkg')
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=DynamoPackage),
make_package(version='1.3',
filename='mypath3',
factory=DynamoPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_search_or(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=DynamoPackage),
make_package('somepackage',
version='1.3',
filename='mypath3',
summary='this is mypkg',
factory=DynamoPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=DynamoPackage),
make_package('package', factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
criteria = {'name': ['mypkg'], 'summary': ['mypkg']}
packages = self.db.search(criteria, 'or')
self.assertItemsEqual(packages, pkgs[:-1])
def test_search_and(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=DynamoPackage),
make_package('somepackage',
version='1.3',
filename='mypath3',
summary='this is mypkg',
factory=DynamoPackage),
make_package('mypkg2',
'1.3.4',
'my/other/path',
factory=DynamoPackage),
make_package('package', factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
criteria = {'name': ['my', 'pkg'], 'summary': ['this', 'mypkg']}
packages = self.db.search(criteria, 'and')
self.assertItemsEqual(packages, pkgs[:-1])
def test_summary(self):
""" summary constructs per-package metadata summary """
self.db.upload('pkg1-0.3a2.tar.gz', None, 'pkg1', '0.3a2')
self.db.upload('pkg1-1.1.tar.gz', None, 'pkg1', '1.1')
p1 = self.db.upload('pkg1a2.tar.gz', None, 'pkg1', '1.1.1a2',
'summary')
p2 = self.db.upload('pkg2.tar.gz', None, 'pkg2', '0.1dev2', 'summary')
summaries = self.db.summary()
self.assertItemsEqual(summaries, [
{
'name': 'pkg1',
'summary': 'summary',
'last_modified': p1.last_modified.replace(tzinfo=UTC),
},
{
'name': 'pkg2',
'summary': 'summary',
'last_modified': p2.last_modified.replace(tzinfo=UTC),
},
])
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, 'allow_overwrite', False):
name, version = 'a', '1'
path1 = 'old_package_path-1.tar.gz'
self.db.upload(path1, None, name, version)
path2 = 'new_path-1.whl'
self.db.upload(path2, None, name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
def test_clear_all_keep_throughput(self):
""" Calling clear_all will keep same table throughput """
throughput = {}
for model in (DynamoPackage, PackageSummary):
tablename = model.meta_.ddb_tablename(self.engine.namespace)
desc = self.dynamo.describe_table(tablename)
self.dynamo.update_table(desc.name, Throughput(7, 7))
for index in desc.global_indexes:
self.dynamo.update_table(
desc.name, global_indexes={index.name: Throughput(7, 7)})
self.db.clear_all()
for model in (DynamoPackage, PackageSummary):
tablename = model.meta_.ddb_tablename(self.engine.namespace)
desc = self.dynamo.describe_table(tablename)
self.assertEqual(desc.throughput.read, 7)
self.assertEqual(desc.throughput.write, 7)
for index in desc.global_indexes:
self.assertEqual(index.throughput.read, 7)
self.assertEqual(index.throughput.write, 7)
def setUp(self):
super(TestDynamoCache, self).setUp()
self.db = DynamoCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
class TestDynamoCache(unittest.TestCase):
""" Tests for the DynamoCache """
dynamo = None
@classmethod
def setUpClass(cls):
super(TestDynamoCache, cls).setUpClass()
host = cls.dynamo.host[cls.dynamo.host.index("//") + 2 :]
host, port = host.split(":")
settings = {
"pypi.storage": "tests.DummyStorage",
"db.host": host,
"db.port": port,
"db.namespace": "test",
"db.access_key": "",
"db.secret_key": "",
}
cls.kwargs = DynamoCache.configure(settings)
cls.engine = cls.kwargs["engine"]
@classmethod
def tearDownClass(cls):
super(TestDynamoCache, cls).tearDownClass()
cls.engine.delete_schema()
def setUp(self):
super(TestDynamoCache, self).setUp()
self.db = DynamoCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestDynamoCache, self).tearDown()
for model in (DynamoPackage, PackageSummary):
self.engine.scan(model).delete()
def _save_pkgs(self, *pkgs):
""" Save a DynamoPackage to the db """
for pkg in pkgs:
self.engine.save(pkg)
summary = self.engine.get(PackageSummary, name=pkg.name) or PackageSummary(pkg)
summary.update_with(pkg)
self.engine.sync(summary)
def test_upload(self):
""" upload() saves package and uploads to storage """
pkg = make_package(factory=DynamoPackage)
self.db.upload(pkg.filename, None, pkg.name, pkg.version)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.engine.scan(DynamoPackage).first()
self.assertEqual(saved_pkg, pkg)
self.storage.upload.assert_called_with(pkg, None)
def test_save(self):
""" save() puts object into database """
pkg = make_package(factory=DynamoPackage)
self.db.save(pkg)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.engine.scan(DynamoPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package(factory=DynamoPackage)
self._save_pkgs(pkg)
self.db.delete(pkg)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 0)
count = self.engine.scan(PackageSummary).count()
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package(factory=DynamoPackage)
self._save_pkgs(pkg)
self.db.delete(pkg)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 0)
count = self.engine.scan(PackageSummary).count()
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=DynamoPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=DynamoPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, keys)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package(factory=DynamoPackage)
self._save_pkgs(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch("missing_pkg-1.2.tar.gz")
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=DynamoPackage),
make_package(version="1.3", filename="mypath3", factory=DynamoPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
saved_pkgs = self.db.all("mypkg")
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=DynamoPackage),
make_package(version="1.3", filename="mypath3", factory=DynamoPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
saved_pkgs = self.db.distinct()
print saved_pkgs
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_summary(self):
""" summary constructs per-package metadata summary """
self.db.upload("pkg1-0.3a2.tar.gz", None, "pkg1", "0.3a2")
self.db.upload("pkg1-1.1.tar.gz", None, "pkg1", "1.1")
p1 = self.db.upload("pkg1a2.tar.gz", None, "pkg1", "1.1.1a2")
p2 = self.db.upload("pkg2.tar.gz", None, "pkg2", "0.1dev2")
summaries = self.db.summary()
self.assertItemsEqual(
summaries,
[
{"name": "pkg1", "stable": "1.1", "unstable": "1.1.1a2", "last_modified": p1.last_modified},
{"name": "pkg2", "stable": None, "unstable": "0.1dev2", "last_modified": p2.last_modified},
],
)
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, "allow_overwrite", False):
name, version = "a", "1"
path1 = "old_package_path-1.tar.gz"
self.db.upload(path1, None, name, version)
path2 = "new_path-1.whl"
self.db.upload(path2, None, name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
def test_clear_all_keep_throughput(self):
""" Calling clear_all will keep same table throughput """
throughput = {}
for model in (DynamoPackage, PackageSummary):
tablename = model.meta_.ddb_tablename(self.engine.namespace)
desc = self.dynamo.describe_table(tablename)
self.dynamo.update_table(desc.name, Throughput(7, 7))
for index in desc.global_indexes:
self.dynamo.update_table(desc.name, global_indexes={index.name: Throughput(7, 7)})
self.db.clear_all()
for model in (DynamoPackage, PackageSummary):
tablename = model.meta_.ddb_tablename(self.engine.namespace)
desc = self.dynamo.describe_table(tablename)
self.assertEqual(desc.throughput.read, 7)
self.assertEqual(desc.throughput.write, 7)
for index in desc.global_indexes:
self.assertEqual(index.throughput.read, 7)
self.assertEqual(index.throughput.write, 7)
def test_update_wrong_summary(self):
""" Updating summary with wrong package doesn't blow up """
pkg1 = make_package("mypkg", "1.0", factory=DynamoPackage)
pkg2 = make_package("mypkg2", "1.3", factory=DynamoPackage)
summary = PackageSummary(pkg1)
summary.update_with(pkg2)
self.assertEqual(summary.stable, pkg1.version)
def test_delete_updates_summary(self):
""" Deleting a package updates the summary """
pkg1 = make_package("mypkg", "1.0", factory=DynamoPackage)
pkg2 = make_package("mypkg", "1.3", factory=DynamoPackage)
self._save_pkgs(pkg1, pkg2)
self.db.delete(pkg2)
summary = self.engine.scan(PackageSummary).first()
self.assertEqual(summary.stable, pkg1.version)
class TestDynamoCache(unittest.TestCase):
""" Tests for the DynamoCache """
dynamo = None
@classmethod
def setUpClass(cls):
super(TestDynamoCache, cls).setUpClass()
host = cls.dynamo.host[cls.dynamo.host.index("//") + 2 :]
host, port = host.split(":")
settings = {
"pypi.storage": "tests.DummyStorage",
"db.region_name": "us-east-1",
"db.host": host,
"db.port": port,
"db.namespace": "test.",
"db.aws_access_key_id": "",
"db.aws_secret_access_key": "",
"db.graceful_reload": True,
}
cls.kwargs = DynamoCache.configure(settings)
cls.engine = cls.kwargs["engine"]
@classmethod
def tearDownClass(cls):
super(TestDynamoCache, cls).tearDownClass()
cls.engine.delete_schema()
def setUp(self):
super(TestDynamoCache, self).setUp()
self.db = DynamoCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestDynamoCache, self).tearDown()
for model in (DynamoPackage, PackageSummary):
self.engine.scan(model).delete()
def _save_pkgs(self, *pkgs):
""" Save a DynamoPackage to the db """
for pkg in pkgs:
self.engine.save(pkg)
summary = PackageSummary(pkg)
self.engine.save(summary, overwrite=True)
def test_add_missing(self):
""" Add missing packages to cache """
keys = [make_package(factory=DynamoPackage)]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, keys)
all_summaries = self.engine.scan(PackageSummary).all()
self.assertEqual(len(all_summaries), 1)
def test_remove_extra(self):
""" Remove extra packages from cache """
keys = [
make_package(factory=DynamoPackage),
make_package("mypkg2", "1.3.4", factory=DynamoPackage),
]
self.db.save(keys[0])
self.db.save(keys[1])
self.storage.list.return_value = keys[:1]
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, keys[:1])
# It should have removed the summary as well
self.assertEqual(self.engine.scan(PackageSummary).count(), 1)
def test_remove_extra_leave_concurrent(self):
""" Removing extra packages will leave packages that were uploaded concurrently """
pkgs = [
make_package(factory=DynamoPackage),
make_package("mypkg2", factory=DynamoPackage),
]
self.db.save(pkgs[0])
self.db.save(pkgs[1])
# Return first pkgs[1], then pkgs[1:] because the second time we list
# we will have "uploaded" pkgs[2]
return_values = [lambda: pkgs[1:2], lambda: pkgs[1:]]
def list_storage(package_class):
""" mocked method for listing storage packages """
# The first time we list from storage, concurrently "upload"
# pkgs[2]
if len(return_values) == 2:
pkg = make_package("mypkg3", factory=DynamoPackage)
pkgs.append(pkg)
self.db.save(pkg)
return return_values.pop(0)()
self.storage.list.side_effect = list_storage
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, pkgs[1:])
self.assertEqual(self.engine.scan(PackageSummary).count(), 2)
def test_remove_extra_concurrent_deletes(self):
""" Remove packages from cache that were concurrently deleted """
pkgs = [
make_package(factory=DynamoPackage),
make_package("mypkg2", factory=DynamoPackage),
]
self.db.save(pkgs[0])
# Return first pkgs[:], then pkgs[:1] because the second time we list
# we will have "deleted" pkgs[1]
return_values = [pkgs[:], pkgs[:1]]
self.storage.list.side_effect = lambda _: return_values.pop(0)
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, pkgs[:1])
self.assertEqual(self.engine.scan(PackageSummary).count(), 1)
def test_add_missing_more_recent(self):
""" If we sync a more recent package, update the summary """
pkgs = [
make_package(
last_modified=datetime.utcnow() - timedelta(hours=1),
factory=DynamoPackage,
),
make_package(version="1.5", factory=DynamoPackage),
]
self.db.save(pkgs[0])
self.storage.list.return_value = pkgs
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, pkgs)
summaries = self.db.summary()
self.assertEqual(len(summaries), 1)
summary = summaries[0]
self.assertEqual(summary["last_modified"], pkgs[1].last_modified)
class TestDynamoCache(unittest.TestCase):
""" Tests for the DynamoCache """
dynamo = None
@classmethod
def setUpClass(cls):
super(TestDynamoCache, cls).setUpClass()
host = cls.dynamo.host[cls.dynamo.host.index("//") + 2 :]
host, port = host.split(":")
settings = {
"pypi.storage": "tests.DummyStorage",
"db.region_name": "us-east-1",
"db.host": host,
"db.port": port,
"db.namespace": "test.",
"db.aws_access_key_id": "",
"db.aws_secret_access_key": "",
}
cls.kwargs = DynamoCache.configure(settings)
cls.engine = cls.kwargs["engine"]
@classmethod
def tearDownClass(cls):
super(TestDynamoCache, cls).tearDownClass()
cls.engine.delete_schema()
def setUp(self):
super(TestDynamoCache, self).setUp()
self.db = DynamoCache(DummyRequest(), **self.kwargs)
self.storage = self.db.storage = MagicMock(spec=IStorage)
def tearDown(self):
super(TestDynamoCache, self).tearDown()
for model in (DynamoPackage, PackageSummary):
self.engine.scan(model).delete()
def _save_pkgs(self, *pkgs):
""" Save a DynamoPackage to the db """
for pkg in pkgs:
self.engine.save(pkg)
summary = PackageSummary(pkg)
self.engine.save(summary, overwrite=True)
def test_upload(self):
""" upload() saves package and uploads to storage """
pkg = make_package(factory=DynamoPackage)
self.db.upload(pkg.filename, None, pkg.name, pkg.version)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.engine.scan(DynamoPackage).first()
self.assertEqual(saved_pkg, pkg)
self.storage.upload.assert_called_with(pkg, None)
def test_save(self):
""" save() puts object into database """
pkg = make_package(factory=DynamoPackage)
self.db.save(pkg)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.engine.scan(DynamoPackage).first()
self.assertEqual(saved_pkg, pkg)
def test_delete(self):
""" delete() removes object from database and deletes from storage """
pkg = make_package(factory=DynamoPackage)
self._save_pkgs(pkg)
self.db.delete(pkg)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 0)
count = self.engine.scan(PackageSummary).count()
self.assertEqual(count, 0)
self.storage.delete.assert_called_with(pkg)
def test_clear(self):
""" clear() removes object from database """
pkg = make_package(factory=DynamoPackage)
self._save_pkgs(pkg)
self.db.delete(pkg)
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 0)
count = self.engine.scan(PackageSummary).count()
self.assertEqual(count, 0)
def test_reload(self):
""" reload_from_storage() inserts packages into the database """
keys = [
make_package(factory=DynamoPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=DynamoPackage),
]
self.storage.list.return_value = keys
self.db.reload_from_storage()
all_pkgs = self.engine.scan(DynamoPackage).all()
self.assertItemsEqual(all_pkgs, keys)
def test_fetch(self):
""" fetch() retrieves a package from the database """
pkg = make_package(factory=DynamoPackage)
self._save_pkgs(pkg)
saved_pkg = self.db.fetch(pkg.filename)
self.assertEqual(saved_pkg, pkg)
def test_fetch_missing(self):
""" fetch() returns None if no package exists """
saved_pkg = self.db.fetch("missing_pkg-1.2.tar.gz")
self.assertIsNone(saved_pkg)
def test_all_versions(self):
""" all() returns all versions of a package """
pkgs = [
make_package(factory=DynamoPackage),
make_package(version="1.3", filename="mypath3", factory=DynamoPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
saved_pkgs = self.db.all("mypkg")
self.assertItemsEqual(saved_pkgs, pkgs[:2])
def test_distinct(self):
""" distinct() returns all unique package names """
pkgs = [
make_package(factory=DynamoPackage),
make_package(version="1.3", filename="mypath3", factory=DynamoPackage),
make_package("mypkg2", "1.3.4", "my/other/path", factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
saved_pkgs = self.db.distinct()
self.assertItemsEqual(saved_pkgs, set([p.name for p in pkgs]))
def test_search_or(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=DynamoPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=DynamoPackage,
),
make_package("mypkg2", "1.3.4", "my/other/path", factory=DynamoPackage),
make_package("package", factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
criteria = {"name": ["mypkg"], "summary": ["mypkg"]}
packages = self.db.search(criteria, "or")
self.assertItemsEqual(packages, pkgs[:-1])
def test_search_and(self):
""" search() returns packages that match the query """
pkgs = [
make_package(factory=DynamoPackage),
make_package(
"somepackage",
version="1.3",
filename="mypath3",
summary="this is mypkg",
factory=DynamoPackage,
),
make_package("mypkg2", "1.3.4", "my/other/path", factory=DynamoPackage),
make_package("package", factory=DynamoPackage),
]
self._save_pkgs(*pkgs)
criteria = {"name": ["my", "pkg"], "summary": ["this", "mypkg"]}
packages = self.db.search(criteria, "and")
self.assertItemsEqual(packages, pkgs[:-1])
def test_summary(self):
""" summary constructs per-package metadata summary """
self.db.upload("pkg1-0.3a2.tar.gz", None, "pkg1", "0.3a2")
self.db.upload("pkg1-1.1.tar.gz", None, "pkg1", "1.1")
p1 = self.db.upload("pkg1a2.tar.gz", None, "pkg1", "1.1.1a2", "summary")
p2 = self.db.upload("pkg2.tar.gz", None, "pkg2", "0.1dev2", "summary")
summaries = self.db.summary()
self.assertItemsEqual(
summaries,
[
{
"name": "pkg1",
"summary": "summary",
"last_modified": p1.last_modified.replace(tzinfo=UTC),
},
{
"name": "pkg2",
"summary": "summary",
"last_modified": p2.last_modified.replace(tzinfo=UTC),
},
],
)
def test_multiple_packages_same_version(self):
""" Can upload multiple packages that have the same version """
with patch.object(self.db, "allow_overwrite", False):
name, version = "a", "1"
path1 = "old_package_path-1.tar.gz"
self.db.upload(path1, None, name, version)
path2 = "new_path-1.whl"
self.db.upload(path2, None, name, version)
all_versions = self.db.all(name)
self.assertEqual(len(all_versions), 2)
def test_clear_all_keep_throughput(self):
""" Calling clear_all will keep same table throughput """
throughput = {}
for model in (DynamoPackage, PackageSummary):
tablename = model.meta_.ddb_tablename(self.engine.namespace)
desc = self.dynamo.describe_table(tablename)
self.dynamo.update_table(desc.name, Throughput(7, 7))
for index in desc.global_indexes:
self.dynamo.update_table(
desc.name, global_indexes={index.name: Throughput(7, 7)}
)
self.db.clear_all()
for model in (DynamoPackage, PackageSummary):
tablename = model.meta_.ddb_tablename(self.engine.namespace)
desc = self.dynamo.describe_table(tablename)
self.assertEqual(desc.throughput.read, 7)
self.assertEqual(desc.throughput.write, 7)
for index in desc.global_indexes:
self.assertEqual(index.throughput.read, 7)
self.assertEqual(index.throughput.write, 7)
def test_upload_no_summary(self):
""" upload() saves package even when there is no summary """
pkg = make_package(factory=DynamoPackage)
self.db.upload(pkg.filename, None, pkg.name, pkg.version, summary="")
count = self.engine.scan(DynamoPackage).count()
self.assertEqual(count, 1)
saved_pkg = self.engine.scan(DynamoPackage).first()
self.assertEqual(saved_pkg, pkg)
self.storage.upload.assert_called_with(pkg, None)
def test_check_health_success(self):
""" check_health returns True for good connection """
ok, msg = self.db.check_health()
self.assertTrue(ok)
def test_check_health_fail(self):
""" check_health returns False for bad connection """
dbmock = self.db.engine = MagicMock()
def throw(*_, **__):
""" Throw an exception """
raise Exception("DB exception")
dbmock.scan.side_effect = throw
ok, msg = self.db.check_health()
self.assertFalse(ok)
