def test_rebuild_quotas_doesnt_update_if_dry_run(self):
paginated_data = [
# get buckets
iter([{
"id": "bucket-1",
"last_modified": 10
}]),
# get collections for first bucket
iter([{
"id": "collection-1",
"last_modified": 100
}]),
# get records for first collection
iter([{
"id": "record-1",
"last_modified": 110
}]),
]
def paginated_mock(*args, **kwargs):
return paginated_data.pop(0)
with mock.patch('kinto.plugins.quotas.scripts.logger') as mocked:
with mock.patch('kinto.plugins.quotas.scripts.paginated',
side_effect=paginated_mock):
scripts.rebuild_quotas(self.storage, dry_run=True)
assert not self.storage.update.called
mocked.info.assert_any_call(
'Bucket bucket-1, collection collection-1. '
'Final size: 1 records, 78 bytes.')
mocked.info.assert_any_call(
'Bucket bucket-1. Final size: 1 collections, '
'1 records, 114 bytes.')
def rebuild_quotas(env, dry_run=False):
"""Administrative command to rebuild quota usage information.
This command recomputes the amount of space used by all
collections and all buckets and updates the quota records in the
storage backend to their correct values. This can be useful when
cleaning up after a bug like e.g.
https://github.com/Kinto/kinto/issues/1226.
"""
registry = env["registry"]
settings = registry.settings
readonly_mode = asbool(settings.get("readonly", False))
# FIXME: readonly_mode is not meant to be a "maintenance mode" but
# rather used with a database user that has read-only permissions.
# If we ever introduce a maintenance mode, we should maybe enforce
# it here.
if readonly_mode:
message = "Cannot rebuild quotas while in readonly mode."
logger.error(message)
return 41
if "kinto.plugins.quotas" not in settings["includes"]:
message = "Cannot rebuild quotas when quotas plugin is not installed."
logger.error(message)
return 42
quotas.rebuild_quotas(registry.storage, dry_run=dry_run)
current_transaction.commit()
return 0
def rebuild_quotas(env, dry_run=False):
"""Administrative command to rebuild quota usage information.
This command recomputes the amount of space used by all
collections and all buckets and updates the quota records in the
storage backend to their correct values. This can be useful when
cleaning up after a bug like e.g.
https://github.com/Kinto/kinto/issues/1226.
"""
registry = env["registry"]
settings = registry.settings
readonly_mode = asbool(settings.get("readonly", False))
# FIXME: readonly_mode is not meant to be a "maintenance mode" but
# rather used with a database user that has read-only permissions.
# If we ever introduce a maintenance mode, we should maybe enforce
# it here.
if readonly_mode:
message = "Cannot rebuild quotas while in readonly mode."
logger.error(message)
return 41
if "kinto.plugins.quotas" not in settings["includes"]:
message = "Cannot rebuild quotas when quotas plugin is not installed."
logger.error(message)
return 42
quotas.rebuild_quotas(registry.storage, dry_run=dry_run)
current_transaction.commit()
return 0
def test_rebuild_quotas_doesnt_update_if_dry_run(self):
paginated_data = [
# get buckets
iter([{"id": "bucket-1", "last_modified": 10}]),
# get collections for first bucket
iter([{"id": "collection-1", "last_modified": 100}]),
# get records for first collection
iter([{"id": "record-1", "last_modified": 110}]),
]
def paginated_mock(*args, **kwargs):
return paginated_data.pop(0)
with mock.patch("kinto.plugins.quotas.scripts.logger") as mocked:
with mock.patch("kinto.plugins.quotas.scripts.paginated", side_effect=paginated_mock):
scripts.rebuild_quotas(self.storage, dry_run=True)
assert not self.storage.update.called
mocked.info.assert_any_call(
"Bucket bucket-1, collection collection-1. " "Final size: 1 records, 78 bytes."
)
mocked.info.assert_any_call(
"Bucket bucket-1. Final size: 1 collections, " "1 records, 114 bytes."
)
def test_rebuild_quotas_updates_records(self):
paginated_data = [
# get buckets
iter([{
"id": "bucket-1",
"last_modified": 10
}]),
# get collections for first bucket
iter([{
"id": "collection-1",
"last_modified": 100
}, {
"id": "collection-2",
"last_modified": 200
}]),
# get records for first collection
iter([{
"id": "record-1",
"last_modified": 110
}]),
# get records for second collection
iter([{
"id": "record-1b",
"last_modified": 210
}]),
]
def paginated_mock(*args, **kwargs):
return paginated_data.pop(0)
with mock.patch(
'kinto.plugins.quotas.scripts.logger') as mocked_logger:
with mock.patch('kinto.plugins.quotas.scripts.paginated',
side_effect=paginated_mock) as mocked_paginated:
scripts.rebuild_quotas(self.storage)
mocked_paginated.assert_any_call(
self.storage,
collection_id='bucket',
parent_id='',
sorting=[self.OLDEST_FIRST],
)
mocked_paginated.assert_any_call(
self.storage,
collection_id='collection',
parent_id='/buckets/bucket-1',
sorting=[self.OLDEST_FIRST],
)
mocked_paginated.assert_any_call(
self.storage,
collection_id='record',
parent_id='/buckets/bucket-1/collections/collection-1',
sorting=[self.OLDEST_FIRST],
)
mocked_paginated.assert_any_call(
self.storage,
collection_id='record',
parent_id='/buckets/bucket-1/collections/collection-2',
sorting=[self.OLDEST_FIRST],
)
self.storage.update.assert_any_call(collection_id='quota',
parent_id='/buckets/bucket-1',
object_id='bucket_info',
record={
'record_count': 2,
'storage_size': 193,
'collection_count': 2
})
self.storage.update.assert_any_call(
collection_id='quota',
parent_id='/buckets/bucket-1/collections/collection-1',
object_id='collection_info',
record={
'record_count': 1,
'storage_size': 78
})
self.storage.update.assert_any_call(
collection_id='quota',
parent_id='/buckets/bucket-1/collections/collection-2',
object_id='collection_info',
record={
'record_count': 1,
'storage_size': 79
})
mocked_logger.info.assert_any_call(
'Bucket bucket-1, collection collection-1. '
'Final size: 1 records, 78 bytes.')
mocked_logger.info.assert_any_call(
'Bucket bucket-1, collection collection-2. '
'Final size: 1 records, 79 bytes.')
mocked_logger.info.assert_any_call(
'Bucket bucket-1. Final size: '
'2 collections, 2 records, 193 bytes.')
def test_rebuild_quotas_updates_records(self):
paginated_data = [
# get buckets
iter([{"id": "bucket-1", "last_modified": 10}]),
# get collections for first bucket
iter(
[
{"id": "collection-1", "last_modified": 100},
{"id": "collection-2", "last_modified": 200},
]
),
# get records for first collection
iter([{"id": "record-1", "last_modified": 110}]),
# get records for second collection
iter([{"id": "record-1b", "last_modified": 210}]),
]
def paginated_mock(*args, **kwargs):
return paginated_data.pop(0)
with mock.patch("kinto.plugins.quotas.scripts.logger") as mocked_logger:
with mock.patch(
"kinto.plugins.quotas.scripts.paginated", side_effect=paginated_mock
) as mocked_paginated:
scripts.rebuild_quotas(self.storage)
mocked_paginated.assert_any_call(
self.storage, resource_name="bucket", parent_id="", sorting=[self.OLDEST_FIRST]
)
mocked_paginated.assert_any_call(
self.storage,
resource_name="collection",
parent_id="/buckets/bucket-1",
sorting=[self.OLDEST_FIRST],
)
mocked_paginated.assert_any_call(
self.storage,
resource_name="record",
parent_id="/buckets/bucket-1/collections/collection-1",
sorting=[self.OLDEST_FIRST],
)
mocked_paginated.assert_any_call(
self.storage,
resource_name="record",
parent_id="/buckets/bucket-1/collections/collection-2",
sorting=[self.OLDEST_FIRST],
)
self.storage.update.assert_any_call(
resource_name="quota",
parent_id="/buckets/bucket-1",
object_id="bucket_info",
obj={"record_count": 2, "storage_size": 193, "collection_count": 2},
)
self.storage.update.assert_any_call(
resource_name="quota",
parent_id="/buckets/bucket-1/collections/collection-1",
object_id="collection_info",
obj={"record_count": 1, "storage_size": 78},
)
self.storage.update.assert_any_call(
resource_name="quota",
parent_id="/buckets/bucket-1/collections/collection-2",
object_id="collection_info",
obj={"record_count": 1, "storage_size": 79},
)
mocked_logger.info.assert_any_call(
"Bucket bucket-1, collection collection-1. " "Final size: 1 records, 78 bytes."
)
mocked_logger.info.assert_any_call(
"Bucket bucket-1, collection collection-2. " "Final size: 1 records, 79 bytes."
)
mocked_logger.info.assert_any_call(
"Bucket bucket-1. Final size: " "2 collections, 2 records, 193 bytes."
)
def test_rebuild_quotas_updates_records(self):
paginated_data = [
# get buckets
iter([{"id": "bucket-1", "last_modified": 10}]),
# get collections for first bucket
iter(
[
{"id": "collection-1", "last_modified": 100},
{"id": "collection-2", "last_modified": 200},
]
),
# get records for first collection
iter([{"id": "record-1", "last_modified": 110}]),
# get records for second collection
iter([{"id": "record-1b", "last_modified": 210}]),
]
def paginated_mock(*args, **kwargs):
return paginated_data.pop(0)
with mock.patch("kinto.plugins.quotas.scripts.logger") as mocked_logger:
with mock.patch(
"kinto.plugins.quotas.scripts.paginated", side_effect=paginated_mock
) as mocked_paginated:
scripts.rebuild_quotas(self.storage)
mocked_paginated.assert_any_call(
self.storage, collection_id="bucket", parent_id="", sorting=[self.OLDEST_FIRST]
)
mocked_paginated.assert_any_call(
self.storage,
collection_id="collection",
parent_id="/buckets/bucket-1",
sorting=[self.OLDEST_FIRST],
)
mocked_paginated.assert_any_call(
self.storage,
collection_id="record",
parent_id="/buckets/bucket-1/collections/collection-1",
sorting=[self.OLDEST_FIRST],
)
mocked_paginated.assert_any_call(
self.storage,
collection_id="record",
parent_id="/buckets/bucket-1/collections/collection-2",
sorting=[self.OLDEST_FIRST],
)
self.storage.update.assert_any_call(
collection_id="quota",
parent_id="/buckets/bucket-1",
object_id="bucket_info",
record={"record_count": 2, "storage_size": 193, "collection_count": 2},
)
self.storage.update.assert_any_call(
collection_id="quota",
parent_id="/buckets/bucket-1/collections/collection-1",
object_id="collection_info",
record={"record_count": 1, "storage_size": 78},
)
self.storage.update.assert_any_call(
collection_id="quota",
parent_id="/buckets/bucket-1/collections/collection-2",
object_id="collection_info",
record={"record_count": 1, "storage_size": 79},
)
mocked_logger.info.assert_any_call(
"Bucket bucket-1, collection collection-1. " "Final size: 1 records, 78 bytes."
)
mocked_logger.info.assert_any_call(
"Bucket bucket-1, collection collection-2. " "Final size: 1 records, 79 bytes."
)
mocked_logger.info.assert_any_call(
"Bucket bucket-1. Final size: " "2 collections, 2 records, 193 bytes."
)