def rebuild_quotas(storage, dry_run=False):
for bucket in paginated(storage, collection_id='bucket',
parent_id='', sorting=[OLDEST_FIRST]):
bucket_id = bucket['id']
bucket_path = '/buckets/{}'.format(bucket['id'])
bucket_collection_count = 0
bucket_record_count = 0
bucket_storage_size = record_size(bucket)
for collection in paginated(storage, collection_id='collection',
parent_id=bucket_path, sorting=[OLDEST_FIRST]):
collection_info = rebuild_quotas_collection(storage, bucket_id, collection, dry_run)
(collection_record_count, collection_storage_size) = collection_info
bucket_collection_count += 1
bucket_record_count += collection_record_count
bucket_storage_size += collection_storage_size
bucket_record = {
'record_count': bucket_record_count,
'storage_size': bucket_storage_size,
'collection_count': bucket_collection_count,
}
if not dry_run:
storage.update(collection_id='quota', parent_id=bucket_path,
object_id=BUCKET_QUOTA_OBJECT_ID, record=bucket_record)
logger.info('Bucket {}. Final size: {} collections, {} records, {} bytes.'.format(
bucket_id, bucket_collection_count, bucket_record_count, bucket_storage_size))
def rebuild_quotas(storage, dry_run=False):
for bucket in paginated(storage, collection_id="bucket", parent_id="", sorting=[OLDEST_FIRST]):
bucket_id = bucket["id"]
bucket_path = "/buckets/{}".format(bucket["id"])
bucket_collection_count = 0
bucket_record_count = 0
bucket_storage_size = record_size(bucket)
for collection in paginated(
storage, collection_id="collection", parent_id=bucket_path, sorting=[OLDEST_FIRST]
):
collection_info = rebuild_quotas_collection(storage, bucket_id, collection, dry_run)
(collection_record_count, collection_storage_size) = collection_info
bucket_collection_count += 1
bucket_record_count += collection_record_count
bucket_storage_size += collection_storage_size
bucket_record = {
"record_count": bucket_record_count,
"storage_size": bucket_storage_size,
"collection_count": bucket_collection_count,
}
if not dry_run:
storage.update(
collection_id="quota",
parent_id=bucket_path,
object_id=BUCKET_QUOTA_OBJECT_ID,
record=bucket_record,
)
logger.info(
"Bucket {}. Final size: {} collections, {} records, {} bytes.".format(
bucket_id, bucket_collection_count, bucket_record_count, bucket_storage_size
)
)
def rebuild_quotas(storage, dry_run=False):
for bucket in paginated(storage, resource_name="bucket", parent_id="", sorting=[OLDEST_FIRST]):
bucket_id = bucket["id"]
bucket_path = f"/buckets/{bucket['id']}"
bucket_collection_count = 0
bucket_record_count = 0
bucket_storage_size = record_size(bucket)
for collection in paginated(
storage, resource_name="collection", parent_id=bucket_path, sorting=[OLDEST_FIRST]
):
collection_info = rebuild_quotas_collection(storage, bucket_id, collection, dry_run)
(collection_record_count, collection_storage_size) = collection_info
bucket_collection_count += 1
bucket_record_count += collection_record_count
bucket_storage_size += collection_storage_size
bucket_record = {
"record_count": bucket_record_count,
"storage_size": bucket_storage_size,
"collection_count": bucket_collection_count,
}
if not dry_run:
storage.update(
resource_name="quota",
parent_id=bucket_path,
object_id=BUCKET_QUOTA_OBJECT_ID,
obj=bucket_record,
)
logger.info(
f"Bucket {bucket_id}. Final size: {bucket_collection_count} collections, {bucket_record_count} records, {bucket_storage_size} bytes."
)
def rebuild_quotas_collection(storage, bucket_id, collection, dry_run=False):
"""Helper method for rebuild_quotas that updates a single collection."""
collection_id = collection["id"]
collection_record_count = 0
collection_storage_size = record_size(collection)
collection_path = "/buckets/{}/collections/{}".format(bucket_id, collection_id)
for record in paginated(
storage, collection_id="record", parent_id=collection_path, sorting=[OLDEST_FIRST]
):
collection_record_count += 1
collection_storage_size += record_size(record)
logger.info(
"Bucket {}, collection {}. Final size: {} records, {} bytes.".format(
bucket_id, collection_id, collection_record_count, collection_storage_size
)
)
new_quota_info = {
"record_count": collection_record_count,
"storage_size": collection_storage_size,
}
if not dry_run:
storage.update(
collection_id="quota",
parent_id=collection_path,
object_id=COLLECTION_QUOTA_OBJECT_ID,
record=new_quota_info,
)
return (collection_record_count, collection_storage_size)
def rebuild_quotas_collection(storage, bucket_id, collection, dry_run=False):
"""Helper method for rebuild_quotas that updates a single collection."""
collection_id = collection["id"]
collection_record_count = 0
collection_storage_size = record_size(collection)
collection_path = "/buckets/{}/collections/{}".format(
bucket_id, collection_id)
for record in paginated(storage,
collection_id="record",
parent_id=collection_path,
sorting=[OLDEST_FIRST]):
collection_record_count += 1
collection_storage_size += record_size(record)
logger.info(
"Bucket {}, collection {}. Final size: {} records, {} bytes.".format(
bucket_id, collection_id, collection_record_count,
collection_storage_size))
new_quota_info = {
"record_count": collection_record_count,
"storage_size": collection_storage_size,
}
if not dry_run:
storage.update(
collection_id="quota",
parent_id=collection_path,
object_id=COLLECTION_QUOTA_OBJECT_ID,
record=new_quota_info,
)
return (collection_record_count, collection_storage_size)
def test_paginated_fetches_next_page(self):
records = self.sample_records
records.reverse()
def get_all_mock(*args, **kwargs):
this_records = records[:3]
del records[:3]
return this_records, len(this_records)
self.storage.get_all.side_effect = get_all_mock
list(paginated(self.storage, sorting=[Sort('id', -1)]))
assert self.storage.get_all.call_args_list == [
mock.call(sorting=[Sort('id', -1)],
limit=25,
pagination_rules=None),
mock.call(
sorting=[Sort('id', -1)],
limit=25,
pagination_rules=[[Filter('id', 'record-03', COMPARISON.LT)]]),
mock.call(
sorting=[Sort('id', -1)],
limit=25,
pagination_rules=[[Filter('id', 'record-01', COMPARISON.LT)]]),
]
def test_paginated_fetches_next_page(self):
objects = self.sample_objects
objects.reverse()
def list_all_mock(*args, **kwargs):
this_objects = objects[:3]
del objects[:3]
return this_objects
self.storage.list_all.side_effect = list_all_mock
list(paginated(self.storage, sorting=[Sort("id", -1)]))
assert self.storage.list_all.call_args_list == [
mock.call(sorting=[Sort("id", -1)],
limit=25,
pagination_rules=None),
mock.call(
sorting=[Sort("id", -1)],
limit=25,
pagination_rules=[[Filter("id", "object-03", COMPARISON.LT)]],
),
mock.call(
sorting=[Sort("id", -1)],
limit=25,
pagination_rules=[[Filter("id", "object-01", COMPARISON.LT)]],
),
]
def rebuild_quotas_collection(storage, bucket_id, collection, dry_run=False):
"""Helper method for rebuild_quotas that updates a single collection."""
collection_id = collection["id"]
collection_record_count = 0
collection_storage_size = record_size(collection)
collection_uri = f"/buckets/{bucket_id}/collections/{collection_id}"
for record in paginated(storage,
resource_name="record",
parent_id=collection_uri,
sorting=[OLDEST_FIRST]):
collection_record_count += 1
collection_storage_size += record_size(record)
logger.info(
f"Bucket {bucket_id}, collection {collection_id}. Final size: {collection_record_count} records, {collection_storage_size} bytes."
)
new_quota_info = {
"record_count": collection_record_count,
"storage_size": collection_storage_size,
}
if not dry_run:
storage.update(
resource_name="quota",
parent_id=collection_uri,
object_id=COLLECTION_QUOTA_OBJECT_ID,
obj=new_quota_info,
)
return (collection_record_count, collection_storage_size)
def test_paginated_fetches_next_page(self):
records = self.sample_records
records.reverse()
def get_all_mock(*args, **kwargs):
this_records = records[:3]
del records[:3]
return this_records, len(this_records)
self.storage.get_all.side_effect = get_all_mock
list(paginated(self.storage, sorting=[Sort("id", -1)]))
assert self.storage.get_all.call_args_list == [
mock.call(sorting=[Sort("id", -1)], limit=25, pagination_rules=None),
mock.call(
sorting=[Sort("id", -1)],
limit=25,
pagination_rules=[[Filter("id", "record-03", COMPARISON.LT)]],
),
mock.call(
sorting=[Sort("id", -1)],
limit=25,
pagination_rules=[[Filter("id", "record-01", COMPARISON.LT)]],
),
]
def test_paginated_yields_records(self):
iter = paginated(self.storage, sorting=[Sort('id', -1)])
assert next(iter) == {"id": "record-01", "flavor": "strawberry"}
def test_paginated_passes_batch_size(self):
i = paginated(self.storage, sorting=[Sort('id', -1)], batch_size=17)
next(i) # make the generator do anything
self.storage.get_all.assert_called_with(sorting=[Sort('id', -1)],
limit=17,
pagination_rules=None)
def test_paginated_yields_objects(self):
iter = paginated(self.storage, sorting=[Sort("id", -1)])
assert next(iter) == {"id": "object-01", "flavor": "strawberry"}
def test_paginated_passes_sort(self):
i = paginated(self.storage, sorting=[Sort("id", -1)])
next(i) # make the generator do anything
self.storage.list_all.assert_called_with(sorting=[Sort("id", -1)],
limit=25,
pagination_rules=None)
def test_paginated_yields_records(self):
iter = paginated(self.storage, sorting=[Sort("id", -1)])
assert next(iter) == {"id": "record-01", "flavor": "strawberry"}
def test_paginated_passes_batch_size(self):
i = paginated(self.storage, sorting=[Sort("id", -1)], batch_size=17)
next(i) # make the generator do anything
self.storage.get_all.assert_called_with(
sorting=[Sort("id", -1)], limit=17, pagination_rules=None
)
def test_paginated_passes_sort(self):
i = paginated(self.storage, sorting=[Sort('id', -1)])
next(i) # make the generator do anything
self.storage.get_all.assert_called_with(sorting=[Sort('id', -1)],
limit=25, pagination_rules=None)