def generate_raw_uploads_for_processing(attempt_reprocessing, limit=None):
count = 0
for object in aws.list_all_objects_in(settings.S3_RAW_LOG_UPLOAD_BUCKET, prefix="raw"):
key = object["Key"]
if key.endswith(".log"): # Don't queue the descriptor files, just the .logs
raw_upload = RawUpload(settings.S3_RAW_LOG_UPLOAD_BUCKET, key)
raw_upload.attempt_reprocessing = attempt_reprocessing
yield raw_upload
count += 1
if limit and limit >= count:
return
def _list_raw_uploads_by_prefix(prefix):
for object in aws.list_all_objects_in(settings.S3_RAW_LOG_UPLOAD_BUCKET, prefix=prefix):
key = object["Key"]
if key.endswith(".log"): # Just emit one message per power.log / canary.log
yield RawUpload(settings.S3_RAW_LOG_UPLOAD_BUCKET, key)
