def generate_raw_uploads_for_processing(attempt_reprocessing, limit=None):
count = 0
for object in aws.list_all_objects_in(settings.S3_RAW_LOG_UPLOAD_BUCKET, prefix="raw"):
key = object["Key"]
if key.endswith(".log"): # Don't queue the descriptor files, just the .logs
raw_upload = RawUpload(settings.S3_RAW_LOG_UPLOAD_BUCKET, key)
raw_upload.attempt_reprocessing = attempt_reprocessing
yield raw_upload
count += 1
if limit and limit >= count:
return
def generate_raw_uploads_for_processing(attempt_reprocessing, limit=None):
count = 0
for object in aws.list_all_objects_in(settings.S3_RAW_LOG_UPLOAD_BUCKET, prefix="raw"):
key = object["Key"]
if key.endswith(".log"): # Don't queue the descriptor files, just the .logs
raw_upload = RawUpload(settings.S3_RAW_LOG_UPLOAD_BUCKET, key)
raw_upload.attempt_reprocessing = attempt_reprocessing
yield raw_upload
count += 1
if limit and limit >= count:
return
