def queue_upload_events_for_reprocessing(events, use_kinesis=False):
if settings.ENV_AWS or use_kinesis:
from hsreplaynet.utils.aws.streams import fill_stream_from_iterable
iterable = _generate_raw_uploads_from_events(events)
publisher_func = aws.publish_raw_upload_to_processing_stream
stream_name = settings.KINESIS_UPLOAD_PROCESSING_STREAM_NAME
fill_stream_from_iterable(stream_name, iterable, publisher_func)
else:
for event in events:
logger.info("Processing UploadEvent %r locally", event)
event.process()
def queue_upload_events_for_reprocessing(events, use_kinesis=False):
if settings.ENV_AWS or use_kinesis:
from hsreplaynet.utils.aws.streams import fill_stream_from_iterable
iterable = _generate_raw_uploads_from_events(events)
publisher_func = aws.publish_raw_upload_batch_to_processing_stream
stream_name = settings.KINESIS_UPLOAD_PROCESSING_STREAM_NAME
fill_stream_from_iterable(stream_name, iterable, publisher_func)
else:
for event in events:
logger.info("Processing UploadEvent %r locally", event)
event.process()
def queue_raw_uploads_for_processing(attempt_reprocessing, limit=None):
"""
Queue all raw logs to attempt processing them into UploadEvents.
The primary use for this is for when we deploy code. The intended deploy process is:
- Notify S3 to suspend triggering lambda upon log upload
- Perform the Deploy
- Notify S3 to resume triggering lambda upon log upload
- Invoke this function to queue for processing any logs uploaded during the deploy
This method is not intended to requeue uploads that have previously failed.
For that see the requeue_failed_* family of methods.
"""
from hsreplaynet.utils.aws.streams import fill_stream_from_iterable
logger.info("Starting - Queue all raw uploads for processing")
publisher_func = aws.publish_raw_upload_to_processing_stream
iterable = generate_raw_uploads_for_processing(attempt_reprocessing, limit)
stream_name = settings.KINESIS_UPLOAD_PROCESSING_STREAM_NAME
fill_stream_from_iterable(stream_name, iterable, publisher_func)
def queue_raw_uploads_for_processing(attempt_reprocessing, limit=None):
"""
Queue all raw logs to attempt processing them into UploadEvents.
The primary use for this is for when we deploy code. The intended deploy process is:
- Notify S3 to suspend triggering lambda upon log upload
- Perform the Deploy
- Notify S3 to resume triggering lambda upon log upload
- Invoke this function to queue for processing any logs uploaded during the deploy
This method is not intended to requeue uploads that have previously failed.
For that see the requeue_failed_* family of methods.
"""
from hsreplaynet.utils.aws.streams import fill_stream_from_iterable
logger.info("Starting - Queue all raw uploads for processing")
publisher_func = aws.publish_raw_upload_batch_to_processing_stream
iterable = generate_raw_uploads_for_processing(attempt_reprocessing, limit)
stream_name = settings.KINESIS_UPLOAD_PROCESSING_STREAM_NAME
fill_stream_from_iterable(stream_name, iterable, publisher_func)
