async def flush_opens():
"""
Code called when flushing the opens currently stored in state.
This will fetch data from the State (without mutating it), format, attempt to POST
to the URL, and remove it on success.
"""
to_flush, number_being_flushed = open_state.get_records_to_pop(
number_of_records=MAX_BATCH_TO_FLUSH)
# Format data
formatted_to_flush = [{
'tracking_id': open_data.tracking_id,
'timestamp': open_data.timestamp.isoformat()
} for open_data in to_flush]
# Attempt to make request
try:
await post_request(url=OPEN_DATA_POST_URL, body=formatted_to_flush)
except RequestError:
# TODO we could implement a retry policy here.
LOG.getChild('flush_opens').exception('Failed to POST Opens')
return
# Remove the records from state on successful request
open_state.pop_records(number_to_pop=number_being_flushed)
async def monitor():
logger = LOG.getChild('monitor')
logger.info('Running Monitor')
awaiting_open_batch = len(open_state)
awaiting_click_batch = len(click_state)
# Opens
if awaiting_open_batch > 0:
logger.info('Batch of OpenData currently awaiting processing',
extra={'length': awaiting_open_batch})
if awaiting_open_batch > MAX_OPENS_ALLOWED_IN_STATE:
logger.error(
'Too many OpenData stuck in memory. Dropping all of them.',
extra={'length': awaiting_open_batch})
await open_state.flush_all()
# Clicks
if awaiting_click_batch > 0:
logger.info('Batch of ClickData currently awaiting processing',
extra={'length': awaiting_click_batch})
if awaiting_click_batch > MAX_CLICKS_ALLOWED_IN_STATE:
logger.error(
'Too many ClickData stuck in memory. Dropping all of them.',
extra={'length': awaiting_click_batch})
await click_state.flush_all()
