def test_clear_cache(self):
"""Test cache clearing"""
FailedCache.upload_cache.append("failed1")
FailedCache.upload_cache.append("failed2")
assert "failed1" in FailedCache.upload_cache
assert "failed2" in FailedCache.upload_cache
FailedCache.clear_cache(FailedCache.upload_cache)
assert FailedCache.upload_cache == []
async def process_message(message):
"""Message procession logic"""
try:
msg_dict = json.loads(message.value.decode('utf-8'))
# Can't use FailedCache.process_failed_cache here because it's tied
# to ThreadExecutor. So do it the asyncio way
if FailedCache.upload_cache:
cache = FailedCache.upload_cache
LOGGER.info("Start processing %d failed uploads", len(cache))
for msg in cache:
LOGGER.info("Processing failed upload: %s", str(msg))
await process_upload_or_re_evaluate(msg)
LOGGER.info("Cleared failed cache")
FailedCache.clear_cache(cache)
except json.decoder.JSONDecodeError:
MESSAGE_PARSE_ERROR.inc()
LOGGER.exception("Unable to parse message: ")
return
if message.topic in CFG.evaluator_topics:
if 'type' not in msg_dict:
LOGGER.error("Received message is missing type field: %s",
msg_dict)
return
if msg_dict['type'] in ['upload_new_file', 're-evaluate_system']:
await process_upload_or_re_evaluate(msg_dict)
if msg_dict['type'] == 'upload_new_file':
send_msg_to_payload_tracker(
PAYLOAD_TRACKER_PRODUCER,
msg_dict,
'processing',
status_msg='Scheduled for evaluation',
loop=MAIN_LOOP)
else:
UNKNOWN_MSG.inc()
LOGGER.error("Received unknown message type: %s", msg_dict['type'])
else:
UNKNOWN_TOPIC.inc()
LOGGER.error("Received message on unsupported topic: %s",
message.topic)