def test_parse_results(records, res_status, res_lsd):
status, lsd, _ = parse_results(records)
assert status == res_status
assert lsd == res_lsd
def test_parse_results(records, res_status, res_lsd):
status, lsd, _ = parse_results(records)
assert status == res_status
assert lsd == res_lsd
def run(self):
"""Main entry point for the worker. Queries an SQS queue for messages
and performs the appropriate action on each message received.
Swallows all exceptions and logs them.
"""
queue_name = str(self._get_queue_name())
sqs = self._get_sqs_wrapper(queue_name, JSONMessage)
scanner_queue_name = str(self._get_scanner_queue_name())
scanner_sqs = self._get_sqs_wrapper(scanner_queue_name, JSONMessage)
dummy_message = {"message": "dummy"} # TODO: make this message meaningful
while(not self._stop_requested): # Loop forever while this variable is set.
try: # Main try-except
for msg in sqs.get_messages_from_queue():
msg_body = msg.get_body()
log({
"status": "new message",
"queue": queue_name,
"msg": msg_body,
})
results = None
final_status = JOBS_ETL_STATUS_ERROR
lsd = None
try:
self._update_scheduled_jobs_on_etl_start(msg_body)
# safe to delete message. if worker dies, scanner will resubmit
sqs.delete_message_from_queue(msg)
try:
# Execute etl
results, action_dict = self._process_msg(msg)
# Parse results
final_status, lsd, extra_info = \
parse_results(results, msg_body['end_date'])
if final_status != JOBS_ETL_STATUS_COMPLETE:
if action_dict['delete_requested']:
final_status = JOBS_ETL_STATUS_DELETED
elif action_dict['cancel_requested']:
final_status = JOBS_ETL_STATUS_CANCELLED
elif action_dict['pause_requested']:
final_status = JOBS_ETL_STATUS_PAUSED
log({
"status": "processed message OK",
"queue": queue_name,
"msg": msg_body,
"results": results,
"job status": final_status,
"last OK date": lsd,
})
except Exception:
final_status = JOBS_ETL_STATUS_ERROR
log_exception(
"Exception in processing msg from queue: " +
queue_name + " msg body:" + str(msg_body)
)
if final_status != JOBS_ETL_STATUS_DELETED:
self._update_scheduled_jobs_on_etl_complete(
msg_body, final_status, lsd
)
scanner_sqs.write_message_to_queue(dummy_message)
try:
self.emailer.mail_result(
final_status, msg_body, additional_info=extra_info
)
log(
"Sent emails to:" + str(msg_body['contact_emails'])
)
except Exception:
log_exception(
"Exception in sending emails of job:" +
str(msg_body)
)
except Exception:
log_exception(
"Failed to update scheduled jobs on etl"
" start/complete, msg body: " + str(msg_body)
)
except Exception: # end of main try-except
log_exception(
"Exception in fetching messages from queue:"
+ queue_name
)
# if sqs queue fails, throttle retry
time.sleep(sqs.get_wait_time())
if self._run_once:
break
self._stop_requested = False
def run(self):
"""Main entry point for the worker. Queries an SQS queue for messages
and performs the appropriate action on each message received.
Swallows all exceptions and logs them.
"""
queue_name = str(self._get_queue_name())
sqs = self._get_sqs_wrapper(queue_name, JSONMessage)
scanner_queue_name = str(self._get_scanner_queue_name())
scanner_sqs = self._get_sqs_wrapper(scanner_queue_name, JSONMessage)
dummy_message = {
"message": "dummy"
} # TODO: make this message meaningful
while (not self._stop_requested
): # Loop forever while this variable is set.
try: # Main try-except
for msg in sqs.get_messages_from_queue():
msg_body = msg.get_body()
log({
"status": "new message",
"queue": queue_name,
"msg": msg_body,
})
results = None
final_status = JOBS_ETL_STATUS_ERROR
lsd = None
try:
self._update_scheduled_jobs_on_etl_start(msg_body)
# safe to delete message. if worker dies, scanner will resubmit
sqs.delete_message_from_queue(msg)
try:
# Execute etl
results, action_dict = self._process_msg(msg)
# Parse results
final_status, lsd, extra_info = \
parse_results(results, msg_body['end_date'])
if final_status != JOBS_ETL_STATUS_COMPLETE:
if action_dict['delete_requested']:
final_status = JOBS_ETL_STATUS_DELETED
elif action_dict['cancel_requested']:
final_status = JOBS_ETL_STATUS_CANCELLED
elif action_dict['pause_requested']:
final_status = JOBS_ETL_STATUS_PAUSED
log({
"status": "processed message OK",
"queue": queue_name,
"msg": msg_body,
"results": results,
"job status": final_status,
"last OK date": lsd,
})
except Exception:
final_status = JOBS_ETL_STATUS_ERROR
log_exception(
"Exception in processing msg from queue: " +
queue_name + " msg body:" + str(msg_body))
if final_status != JOBS_ETL_STATUS_DELETED:
self._update_scheduled_jobs_on_etl_complete(
msg_body, final_status, lsd)
scanner_sqs.write_message_to_queue(dummy_message)
try:
self.emailer.mail_result(
final_status,
msg_body,
additional_info=extra_info)
log("Sent emails to:" +
str(msg_body['contact_emails']))
except Exception:
log_exception(
"Exception in sending emails of job:" +
str(msg_body))
except Exception:
log_exception("Failed to update scheduled jobs on etl"
" start/complete, msg body: " +
str(msg_body))
except Exception: # end of main try-except
log_exception("Exception in fetching messages from queue:" +
queue_name)
# if sqs queue fails, throttle retry
time.sleep(sqs.get_wait_time())
if self._run_once:
break
self._stop_requested = False