def send_run_alert_mails():
"""
A Celery task that sends an email to the owner when a Spark job run has
failed and records a datetime when it was sent.
"""
with transaction.atomic():
failed_run_alerts = (
SparkJobRunAlert.objects.select_for_update().filter(
reason_code__in=Cluster.FAILED_STATE_CHANGE_REASON_LIST,
mail_sent_date__isnull=True,
).prefetch_related("run__spark_job__created_by"))
failed_jobs = []
for alert in failed_run_alerts:
with transaction.atomic():
failed_jobs.append(alert.run.spark_job.identifier)
message = mail_builder.build_message(
"atmo/jobs/mails/failed_run_alert.mail",
{
"alert": alert,
"settings": settings
},
)
message.send()
alert.mail_sent_date = timezone.now()
alert.save()
return failed_jobs
def send_expired_mails():
"""
A Celery task the send emails for when a Spark job as expired
(the end_date has passed) to the owner.
"""
expired_spark_jobs = SparkJob.objects.filter(expired_date__isnull=False, )
for spark_job in expired_spark_jobs:
message = mail_builder.build_message('atmo/jobs/mails/expired.mail', {
'settings': settings,
'spark_job': spark_job,
})
message.send()
def terminate_and_notify(self, spark_job):
"""
When the Spark job has timed out because it has run longer
than the maximum runtime we will terminate it (and its cluster)
and notify the owner to optimize the Spark job code.
"""
logger.debug(
'The last run of Spark job %s has not finished yet and timed out, '
'terminating it and notifying owner.',
spark_job,
)
spark_job.terminate()
message = mail_builder.build_message('atmo/jobs/mails/timed_out.mail',
{
'settings': settings,
'spark_job': spark_job,
})
message.send()
def send_expiration_mails():
"""Send expiration emails an hour before the cluster expires."""
deadline = timezone.now() + timedelta(hours=1)
with transaction.atomic():
soon_expired = (Cluster.objects.select_for_update().active().filter(
expires_at__lte=deadline, expiration_mail_sent=False))
for cluster in soon_expired:
with transaction.atomic():
message = mail_builder.build_message(
"atmo/clusters/mails/expiration.mail",
{
"cluster": cluster,
"deadline": deadline,
"settings": settings
},
)
message.send()
cluster.expiration_mail_sent = True
cluster.save()
def test_empty(self):
msg = build_message('empty.email')
self.assertEqual(msg.to, [])
def test_handle_template_exception(self):
msg = build_message('exception.email')
def test_html(self):
msg = build_message('html.email')
alts = [alt for alt, mtype in msg.alternatives if mtype == 'text/html']
self.assertEqual(len(alts), 1)
self.assertHTMLEqual(alts[0], '<h1> Welcome! </h1>')
def test_to(self):
msg = build_message('to.email')
self.assertEqual(msg.to, ['*****@*****.**'])
def test_subject(self):
msg = build_message('subject.email')
self.assertEqual(msg.subject, 'Test Subject')
def test_defaults(self):
msg = build_message('empty.email', from_email='*****@*****.**')
self.assertEqual(msg.to, [])
self.assertEqual(msg.from_email, '*****@*****.**')
def test_empty(self):
msg = build_message('empty.email')
self.assertEqual(msg.to, [])
def test_handle_template_exception(self):
msg = build_message('exception.email')
def test_html(self):
msg = build_message('html.email')
alts = [alt for alt, mtype in msg.alternatives if mtype == 'text/html']
self.assertEqual(len(alts), 1)
self.assertHTMLEqual(alts[0], '<h1> Welcome! </h1>')
def test_to(self):
msg = build_message('to.email')
self.assertEqual(msg.to, ['*****@*****.**'])
def test_subject(self):
msg = build_message('subject.email')
self.assertEqual(msg.subject, 'Test Subject')
def test_defaults(self):
msg = build_message('empty.email', from_email='*****@*****.**')
self.assertEqual(msg.to, [])
self.assertEqual(msg.from_email, '*****@*****.**')
