def send_email(to, subject, html_content, files=None, dryrun=False, cc=None, bcc=None, mime_subtype='mixed', **kwargs): """ Send an email with html content using sendgrid. To use this plugin: 0. include sendgrid subpackage as part of your Airflow installation, e.g., pip install airflow[sendgrid] 1. update [email] backend in airflow.cfg, i.e., [email] email_backend = airflow.contrib.utils.sendgrid.send_email 2. configure Sendgrid specific environment variables at all Airflow instances: SENDGRID_MAIL_FROM={your-mail-from} SENDGRID_API_KEY={your-sendgrid-api-key}. """ mail = Mail() mail.from_email = Email(os.environ.get('SENDGRID_MAIL_FROM')) mail.subject = subject # Add the recipient list of to emails. personalization = Personalization() to = get_email_address_list(to) for to_address in to: personalization.add_to(Email(to_address)) if cc: cc = get_email_address_list(cc) for cc_address in cc: personalization.add_cc(Email(cc_address)) if bcc: bcc = get_email_address_list(bcc) for bcc_address in bcc: personalization.add_bcc(Email(bcc_address)) # Add custom_args to personalization if present pers_custom_args = kwargs.get('personalization_custom_args', None) if isinstance(pers_custom_args, dict): for key in pers_custom_args.keys(): personalization.add_custom_arg(CustomArg(key, pers_custom_args[key])) mail.add_personalization(personalization) mail.add_content(Content('text/html', html_content)) categories = kwargs.get('categories', []) for cat in categories: mail.add_category(Category(cat)) # Add email attachment. for fname in files or []: basename = os.path.basename(fname) attachment = Attachment() with open(fname, "rb") as f: attachment.content = base64.b64encode(f.read()) attachment.type = mimetypes.guess_type(basename)[0] attachment.filename = basename attachment.disposition = "attachment" attachment.content_id = '<%s>' % basename mail.add_attachment(attachment) _post_sendgrid_mail(mail.get())
def execute(self, context): s3_hook = S3Hook(self.aws_conn_id) file_list = s3_hook.list_keys( bucket_name=self.s3_bucket, prefix=self.s3_key, delimiter='/' ) header = '' body = '' for file in file_list: if 'header' in file: header = s3_hook.read_key( key=file, bucket_name=self.s3_bucket ) else: body_text = s3_hook.read_key( key=file, bucket_name=self.s3_bucket ) body = body + body_text if header == '': contents = body else: contents = header + body SMTP_MAIL_FROM = configuration.conf.get('smtp', 'SMTP_MAIL_FROM') to = get_email_address_list(self.to) msg = MIMEMultipart(self.mime_subtype) msg['Subject'] = self.subject msg['From'] = SMTP_MAIL_FROM msg['To'] = ", ".join(self.to) recipients = self.to if self.cc: cc = get_email_address_list(self.cc) msg['CC'] = ", ".join(cc) recipients = recipients + cc if self.bcc: # don't add bcc in header bcc = get_email_address_list(self.bcc) recipients = recipients + bcc msg['Date'] = formatdate(localtime=True) mime_text = MIMEText(self.html_content, 'html', self.mime_charset) msg.attach(mime_text) filename = '{filename}{attachment_extension}'.format(filename=self.filename, attachment_extension=self.attachment_extension) attachment = MIMEText(contents, 'plain') attachment.add_header('Content-Disposition', 'attachment', filename=filename) msg.attach(attachment) send_MIME_email(SMTP_MAIL_FROM, recipients, msg)
def send_email(to, subject, html_content, files=None, dryrun=False, cc=None, bcc=None, mime_subtype='mixed', **kwargs): """ Send an email with html content using sendgrid. To use this plugin: 0. include sendgrid subpackage as part of your Airflow installation, e.g., pip install airflow[sendgrid] 1. update [email] backend in airflow.cfg, i.e., [email] email_backend = airflow.contrib.utils.sendgrid.send_email 2. configure Sendgrid specific environment variables at all Airflow instances: SENDGRID_MAIL_FROM={your-mail-from} SENDGRID_API_KEY={your-sendgrid-api-key}. """ mail = Mail() mail.from_email = Email(os.environ.get('SENDGRID_MAIL_FROM')) mail.subject = subject # Add the recipient list of to emails. personalization = Personalization() to = get_email_address_list(to) for to_address in to: personalization.add_to(Email(to_address)) if cc: cc = get_email_address_list(cc) for cc_address in cc: personalization.add_cc(Email(cc_address)) if bcc: bcc = get_email_address_list(bcc) for bcc_address in bcc: personalization.add_bcc(Email(bcc_address)) mail.add_personalization(personalization) mail.add_content(Content('text/html', html_content)) # Add custom_args to personalization if present pers_custom_args = kwargs.get('personalization_custom_args', None) if isinstance(pers_custom_args, dict): for key in pers_custom_args.keys(): personalization.add_custom_arg(CustomArg(key, pers_custom_args[key])) # Add email attachment. for fname in files or []: basename = os.path.basename(fname) attachment = Attachment() with open(fname, "rb") as f: attachment.content = base64.b64encode(f.read()) attachment.type = mimetypes.guess_type(basename)[0] attachment.filename = basename attachment.disposition = "attachment" attachment.content_id = '<%s>' % basename mail.add_attachment(attachment) _post_sendgrid_mail(mail.get())
def log_email_backend(to, subject, html_content, files=None, dryrun=False, cc=None, bcc=None, mime_subtype='mixed', mime_charset='utf-8', **kwargs): smtp_mail_from = configuration.conf.get('smtp', 'SMTP_MAIL_FROM') to = get_email_address_list(to) msg = MIMEMultipart(mime_subtype) msg['Subject'] = subject msg['From'] = smtp_mail_from msg['To'] = ", ".join(to) if cc: cc = get_email_address_list(cc) msg['CC'] = ", ".join(cc) if bcc: # don't add bcc in header bcc = get_email_address_list(bcc) msg['Date'] = formatdate(localtime=True) mime_text = MIMEText(None, 'plain', mime_charset) mime_text.replace_header('content-transfer-encoding', 'quoted-printable') mime_text.set_payload(html_content) msg.attach(mime_text) for fname in files or []: basename = os.path.basename(fname) with open(fname, "rb") as f: part = MIMEApplication(f.read(), Name=basename) part[ 'Content-Disposition'] = 'attachment; filename="%s"' % basename part['Content-ID'] = '<%s>' % basename msg.attach(part) logger = logging.getLogger("plugin.log_email_backend") logger.info("\n" + msg.as_string())
def log_email_backend(to, subject, html_content, files=None, dryrun=False, cc=None, bcc=None, mime_subtype='mixed', mime_charset='utf-8', **kwargs): smtp_mail_from = configuration.conf.get('smtp', 'SMTP_MAIL_FROM') to = get_email_address_list(to) msg = MIMEMultipart(mime_subtype) msg['Subject'] = subject msg['From'] = smtp_mail_from msg['To'] = ", ".join(to) if cc: cc = get_email_address_list(cc) msg['CC'] = ", ".join(cc) if bcc: # don't add bcc in header bcc = get_email_address_list(bcc) msg['Date'] = formatdate(localtime=True) mime_text = MIMEText(None, 'plain', mime_charset) mime_text.replace_header('content-transfer-encoding', 'quoted-printable') mime_text.set_payload(html_content) msg.attach(mime_text) for fname in files or []: basename = os.path.basename(fname) with open(fname, "rb") as f: part = MIMEApplication( f.read(), Name=basename ) part['Content-Disposition'] = 'attachment; filename="%s"' % basename part['Content-ID'] = '<%s>' % basename msg.attach(part) logger = logging.getLogger("plugin.log_email_backend") logger.info("\n" + msg.as_string())
def test_get_email_address_colon_sep_string(self): emails_string = '[email protected]; [email protected]' self.assertEqual( get_email_address_list(emails_string), EMAILS)
def test_get_email_address_invalid_type_in_iterable(self): emails_list = ['*****@*****.**', 2] with pytest.raises(TypeError): get_email_address_list(emails_list)
def test_get_email_address_invalid_type(self): emails_string = 1 with pytest.raises(TypeError): get_email_address_list(emails_string)
def test_get_email_address_list(self): emails_list = ['*****@*****.**', '*****@*****.**'] self.assertEqual( get_email_address_list(emails_list), EMAILS)
def send_email(to, subject, html_content, files=None, dryrun=False, cc=None, bcc=None, mime_subtype='mixed', sandbox_mode=False, **kwargs): """ Send an email with html content using sendgrid. To use this plugin: 0. include sendgrid subpackage as part of your Airflow installation, e.g., pip install 'apache-airflow[sendgrid]' 1. update [email] backend in airflow.cfg, i.e., [email] email_backend = airflow.contrib.utils.sendgrid.send_email 2. configure Sendgrid specific environment variables at all Airflow instances: SENDGRID_MAIL_FROM={your-mail-from} SENDGRID_API_KEY={your-sendgrid-api-key}. """ if files is None: files = [] mail = Mail() from_email = kwargs.get('from_email') or os.environ.get('SENDGRID_MAIL_FROM') from_name = kwargs.get('from_name') or os.environ.get('SENDGRID_MAIL_SENDER') mail.from_email = Email(from_email, from_name) mail.subject = subject mail.mail_settings = MailSettings() if sandbox_mode: mail.mail_settings.sandbox_mode = SandBoxMode(enable=True) # Add the recipient list of to emails. personalization = Personalization() to = get_email_address_list(to) for to_address in to: personalization.add_to(Email(to_address)) if cc: cc = get_email_address_list(cc) for cc_address in cc: personalization.add_cc(Email(cc_address)) if bcc: bcc = get_email_address_list(bcc) for bcc_address in bcc: personalization.add_bcc(Email(bcc_address)) # Add custom_args to personalization if present pers_custom_args = kwargs.get('personalization_custom_args', None) if isinstance(pers_custom_args, dict): for key in pers_custom_args.keys(): personalization.add_custom_arg(CustomArg(key, pers_custom_args[key])) mail.add_personalization(personalization) mail.add_content(Content('text/html', html_content)) categories = kwargs.get('categories', []) for cat in categories: mail.add_category(Category(cat)) # Add email attachment. for fname in files: basename = os.path.basename(fname) attachment = Attachment() attachment.type = mimetypes.guess_type(basename)[0] attachment.filename = basename attachment.disposition = "attachment" attachment.content_id = '<{0}>'.format(basename) with open(fname, "rb") as f: attachment.content = base64.b64encode(f.read()).decode('utf-8') mail.add_attachment(attachment) _post_sendgrid_mail(mail.get())
def test_get_email_address_single_email(self): emails_string = '*****@*****.**' assert get_email_address_list(emails_string) == [emails_string]
def test_get_email_address_tuple(self): emails_tuple = ('*****@*****.**', '*****@*****.**') self.assertEqual(get_email_address_list(emails_tuple), EMAILS)
def test_get_email_address_single_email(self): emails_string = '*****@*****.**' self.assertEqual(get_email_address_list(emails_string), [emails_string])
def send_email(to, subject, html_content, files=None, cc=None, bcc=None, sandbox_mode=False, **kwargs): """ Send an email with html content using `Sendgrid <https://sendgrid.com/>`__. To use this plugin: 0. Include ``sendgrid`` subpackage as part of your Airflow installation, e.g.,:: pip install 'apache-airflow[sendgrid]' 1. Update ``email_backend`` property in `[email]`` section in ``airflow.cfg``, i.e.,:: [email] email_backend = airflow.contrib.utils.sendgrid.send_email 2. Configure Sendgrid specific environment variables at all Airflow instances::: SENDGRID_MAIL_FROM={your-mail-from} SENDGRID_API_KEY={your-sendgrid-api-key}. """ if files is None: files = [] mail = Mail() from_email = kwargs.get('from_email') or os.environ.get( 'SENDGRID_MAIL_FROM') from_name = kwargs.get('from_name') or os.environ.get( 'SENDGRID_MAIL_SENDER') mail.from_email = Email(from_email, from_name) mail.subject = subject mail.mail_settings = MailSettings() if sandbox_mode: mail.mail_settings.sandbox_mode = SandBoxMode(enable=True) # Add the recipient list of to emails. personalization = Personalization() to = get_email_address_list(to) for to_address in to: personalization.add_to(Email(to_address)) if cc: cc = get_email_address_list(cc) for cc_address in cc: personalization.add_cc(Email(cc_address)) if bcc: bcc = get_email_address_list(bcc) for bcc_address in bcc: personalization.add_bcc(Email(bcc_address)) # Add custom_args to personalization if present pers_custom_args = kwargs.get('personalization_custom_args', None) if isinstance(pers_custom_args, dict): for key in pers_custom_args.keys(): personalization.add_custom_arg( CustomArg(key, pers_custom_args[key])) mail.add_personalization(personalization) mail.add_content(Content('text/html', html_content)) categories = kwargs.get('categories', []) for cat in categories: mail.add_category(Category(cat)) # Add email attachment. for fname in files: basename = os.path.basename(fname) with open(fname, "rb") as file: content = base64.b64encode(file.read()).decode('utf-8') attachment = Attachment(file_content=content, file_type=mimetypes.guess_type(basename)[0], file_name=basename, disposition="attachment", content_id=f"<{basename}>") mail.add_attachment(attachment) _post_sendgrid_mail(mail.get())
def test_get_email_address_list(self): emails_list = ['*****@*****.**', '*****@*****.**'] self.assertEqual( get_email_address_list(emails_list), EMAILS)
def send_email( # pylint: disable=too-many-locals to: AddressesType, subject: str, html_content: str, files: Optional[AddressesType] = None, cc: Optional[AddressesType] = None, bcc: Optional[AddressesType] = None, sandbox_mode: bool = False, conn_id: str = "sendgrid_default", **kwargs, ) -> None: """ Send an email with html content using `Sendgrid <https://sendgrid.com/>`__. .. note:: For more information, see :ref:`email-configuration-sendgrid` """ if files is None: files = [] mail = Mail() from_email = kwargs.get('from_email') or os.environ.get( 'SENDGRID_MAIL_FROM') from_name = kwargs.get('from_name') or os.environ.get( 'SENDGRID_MAIL_SENDER') mail.from_email = Email(from_email, from_name) mail.subject = subject mail.mail_settings = MailSettings() if sandbox_mode: mail.mail_settings.sandbox_mode = SandBoxMode(enable=True) # Add the recipient list of to emails. personalization = Personalization() to = get_email_address_list(to) for to_address in to: personalization.add_to(Email(to_address)) if cc: cc = get_email_address_list(cc) for cc_address in cc: personalization.add_cc(Email(cc_address)) if bcc: bcc = get_email_address_list(bcc) for bcc_address in bcc: personalization.add_bcc(Email(bcc_address)) # Add custom_args to personalization if present pers_custom_args = kwargs.get('personalization_custom_args') if isinstance(pers_custom_args, dict): for key in pers_custom_args.keys(): personalization.add_custom_arg( CustomArg(key, pers_custom_args[key])) mail.add_personalization(personalization) mail.add_content(Content('text/html', html_content)) categories = kwargs.get('categories', []) for cat in categories: mail.add_category(Category(cat)) # Add email attachment. for fname in files: basename = os.path.basename(fname) with open(fname, "rb") as file: content = base64.b64encode(file.read()).decode('utf-8') attachment = Attachment( file_content=content, file_type=mimetypes.guess_type(basename)[0], file_name=basename, disposition="attachment", content_id=f"<{basename}>", ) mail.add_attachment(attachment) _post_sendgrid_mail(mail.get(), conn_id)
def test_get_email_address_colon_sep_string(self): emails_string = '[email protected]; [email protected]' assert get_email_address_list(emails_string) == EMAILS
def test_get_email_address_colon_sep_string(self): emails_string = '[email protected]; [email protected]' self.assertEqual( get_email_address_list(emails_string), EMAILS)
def test_get_email_address_list(self): emails_list = ['*****@*****.**', '*****@*****.**'] assert get_email_address_list(emails_list) == EMAILS
def manage_slas(self, dag: DAG, session: Session = None) -> None: """ Finding all tasks that have SLAs defined, and sending alert emails where needed. New SLA misses are also recorded in the database. We are assuming that the scheduler runs often, so we only check for tasks that should have succeeded in the past hour. """ self.log.info("Running SLA Checks for %s", dag.dag_id) if not any(isinstance(ti.sla, timedelta) for ti in dag.tasks): self.log.info( "Skipping SLA check for %s because no tasks in DAG have SLAs", dag) return qry = (session.query( TI.task_id, func.max(TI.execution_date).label('max_ti')).with_hint( TI, 'USE INDEX (PRIMARY)', dialect_name='mysql').filter(TI.dag_id == dag.dag_id).filter( or_(TI.state == State.SUCCESS, TI.state == State.SKIPPED)).filter( TI.task_id.in_(dag.task_ids)).group_by( TI.task_id).subquery('sq')) max_tis: List[TI] = (session.query(TI).filter( TI.dag_id == dag.dag_id, TI.task_id == qry.c.task_id, TI.execution_date == qry.c.max_ti, ).all()) ts = timezone.utcnow() for ti in max_tis: task = dag.get_task(ti.task_id) if task.sla and not isinstance(task.sla, timedelta): raise TypeError( f"SLA is expected to be timedelta object, got " f"{type(task.sla)} in {task.dag_id}:{task.task_id}") dttm = dag.following_schedule(ti.execution_date) while dttm < timezone.utcnow(): following_schedule = dag.following_schedule(dttm) if following_schedule + task.sla < timezone.utcnow(): session.merge( SlaMiss(task_id=ti.task_id, dag_id=ti.dag_id, execution_date=dttm, timestamp=ts)) dttm = dag.following_schedule(dttm) session.commit() # pylint: disable=singleton-comparison slas: List[SlaMiss] = ( session.query(SlaMiss).filter(SlaMiss.notification_sent == False, SlaMiss.dag_id == dag.dag_id) # noqa .all()) # pylint: enable=singleton-comparison if slas: # pylint: disable=too-many-nested-blocks sla_dates: List[datetime.datetime] = [ sla.execution_date for sla in slas ] fetched_tis: List[TI] = (session.query(TI).filter( TI.state != State.SUCCESS, TI.execution_date.in_(sla_dates), TI.dag_id == dag.dag_id).all()) blocking_tis: List[TI] = [] for ti in fetched_tis: if ti.task_id in dag.task_ids: ti.task = dag.get_task(ti.task_id) blocking_tis.append(ti) else: session.delete(ti) session.commit() task_list = "\n".join(sla.task_id + ' on ' + sla.execution_date.isoformat() for sla in slas) blocking_task_list = "\n".join(ti.task_id + ' on ' + ti.execution_date.isoformat() for ti in blocking_tis) # Track whether email or any alert notification sent # We consider email or the alert callback as notifications email_sent = False notification_sent = False if dag.sla_miss_callback: # Execute the alert callback self.log.info('Calling SLA miss callback') try: dag.sla_miss_callback(dag, task_list, blocking_task_list, slas, blocking_tis) notification_sent = True except Exception: # pylint: disable=broad-except self.log.exception( "Could not call sla_miss_callback for DAG %s", dag.dag_id) email_content = f"""\ Here's a list of tasks that missed their SLAs: <pre><code>{task_list}\n<code></pre> Blocking tasks: <pre><code>{blocking_task_list}<code></pre> Airflow Webserver URL: {conf.get(section='webserver', key='base_url')} """ tasks_missed_sla = [] for sla in slas: try: task = dag.get_task(sla.task_id) except TaskNotFound: # task already deleted from DAG, skip it self.log.warning( "Task %s doesn't exist in DAG anymore, skipping SLA miss notification.", sla.task_id) continue tasks_missed_sla.append(task) emails: Set[str] = set() for task in tasks_missed_sla: if task.email: if isinstance(task.email, str): emails |= set(get_email_address_list(task.email)) elif isinstance(task.email, (list, tuple)): emails |= set(task.email) if emails: try: send_email(emails, f"[airflow] SLA miss on DAG={dag.dag_id}", email_content) email_sent = True notification_sent = True except Exception: # pylint: disable=broad-except Stats.incr('sla_email_notification_failure') self.log.exception( "Could not send SLA Miss email notification for DAG %s", dag.dag_id) # If we sent any notification, update the sla_miss table if notification_sent: for sla in slas: sla.email_sent = email_sent sla.notification_sent = True session.merge(sla) session.commit()
def test_get_email_address_tuple(self): emails_tuple = ('*****@*****.**', '*****@*****.**') assert get_email_address_list(emails_tuple) == EMAILS