def slack_failed_notification(context):
"""
Define the callback to post on Slack if a failure is detected in the Workflow
set on 'on_failure_callback'
"""
operator = SlackAPIPostOperator(task_id='slack_failed_notification',
text=str(context['task_instance']) + '\n' +
'exception: ' + str(context['exception']),
token=Variable.get("slack_access_token"),
channel='#channel_name')
return operator.execute(context=context)
def notify(
text: str,
channel: str,
status: bool,
username: Optional[str] = "airflow-bot",
task_id: Optional[str] = "slack_notification",
attachments: Optional[Any] = None,
):
if status:
status_emoji = ":tada:"
else:
status_emoji = ":skull_and_crossbones:"
# place dict into a list
if isinstance(attachments, dict):
attachments = [attachments]
return SlackAPIPostOperator(
task_id=task_id,
username=username,
channel=channel,
token=SLACK_TOKEN,
text=f"{status_emoji} {text}",
attachments=attachments,
).execute()
def test_api_call_params_with_default_args(self, mock_hook):
test_slack_conn_id = 'test_slack_conn_id'
slack_api_post_operator = SlackAPIPostOperator(
task_id='slack',
username=self.test_username,
slack_conn_id=test_slack_conn_id,
)
slack_api_post_operator.execute()
expected_api_params = {
'channel': "#general",
'username': self.test_username,
'text': 'No message has been set.\n'
'Here is a cat video instead\n'
'https://www.youtube.com/watch?v=J---aiyznGQ',
'icon_url': "https://raw.githubusercontent.com/apache/"
"airflow/master/airflow/www/static/pin_100.png",
'attachments': '[]',
'blocks': '[]',
}
assert expected_api_params == slack_api_post_operator.api_params
def __construct_operator(self,
test_token,
test_slack_conn_id,
test_api_params=None):
return SlackAPIPostOperator(
task_id='slack',
username=self.test_username,
token=test_token,
slack_conn_id=test_slack_conn_id,
channel=self.test_channel,
text=self.test_text,
icon_url=self.test_icon_url,
attachments=self.test_attachments,
blocks=self.test_blocks,
api_params=test_api_params,
)
forex_processing = SparkSubmitOperator(
task_id='forex_processing',
conn_id='spark_conn',
application="/User/arunraja/airflow/dags/scripts/forex_processing.py",
verbose=False,
executor_cores=2,
num_executors=2,
executor_memory='256M',
driver_memory='1G'
)
sending_email_notification = EmailOperator(
task_id="sending_email",
to="*****@*****.**",
subject="forex_data_pipeline",
html_content="""
<h3>forex_data_pipeline succeeded</h3>
"""
)
sending_slack_notification = SlackAPIPostOperator(
task_id="sending_slack",
token='asdljfwiei3234kndoiff2309e',
username="******",
text="DAG forex_data_pipeline: DONE",
channel="#airflow-exploit"
)
is_forex_rates_available >> is_forex_currencies_file_available >> downloading_rates >> saving_rates >> creating_forex_rates_table >> forex_processing
forex_processing >> sending_email_notification >> sending_slack_notification
path='output/json',
tables=['ports'],
filenames={'ports': 'ports.json'},
dag=dag)
send_notification_email = EmailOperator(
task_id='send_notification_email',
to='*****@*****.**',
subject='Notification for unece_dag',
html_content='<h3>unece_data_pipeline executed successfully</h3>',
dag=dag)
send_notification_slack = SlackAPIPostOperator(
task_id='send_notification_slack',
username='******',
token=Variable.get('slack_api_key'),
text='unece_dag executed successfully',
channel='#automationcase',
dag=dag)
end_execution = DummyOperator(task_id='end_execution', dag=dag)
begin_execution >> check_ports_scraper_contracts >> scrape_unece_data
scrape_unece_data >> stage_to_mongodb
stage_to_mongodb >> transform_and_load_to_postgres
transform_and_load_to_postgres >> check_data_quality
check_data_quality >> [export_to_json, load_to_analytics_dw]
load_to_analytics_dw >> check_dw_data_quality
check_dw_data_quality >> [send_notification_email, send_notification_slack]
export_to_json >> [send_notification_email, send_notification_slack]
send_notification_email >> end_execution