def slack_succeeded_task(context_dictionary, **kwargs):
"""
Connects to Slack and sends a success message with the table, the timestamp the task failed and params
:param context_dictionary: Airflow keyword arguments (see all here: https://airflow.apache.org/code.html#macros)
:return: Returns the execution for the passed function
"""
try:
from config.secret import token_slack
except ModuleNotFoundError:
logging.warning(
"There is no secret.py file inside dags/config/ with token_auth defined! No messages will be send to Slack"
)
return
# unpack context
s_dag = context_dictionary['dag'].dag_id
s_task = context_dictionary["task"]
succeed_msg = SlackAPIPostOperator(
task_id='Success',
channel="#airflow-test",
token=token_slack,
text=":green_heart: Completed task {} from DAG '{}'".format(
s_task, s_dag),
username="******",
owner='_owner',
)
return succeed_msg.execute()
def slack_failed_task(context_dictionary, **kwargs):
"""
Connects to Slack and sends a failed message with the table, the timestamp the task failed and params
:param context_dictionary: Airflow keyword arguments (see all here: https://airflow.apache.org/code.html#macros)
:return: Returns the execution for the failed function
"""
try:
from config.secret import token_slack
except ModuleNotFoundError:
logging.warning(
"There is no secret.py file inside dags/config/ with token_auth defined! No messages will be send to Slack"
)
return
# unpack context
s_dag = context_dictionary['dag'].dag_id
timestamp = context_dictionary['ts']
params = context_dictionary["params"]
s_task = context_dictionary["task"]
failed_alert = SlackAPIPostOperator(
task_id='Failure',
channel="#airflow-test",
token=token_slack,
text=":red_circle: DAG '{}'\nTask: '{}'\n Timestamp: {}\nParams '{}'".
format(s_dag, s_task, timestamp, params),
username="******",
owner='_owner',
)
return failed_alert.execute()
def send_to_slack_func(execution_date, **context):
operator = SlackAPIPostOperator(
task_id='postTopCommitter',
text=execution_date.strftime("%a")+": "+str(context['task_instance'].xcom_pull(task_ids='bq_fetch_data')),
token=Variable.get('slack_token'),
channel='general'
)
return operator.execute(context=context)
def slack_success_task(context):
success_alert = SlackAPIPostOperator(
task_id='slack_success',
channel="#data",
token="MySlackApi",
text=':red_circle: Airflow Task Success',
username='******',
)
return success_alert.execute(context=context)
def send_to_slack_func(**context):
lista = context.get('ti').xcom_pull(key='return_value', task_ids='bq_fetch_data')
lista = ['date: {}'.format(context.get('ds'))] + lista
operator = SlackAPIPostOperator(
task_id='slack_api',
text=str(lista),
token=Variable.get('token'),
channel=Variable.get("slack_channel")
)
return operator.execute(context=context)
def send_to_slack_func(**context):
operator = SlackAPIPostOperator(
task_id='send_to_slack',
text=context.get('ds') + " >> " +
str(context.get('ti').xcom_pull(key=None, task_ids='bq_fetch_data')),
token=Variable.get('slack_key'),
# todo: should be passed into a variable from Airflow
channel="general",
)
return operator.execute(context=context)
def send_to_slack_func(execution_date, **context):
slack_message = context['ti'].xcom_pull(task_ids='bq_fetch_data')
slack_operator = SlackAPIPostOperator(
task_id='slack_operator',
text=slack_message,
username="******",
token=Variable.get("slack_access_token"),
channel=Variable.get("slack_channel"))
slack_operator.execute(context=context)
def slack_success_task(contextDictionary, **kwargs):
success_alert = SlackAPIPostOperator(
task_id='slack_success',
channel="#{}".format(_slack_channel),
token= "{}".format(_slack_token),
text =
""":sunny: [ DAG : {} ]\nSucceeded on: {}
""".format(contextDictionary["dag"].dag_id,
datetime.utcnow().isoformat()+"Z"),
owner = 'Farah.C',)
return success_alert.execute()
def slack_failed_task(contextDictionary, **kwargs):
failed_alert = SlackAPIPostOperator(
task_id = 'slack_failed',
channel = "#{}".format(_slack_channel),
token = "{}".format(_slack_token),
text =
""":rain_cloud: [ DAG : {} - Task : {} ]\nFailed on : {}\n------------- Exception -------------\n{}""".format(
contextDictionary["dag"].dag_id,
contextDictionary["task"].task_id,
datetime.utcnow().isoformat()+"Z",
contextDictionary["exception"]),
owner = 'Farah.C',)
return failed_alert.execute()
def send_to_slack_func(**context):
task_id = 'top3_committers_airflow'
text = context['task_instance'].xcom_pull(task_ids='bq_fetch_data')
channel = "general"
operator = SlackAPIPostOperator(
task_id=task_id,
token=Variable.get("secret_token"),
channel=channel,
username="******",
text=text,
icon_url="https://media.makeameme.org/created/i-am-your-5c16f0.jpg")
return operator.execute(context=context)
def get_slack_notification(slack_notification_message):
logging.info("> started executing get_slack_notification() ")
try:
print("*************** > ",slack_notification_message)
requests.post(slack_token , json={"channel": slack_channel , "text": slack_notification_message })
slack_push_notification = SlackAPIPostOperator(
task_id='slack_notification_records',
channel= slack_channel,
token= slack_token,
username= slack_user ,
text="Hi Update from Airflow \n "+slack_notification_message,
dag=dag
)
slack_push_notification.execute()
return True
except Exception as e:
logging.error('ERROR: some issue with Slack configuration'.format(local_file_path, str(e)))
raise
def task_fail_slack_alert(context):
failed_alert = SlackAPIPostOperator(
task_id='slack_failed',
channel=slack_channel,
token=slack_token,
text="""
:red_circle: Task Failed.
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get('task_instance').task_id,
dag=context.get('task_instance').dag_id,
ti=context.get('task_instance'),
exec_date=context.get('execution_date'),
log_url=context.get('task_instance').log_url,
))
return failed_alert.execute(context=context)
def failure_slack_message(context):
slack_alert = SlackAPIPostOperator(
task_id = 'task_failure_slack_message',
channel = "#data_notifications",
token = slack_token,
username = slack_username
text = (
":heavy_exclamation_mark: *Daily Update* \n"
"*Date*: %s \n"
"*Alert*: `DAG: %s, Task: %s`"
% (
yesterday
, context.get('task_instance').dag_id
, context.get('task_instance').task_id
)
)
)
return(slack_alert.execute(context=context))
def __construct_operator(self, test_token, test_slack_conn_id, test_api_params=None):
return SlackAPIPostOperator(
task_id='slack',
username=self.test_username,
token=test_token,
slack_conn_id=test_slack_conn_id,
channel=self.test_channel,
text=self.test_text,
icon_url=self.test_icon_url,
attachments=self.test_attachments,
blocks=self.test_blocks,
api_params=test_api_params,
)
def test_api_call_params_with_default_args(self, mock_hook):
test_slack_conn_id = 'test_slack_conn_id'
slack_api_post_operator = SlackAPIPostOperator(
task_id='slack',
username=self.test_username,
slack_conn_id=test_slack_conn_id,
)
slack_api_post_operator.execute()
expected_api_params = {
'channel': "#general",
'username': self.test_username,
'text': 'No message has been set.\n'
'Here is a cat video instead\n'
'https://www.youtube.com/watch?v=J---aiyznGQ',
'icon_url': "https://raw.githubusercontent.com/apache/"
"airflow/master/airflow/www/static/pin_100.png",
'attachments': '[]',
'blocks': '[]',
}
self.assertEqual(expected_api_params,
slack_api_post_operator.api_params)
def send_to_slack_func(execution_date, **context):
ti = context['ti']
v1 = ti.xcom_pull(key=None, task_ids='bq_fetch_data')
print(type(v1))
res = []
for x in v1:
res.append("*" + x[0].encode('utf-8') + "*")
execution = execution_date.to_date_string()
execution_start = execution_date.subtract(days=7).to_date_string()
print(execution)
op = SlackAPIPostOperator(
task_id="slack_post",
text=", ".join(res) + " were _really_ active last week! " +
str(execution_start) + " - " + str(execution),
username="******",
icon_url=
"https://www.petmd.com/sites/default/files/Acute-Dog-Diarrhea-47066074.jpg",
token=Variable.get("token"),
dag=dag)
op.execute(context=context)
def failed(self, context):
self.conf = context["conf"]
self.task = context["task"]
self.execution_date = context["execution_date"]
self.dag = context["dag"]
self.errors = SlackAPIPostOperator(
task_id='task_failed',
token=Variable.get('slack_token'),
channel='C1SRU2R33',
text="Your DAG has encountered an error, please follow the link to view the log details: " + "http://localhost:8080/admin/airflow/log?" + "task_id=" + task.task_id + "&" +\
"execution_date=" + execution_date.isoformat() + "&" + "dag_id=" + dag.dag_id,
dag=pipeline
)
errors.execute()
def task_fail_slack_alert(context):
slack_msg = """
:red_circle: Task Failure
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get("task_instance").task_id,
dag=context.get("task_instance").dag_id,
ti=context.get("task_instance"),
exec_date=context.get("execution_date"),
log_url=context.get("task_instance").log_url,
)
notification = SlackAPIPostOperator(
task_id="successnotification",
channel="#bidata",
username="******",
text=slack_msg,
token=SLACK_TOKEN)
return notification.execute()
default_args = {
'owner': 'mikeghen',
'start_date': datetime(2017, 8, 11),
'depends_on_past': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
dag = DAG('sakila_main_tables',
default_args=default_args,
schedule_interval=timedelta(days=1))
slack_notify = SlackAPIPostOperator(
task_id='slack_notify',
token='xxxxxx',
channel='data-status',
username='******',
text='Successfully performed sakila ETL operation',
dag=dag)
for connection in sakila_connections:
for table in sakila_tables:
extract = MySqlToGoogleCloudStorageOperator(
task_id="extract_mysql_%s_%s" % (connection, table),
mysql_conn_id=connection,
google_cloud_storage_conn_id='gcp_test',
sql="SELECT *, '%s' as source FROM sakila.%s" %
(connection, table),
bucket='ghen-airflow',
filename="%s/%s/%s{}.json" % (connection, table, table),
schema_filename="%s/schemas/%s.json" % (connection, table),
)
update_read_history = PythonOperator(
task_id="update_read_history",
python_callable=process_metadata,
op_args=["write"],
provide_context=True,
)
generate_notification = PythonOperator(
task_id="yes_generate_notification",
python_callable=generate_message,
provide_context=True,
)
send_notification = SlackAPIPostOperator(
task_id="send_notification",
token="YOUR_SLACK_TOKEN",
channel="#comic-notification",
text="[{{ ds }}] 海賊王有新番了!",
icon_url="http://airbnb.io/img/projects/airflow3.png",
)
do_nothing = DummyOperator(task_id="no_do_nothing")
# define workflow
get_read_history >> check_comic_info >> decide_what_to_do
decide_what_to_do >> generate_notification
decide_what_to_do >> do_nothing
generate_notification >> send_notification >> update_read_history
provide_context=True)
decide_what_to_do = BranchPythonOperator(task_id='new_comic_available',
python_callable=decide_what_to_do,
provide_context=True)
update_read_history = PythonOperator(task_id='update_read_history',
python_callable=process_metadata,
op_args=['write'],
provide_context=True)
generate_notification = PythonOperator(task_id='yes_generate_notification',
python_callable=generate_message,
provide_context=True)
send_notification = SlackAPIPostOperator(
task_id='send_notification',
token=get_token(),
channel='#comic-notification',
text=get_message_text(),
icon_url='http://airbnb.io/img/projects/airflow3.png')
do_nothing = DummyOperator(task_id='no_do_nothing')
# define workflow
latest_only >> get_read_history
get_read_history >> check_comic_info >> decide_what_to_do
decide_what_to_do >> generate_notification
decide_what_to_do >> do_nothing
generate_notification >> send_notification >> update_read_history
from airflow.operators.slack_operator import SlackAPIPostOperator
from roi_preprocess import run_preprocess
from transform_positives import run_transform
default_args = {
'owner': 'hal',
'start_date': datetime(2018, 7, 17, 5),
}
dag = DAG(dag_id='roi_chunk_dag', default_args=default_args)
preprocess_op = PythonOperator(task_id='run_preprocess',
python_callable=run_preprocess,
dag=dag)
transform_op = PythonOperator(task_id='run_transform',
python_callable=run_transform,
dag=dag)
slack_confirmation = SlackAPIPostOperator(
task_id='slack_confirmation',
channel='i-utra',
username='******',
token=os.environ['SLACK_TOKEN'],
text=f'ROI chunk data processed + uploaded',
dag=dag,
)
preprocess_op >> transform_op >> slack_confirmation
cad DOUBLE
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
STORED AS TEXTFILE
"""
)
forex_processing = SparkSubmitOperator(
task_id="forex_processing",
conn_id="spark_conn",
application="/usr/local/airflow/dags/scripts/forex_processing.py",
verbose=False
)
sending_email_notification = EmailOperator(
task_id="sending_email",
to="*****@*****.**",
subject="forex_data_pipeline",
html_content="""
<h3>forex_data_pipeline succeeded</h3>
"""
)
sending_slack_notification = SlackAPIPostOperator(
task_id="sending_slack",
slack_conn_id="slack_conn",
username="******",
text="DAG forex_data_pipeline: DONE",
channel="#airflow-exploit"
)
nzd DOUBLE,
gbp DOUBLE,
jpy DOUBLE,
cad DOUBLE
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
STORED AS TEXTFILE
""")
forex_processing = SparkSubmitOperator(
task_id="forex_processing",
conn_id="spark_conn",
application="/usr/local/airflow/dags/scripts/forex_processing.py",
verbose=False,
)
sending_email_notification = EmailOperator(
to="*****@*****.**",
task_id="sending_email",
subject="forex_data_pipeline_subject",
html_content="<h3> some content forex data pipeline </h3>")
sending_slack_notification = SlackAPIPostOperator(
task_id="sending_slack",
token="xoxb-1858294503888-1902170231477-FvCA58i8jN0DDwzdt3iOYgCJ",
username="******",
text="DAG TEST",
channel="#flash-dance")
is_forex_rates_available >> is_forex_file_available >> downloading_rates >> saving_rates >> creating_forex_rates_table >> forex_processing >> sending_email_notification >> sending_slack_notification
'owner': 'luke',
'email': '*****@*****.**',
'start_date': datetime.datetime(2018, 7, 20, 2),
}
train_dag_id = 'train_model'
train_dag = DAG(dag_id='train_model',
description='Trains a ML model on gpu1708',
default_args=args,
schedule_interval=None,
max_active_runs=1)
run_bluenot_op = RunBluenotOperator(task_id='run_bluenot', dag=train_dag)
sense_complete_op = WebTrainerSensor(task_id='sense_complete',
fragment='bluenow',
dag=train_dag,
retries=3)
slack_confirm_op = SlackAPIPostOperator(
task_id='slack_confirmation',
channel='tests',
username='******',
token=os.environ['SLACK_TOKEN'],
text=f'DAG {train_dag_id} has finished',
dag=train_dag,
)
run_bluenot_op >> sense_complete_op
sense_complete_op >> slack_confirm_op
cur.execute(sql)
result = cur.fetchall()
return result[0][0]
result = get_vertica_status()
test_path = '/tmp/tmp_vertica_monitoring/fail.txt'
# если вертика жива и файлик существует
if result == '1':
if os.path.isfile(test_path):
alive_vertica = SlackAPIPostOperator(
dag=dag,
task_id='alive_vertica',
token=slack_token,
channel="#airflow",
username='******',
text='Vertica is alive :3',
icon_url=
'https://github.com/E7su/lapras/blob/master/images/vertica.png?raw=true'
)
os.remove(test_path) # удалить файлик
# если вертика жива и файл не существует
else:
# если вертика прилегла и файлик существует
if os.path.isfile(test_path):
pass
# если вертика прилегла и файлик НЕ существует
else:
open(test_path, "w") # создать файл
fail_vertica = SlackAPIPostOperator(
dag=dag,
FIELDS TERMINATED BY ','
STORED AS TEXTFILE
""")
forex_processing = SparkSubmitOperator(
task_id="forex_processing",
conn_id="spark_conn",
application="/usr/local/airflow/dags/scripts/forex_processing.py",
verbose=False)
sending_email_notification = EmailOperator(
task_id="sending_email_notification",
to="*****@*****.**",
subject="forex_data_pipeline",
html_content="<h3> forex_data_pipeline_succeeded </h3>")
sending_slack_notification = SlackAPIPostOperator(
task_id="sending_slack",
token="xoxb-1680149573351-1707585079553-IQ0poUalTn95fwoaquymkLDN",
username="******",
text="DAG forex_data_pipeline: DONE",
channel="#airflow-exploit")
# sending_slack_notification = SlackAPIPostOperator(
# task_id = "sending_slack_notification",
# token = "xoxb-1680149573351-1707585079553-IQ0poUalTn95fwoaquymkLDN",
# username = "******",
# text = "DAG forex_data_pipeline: DONE",
# channel = "#airflow-exploit"
# )
is_forex_rates_available >> forex_currencies_file_available >> downloading_rates >> saving_rates >> creating_forex_rates_table >> forex_processing >> sending_email_notification >> sending_slack_notification
dag = DAG(dag_id='mip_dag', default_args=default_args)
axial_to_coronal_and_sagittal_op = \
PythonOperator(task_id='axial_to_coronal_and_sagittal',
python_callable=axial_to_coronal_and_sagittal,
dag=dag)
normal_mip_op = PythonOperator(task_id='normal_mip',
python_callable=normal_mip,
dag=dag)
multichannel_mip_op = PythonOperator(task_id='multichannel_mip',
python_callable=multichannel_mip,
dag=dag)
overlap_mip_op = PythonOperator(task_id='overlap_mip',
python_callable=overlap_mip,
dag=dag)
slack_confirmation = SlackAPIPostOperator(
task_id='slack_confirmation',
channel='i-utra',
username='******',
token=os.environ['SLACK_TOKEN'],
text='Coronal and axial scans MIPed',
dag=dag,
)
axial_to_coronal_and_sagittal_op >> normal_mip_op >> \
multichannel_mip_op >> overlap_mip_op >> slack_confirmation
'gs://us-central1-hiroki-kurasawa-772758ea-bucket/temp',
'runner': 'DataflowRunner'
}
}
with airflow.DAG('asl_ml_pipeline',
'catchup=False',
default_args=default_args,
schedule_interval=datetime.timedelta(days=1)) as dag:
# Print the dag_run id from the Airflow logs
start = DummyOperator(task_id='start')
post_success_slack = SlackAPIPostOperator(
task_id='post-success-to-slack',
token=Variable.get('slack_access_token'),
text='Hello Airflow',
channel='#feed')
post__fail_slack = SlackAPIPostOperator(
task_id='post-fail-to-slack',
token=Variable.get('slack_access_token'),
trigger_rule=TriggerRule.ONE_FAILED,
text='Hello World!',
channel='#feed')
job_args = {'output': 'gs://dev-recommend/preprocess'}
data_flow = DataFlowPythonOperator(task_id='submit-job-data-flow',
py_file=DATAFLOW_FILE,
options=job_args)
decide_what_to_do = BranchPythonOperator(task_id='new_comic_available',
python_callable=decide_what_to_do,
provide_context=True)
update_read_history = PythonOperator(task_id='update_read_history',
python_callable=process_metadata,
op_args=['write'],
provide_context=True)
generate_notification = PythonOperator(task_id='yes_generate_notification',
python_callable=generate_message,
provide_context=True)
send_notification = SlackAPIPostOperator(
task_id='send_notification',
token="YOUR_SLACK_TOKEN",
channel='#comic-notification',
text="[{{ ds }}] New chapter is released",
icon_url='http://airbnb.io/img/projects/airflow3.png')
do_nothing = DummyOperator(task_id='no_do_nothing')
# ワークフローの定義
get_read_history >> check_comic_info >> decide_what_to_do
decide_what_to_do >> generate_notification
decide_what_to_do >> do_nothing
generate_notification >> send_notification >> update_read_history
