def _run_as_operator(self, **kwargs):
task = EmailOperator(
to='*****@*****.**',
subject='Test Run',
html_content='The quick brown fox jumps over the lazy dog',
task_id='task',
dag=self.dag,
**kwargs)
task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
def _run_as_operator(self, **kwargs):
task = EmailOperator(
to='*****@*****.**',
subject='Test Run',
html_content='The quick brown fox jumps over the lazy dog',
task_id='task',
dag=self.dag,
files=[
"/tmp/Report-A-{{ execution_date.strftime('%Y-%m-%d') }}.csv"
],
**kwargs,
)
task.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE)
def example_dag_decorator(email: str = '*****@*****.**'):
"""
DAG to send server IP to email.
:param email: Email to send IP to. Defaults to [email protected].
:type email: str
"""
# Using default connection as it's set to httpbin.org by default
get_ip = SimpleHttpOperator(task_id='get_ip', endpoint='get', method='GET')
@task(multiple_outputs=True)
def prepare_email(raw_json: str) -> Dict[str, str]:
external_ip = json.loads(raw_json)['origin']
return {
'subject':
f'Server connected from {external_ip}',
'body':
f'Seems like today your server executing Airflow is connected from IP {external_ip}<br>'
}
email_info = prepare_email(get_ip.output)
EmailOperator(task_id='send_email',
to=email,
subject=email_info['subject'],
html_content=email_info['body'])
def example_dag_decorator(email: str = '*****@*****.**'):
"""
DAG to send server IP to email.
:param email: Email to send IP to. Defaults to [email protected].
:type email: str
"""
get_ip = GetRequestOperator(task_id='get_ip', url="http://httpbin.org/get")
@task(multiple_outputs=True)
def prepare_email(raw_json: Dict[str, Any]) -> Dict[str, str]:
external_ip = raw_json['origin']
return {
'subject':
f'Server connected from {external_ip}',
'body':
f'Seems like today your server executing Airflow is connected from IP {external_ip}<br>',
}
email_info = prepare_email(get_ip.output)
EmailOperator(task_id='send_email',
to=email,
subject=email_info['subject'],
html_content=email_info['body'])
def send_email(email: str = "*****@*****.**"):
"""
DAG to send server IP to email.
:param email: Email to send IP to. Defaults to [email protected].
:type email: str
"""
get_ip = GetRequestOperator(task_id="get_ip", url="http://httpbin.org/get")
@task(multiple_outputs=True)
def prepare_email(raw_json: Dict[str, Any]) -> Dict[str, str]:
external_ip = raw_json["origin"]
return {
"subject": f"Hello from the Apache Aiflow",
"body": f"This email is send by IoToI workflow.<br>Seems like today your server executing Airflow is connected from IP {external_ip}<br>",
}
email_info = prepare_email(get_ip.output)
EmailOperator(
task_id="send_email",
to=email,
subject=email_info["subject"],
html_content=email_info["body"],
)
schedule_interval='0 8 * * 1-5',
#dagrun_timeout=timedelta(minutes=5),
start_date=datetime(2019, 1, 1),
tags=['acoes', 'swing', 'b3'])
tsk_B3 = BashOperator(
task_id='analisa_papeis',
bash_command=
'cd /home/fm/Documents/swing_bbot3/Scripts && /home/fm/miniconda3/envs/b3/bin/python /home/fm/Documents/swing_bbot3/Scripts/app.py',
dag=dag,
depends_on_past=False)
mail_results = EmailOperator(
task_id='send_result',
to=['*****@*****.**'],
subject='Análise Gráfica Diária',
html_content=""" <h3>Bom dia, humano. Seguem resultados anexos.</h3> """,
files=[
f"/home/fm/Documents/swing_bbot3/Dados/analise_{datetime.now().strftime('%m.%d.%Y')}.xlsx"
],
dag=dag)
check_results_storage = FileSensor(
task_id='sense_result',
filepath=
f"/home/fm/Documents/swing_bbot3/Dados/analise_{datetime.now().strftime('%m.%d.%Y')}.xlsx",
poke_interval=3,
dag=dag)
tsk_B3 >> check_results_storage >> mail_results
sqlite_conn_id="sqlite_default",
sql="""
CREATE TABLE IF NOT EXISTS covid19 (
NewConfirmed TEXT NOT NULL
);
""")
load_data_to_db = BashOperator(task_id="load_data_to_db",
bash_command="""
sqlite3 -separator "," /Users/atb/covid19-{{ ds }}.db ".import /Users/atb/covid19-{{ ds }}.csv covid19"
""")
send_mail = EmailOperator(
task_id="send_email",
to=[
"*****@*****.**",
],
subject="Finished loading COVID-19 data to db!",
html_content="Yeah!",
)
load_data_to_s3 = PythonOperator(
task_id="load_data_to_s3",
python_callable=_load_data_to_s3,
)
end = DummyOperator(task_id="end")
start >> print_prev_ds >> check_api >> download_covid19_data \
>> create_table >> load_data_to_db >> send_mail >> end
download_covid19_data >> load_data_to_s3
def make_email_operator(email: str, email_info):
return EmailOperator(task_id='send_email',
to=email,
subject=email_info['subject'],
html_content=email_info['body'])
# -E trusted connection, -i input file
# -o output file, -s, use comma to separate fields
# -W trim white space, -X security measure for automated envs
query_cmd = (f'sqlcmd -S {ebi_db_server_prod} -d FI_DM_EBI -E '
f'-i {path}\\bmt_query.sql '
f'-o {path}\\bmt_results.csv '
'-s, -W -X')
copy_cmd = (f'pscp -pw {pw} {path}\\bmt_results.csv '
f'{airflow_server_prod}:/var/nfsshare/files')
query = SSHOperator(ssh_conn_id='tableau_server',
task_id='query_bmt',
command=query_cmd,
dag=dag)
copy = SSHOperator(ssh_conn_id='tableau_server',
task_id='copy_bmt',
command=copy_cmd,
dag=dag)
email = EmailOperator(task_id='email_bmt',
to=['*****@*****.**'],
cc=['*****@*****.**'],
subject='BMT Data {{ ds }}',
html_content='See attached.',
files=['/var/nfsshare/files/bmt_results.csv'],
dag=dag)
tasks >> query >> copy >> email
forex_processing = SparkSubmitOperator(
task_id='forex_processing',
conn_id='spark_conn',
application="/User/arunraja/airflow/dags/scripts/forex_processing.py",
verbose=False,
executor_cores=2,
num_executors=2,
executor_memory='256M',
driver_memory='1G'
)
sending_email_notification = EmailOperator(
task_id="sending_email",
to="*****@*****.**",
subject="forex_data_pipeline",
html_content="""
<h3>forex_data_pipeline succeeded</h3>
"""
)
sending_slack_notification = SlackAPIPostOperator(
task_id="sending_slack",
token='asdljfwiei3234kndoiff2309e',
username="******",
text="DAG forex_data_pipeline: DONE",
channel="#airflow-exploit"
)
is_forex_rates_available >> is_forex_currencies_file_available >> downloading_rates >> saving_rates >> creating_forex_rates_table >> forex_processing
forex_processing >> sending_email_notification >> sending_slack_notification
from airflow import DAG
from datetime import datetime
from airflow.operators.email import EmailOperator
default_args = {'start_date': datetime(2021, 1, 1)}
with DAG(dag_id='my_send_email', default_args=default_args,
catchup=False) as dag:
send_email = EmailOperator(
task_id='send_to_icloud',
#to
to=['*****@*****.**'],
#cc
cc=['*****@*****.**'],
#subject
subject='my send email',
#content
html_content='''
Test email at {}
'''.format(datetime.now())
#attachedment
)
send_email
for t in proc_map.loc[proc_map.proc_name == p].dependency_name.values[0]:
sql_ops[t + '_logic'] >> sql_ops[p]
# set ds refreshes downstream from all their procedure dependencies
for ds in unique_ds.ds_name:
for p in ds_map.loc[ds_map.ds_name == ds].proc_name.values[0]:
sql_ops[p] >> python_ops[ds]
# create sensor to wait for etl dependencies to be in json
# deps = ExternalTaskSensor(
# external_dag_id='get_etl_deps',
# external_task_id='query_and_save_deps',
# task_id='wait_for_dependencies_file',
# dag=dag
# )
# for p in no_dep_procs.proc_name:
# deps >> sql_ops[p]
# create final email task
email = EmailOperator(task_id='email_edw',
to=['*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**'],
cc=['*****@*****.**'],
subject='EBI ETL {{ next_ds }} Complete',
html_content='-',
dag=dag)
for ds in python_ops:
python_ops[ds] >> email
sensor = ExternalTaskSensor(task_id='sensor',
external_dag_id='nyt_bestsellers_ETL',
external_task_id='load_nyt_best_sellers')
'''
def build_email():
mysql_hook = MySqlHook(conn_name_attr='nyt_db_id')
best_sellers_data_ = mysql_hook.get_pandas_df(
sql= "SELECT title, author, ranking\
FROM nyt_bestsellers_fiction\
UNION SELECT title, author, ranking,\
FROM nyt_bestsellers_nonfiction")
email_body = '<h3> The New York Times Best Seller List Has Recently Been Updated! </h3>'
return email_body
task = PythonOperator(
task_id='create_email',
python_callable=build_email,
do_xcom_push=True,
dag=dag
)
'''
email = EmailOperator(
task_id='send_email',
to=['*****@*****.**', "{{dag_run.conf['email']}}"],
subject='New York Times Best Seller List',
html_content='templates/email_body.html')
sensor >> email
'owner': "airflow",
'depends_on_past': False,
'start_date': datetime.datetime(2021, 7, 1),
'email': ['*****@*****.**'],
'email_on_retry': True,
'email_on_failure': True,
'retries': 3,
'retry_delay': timedelta(minutes=1),
}
dag = DAG(
'spotify_dag',
default_args=default_args,
description='Dag with Spotify ETL process',
schedule_interval='@monthly',
)
run_etl = PythonOperator(task_id='spotify_etl',
python_callable=run_spotify_etl,
dag=dag)
email = EmailOperator(
task_id='send_email',
to='*****@*****.**',
subject='Airflow Alert spotify etl',
html_content=
"""<h3> Email Trigger batch spotify etl for {currentMonth}. Thank you </h3>""",
dag=dag)
run_etl >> email
sql='''
CREATE TABLE IF NOT EXISTS covid19 (
NewConfirmed TEXT NOT NULL
);
''')
load_data_to_db = BashOperator(task_id='load_data_to_db',
bash_command='''
sqlite3 -separator "," /Users/yothinix/projects/_class/my-airflow/covid19.db ".import /Users/yothinix/projects/_class/my-airflow/covid19-{{ ds }}.csv covid19"
''')
# if we have f-string we need to use 4: {{{{ XXX }}}} but template will use just 2 {{ ds }}
send_email = EmailOperator(
task_id='send_email',
to=[
'*****@*****.**',
],
subject='Finished loading COVID-19 data to db!',
html_content='<h1 style="color: red">Yeah!</h1>')
load_data_to_s3 = PythonOperator(task_id='load_data_to_s3',
python_callable=_load_data_to_s3)
end = DummyOperator(task_id='end')
# Defining how task ordered
start >> print_prev_ds \
>> check_api \
>> download_covid19_data \
>> create_table \
>> load_data_to_db \
}
with DAG("exhange_rates_data_processing",
schedule_interval="@daily",
default_args=default_args,
start_date=timezone.datetime(2021, 3, 19),
tags=["exchange_rates", "odds"]) as dag:
start = DummyOperator(task_id='start')
download_exchange_rates_data = PythonOperator(
task_id="download_exchange_rates_data",
python_callable=_download_exchange_rates_data,
)
load_data_to_s3 = PythonOperator(
task_id="load_data_to_s3",
python_callable=_load_data_to_s3,
)
send_email = EmailOperator(
task_id="send_email",
to=["*****@*****.**",],
subject="Finished loading exchange rates data to s3",
html_content="Logi Yeah!",
)
end = DummyOperator(task_id="end")
start >> download_exchange_rates_data >> load_data_to_s3 \
>> send_email >> end
load_to_analytics_dw = DummyOperator(task_id='load_to_analytics_dw', dag=dag)
check_dw_data_quality = DummyOperator(task_id='check_dw_data_quality', dag=dag)
export_to_json = LoadToJsonOperator(task_id='export_to_json',
postgres_config=postgres_master_config,
query=SqlQueries.select_all_query_to_json,
path='output/json',
tables=['ports'],
filenames={'ports': 'ports.json'},
dag=dag)
send_notification_email = EmailOperator(
task_id='send_notification_email',
to='*****@*****.**',
subject='Notification for unece_dag',
html_content='<h3>unece_data_pipeline executed successfully</h3>',
dag=dag)
send_notification_slack = SlackAPIPostOperator(
task_id='send_notification_slack',
username='******',
token=Variable.get('slack_api_key'),
text='unece_dag executed successfully',
channel='#automationcase',
dag=dag)
end_execution = DummyOperator(task_id='end_execution', dag=dag)
begin_execution >> check_ports_scraper_contracts >> scrape_unece_data
scrape_unece_data >> stage_to_mongodb
'start_date': airflow.utils.dates.days_ago(2),
'retries': 3,
'retry_delay': timedelta(minutes=1),
'on_failure_callback': dag_failure_notification,
'on_success_callback': dag_success_notification
}
with DAG('dag_km_email',
default_args=default_args,
schedule_interval=None,
catchup=False,
dagrun_timeout=timedelta(minutes=90)) as dag:
dummy_task = DummyOperator(task_id='dummy_task')
email_task_emailop = EmailOperator(
task_id='email_task_emailop',
to=to_email_address,
subject='Airflow Alert',
html_content=""" <h3>Email Test</h3> """,
)
dummy_task >> email_task_emailop
send_af_email = PythonOperator(task_id='send_af_email',
python_callable=send_af_email,
provide_context=True,
op_kwargs={
'to_email_address': to_email_address,
})
dummy_task >> send_af_email
bash_command='rm -f /home/repl/*.tmp',
dag=dag)
python_task = PythonOperator(task_id='run_processing',
python_callable=process_data,
provide_context=True,
dag=dag)
email_subject = """
Email report for {{ params.department }} on {{ ds_nodash }}
"""
email_report_task = EmailOperator(
task_id='email_report_task',
to='*****@*****.**',
subject=email_subject,
html_content='',
params={'department': 'Data subscription services'},
dag=dag)
no_email_task = DummyOperator(task_id='no_email_task', dag=dag)
def check_weekend(**kwargs):
dt = datetime.strptime(kwargs['execution_date'], "%Y-%m-%d")
# If dt.weekday() is 0-4, it's Monday - Friday. If 5 or 6, it's Sat / Sun.
if dt.weekday() < 5:
return 'email_report_task'
else:
return 'no_email_task'
def airflow2_good_example():
begin = DummyOperator(task_id="begin")
end = DummyOperator(task_id="end", trigger_rule=TriggerRule.NONE_FAILED)
check_day_of_week = BranchDayOfWeekOperator(
task_id="check_day_of_week",
week_day={WeekDay.SATURDAY, WeekDay.SUNDAY},
follow_task_ids_if_true="weekend",
follow_task_ids_if_false="weekday",
use_task_execution_day=True,
)
weekend = DummyOperator(task_id="weekend")
weekday = DummyOperator(task_id="weekday")
# Templated value for determining the name of the day of week based on the start date of the DagRun.
day_name = "{{ dag_run.start_date.strftime('%A').lower() }}"
# Begin weekday tasks.
with TaskGroup("weekday_activities") as weekday_activities:
which_weekday_activity_day = BranchPythonOperator(
task_id="which_weekday_activity_day",
python_callable=_get_activity,
op_args=[day_name],
)
for day, day_info in DAY_ACTIVITY_MAPPING.items():
if day_info["is_weekday"]:
day_of_week = Label(label=day)
activity = day_info["activity"]
do_activity = BashOperator(
task_id=activity.replace(" ", "_"),
bash_command=
f"echo It's {day.capitalize()} and I'm busy with {activity}.",
)
# Declaring task dependencies within the `TaskGroup` via the classic bitshift operator.
which_weekday_activity_day >> day_of_week >> do_activity
# Begin weekend tasks.
with TaskGroup("weekend_activities") as weekend_activities:
which_weekend_activity_day = BranchPythonOperator(
task_id="which_weekend_activity_day",
python_callable=_get_activity,
op_args=[day_name],
)
saturday = Label(label="saturday")
sunday = Label(label="sunday")
sleeping_in = BashOperator(
task_id="sleeping_in",
bash_command="sleep $[ ( $RANDOM % 30 ) + 1 ]s")
going_to_the_beach = _going_to_the_beach()
# Because the ``going_to_the_beach()`` function has ``multiple_outputs`` enabled, each dict key is
# accessible as their own `XCom` key.
inviting_friends = EmailOperator(
task_id="inviting_friends",
to="*****@*****.**",
subject=going_to_the_beach["subject"],
html_content=going_to_the_beach["body"],
)
# Using ``chain()`` here for list-to-list dependencies which are not supported by the bitshift
# operator and to simplify the notation for the desired dependency structure.
chain(which_weekend_activity_day, [saturday, sunday],
[going_to_the_beach, sleeping_in])
# High-level dependencies.
chain(begin, check_day_of_week, [weekday, weekend],
[weekday_activities, weekend_activities], end)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
STORED AS TEXTFILE
""")
#Spark operator
forex_processing = SparkSubmitOperator(
task_id="forex_processing",
application="/home/enes/airflow2/dags/scripts/forex_processing.py",
conn_id="spark_conn",
verbose=False)
#Email operator
send_email_notification = EmailOperator(
task_id="send_email_notification",
to="*****@*****.**",
subject="forex_data_pipeline",
html_content="<h3>forex_data_pipeline</h3>")
#Slack notification operator
send_slack_notification = SlackWebhookOperator(
task_id="send_slack_notification",
http_conn_id="slack_conn",
message=_get_message(),
channel="#monitoring")
#Define dependencies
is_forex_rates_available >> is_file_available >> downloading_rates >> saving_rates
saving_rates >> creating_forex_rates_table >> forex_processing >> send_email_notification
send_email_notification >> send_slack_notification
airflow_server_prod = Variable.get('airflow_server_prod')
query_cmd = (f'sqlcmd -S {ebi_db_server_prod} -d FI_DM_EBI -E '
f'-i {path}\\hims_query.sql '
f'-o {path}\\hims_results.csv '
'-s"|" -W -X -I')
copy_cmd = (f'pscp -pw {pw} {path}\\hims_results.csv '
f'{airflow_server_prod}:/var/nfsshare/files')
query = SSHOperator(ssh_conn_id='tableau_server',
task_id='query_hims',
command=query_cmd,
dag=dag)
copy = SSHOperator(ssh_conn_id='tableau_server',
task_id='copy_hims',
command=copy_cmd,
dag=dag)
email = EmailOperator(
task_id='email_hims',
to=['*****@*****.**', '*****@*****.**', '*****@*****.**'],
cc=['*****@*****.**'],
subject='HIMS Data {{ ds }}',
html_content='See attached.',
files=['/var/nfsshare/files/hims_results.csv'],
dag=dag)
query >> copy >> email
'retries': 1,
'retry_delay': timedelta(seconds=10),
}
dag = DAG('MYDAG', default_args=default_args, schedule_interval='0 11 * * *')
def DataFetch_Transform_Clean():
pm.execute_notebook('/c/users/leand/AirflowHome/dags/stock.ipynb',
'/c/users/leand/AirflowHome/dags/stock_out.ipynb',
parameters=dict(alpha=0.6, ratio=0.1))
DataFetch_Transform_Clean = PythonOperator(
task_id="run_notebook", python_callable=DataFetch_Transform_Clean, dag=dag)
move_file = BashOperator(
task_id='move-file',
bash_command=
'tail -7 /c/users/leand/AirflowHome/dags/stocks.csv>> /c/users/leand/AirflowHome/dags/stocks.csv',
dag=dag)
send_email = EmailOperator(
task_id='send_email',
to='*****@*****.**',
subject='Airflow Alert',
html_content=""" <h3>HELLO,</h3><br><h3> PIPELINE SUCESSFULL</h3> """,
dag=dag)
DataFetch_Transform_Clean >> move_file >> send_email
