def run_validate(*args, **kwargs):
start_time, end_time = get_start_end_time(kwargs['execution_date'])
# set environment variables
os.environ.update(get_environments())
return validate_exports(start_time, end_time)
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
'on_failure_callback': email_notify
}
dag = DAG(
'daily_customer_selections',
catchup=False,
default_args=default_args,
# run every day at 4:30am PST after conversation closure
schedule_interval='30 04 * * 1-7')
dag.doc_md = __doc__
# It is not recommanded to use Variable with global scope
# but not sure if there is another way to inject airflow variables
# into envionment variables.
env = os.environ.copy()
env.update(get_environments())
daily_customer_selections = BashOperator(
task_id='customer_selections_script',
bash_command=
'python -m tools.analysis.customer_selections.populate_customer_selections \
--start_date="{{ execution_date.format("%d-%m-%Y") }} 00:00:00" \
--end_date="{{ execution_date.format("%d-%m-%Y") }} 23:59:59" \
--timezone="{{ var.value.TIMEZONE }}"',
retries=1,
env=env,
dag=dag)
def run_export(*args, **kwargs):
start_time, end_time = get_start_end_time(kwargs['execution_date'])
env = Variable.get('ENVIRONMENT')
# set environment variables
os.environ.update(get_environments())
return run_exports(start_time, end_time, env)
