def read_data(**kwargs):
cluster = cl(['10.103.5.51', '10.103.5.52', '10.103.5.53'])
session = cluster.connect('darbiz')
conn = PostgresHook(postgres_conn_id='pgConn_pg').get_conn()
cur = conn.cursor()
rows = session.execute("SELECT * from darbiz.forte_express_loan_requests where created_on>='2020-09-20' allow filtering")
cur.execute ("select distinct owner profile_id, uid order_id, pay_title from dar_group.bazar_orders1 where created_on>=now()-interval '24' hour")
res = cur.fetchall()
for user_row in rows:
d = json.loads(user_row.loan_request)
id0 = user_row.profile_id
id1 = user_row.order_id
id2 = user_row.created_on
pp = d['person_info']['financing_info']['period'] if 'period' in d['person_info']['financing_info'] else None
lh = datetime.now() - timedelta(hours = 12)
if id2>=lh:
for a,b,c in res:
ll=c.split()
if id1==b:
if pp!=int(ll[2]):
email = EmailOperator(\
task_id='send_email',\
to=['*****@*****.**','*****@*****.**'],\
subject='Ошибка в Fortemarket',\
html_content='Error in order_id: {} created at: {}, profile_id: {}, months in request: {}, months in orders: {}\n' \
.format(a, id2, b, pp, ll[2])\
)
email.execute(context=kwargs)
t3 = SlackWebhookOperator(
task_id='send_slack_notification',
http_conn_id='slack_connection',
message='Error in order_id: {} created at: {}, profile_id: {}, months in request: {}, months in orders: {}\n' \
.format(a, id2, b, pp, ll[2]),\
# files = '/tmp/BPM_report.xlsx',
channel='#reports',\
dag=dag
)
t3.execute(context=kwargs)
else:
continue
else:
continue
# lt = d['person_info']['financing_info']['loan_type'] if 'loan_type' in d['person_info']['financing_info'] else None
cur.close()
conn.close()
def task_fail_slack_alert(self, context):
slack_webhook_token = BaseHook.get_connection(
self.SLACK_CONN_ID).password
slack_msg = """
:red_circle: Task Failed.
*Task*: {task}
*Dag*: {dag}
*Input*: {input}
*Environemnt*: {env}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get('task_instance').task_id,
dag=context.get('task_instance').dag_id,
ti=context.get('task_instance'),
exec_date=context.get('execution_date'),
log_url=context.get('task_instance').log_url,
input="As specified in dag" if context.get('dag_run').conf is None
or context.get('dag_run').conf.get('file') is None else
context.get('dag_run').conf.get('file'),
env="prod" if context.get('dag_run').conf is None
or context.get('dag_run').conf.get('env') is None else
context.get('dag_run').conf.get('env'))
failed_alert = SlackWebhookOperator(task_id='slack_test',
http_conn_id='slack',
webhook_token=slack_webhook_token,
message=slack_msg,
username='******')
return failed_alert.execute(context=context)
def task_fail_slack_alert(context):
"""
Callback task that can be used in DAG to alert of failure task completion
Args:
context (dict): Context variable passed in from Airflow
Returns:
None: Calls the SlackWebhookOperator execute method internally
"""
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
slack_msg = """
:red_circle: Task Failed.
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get("task_instance").task_id,
dag=context.get("task_instance").dag_id,
ti=context.get("task_instance"),
exec_date=context.get("execution_date"),
log_url=context.get("task_instance").log_url,
)
failed_alert = SlackWebhookOperator(
task_id="slack_test",
http_conn_id="anitha_slack",
webhook_token=slack_webhook_token,
message=slack_msg,
username="******",
channel="airflowteam"
)
return failed_alert.execute(context=context)
def alert_job_requester(context):
"""
The alert will be send to slack, about the begining of the task
"""
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
slack_msg = """
:large_blue_circle: Task requested!
*Requester*: {requester}
*Goal*: {goal}
*Task*: {task}
*Dag*: {dag1}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
requester=requester,
goal=goal,
task=context.get("task_instance").task_id,
dag1=context.get("task_instance").dag_id,
ti=context.get("task_instance"),
exec_date=context.get("execution_date"),
log_url=context.get("task_instance").log_url,
)
request_job = SlackWebhookOperator(
task_id="slack_test",
http_conn_id="slack",
webhook_token=slack_webhook_token,
message=slack_msg,
username="******",
)
return request_job.execute(context=context)
def task_success_slack_alert(context, usr=None):
slack_msg = """
:scream_cat: *Task Success:* {usr}
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
usr=get_users(usr),
task=context.get('task_instance').task_id,
dag=context.get('task_instance').dag_id,
exec_date=context.get('execution_date'),
log_url=context.get('task_instance').log_url,
)
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
success_alert = SlackWebhookOperator(task_id='slack_success_notification',
http_conn_id=SLACK_CONN_ID,
webhook_token=slack_webhook_token,
message=slack_msg,
username='******',
link_names=True,
icon_emoji=':robot_face:')
return success_alert.execute(context=context)
def task_success_slack_alert(context):
"""
Callback task that can be used in DAG to alert of successful task completion
Args:
context (dict): Context variable passed in from Airflow
Returns:
None: Calls the SlackWebhookOperator execute method internally
"""
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
slack_msg = """
:large_blue_circle: Task Succeeded!
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get("task_instance").task_id,
dag=context.get("task_instance").dag_id,
ti=context.get("task_instance"),
exec_date=context.get("execution_date"),
log_url=context.get("task_instance").log_url,
)
success_alert = SlackWebhookOperator(
task_id="slack_test",
http_conn_id="slack",
webhook_token=slack_webhook_token,
message=slack_msg,
username="******",
)
return success_alert.execute(context=context)
def task_success_slack_alert(context):
slack_webhook_token = get_slack_webhook_token()
slack_msg = """
:heavy_check_mark: Task Succeeded.
*Task*: {task}
*Task Instance*: {ti}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get("task_instance").task_id,
dag=context.get("task_instance").dag_id,
ti=context.get("task_instance"),
exec_date=context.get("execution_date"),
log_url=context.get("task_instance").log_url,
)
success_alert = SlackWebhookOperator(
task_id="slack_success_notifier",
http_conn_id="slack",
webhook_token=slack_webhook_token,
message=slack_msg,
username="******",
)
return success_alert.execute(context=context)
def task_failure_slack_alert(context):
"""
Taken largely from https://github.com/tszumowski/airflow_slack_operator/blob/master/dags/slack_operator.py
Callback task that can be used in DAG to alert of success task failure
Input:
context (dict): Context variable passed in from Airflow - not needed if uses as a config option
Returns:
Calls the SlackWebhookOperator execute method internally
"""
slack_msg = """
:red_circle: Task Failed
*Task*: {task}
*Dag*: {dag}
*Log Url*:\n {log_url}
""".format(
task=context.get('task_instance').task_id,
dag=context.get('task_instance').dag_id,
log_url=context.get('task_instance').log_url.replace( "http://localhost:8080/",AIRFLOW_URL ) ,
)
failed_alert = SlackWebhookOperator(
task_id="slack",
http_conn_id=SLACK_CONN_ID,
webhook_token=slack_webhook_token,
message=slack_msg,
username=USERNAME,
)
return failed_alert.execute(context=context)
def execute(self, context):
ti = context['ti']
if self.message_content_task_id and self.message_content_task_key:
self.message_content = ti.xcom_pull(task_ids=self.message_content_task_id)[self.message_content_task_key]
slack_message = """
:robot_face: *{header}*
{dag}
{content} {body}
""".format(
header=self.message_header,
dag=context.get('task_instance').dag_id,
content=self.message_content,
body=self.message_body
)
send_to_slack = SlackWebhookOperator(
task_id="slack",
http_conn_id=SLACK_CONN_ID,
webhook_token=slack_webhook_token,
message=slack_message,
username=USERNAME
)
return send_to_slack.execute(context=context)
# task_instance.xcom_pull(‘other_task’, key=’return_value’)
def task_success_slack_alert(self, context):
slack_webhook_token = BaseHook.get_connection(
self.SLACK_CONN_ID).password
slack_msg = """
:large_blue_circle: Success.
*Task*: {task}
*Dag*: {dag}
*Input*: {input}
*Environemnt*: {env}
*Execution Time*: {exec_date}
*Message*: All the tasks of this dag has been passed.
""".format(
task=context.get('task_instance').task_id,
dag=context.get('task_instance').dag_id,
ti=context.get('task_instance'),
exec_date=context.get('execution_date'),
input="As specified in dag" if context.get('dag_run').conf is None
or context.get('dag_run').conf.get('file') is None else
context.get('dag_run').conf.get('file'),
env="prod" if context.get('dag_run').conf is None
or context.get('dag_run').conf.get('env') is None else
context.get('dag_run').conf.get('env'))
success_alert = SlackWebhookOperator(task_id='slack_test',
http_conn_id='slack',
webhook_token=slack_webhook_token,
message=slack_msg,
username='******')
return success_alert.execute(context=context)
def task_successed_slack_alert(context):
ti = context.get("task_instance")
task_id = ti.task_id
dag_id = ti.dag_id
exec_date = context.get("execution_date")
log_url = ti.log_url
slack_webhook_token = BaseHook.get_connection("slack").password
slack_msg = f"""
Task Successed.
---------------
*Task*: {task_id}
*Dag*: {dag_id}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
"""
failed_alert = SlackWebhookOperator(
task_id="slack_test",
http_conn_id="slack",
webhook_token=slack_webhook_token,
message=slack_msg,
username="******",
retries=4,
retry_delay=datetime.timedelta(seconds=30),
)
return failed_alert.execute(context=context)
def task_fail_slack_alert(context):
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
if context.get('task_instance').task_id == 't1':
task_msg = """:cat_shock: The Task {task} in Pull WYS dag failed,
{slack_name} please check.""".format(
task=context.get('task_instance').task_id,
slack_name=list_names,
)
# else other msg for task2
else:
task_msg = """ :eyes: The Task {task} in Pull WYS dag failed,
{slack_name} please check.""".format(
task=context.get('task_instance').task_id,
slack_name=list_names,
)
# this adds the error log url at the end of the msg
slack_msg = task_msg + """ (<{log_url}|log>)""".format(
log_url=context.get('task_instance').log_url, )
failed_alert = SlackWebhookOperator(
task_id='slack_test',
http_conn_id='slack',
webhook_token=slack_webhook_token,
message=slack_msg,
username='******',
)
return failed_alert.execute(context=context)
def task_fail_slack_alert(context):
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
slack_msg = """
:red_circle: Task Failed.
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get("task_instance").task_id,
dag=context.get("task_instance").dag_id,
ti=context.get("task_instance"),
exec_date=context.get("execution_date"),
log_url=context.get("task_instance").log_url,
)
failed_alert = SlackWebhookOperator(
task_id="slack_test",
http_conn_id=SLACK_CONN_ID,
webhook_token=slack_webhook_token,
message=slack_msg,
username="******",
)
return failed_alert.execute(context=context)
def tell_slack(**context):
webhook = BaseHook.get_connection('Slack2').password
message = "hey there! we connected to slack"
alterHook = SlackWebhookOperator(
task_id = 'integrate_slack',
http_conn_id='Slack2',
webhook_token=webhook,
message = message,
username='******',
dag=dag)
return alterHook.execute(context=context)
def slack_alert(**kwargs):
slack_msg = kwargs['slack_msg']
context = kwargs['context']
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
alert = SlackWebhookOperator(
task_id='slack_msg',
http_conn_id='slack_alert',
webhook_token=slack_webhook_token,
message=slack_msg,
channel='#tech-alerts',
username='******')
return alert.execute(context=context)
def execute(self, context):
self.webhook_token = BaseHook.get_connection(
self.slack_conn_id).password
self.message = """
*Dag:* {dag}
*Task:* {task}
*Scheduled:* {scheduler_exec_date}
{message}
*Log Url:* {log_url}
""".format(
dag=context.get('task_instance').dag_id,
task=context.get('task_instance').task_id,
scheduler_exec_date=context.get('execution_date'),
message=self.slack_msg,
log_url=context.get('task_instance').log_url,
)
SlackWebhookOperator.execute(self, context)
def send_release_to_slack(release_file, **context):
with open(release_file, 'r') as msg_file:
slack_msg = msg_file.read()
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
slack_send_message = SlackWebhookOperator(
task_id='slack_send_message',
http_conn_id=SLACK_CONN_ID,
webhook_token=slack_webhook_token,
message=slack_msg,
username='******')
return slack_send_message.execute(context=context)
def tell_slack_success(context):
webhook = BaseHook.get_connection('Slack').password
message = ':white_check_mark: AIRFLOW TASK FINISH TIPS:\n' \
'DAG: {}\n' \
'TASKS: {}\n' \
.format(context['task_instance'].dag_id,
context['task_instance'].task_id)
alterHook = SlackWebhookOperator(task_id='integrate_slack',
http_conn_id='Slack',
webhook_token=webhook,
message=message,
username='******')
return alterHook.execute(context=context)
def task_fail_slack_alert(context):
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
# print this task_msg and tag these users
task_msg = """The Task {task} failed. {slack_name} please fix it """.format(
task=context.get('task_instance').task_id, slack_name = list_names,)
# this adds the error log url at the end of the msg
slack_msg = task_msg + """ (<{log_url}|log>)""".format(
log_url=context.get('task_instance').log_url,)
failed_alert = SlackWebhookOperator(
task_id='slack_test',
http_conn_id='slack',
webhook_token=slack_webhook_token,
message=slack_msg,
username='******',
)
return failed_alert.execute(context=context)
def tell_slack_failed(context):
#return SlackNequiOperator.tell_slack_failed(context)
webhook = BaseHook.get_connection('Slack').password
message = ':red_circle: AIRFLOW TASK FAILURE TIPS:\n' \
'DAG: {}\n' \
'TASKS: {}\n' \
'Reason: {}\n' \
.format(context['task_instance'].dag_id,
context['task_instance'].task_id,
context['exception'])
alterHook = SlackWebhookOperator(task_id='integrate_slack',
http_conn_id='Slack',
webhook_token=webhook,
message=message,
username='******')
return alterHook.execute(context=context)
def task_fail_slack_alert(context):
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
task_msg = 'The {task} in Refreshing the WYS Open Data failed, {slack_name} go fix it meow :meow_headache: '.format(
task=context.get('task_instance').task_id,
slack_name=list_names,
)
slack_msg = task_msg + """(<{log_url}|log>)""".format(
log_url=context.get('task_instance').log_url, )
failed_alert = SlackWebhookOperator(
task_id='slack_test',
http_conn_id='slack',
webhook_token=slack_webhook_token,
message=slack_msg,
username='******',
)
return failed_alert.execute(context=context)
def dag_success_slack_alert(context):
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
slack_msg = """
:large_green_circle: DAG Succeeded.
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
dag=context.get('task_instance').dag_id,
ti=context.get('task_instance'),
exec_date=context.get('execution_date'),
log_url=context.get('task_instance').log_url,
)
failed_alert = SlackWebhookOperator(task_id='slack_test',
http_conn_id='slack',
webhook_token=slack_webhook_token,
message=slack_msg,
username='******')
return failed_alert.execute(context=context)
def task_fail_slack_alert(context):
"""
Callback task that can be used in DAG to alert of failure task completion
Args:
context (dict): Context variable passed in from Airflow
Returns:
None: Calls the SlackWebhookOperator execute method internally
"""
if ENV == "datatst":
return
if context["dag_run"].external_trigger is True:
return
if context["dag"].is_paused is True:
return
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
slack_msg = """
:red_circle: Task Failed.
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Running For*: {run_time} secs
*Log Url*: {log_url}
""".format(
task=context["task_instance"].task_id,
dag=context["task_instance"].dag_id,
ti=context["task_instance"],
exec_date=context["execution_date"],
run_time=get_task_run_time(context["task_instance"]),
log_url=context["task_instance"].log_url,
)
failed_alert = SlackWebhookOperator(
task_id=context["task_instance"].task_id,
http_conn_id=SLACK_CONN_ID,
webhook_token=slack_webhook_token,
message=slack_msg,
username="******",
)
return failed_alert.execute(context=context)
def task_success_alert(context):
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
slack_msg = """
:heavy_check_mark: Tableau Subscription Succeeded!
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get('task_instance').task_id,
dag=context.get('task_instance').dag_id,
ti=context.get('task_instance'),
exec_date=context.get('execution_date'),
log_url=context.get('task_instance').log_url,
)
failed_alert = SlackWebhookOperator(task_id='slack_test',
http_conn_id='slack-coffeebiscuits',
webhook_token=slack_webhook_token,
message=slack_msg,
username='******')
return failed_alert.execute(context=context)
def execute_slackpostonfail(context, conn_id="AIRFLOW_CONN_SLACK_WEBHOOK", message=None):
"""Task Method to Post Failed DAG or Task Completion on Slack."""
conn = BaseHook.get_connection(conn_id)
task_instance = context.get("task_instance")
log_url = task_instance.log_url
task_id = task_instance.task_id
dag_id = task_instance.dag_id
task_date = context.get("execution_date")
if not message:
message = "Task failed: {} {} {} {}".format(dag_id, task_id, task_date, log_url)
slack_post = SlackWebhookOperator(
task_id="slackpostonfail",
http_conn_id=conn_id,
webhook_token=conn.password,
message=":poop: " + message,
username="******",
dag=context.get("dag")
)
return slack_post.execute(context=context)
def task_fail_slack_alert_critical(context):
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
slack_msg = """
<!channel> :red_circle: Critical Failure
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get('task_instance').task_id,
dag=context.get('task_instance').dag_id,
ti=context.get('task_instance'),
exec_date=context.get('execution_date'),
log_url=context.get('task_instance').log_url,
)
failed_alert = SlackWebhookOperator(task_id='slack_test',
http_conn_id='slack',
webhook_token=slack_webhook_token,
message=slack_msg,
username='******')
return failed_alert.execute(context=context)
def task_success_slack_alert(context):
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
slack_msg = """
:white_check_mark: Task Sucessfully Completed.
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get('task_instance').task_id,
dag=context.get('task_instance').dag_id,
ti=context.get('task_instance'),
exec_date=context.get('execution_date'),
log_url=context.get('task_instance').log_url,
)
success_alert = SlackWebhookOperator(task_id='slack_test',
http_conn_id='slack',
webhook_token=slack_webhook_token,
message=slack_msg,
username='******')
return success_alert.execute(context=context)
def task_fail_slack_alert(context):
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
# print this task_msg and tag these users
task_msg = """The Task {task} failed, the total volume is too low.
Either a lot of loop detectors are down or there's a problem in the pipeline.
{slack_name} please fix it :thanks_japanese: """.format(
task=context.get('task_instance').task_id,
slack_name=list_names,
)
# this adds the error log url at the end of the msg
slack_msg = task_msg + """ (<{log_url}|log>)""".format(
log_url=context.get('task_instance').log_url, )
failed_alert = SlackWebhookOperator(
task_id='slack_test',
http_conn_id='slack',
webhook_token=slack_webhook_token,
message=slack_msg,
username='******',
)
return failed_alert.execute(context=context)
def task_fail_slack_alert(context):
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
slack_msg = """
:red_circle: Task Failed.
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get('task_instance').task_id,
dag=context.get('task_instance').dag_id,
ti=context.get('task_instance'),
exec_date=context.get('execution_date'),
log_url=(context.get('task_instance').log_url).replace(
'localhost', Variable.get('server_log_url')),
)
failed_alert = SlackWebhookOperator(task_id='slack_test',
http_conn_id=SLACK_CONN_ID,
webhook_token=slack_webhook_token,
message=slack_msg,
username='******')
return failed_alert.execute(context=context)
def slack_failed_task(context):
slack_webhook_token = BaseHook.get_connection(SLACK_CONN_ID).password
slack_msg = """
:ballmer: Task Failed.
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get('task_instance').task_id,
dag=context.get('task_instance').dag_id,
ti=context.get('task_instance'),
exec_date=context.get('execution_date').astimezone(
timezone('US/Pacific')),
log_url=replace_airflow_url_prefix(
context.get('task_instance').log_url, 'http://localhost'),
)
failed_alert = SlackWebhookOperator(task_id='slack_test',
http_conn_id='slack',
webhook_token=slack_webhook_token,
message=slack_msg,
username='******')
return failed_alert.execute(context=context)
