def test_poke_exception(self):
"""
Exception occurs in poke function should not be ignored.
"""
def resp_check(resp):
raise AirflowException('AirflowException raised here!')
task = HttpSensor(task_id='http_sensor_poke_exception',
http_conn_id='http_default',
endpoint='',
request_params={},
response_check=resp_check,
poke_interval=5)
with self.assertRaisesRegexp(AirflowException,
'AirflowException raised here!'):
task.execute(None)
task_id='put_op',
method='PUT',
endpoint='put',
data=json.dumps({"priority": 5}),
headers={"Content-Type": "application/json"},
dag=dag,
)
# [END howto_operator_http_task_put_op]
# [START howto_operator_http_task_del_op]
task_del_op = SimpleHttpOperator(
task_id='del_op',
method='DELETE',
endpoint='delete',
data="some=data",
headers={"Content-Type": "application/x-www-form-urlencoded"},
dag=dag,
)
# [END howto_operator_http_task_del_op]
# [START howto_operator_http_http_sensor_check]
task_http_sensor_check = HttpSensor(
task_id='http_sensor_check',
http_conn_id='http_default',
endpoint='',
request_params={},
response_check=lambda response: "httpbin" in response.text,
poke_interval=5,
dag=dag,
)
# [END howto_operator_http_http_sensor_check]
task_http_sensor_check >> task_post_op >> task_get_op >> task_get_op_response_filter
task_get_op_response_filter >> task_put_op >> task_del_op >> task_post_op_formenc
'geonames_endpoint': 'export/dump/allCountries.zip',
},
schedule_interval='30 1 * * 0',
tags=['k8s', 'nemo', 'psc', 'egg'],
)
# [END instantiate_dag]
with pipeline:
# [START task_http_geonames_org_sensor_check]
task_http_geonames_org_sensor_check = HttpSensor(
task_id='http_geonames_org_sensor_check',
http_conn_id='http_geonames_org',
endpoint='{{ params.geonames_endpoint }}',
method='HEAD',
response_check=lambda response: True if response.ok else False,
poke_interval=2,
# Extra options for the ‘requests’ library, see the ‘requests’ documentation (options to modify timeout, ssl, etc.)
extra_options={
'verify': False,
},
)
# [START task_http_egg_svc_check]
task_http_egg_svc_check = KubernetesPodOperator(
namespace='processing',
name='dea-access-egg-svc-check',
task_id='http_egg_svc_sensor_check',
image_pull_policy='IfNotPresent',
image=CURL_SVC_IMAGE,
is_delete_operator_pod=True,
arguments=["--verbose", "http://{{ params.egg_svc_name }}:9200"],
t3 = SimpleHttpOperator(task_id='put_op',
method='PUT',
endpoint='api/v1.0/nodes',
data=json.dumps({"priority": 5}),
headers={"Content-Type": "application/json"},
dag=dag)
t4 = SimpleHttpOperator(
task_id='del_op',
method='DELETE',
endpoint='api/v1.0/nodes',
data="some=data",
headers={"Content-Type": "application/x-www-form-urlencoded"},
dag=dag)
sensor = HttpSensor(task_id='http_sensor_check',
http_conn_id='http_default',
endpoint='',
params={},
response_check=lambda response: True
if "Google" in response.content else False,
poke_interval=5,
dag=dag)
t1.set_upstream(sensor)
t2.set_upstream(t1)
t3.set_upstream(t2)
t4.set_upstream(t3)
t5.set_upstream(t4)
'email_on_retry': False,
'retries': 5,
'retry_delay': timedelta(minutes=5)
# 'concurrency': 4
}
dag = DAG('extract_github_commits',
default_args=default_args,
schedule_interval='@daily')
hdfs_dir = 'hdfs://10.0.0.9:9000'
# Check if the new dump exists yet, retry every hour until it does
check_for_new_dump = HttpSensor(task_id='check_for_new_dump',
http_conn_id='ghtorrent',
method='HEAD',
poke_interval=60 * 60,
timeout=60 * 60 * 24,
endpoint="""mongo-dump-{{ ds }}.tar.gz""",
dag=dag)
# Download the bson file
download = BashOperator(task_id='download',
bash_command="""
wget -qO- http://ghtorrent-downloads.ewi.tudelft.nl/mongo-daily/mongo-dump-{{ ds }}.tar.gz | tar xvz dump/github/commits.bson --strip-components=2
wait
mv commits.bson ~/staging/commits_{{ ds }}.bson
""",
params={'hdfs_dir': hdfs_dir},
dag=dag)
# this extracts the bson file
# http_conn_id='rest-connection',
# endpoint="/update?id={empId}".format(empId = Variable.get("id")),
# method="PUT",
# headers={"Content-Type": "application/json"},
# response_filter=lambda response: response.json(),
# xcom_push=True,
# dag=dag,
# )
# [END howto_operator_http_task_del_op]
# [START howto_operator_http_http_sensor_check]
task_http_sensor_check = HttpSensor(
task_id='api_health_check',
http_conn_id=conn_id,
endpoint='/',
request_params={},
# response_check=lambda response: "httpbin" in response.text,
poke_interval=5,
# on_failure_callback=notify_email,
dag=dag,
)
# Task 3: Save JSON data locally
# save_and_transform = PythonOperator(
# task_id="save_and_transform",
# python_callable=transform_json,
# provide_context=True,
# )
save_employee = PythonOperator(task_id="save_employee_transform",
python_callable=save_emp_json,
provide_context=True)
# 'pool': 'backfill',
# 'priority_weight': 10,
# 'end_date': datetime(2016, 1, 1),
}
dag_x = DAG("segmentstream_demo",
default_args=default_args,
schedule_interval=timedelta(days=1))
initial_task = DummyOperator(task_id='start', dag=dag_x)
wait_for_currency_rates_service = HttpSensor(
task_id="wait_for_currency_rates_service",
dag=dag_x,
http_conn_id='currency_service',
method='GET',
endpoint='get_rates',
headers={"Content-Type": "application/json"},
request_params={'date': datetime.now().strftime('%d.%m.%Y')},
response_check=check_currency_response,
)
wait_for_currency_rates_service << initial_task
get_daily_conversion_rates = PythonOperator(
task_id="get_daily_conversion_rates",
python_callable=get_daily_conversion_rates_callback,
provide_context=True,
dag=dag_x)
get_daily_conversion_rates.set_upstream(
task_or_task_list=wait_for_currency_rates_service)
from datetime import datetime
default_args = {
'owner': 'airflow',
'depends_on_past': True,
'start_date': datetime(2019, 1, 1),
'email': ['*****@*****.**'],
'email_on_failure': False,
'email_on_retry': False
}
dag = DAG('workshop_airflow_exo_2',
default_args=default_args,
schedule_interval="0 6 * * *")
wait_for_right_time = HttpSensor(
task_id='wait_for_right_time',
http_conn_id='navitia',
endpoint='journeys?from=2.2728894%3B48.8812988&to=2.2950275%3B48.8737917&',
headers={'Authorization': '9cdfa8dd-4ed8-4411-a6eb-690d361fddf6'},
response_check=check_if_time_to_leave,
dag=dag)
send_mail = EmailOperator(task_id='send_mail',
to=['*****@*****.**'],
subject="You need to leave now!",
html_content="Leave now if you want to be on time!",
dag=dag)
wait_for_right_time >> send_mail
xcom_push=True,
dag=dag)
# retrieve the job id associated with the async call in t1
t2 = PythonOperator(
task_id='weekly_dbm_advertiser_sync_jobid',
python_callable=setSyncEndPoint,
provide_context=True,
dag=dag
)
t3 = HttpSensor(
task_id='weekly_dbm_advertiser_sync_status',
http_conn_id='i2ap_processor',
endpoint=Variable.get('weekly_dbm_advertiser_sync-statusEndpoint'),
headers={"Content-Type": "application/json",
"Tt-I2ap-Id": "*****@*****.**",
"Tt-I2ap-Sec": "E8OLhEWWihzdpIz5"},
response_check=responseCheck,
poke_interval=60,
dag=dag)
# Make the asynchronous call to the i2ap data job
t4 = SimpleHttpOperator(
task_id='weekly_dbm_partner_pull',
endpoint='/Partner',
method='POST',
data=json.dumps({"start-date": startDate,
"end-date": endDate,
"restrict": "True",
"history": "False",
"version": Variable.get('weekly_dbm_partner_pull-version')}),
task_id='cms_data_pull',
python_callable=cdp.run_cms_data_pull,
op_kwargs={"website_link": "data.cms.gov",
"token": None,
"dataset_identifier": "xbte-dn4t",
"crawl_limit": 5000,
"db_url_full": constants.LOCAL_DB_URL,
"db_url": "airflow_works",
"schema": "sandbox",
"table_name": "cms_drug_file"},
dag=dag_game_1)
s1 = HttpSensor(
task_id='http_sensor_check',
http_conn_id='http_default',
endpoint='',
request_params={},
response_check=lambda response: True if "Google" in response.text else False,
dag=dag_game_1,
)
s2 = HttpSensor(
task_id='cms_http_sensor',
http_conn_id='cms_gov_http_id',
endpoint='',
request_params={},
dag=dag_game_1,
)
t2.set_upstream(t1)
t1.set_upstream(s1)
data=json.dumps(job_param_dict),
xcom_push=True,
response_check=lambda response: response.json().get('run_id') is not None,
dag=dag)
run_id_extractor = PythonOperator(task_id='extract_run_id',
provide_context=True,
python_callable=extract_run_id,
dag=dag)
state_http_sensor = HttpSensor(
task_id='sensor_job_state',
http_conn_id='databricks',
timeout=timeout,
method='GET',
endpoint='/api/2.0/jobs/runs/get',
request_params={
'run_id': """{{ ti.xcom_pull(task_ids='extract_run_id') }}"""
},
response_check=check_state,
poke_interval=30,
dag=dag)
fetch_result_http_op = SimpleHttpOperator(
task_id='http_get_to_databricks',
http_conn_id='databricks',
method='GET',
data={'run_id': """{{ ti.xcom_pull(task_ids='extract_run_id') }}"""},
endpoint='/api/2.0/jobs/runs/get-output',
xcom_push=True,
response_check=lambda response: response.json()['metadata']['state'].get(
'result_state') == 'SUCCESS',