t3 = SimpleHttpOperator(
task_id='put_op',
method='PUT',
endpoint='api/v1.0/nodes',
data=json.dumps({"priority": 5}),
headers={"Content-Type": "application/json"},
dag=dag)
t4 = SimpleHttpOperator(
task_id='del_op',
method='DELETE',
endpoint='api/v1.0/nodes',
data="some=data",
headers={"Content-Type": "application/x-www-form-urlencoded"},
dag=dag)
sensor = HttpSensor(
task_id='http_sensor_check',
http_conn_id='http_default',
endpoint='',
request_params={},
response_check=lambda response: True if "Google" in response.content else False,
poke_interval=5,
dag=dag)
t1.set_upstream(sensor)
t2.set_upstream(t1)
t3.set_upstream(t2)
t4.set_upstream(t3)
t5.set_upstream(t4)
method = 'POST'
data = {},
#data={"sql":"SELECT a.shape_name, a.num_sides, b.color_name, b.red_value, b.green_value, b.blue_value FROM shapes_production a, colors_production b WHERE a.color_name = b.color_name LIMIT 5;"},
#data=json.dumps({'ldap' : 'tangupta'}),
endpoint = ''
#endpoint='/lookup/dataSets/testCollection/keys/ldap?imsOrg=testDb',
headers = {
"Content-Type": "application/x-www-form-urlencoded",
"accept": "application/json"
}
http = HttpHook(method, http_conn_id='ims_default')
logging.info('Calling HTTP method')
print(os.environ['PATH'])
response = http.run(endpoint, data, headers)
print(response)
print(response.text)
print(configuration.get('testing', 'tanuj').encode('utf-8'))
return 'Hello world!'
#from IPython import embed; embed()
hello_operator = PythonOperator(task_id='hello_task',
python_callable=print_hello,
dag=dag)
#t1.set_upstream(t2)
t2.set_upstream(hello_operator)
schedule_interval='@once',
default_args=default_args)
def response_check(response):
"""
Dumps the http response and returns True when the http call status is 200/success
"""
print(response)
print(response.text)
return response.status_code == 200
t1 = SimpleHttpOperator(task_id='get_labrador',
method='GET',
http_conn_id='http_default',
endpoint='api/breed/labrador/images',
headers={"Content-Type": "application/json"},
xcom_push=True,
response_check=response_check,
dag=dag)
t2 = SimpleHttpOperator(task_id='get_breeds',
method='GET',
http_conn_id='http_default',
endpoint='api/breeds/list',
headers={"Content-Type": "application/json"},
dag=dag)
t2.set_upstream(t1)
"Tt-I2ap-Id": "*****@*****.**",
"Tt-I2ap-Sec": "E8OLhEWWihzdpIz5"},
http_conn_id='i2ap_processor',
xcom_push=True,
dag=dag)
# retrieve the job id associated with the async call in t1
t5 = PythonOperator(
task_id='weekly_dbm_partner_pull_jobid',
python_callable=setPartnerEndPoint,
provide_context=True,
dag=dag
)
# loop on the status pull until the status is ERROR or COMPLETE
t6 = HttpSensor(
task_id='weekly_dbm_partner_pull_status',
http_conn_id='i2ap_processor',
endpoint=Variable.get('weekly_dbm_partner_pull-statusEndpoint'),
headers={"Content-Type": "application/json",
"Tt-I2ap-Id": "*****@*****.**",
"Tt-I2ap-Sec": "E8OLhEWWihzdpIz5"},
response_check=responseCheck,
poke_interval=60,
dag=dag)
t2.set_upstream(t1)
t3.set_upstream(t2)
t4.set_upstream(t3)
t5.set_upstream(t4)
t6.set_upstream(t5)
provide_context=True,
python_callable=update_variables,
op_kwargs={
"UPDATE": {
"CATALOG_PRE_PRODUCTION_HARVEST_FROM_DATE": CATALOG_HARVEST_FROM_DATE_NEW
}
},
dag=DAG
)
POST_SLACK = PythonOperator(
task_id="slack_post_succ",
python_callable=catalog_slackpostonsuccess,
provide_context=True,
dag=DAG
)
# SET UP TASK DEPENDENCIES
SET_S3_NAMESPACE.set_upstream(SAFETY_CHECK)
GET_NUM_SOLR_DOCS_PRE.set_upstream(SET_S3_NAMESPACE)
BW_OAI_HARVEST.set_upstream(GET_NUM_SOLR_DOCS_PRE)
LIST_CATALOG_BW_S3_DATA.set_upstream(BW_OAI_HARVEST)
PREPARE_BOUNDWITHS.set_upstream(LIST_CATALOG_BW_S3_DATA)
OAI_HARVEST.set_upstream(PREPARE_BOUNDWITHS)
INDEX_UPDATES_OAI_MARC.set_upstream(OAI_HARVEST)
INDEX_DELETES_OAI_MARC.set_upstream(INDEX_UPDATES_OAI_MARC)
SOLR_COMMIT.set_upstream(INDEX_DELETES_OAI_MARC)
GET_NUM_SOLR_DOCS_POST.set_upstream(SOLR_COMMIT)
UPDATE_DATE_VARIABLES.set_upstream(GET_NUM_SOLR_DOCS_POST)
POST_SLACK.set_upstream(UPDATE_DATE_VARIABLES)
COLLECTION_NAME,
"get_num_solr_docs_post",
conn_id=SOLR_CLOUD.conn_id)
GET_NUM_SOLR_DOCS_CURRENT_PROD = task_solrgetnumdocs(
DAG,
PROD_COLLECTION_NAME,
"get_num_solr_docs_current_prod",
conn_id=SOLR_CLOUD.conn_id)
POST_SLACK = PythonOperator(task_id="slack_post_succ",
python_callable=catalog_slackpostonsuccess,
provide_context=True,
dag=DAG)
# SET UP TASK DEPENDENCIES
SET_S3_NAMESPACE.set_upstream(SAFETY_CHECK)
LIST_ALMA_S3_DATA.set_upstream(SET_S3_NAMESPACE)
LIST_BOUNDWITH_S3_DATA.set_upstream(SET_S3_NAMESPACE)
PREPARE_BOUNDWITHS.set_upstream(LIST_BOUNDWITH_S3_DATA)
PREPARE_ALMA_DATA.set_upstream(LIST_ALMA_S3_DATA)
PREPARE_ALMA_DATA.set_upstream(PREPARE_BOUNDWITHS)
CREATE_COLLECTION.set_upstream(PREPARE_ALMA_DATA)
CREATE_COLLECTION.set_upstream(PREPARE_BOUNDWITHS)
GET_NUM_SOLR_DOCS_PRE.set_upstream(CREATE_COLLECTION)
INDEX_SFTP_MARC.set_upstream(GET_NUM_SOLR_DOCS_PRE)
SOLR_COMMIT.set_upstream(INDEX_SFTP_MARC)
GET_NUM_SOLR_DOCS_POST.set_upstream(SOLR_COMMIT)
POST_SLACK.set_upstream(GET_NUM_SOLR_DOCS_POST)
POST_SLACK.set_upstream(GET_NUM_SOLR_DOCS_CURRENT_PROD)