'team_col_name': TEAM_COL_NAME,
'validation_results_path': VALIDATION_RESULTS_PATH,
'parse_results_path': PARSE_RESULTS_PATH,
'error_stats_path': ERROR_STATS_PATH,
'blacklist_conn_id': PROJECT_DB_CONN_ID,
'temp_table_name': TEMP_TABLE_NAME
},
templates_dict={'load_data_query': 'load_data.sql'},
provide_context=True,
dag=dag)
send_report_task = PythonOperator(task_id='send_report',
python_callable=send_report,
op_kwargs={
'token_id':
Variable.get('HW3_TELEGRAM_BOT_TOKEN'),
'chat_id':
Variable.get('HW3_TELEGRAM_CHAT_ID'),
'error_stats_path':
ERROR_STATS_PATH
},
provide_context=True,
dag=dag)
(process_urls_task >> parse_urls_task >> calculate_stats_task >>
send_report_task)
if __name__ == '__main__':
dag.clear(reset_dag_runs=True)
dag.run()
get_product_catalog_db_json['rows']):
product_catalog_document = product_catalog['value']
product_catalog_document.pop('_rev', None)
product_catalog_document['_id'] = f"{product_catalog_document['type']}_{product_catalog_document['subtype']}" \
f":{product_catalog_document['_id']}"
print("request", product_catalog_document)
print("request", json.dumps(product_catalog_document).encode('utf-8'))
print("request", json.loads(json.dumps(product_catalog_document)))
response = http_hook.run(
endpoint='copy_product',
headers={"Content-Type": "application/json; charset=utf-8"},
json=product_catalog_document,
)
print("response", response)
store_product_catalog_with_partition = PythonOperator(
task_id='prepare_product_catalog',
provide_context=True,
python_callable=store_product_catalog_with_partition,
dag=dag,
)
get_product_catalog_db >> store_product_catalog_with_partition
if __name__ == '__main__':
dag.clear(reset_dag_runs=True)
dag.run(start_date=args['start_date'])
