'team_col_name': TEAM_COL_NAME, 'validation_results_path': VALIDATION_RESULTS_PATH, 'parse_results_path': PARSE_RESULTS_PATH, 'error_stats_path': ERROR_STATS_PATH, 'blacklist_conn_id': PROJECT_DB_CONN_ID, 'temp_table_name': TEMP_TABLE_NAME }, templates_dict={'load_data_query': 'load_data.sql'}, provide_context=True, dag=dag) send_report_task = PythonOperator(task_id='send_report', python_callable=send_report, op_kwargs={ 'token_id': Variable.get('HW3_TELEGRAM_BOT_TOKEN'), 'chat_id': Variable.get('HW3_TELEGRAM_CHAT_ID'), 'error_stats_path': ERROR_STATS_PATH }, provide_context=True, dag=dag) (process_urls_task >> parse_urls_task >> calculate_stats_task >> send_report_task) if __name__ == '__main__': dag.clear(reset_dag_runs=True) dag.run()
get_product_catalog_db_json['rows']): product_catalog_document = product_catalog['value'] product_catalog_document.pop('_rev', None) product_catalog_document['_id'] = f"{product_catalog_document['type']}_{product_catalog_document['subtype']}" \ f":{product_catalog_document['_id']}" print("request", product_catalog_document) print("request", json.dumps(product_catalog_document).encode('utf-8')) print("request", json.loads(json.dumps(product_catalog_document))) response = http_hook.run( endpoint='copy_product', headers={"Content-Type": "application/json; charset=utf-8"}, json=product_catalog_document, ) print("response", response) store_product_catalog_with_partition = PythonOperator( task_id='prepare_product_catalog', provide_context=True, python_callable=store_product_catalog_with_partition, dag=dag, ) get_product_catalog_db >> store_product_catalog_with_partition if __name__ == '__main__': dag.clear(reset_dag_runs=True) dag.run(start_date=args['start_date'])