current_time = datetime.today().strftime("%Y%m%d_%H_%M") GCS_dir_archive = "gs://{}/archive/{}/{}".format(bucket_name,cc,current_time) tempGCS_filepath = os.path.join(tempGCS_dir_temp,cc) t4 = FileToGoogleCloudStorageOperator(task_id='uploadToGCS_{}'.format(cc), src=input_t4, dst=tempGCS_filepath, google_cloud_storage_conn_id = storage_connection_id, gzip = False, dag=dag) t4_archive = FileToGoogleCloudStorageOperator(task_id='uploadToGCS_archive_{}'.format(cc), src=input_t4, dst=GCS_dir_archive, google_cloud_storage_conn_id = storage_connection_id, gzip = True, dag=dag) t4.set_upstream(t3) t4_archive.set_upstream(t3) writeToGCS_task.append(t4) tempGCS_dir_paths.append(tempGCS_filepath) schema = None # remember to enter schema! ''' Creating dummy task to merge next set of nodes ''' dummy_task = DummyOperator(task_id="forkMerge", dag=dag)
'email': ['*****@*****.**'], 'email_on_failure': True, 'email_on_retry': False, 'retries': 2, 'retry_delay': timedelta(minutes=5), } dag = DAG('site-visits-dag', default_args=default_args, schedule_interval='@daily') #t1 = file_sensor( # task_id='local_file', # fs_conn_id='fs_default', # file_path='/Users/raghu/git-hub/demo/incoming/site-visits.json', # dag=dag) t1 = DummyOperator(task_id='op1', dag=dag) t2 = PythonOperator(task_id='python_task', python_callable=convert_json_to_csv.main, dag=dag) t3 = FileToGoogleCloudStorageOperator( task_id='fileToGCS', src='/usr/local/demo/outgoing/site-visits.csv', dst='site-visits', bucket='springml-demo', google_cloud_storage_conn_id='google_cloud_default', dag=dag) t3.set_upstream(t2) t2.set_upstream(t1)