"name": "emp_name", "type": "STRING", "mode": "REQUIRED" }, { "name": "salary", "type": "INTEGER", "mode": "NULLABLE" }, ], ) # [END howto_operator_bigquery_create_table] # [START howto_operator_bigquery_delete_table] delete_table = BigQueryDeleteTableOperator( task_id="delete_table", deletion_dataset_table=f"{PROJECT_ID}.{DATASET_NAME}.test_table", ) # [END howto_operator_bigquery_delete_table] # [START howto_operator_bigquery_create_view] create_view = BigQueryCreateEmptyTableOperator( task_id="create_view", dataset_id=DATASET_NAME, table_id="test_view", view={ "query": f"SELECT * FROM `{PROJECT_ID}.{DATASET_NAME}.test_table`", "useLegacySql": False, }, ) # [END howto_operator_bigquery_create_view]
task_id="create_view", dataset_id=LOCATION_DATASET_NAME, table_id="test_view", view={ "query": "SELECT * FROM `{}.test_table`".format(DATASET_NAME), "useLegacySql": False }) get_empty_dataset_tables = BigQueryGetDatasetTablesOperator( task_id="get_empty_dataset_tables", dataset_id=DATASET_NAME) get_dataset_tables = BigQueryGetDatasetTablesOperator( task_id="get_dataset_tables", dataset_id=DATASET_NAME) delete_view = BigQueryDeleteTableOperator( task_id="delete_view", deletion_dataset_table="{}.test_view".format(DATASET_NAME)) delete_table = BigQueryDeleteTableOperator( task_id="delete_table", deletion_dataset_table="{}.test_table".format(DATASET_NAME)) get_dataset = BigQueryGetDatasetOperator(task_id="get-dataset", dataset_id=DATASET_NAME) get_dataset_result = BashOperator( task_id="get_dataset_result", bash_command= "echo \"{{ task_instance.xcom_pull('get-dataset')['id'] }}\"", )
project_id=os.environ['GCP_PROJECT_ID'], region=os.environ['GCP_REGION'], parameters={ 'PATH_TO_ETL_FILE': f"gs://{os.environ['GCP_GCS_BUCKET_ROUTINE']}/etl/etl.py", 'CLUSTER_NAME': 'goodreads-etl', 'ARG_SOURCE_BUCKET': os.environ['GCP_GCS_BUCKET_WORKING'], 'ARG_DESTINATION_BUCKET': os.environ['GCP_GCS_BUCKET_PROCESSED'], 'ARG_OBJECT_PREFIX': '{{ run_id }}', }, gcp_conn_id='gr_dataproc_conn', ) drop_stage_user_data = BigQueryDeleteTableOperator( task_id="drop_stage_user_data", deletion_dataset_table= f"{GCP_PROJECT_ID}.{GCP_BQ_DATASET_STAGE}.{GCP_BQ_TABLE_USERS}", gcp_conn_id='gr_bigquery_conn', ignore_if_missing=True) drop_stage_authors_data = BigQueryDeleteTableOperator( task_id="drop_stage_authors_data", deletion_dataset_table= f"{GCP_PROJECT_ID}.{GCP_BQ_DATASET_STAGE}.{GCP_BQ_TABLE_AUTHORS}", gcp_conn_id='gr_bigquery_conn', ignore_if_missing=True) drop_stage_books_data = BigQueryDeleteTableOperator( task_id="drop_stage_books_data", deletion_dataset_table= f"{GCP_PROJECT_ID}.{GCP_BQ_DATASET_STAGE}.{GCP_BQ_TABLE_BOOKS}", gcp_conn_id='gr_bigquery_conn',