"name": "emp_name",
"type": "STRING",
"mode": "REQUIRED"
},
{
"name": "salary",
"type": "INTEGER",
"mode": "NULLABLE"
},
],
)
# [END howto_operator_bigquery_create_table]
# [START howto_operator_bigquery_delete_table]
delete_table = BigQueryDeleteTableOperator(
task_id="delete_table",
deletion_dataset_table=f"{PROJECT_ID}.{DATASET_NAME}.test_table",
)
# [END howto_operator_bigquery_delete_table]
# [START howto_operator_bigquery_create_view]
create_view = BigQueryCreateEmptyTableOperator(
task_id="create_view",
dataset_id=DATASET_NAME,
table_id="test_view",
view={
"query": f"SELECT * FROM `{PROJECT_ID}.{DATASET_NAME}.test_table`",
"useLegacySql": False,
},
)
# [END howto_operator_bigquery_create_view]
task_id="create_view",
dataset_id=LOCATION_DATASET_NAME,
table_id="test_view",
view={
"query": "SELECT * FROM `{}.test_table`".format(DATASET_NAME),
"useLegacySql": False
})
get_empty_dataset_tables = BigQueryGetDatasetTablesOperator(
task_id="get_empty_dataset_tables", dataset_id=DATASET_NAME)
get_dataset_tables = BigQueryGetDatasetTablesOperator(
task_id="get_dataset_tables", dataset_id=DATASET_NAME)
delete_view = BigQueryDeleteTableOperator(
task_id="delete_view",
deletion_dataset_table="{}.test_view".format(DATASET_NAME))
delete_table = BigQueryDeleteTableOperator(
task_id="delete_table",
deletion_dataset_table="{}.test_table".format(DATASET_NAME))
get_dataset = BigQueryGetDatasetOperator(task_id="get-dataset",
dataset_id=DATASET_NAME)
get_dataset_result = BashOperator(
task_id="get_dataset_result",
bash_command=
"echo \"{{ task_instance.xcom_pull('get-dataset')['id'] }}\"",
)
project_id=os.environ['GCP_PROJECT_ID'],
region=os.environ['GCP_REGION'],
parameters={
'PATH_TO_ETL_FILE':
f"gs://{os.environ['GCP_GCS_BUCKET_ROUTINE']}/etl/etl.py",
'CLUSTER_NAME': 'goodreads-etl',
'ARG_SOURCE_BUCKET': os.environ['GCP_GCS_BUCKET_WORKING'],
'ARG_DESTINATION_BUCKET': os.environ['GCP_GCS_BUCKET_PROCESSED'],
'ARG_OBJECT_PREFIX': '{{ run_id }}',
},
gcp_conn_id='gr_dataproc_conn',
)
drop_stage_user_data = BigQueryDeleteTableOperator(
task_id="drop_stage_user_data",
deletion_dataset_table=
f"{GCP_PROJECT_ID}.{GCP_BQ_DATASET_STAGE}.{GCP_BQ_TABLE_USERS}",
gcp_conn_id='gr_bigquery_conn',
ignore_if_missing=True)
drop_stage_authors_data = BigQueryDeleteTableOperator(
task_id="drop_stage_authors_data",
deletion_dataset_table=
f"{GCP_PROJECT_ID}.{GCP_BQ_DATASET_STAGE}.{GCP_BQ_TABLE_AUTHORS}",
gcp_conn_id='gr_bigquery_conn',
ignore_if_missing=True)
drop_stage_books_data = BigQueryDeleteTableOperator(
task_id="drop_stage_books_data",
deletion_dataset_table=
f"{GCP_PROJECT_ID}.{GCP_BQ_DATASET_STAGE}.{GCP_BQ_TABLE_BOOKS}",
gcp_conn_id='gr_bigquery_conn',
