def test_execute(self, mock_hook):
ignore_if_missing = True
deletion_dataset_table = f'{TEST_DATASET}.{TEST_TABLE_ID}'
operator = BigQueryDeleteTableOperator(
task_id=TASK_ID,
deletion_dataset_table=deletion_dataset_table,
ignore_if_missing=ignore_if_missing)
operator.execute(None)
mock_hook.return_value.delete_table.assert_called_once_with(
table_id=deletion_dataset_table, not_found_ok=ignore_if_missing)
def test_execute(self, mock_hook):
ignore_if_missing = True
deletion_dataset_table = '{}.{}'.format(TEST_DATASET, TEST_TABLE_ID)
operator = BigQueryDeleteTableOperator(
task_id=TASK_ID,
deletion_dataset_table=deletion_dataset_table,
ignore_if_missing=ignore_if_missing)
operator.execute(None)
mock_hook.return_value \
.run_table_delete \
.assert_called_once_with(
deletion_dataset_table=deletion_dataset_table,
ignore_if_missing=ignore_if_missing
)
# Main DAG
# --------------------------------------------------------------------------------
with models.DAG('delete_tables_dag',
default_args=default_args,
schedule_interval=None) as dag:
start = dummy.DummyOperator(task_id='start', trigger_rule='all_success')
end = dummy.DummyOperator(task_id='end', trigger_rule='all_success')
# Bigquery Tables deleted here for demo porpuse.
# Consider a dedicated pipeline or tool for a real life scenario.
with TaskGroup('delete_table') as delte_table:
delete_table_customers = BigQueryDeleteTableOperator(
task_id="delete_table_customers",
deletion_dataset_table=DWH_LAND_PRJ + "." + DWH_LAND_BQ_DATASET +
".customers",
impersonation_chain=[TRF_SA_DF])
delete_table_purchases = BigQueryDeleteTableOperator(
task_id="delete_table_purchases",
deletion_dataset_table=DWH_LAND_PRJ + "." + DWH_LAND_BQ_DATASET +
".purchases",
impersonation_chain=[TRF_SA_DF])
delete_table_customer_purchase_curated = BigQueryDeleteTableOperator(
task_id="delete_table_customer_purchase_curated",
deletion_dataset_table=DWH_CURATED_PRJ + "." +
DWH_CURATED_BQ_DATASET + ".customer_purchase",
impersonation_chain=[TRF_SA_DF])
task_id="create_view",
dataset_id=LOCATION_DATASET_NAME,
table_id="test_view",
view={
"query": "SELECT * FROM `{}.test_table`".format(DATASET_NAME),
"useLegacySql": False
})
get_empty_dataset_tables = BigQueryGetDatasetTablesOperator(
task_id="get_empty_dataset_tables", dataset_id=DATASET_NAME)
get_dataset_tables = BigQueryGetDatasetTablesOperator(
task_id="get_dataset_tables", dataset_id=DATASET_NAME)
delete_view = BigQueryDeleteTableOperator(
task_id="delete_view",
deletion_dataset_table="{}.test_view".format(DATASET_NAME))
delete_table = BigQueryDeleteTableOperator(
task_id="delete_table",
deletion_dataset_table="{}.test_table".format(DATASET_NAME))
get_dataset = BigQueryGetDatasetOperator(task_id="get-dataset",
dataset_id=DATASET_NAME)
get_dataset_result = BashOperator(
task_id="get_dataset_result",
bash_command=
"echo \"{{ task_instance.xcom_pull('get-dataset')['id'] }}\"",
)
schema_fields_updates=[
{
"name": "emp_name",
"description": "Name of employee"
},
{
"name": "salary",
"description": "Monthly salary in USD"
},
],
)
# [END howto_operator_bigquery_update_table_schema]
# [START howto_operator_bigquery_delete_table]
delete_table = BigQueryDeleteTableOperator(
task_id="delete_table",
deletion_dataset_table=f"{PROJECT_ID}.{DATASET_NAME}.test_table",
)
# [END howto_operator_bigquery_delete_table]
# [START howto_operator_bigquery_create_view]
create_view = BigQueryCreateEmptyTableOperator(
task_id="create_view",
dataset_id=DATASET_NAME,
table_id="test_view",
view={
"query": f"SELECT * FROM `{PROJECT_ID}.{DATASET_NAME}.test_table`",
"useLegacySql": False,
},
)
# [END howto_operator_bigquery_create_view]
"ORDER BY avg_rating DESC"),
write_disposition="WRITE_TRUNCATE",
create_disposition="CREATE_IF_NEEDED",
bigquery_conn_id="gcp",
dag=dag,
)
extract_top_ratings = BigQueryToGCSOperator(
task_id="extract_top_ratings",
source_project_dataset_table=(os.environ["GCP_PROJECT"] + ":" +
os.environ["BIGQUERY_DATASET"] + "." +
"rating_results_{{ ds_nodash }}"),
destination_cloud_storage_uris=[
"gs://" + os.environ["RESULT_BUCKET"] + "/{{ ds_nodash }}.csv"
],
export_format="CSV",
bigquery_conn_id="gcp",
dag=dag,
)
delete_result_table = BigQueryDeleteTableOperator(
task_id="delete_result_table",
deletion_dataset_table=(os.environ["GCP_PROJECT"] + ":" +
os.environ["BIGQUERY_DATASET"] + "." +
"rating_results_{{ ds_nodash }}"),
bigquery_conn_id="gcp",
dag=dag,
)
fetch_ratings >> import_in_bigquery >> query_top_ratings >> extract_top_ratings >> delete_result_table
project_id=os.environ['GCP_PROJECT_ID'],
region=os.environ['GCP_REGION'],
parameters={
'PATH_TO_ETL_FILE':
f"gs://{os.environ['GCP_GCS_BUCKET_ROUTINE']}/etl/etl.py",
'CLUSTER_NAME': 'goodreads-etl',
'ARG_SOURCE_BUCKET': os.environ['GCP_GCS_BUCKET_WORKING'],
'ARG_DESTINATION_BUCKET': os.environ['GCP_GCS_BUCKET_PROCESSED'],
'ARG_OBJECT_PREFIX': '{{ run_id }}',
},
gcp_conn_id='gr_dataproc_conn',
)
drop_stage_user_data = BigQueryDeleteTableOperator(
task_id="drop_stage_user_data",
deletion_dataset_table=
f"{GCP_PROJECT_ID}.{GCP_BQ_DATASET_STAGE}.{GCP_BQ_TABLE_USERS}",
gcp_conn_id='gr_bigquery_conn',
ignore_if_missing=True)
drop_stage_authors_data = BigQueryDeleteTableOperator(
task_id="drop_stage_authors_data",
deletion_dataset_table=
f"{GCP_PROJECT_ID}.{GCP_BQ_DATASET_STAGE}.{GCP_BQ_TABLE_AUTHORS}",
gcp_conn_id='gr_bigquery_conn',
ignore_if_missing=True)
drop_stage_books_data = BigQueryDeleteTableOperator(
task_id="drop_stage_books_data",
deletion_dataset_table=
f"{GCP_PROJECT_ID}.{GCP_BQ_DATASET_STAGE}.{GCP_BQ_TABLE_BOOKS}",
gcp_conn_id='gr_bigquery_conn',