# [START howto_operator_bigquery_upsert_table]
update_table = BigQueryUpsertTableOperator(
task_id="update_table",
dataset_id=DATASET_NAME,
table_resource={
"tableReference": {
"tableId": "test_table_id"
},
"expirationTime": (int(time.time()) + 300) * 1000
})
# [END howto_operator_bigquery_upsert_table]
# [START howto_operator_bigquery_check]
check_count = BigQueryCheckOperator(
task_id="check_count",
sql="SELECT COUNT(*) FROM {}.save_query_result".format(DATASET_NAME),
use_legacy_sql=False,
)
# [END howto_operator_bigquery_check]
# [START howto_operator_bigquery_value_check]
check_value = BigQueryValueCheckOperator(
task_id="check_value",
sql="SELECT COUNT(*) FROM {}.save_query_result".format(DATASET_NAME),
pass_value=1000,
use_legacy_sql=False,
)
# [END howto_operator_bigquery_value_check]
# [START howto_operator_bigquery_interval_check]
check_interval = BigQueryIntervalCheckOperator(
table_id=TABLE_1,
max_results=10,
selected_fields="value,name",
location=location,
)
# [END howto_operator_bigquery_get_data]
get_data_result = BashOperator(
task_id="get_data_result",
bash_command="echo \"{{ task_instance.xcom_pull('get_data') }}\"",
)
# [START howto_operator_bigquery_check]
check_count = BigQueryCheckOperator(
task_id="check_count",
sql=f"SELECT COUNT(*) FROM {DATASET_NAME}.{TABLE_1}",
use_legacy_sql=False,
location=location,
)
# [END howto_operator_bigquery_check]
# [START howto_operator_bigquery_value_check]
check_value = BigQueryValueCheckOperator(
task_id="check_value",
sql=f"SELECT COUNT(*) FROM {DATASET_NAME}.{TABLE_1}",
pass_value=4,
use_legacy_sql=False,
location=location,
)
# [END howto_operator_bigquery_value_check]
# [START howto_operator_bigquery_interval_check]
destination_project_dataset_table=
f"{os.environ['GCP_PROJECT_ID']}.{os.environ['GCP_BQ_DATASET_STAGE']}.{os.environ['GCP_BQ_TABLE_REVIEWS']}",
schema_fields=reviews_schema,
source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,
create_disposition=bigquery.CreateDisposition.CREATE_NEVER,
write_disposition=bigquery.WriteDisposition.WRITE_EMPTY,
skip_leading_rows=1,
bigquery_conn_id='gr_bigquery_conn',
google_cloud_storage_conn_id='gr_storage_conn',
)
check_stage_data = DummyOperator(task_id='check_stage_data')
check_stage_users_data = BigQueryCheckOperator(
task_id='check_stage_users_data',
sql=stage.check_stage_users_data_sql,
use_legacy_sql=False,
gcp_conn_id='gr_bigquery_conn')
check_stage_authors_data = BigQueryCheckOperator(
task_id='check_stage_authors_data',
sql=stage.check_stage_authors_data_sql,
use_legacy_sql=False,
gcp_conn_id='gr_bigquery_conn')
check_stage_books_data = BigQueryCheckOperator(
task_id='check_stage_books_data',
sql=stage.check_stage_books_data_sql,
use_legacy_sql=False,
gcp_conn_id='gr_bigquery_conn')
check_stage_reviews_data = BigQueryCheckOperator(
