def check_for_bq_job_and_children_errors(bq_client: bigquery.Client,
job: Union[bigquery.LoadJob,
bigquery.QueryJob]):
"""checks if BigQuery job (or children jobs in case of multi-statement sql)
should be considered failed because there were errors or the query affected
no rows while FAIL_ON_ZERO_DML_ROWS_AFFECTED env var is set to True
(this is the default).
Args:
bq_client: bigquery.Client
job: Union[bigquery.LoadJob, bigquery.QueryJob] job to check for errors.
Raises:
exceptions.BigQueryJobFailure
"""
if job.state != "DONE":
wait_on_bq_job_id(bq_client, job.job_id, 5)
if job.errors:
raise exceptions.BigQueryJobFailure(
f"BigQuery Job {job.job_id} failed during backfill with the "
f"following errors: {job.errors} "
f"{pprint.pformat(job.to_api_repr())}")
if isinstance(job, bigquery.QueryJob):
if (constants.FAIL_ON_ZERO_DML_ROWS_AFFECTED
and job.statement_type in constants.BQ_DML_STATEMENT_TYPES
and job.num_dml_affected_rows < 1):
raise exceptions.BigQueryJobFailure(
f"query job {job.job_id} ran successfully but did not "
f"affect any rows. {pprint.pformat(job.to_api_repr())}")
for child_job in bq_client.list_jobs(parent_job=job):
check_for_bq_job_and_children_errors(bq_client, child_job)
def check_for_bq_job_and_children_errors(
bq_client: bigquery.Client, job: Union[bigquery.LoadJob,
bigquery.QueryJob],
table: Optional[bigquery.TableReference]):
"""checks if BigQuery job (or children jobs in case of multi-statement sql)
should be considered failed because there were errors or the query affected
no rows while FAIL_ON_ZERO_DML_ROWS_AFFECTED env var is set to True
(this is the default).
Args:
bq_client: bigquery.Client
job: Union[bigquery.LoadJob, bigquery.QueryJob] job to check for errors.
table: bigquery.TableReference of table being loaded
Raises:
exceptions.BigQueryJobFailure
"""
if job.state != "DONE":
wait_on_bq_job_id(bq_client, job.job_id, table, 5)
if job.error_result:
logging.log_bigquery_job(job, table)
# Raise any 5xx error codes
exception: Optional[
google.api_core.exceptions.GoogleAPICallError] = job.exception()
if isinstance(exception, (google.api_core.exceptions.ServerError,
google.api_core.exceptions.BadRequest)):
# Raise these two exception types so that the job can be retried
raise exception
raise exceptions.BigQueryJobFailure(
f"BigQuery Job {job.job_id} failed during backfill with the "
f"following errors: {job.error_result} "
f"{pprint.pformat(job.to_api_repr())}")
if isinstance(job, bigquery.QueryJob):
if (constants.FAIL_ON_ZERO_DML_ROWS_AFFECTED
and job.statement_type in constants.BQ_DML_STATEMENT_TYPES
and job.num_dml_affected_rows < 1):
logging.log_bigquery_job(
job,
table,
"BigQuery query job ran successfully "
"but did not affect any rows.",
"ERROR",
)
raise exceptions.BigQueryJobFailure(
f"query job {job.job_id} ran successfully but did not "
f"affect any rows. {pprint.pformat(job.to_api_repr())}")
for child_job in bq_client.list_jobs(parent_job=job):
check_for_bq_job_and_children_errors(bq_client, child_job, table)
