def test_description_external_table(self, bq_hook):
operator = GCSToBigQueryOperator(
task_id=TASK_ID,
bucket=TEST_BUCKET,
source_objects=TEST_SOURCE_OBJECTS,
destination_project_dataset_table=TEST_EXPLICIT_DEST,
description=DESCRIPTION,
external_table=True,
)
operator.execute(None)
# fmt: off
bq_hook.return_value.get_conn.return_value.cursor.return_value.create_external_table. \
assert_called_once_with(
external_project_dataset_table=mock.ANY,
schema_fields=mock.ANY,
source_uris=mock.ANY,
source_format=mock.ANY,
compression=mock.ANY,
skip_leading_rows=mock.ANY,
field_delimiter=mock.ANY,
max_bad_records=mock.ANY,
quote_character=mock.ANY,
ignore_unknown_values=mock.ANY,
allow_quoted_newlines=mock.ANY,
allow_jagged_rows=mock.ANY,
encoding=mock.ANY,
src_fmt_configs=mock.ANY,
encryption_configuration=mock.ANY,
labels=mock.ANY,
description=DESCRIPTION,
)
def test_execute_explicit_project(self, bq_hook):
operator = GCSToBigQueryOperator(
task_id=TASK_ID,
bucket=TEST_BUCKET,
source_objects=TEST_SOURCE_OBJECTS,
destination_project_dataset_table=TEST_EXPLICIT_DEST,
max_id_key=MAX_ID_KEY,
)
# using non-legacy SQL
bq_hook.return_value.get_conn.return_value.cursor.return_value.use_legacy_sql = False
operator.execute(None)
bq_hook.return_value.get_conn.return_value.cursor.return_value.execute.assert_called_once_with(
"SELECT MAX(id) FROM `test-project.dataset.table`")
def test_labels(self, bq_hook):
operator = GCSToBigQueryOperator(
task_id=TASK_ID,
bucket=TEST_BUCKET,
source_objects=TEST_SOURCE_OBJECTS,
destination_project_dataset_table=TEST_EXPLICIT_DEST,
labels=LABELS,
)
operator.execute(None)
bq_hook.return_value.get_conn.return_value.cursor.return_value.run_load.assert_called_once_with(
destination_project_dataset_table=mock.ANY,
schema_fields=mock.ANY,
source_uris=mock.ANY,
source_format=mock.ANY,
autodetect=mock.ANY,
create_disposition=mock.ANY,
skip_leading_rows=mock.ANY,
write_disposition=mock.ANY,
field_delimiter=mock.ANY,
max_bad_records=mock.ANY,
quote_character=mock.ANY,
ignore_unknown_values=mock.ANY,
allow_quoted_newlines=mock.ANY,
allow_jagged_rows=mock.ANY,
encoding=mock.ANY,
schema_update_options=mock.ANY,
src_fmt_configs=mock.ANY,
time_partitioning=mock.ANY,
cluster_fields=mock.ANY,
encryption_configuration=mock.ANY,
labels=LABELS,
description=mock.ANY,
)