def test_execute(self, mock_hook):
operator = BigQueryCreateExternalTableOperator(
task_id=TASK_ID,
destination_project_dataset_table='{}.{}'.format(
TEST_DATASET, TEST_TABLE_ID),
schema_fields=[],
bucket=TEST_GCS_BUCKET,
source_objects=TEST_GCS_DATA,
source_format=TEST_SOURCE_FORMAT)
operator.execute(None)
mock_hook.return_value \
.get_conn.return_value \
.cursor.return_value \
.create_external_table \
.assert_called_once_with(
external_project_dataset_table='{}.{}'.format(
TEST_DATASET, TEST_TABLE_ID
),
schema_fields=[],
source_uris=['gs://{}/{}'.format(TEST_GCS_BUCKET, source_object)
for source_object in TEST_GCS_DATA],
source_format=TEST_SOURCE_FORMAT,
compression='NONE',
skip_leading_rows=0,
field_delimiter=',',
max_bad_records=0,
quote_character=None,
allow_quoted_newlines=False,
allow_jagged_rows=False,
src_fmt_configs={},
labels=None,
encryption_configuration=None
)
max_results="10",
selected_fields="value,to_address",
)
get_data_result = BashOperator(
task_id="get_data_result",
bash_command="echo \"{{ task_instance.xcom_pull('get-data') }}\"")
create_external_table = BigQueryCreateExternalTableOperator(
task_id="create_external_table",
bucket=DATA_SAMPLE_GCS_BUCKET_NAME,
source_objects=[DATA_SAMPLE_GCS_OBJECT_NAME],
destination_project_dataset_table="{}.external_table".format(
DATASET_NAME),
skip_leading_rows=1,
schema_fields=[{
"name": "name",
"type": "STRING"
}, {
"name": "post_abbr",
"type": "STRING"
}],
)
execute_query_external_table = BigQueryExecuteQueryOperator(
task_id="execute_query_external_table",
destination_dataset_table="{}.selected_data_from_external_table".
format(DATASET_NAME),
sql='SELECT * FROM `{}.external_table` WHERE name LIKE "W%"'.format(
DATASET_NAME),
use_legacy_sql=False,