def test_execute(self, mock_hook): operator = BigQueryCreateExternalTableOperator( task_id=TASK_ID, destination_project_dataset_table='{}.{}'.format( TEST_DATASET, TEST_TABLE_ID), schema_fields=[], bucket=TEST_GCS_BUCKET, source_objects=TEST_GCS_DATA, source_format=TEST_SOURCE_FORMAT) operator.execute(None) mock_hook.return_value \ .get_conn() \ .cursor() \ .create_external_table \ .assert_called_once_with( external_project_dataset_table='{}.{}'.format( TEST_DATASET, TEST_TABLE_ID ), schema_fields=[], source_uris=['gs://{}/{}'.format(TEST_GCS_BUCKET, source_object) for source_object in TEST_GCS_DATA], source_format=TEST_SOURCE_FORMAT, compression='NONE', skip_leading_rows=0, field_delimiter=',', max_bad_records=0, quote_character=None, allow_quoted_newlines=False, allow_jagged_rows=False, src_fmt_configs={}, labels=None )
def test_execute(self, mock_hook): operator = BigQueryCreateExternalTableOperator( task_id=TASK_ID, destination_project_dataset_table='{}.{}'.format( TEST_DATASET, TEST_TABLE_ID ), schema_fields=[], bucket=TEST_GCS_BUCKET, source_objects=TEST_GCS_DATA, source_format=TEST_SOURCE_FORMAT ) operator.execute(None) mock_hook.return_value \ .get_conn() \ .cursor() \ .create_external_table \ .assert_called_once_with( external_project_dataset_table='{}.{}'.format( TEST_DATASET, TEST_TABLE_ID ), schema_fields=[], source_uris=['gs://{}/{}'.format(TEST_GCS_BUCKET, source_object) for source_object in TEST_GCS_DATA], source_format=TEST_SOURCE_FORMAT, compression='NONE', skip_leading_rows=0, field_delimiter=',', max_bad_records=0, quote_character=None, allow_quoted_newlines=False, allow_jagged_rows=False, src_fmt_configs={}, labels=None )
max_results="10", selected_fields="value,to_address", ) get_data_result = BashOperator( task_id="get-data-result", bash_command="echo \"{{ task_instance.xcom_pull('get-data') }}\"") create_external_table = BigQueryCreateExternalTableOperator( task_id="create-external-table", bucket=DATA_SAMPLE_GCS_BUCKET_NAME, source_objects=[DATA_SAMPLE_GCS_OBJECT_NAME], destination_project_dataset_table="{}.external_table".format( DATASET_NAME), skip_leading_rows=1, schema_fields=[{ "name": "name", "type": "STRING" }, { "name": "post_abbr", "type": "STRING" }], ) execute_query_external_table = BigQueryOperator( task_id="execute-query-external-table", destination_dataset_table="{}.selected_data_from_external_table". format(DATASET_NAME), sql='SELECT * FROM `{}.external_table` WHERE name LIKE "W%"'.format( DATASET_NAME), use_legacy_sql=False,
use_legacy_sql=False, dag=dag) t5_prime_tableCheck.set_upstream(writeToGCS_task) storageToBQ_task = GoogleCloudStorageToBigQueryOperator(task_id='uploadtoBQ_{}'.format(datetime.now().strptime('%Y%m%d_%H%M')), bucket=bucket_name, source_objects=tempGCS_dir_paths, destination_project_dataset="`{}.{}.{}`".format(project_id,dataset_id,table_id), schema_fields=schema, create_disposition='WRITE_TRUNCATE', dag=dag) storageToBQ_task.set_upstream(t5_prime_tableCheck) dummy_task.set_upstream(storageToBQ_task) except Exception as e : print("BigQueryCheck error = {}".format(e)) t5_gamme_tableCreate = BigQueryCreateExternalTableOperator(task_id='CreateBQtable', bucket=bucket_name, source_objects=tempGCS_dir_paths, destination_project_dataset_table="{}.{}.{}"format(project_id,dataset_id,dataset_id), schema=schema, dag=dag) dummy_task.set_upstream(t5_gamme_tableCreate) ''' Read in queries stored in GCS bucket t5_(gamma,prime) >> t6_(uploadToBQ_TableExists, CreateTableBQ_UploadtoBQ) >> t7_dummy >> [t8_query1Result_BQTable,...] Note : Storing it all in one dataset and deleting dataset at the end! '''