def parse_task(ds, **kwargs):
client = bigquery.Client()
parse(bigquery_client=client,
table_definition=table_definition,
ds=ds,
source_project_id=SOURCE_PROJECT_ID,
source_dataset_name=SOURCE_DATASET_NAME,
destination_project_id=parse_destination_dataset_project_id,
sqls_folder=os.path.join(dags_folder,
'resources/stages/parse/sqls'),
parse_all_partitions=parse_all_partitions)
def test_create_or_update_table_from_table_definition(table_definition_file,
parse_all_partitions):
bigquery_client = MockBigqueryClient()
table_definition = read_json_file(
os.path.join(table_definitions_folder, table_definition_file))
parse(bigquery_client=bigquery_client,
table_definition=table_definition,
ds='2020-01-01',
source_project_id='bigquery-public-data',
source_dataset_name='crypto_ethereum',
destination_project_id='blockchain-etl',
sqls_folder=sqls_folder,
parse_all_partitions=parse_all_partitions,
time_func=lambda: 1587556654.993)
assert len(bigquery_client.queries) > 0
for ind, query in enumerate(bigquery_client.queries):
expected_filename = table_definition_file_to_expected_file(
table_definition_file, parse_all_partitions, ind)
assert trim(query) == trim(read_resource(expected_filename))