def create_or_replace_stitch_view(
bigquery_client,
dataset_name,
table_definition,
ds,
destination_project_id,
internal_project_id,
sqls_folder
):
table_name = table_definition['table']['table_name']
history_table_name = table_name + '_history'
# # # Create view
sql = render_stitch_view_template(
sqls_folder=sqls_folder,
internal_project_id=internal_project_id,
dataset_name=dataset_name,
table_name=table_name,
history_table_name=history_table_name,
ds=ds
)
print('Stitch view: ' + sql)
dataset = create_dataset(bigquery_client, dataset_name, destination_project_id)
dest_view_ref = dataset.table(table_name)
create_view(bigquery_client, sql, dest_view_ref)
def create_view_task(ds, **kwargs):
client = bigquery.Client()
dest_table_name = view_name
dest_table_ref = create_dataset(client, dataset_name, parse_destination_dataset_project_id).table(dest_table_name)
print('View sql: \n' + sql)
create_view(client, sql, dest_table_ref)
def create_or_replace_internal_view(bigquery_client, dataset_name,
table_definition, ds, public_project_id,
public_dataset_name, internal_project_id,
destination_project_id, sqls_folder):
table_name = table_definition['table']['table_name']
parser_type = table_definition['parser'].get('type', 'log')
udf_name = 'parse_{}'.format(table_name)
dataset = create_dataset(bigquery_client, dataset_name,
internal_project_id)
# # # Create UDF
sql = render_parse_udf_template(
sqls_folder,
parser_type,
internal_project_id=internal_project_id,
dataset_name=dataset_name,
udf_name=udf_name,
abi=json.dumps(table_definition['parser']['abi']),
struct_fields=create_struct_string_from_schema(
table_definition['table']['schema']))
query(bigquery_client, sql)
# # # Create view
selector = abi_to_selector(parser_type, table_definition['parser']['abi'])
parse_mode = get_parse_mode(HistoryType.LIVE)
full_source_table_name = get_source_table(
parser_type=parser_type,
parse_mode=parse_mode,
ds=ds,
internal_project_id=internal_project_id,
public_project_id=public_project_id,
public_dataset_name=public_dataset_name,
selector=selector)
sql = generate_parse_sql_template(
sqls_folder,
parser_type,
parse_mode,
full_source_table_name=full_source_table_name,
selector=selector,
internal_project_id=internal_project_id,
destination_project_id=destination_project_id,
dataset_name=dataset_name,
udf_name=udf_name,
table_definition=table_definition,
parse_all_partitions=None,
ds=ds)
dest_view_ref = dataset.table(table_name)
create_view(bigquery_client, sql, dest_view_ref)
def create_view_task(ds, **kwargs):
client = bigquery.Client()
dest_table_name = 'amended_tokens'
dest_table_ref = client.dataset(dataset_name, project=destination_dataset_project_id).table(dest_table_name)
sql_path = os.path.join(dags_folder, 'resources/stages/enrich/sqls/amended_tokens.sql')
sql = read_file(sql_path)
print('amended_tokens view: \n' + sql)
description = 'Tokens amended with data from https://github.com/blockchain-etl/ethereum-etl-airflow/blob/master/dags/resources/stages/seed/data/token_amendments.csv'
create_view(client, sql, dest_table_ref, description=description)