def exec_call_stored_procedure(task, runner: QueryRunner, **kwargs):
airflow_context = kwargs['airflow_context']
sql = airflow_context['templates_dict']['sql']
runner.call_stored_procedure(
query=sql
)
def exec_replace_or_upsert(task: schema.Task, runner: QueryRunner, **kwargs):
airflow_context = kwargs["airflow_context"]
relation = create_relation_from_task(task=task)
sql = airflow_context["templates_dict"]["sql"]
dag_run_conf = (airflow_context["dag_run"].conf
if airflow_context["dag_run"].conf else {})
full_refresh = dag_run_conf.get("full_refresh", False)
if type(full_refresh) != bool:
raise RuntimeError(
"Dag config `full_refresh` must be set to either `true` or `false` (as a boolean value)."
)
logging.info(f"### IS FULL REFRESH ENABLED: {full_refresh}")
runner.replace_or_upsert(
query=sql,
relation=relation,
options={
"partition_key":
task.partitioning.field if task.partitioning else None,
"partition_data_type":
task.partitioning.data_type if task.partitioning else None,
"full_refresh":
full_refresh,
},
)
def exec_recreate_udf(task, runner: QueryRunner, **kwargs):
airflow_context = kwargs["airflow_context"]
relation = create_relation_from_task(task=task)
sql = airflow_context["templates_dict"]["sql"]
arguments = task.options.get("arguments", [])
runner.recreate_udf(arguments=arguments, query=sql, relation=relation)
def exec_recreate_view(task, runner: QueryRunner, **kwargs):
airflow_context = kwargs['airflow_context']
relation = create_relation_from_task(task=task)
sql = airflow_context['templates_dict']['sql']
runner.recreate_view(
query=sql,
relation=relation
)
def exec_recreate_udf(task, runner: QueryRunner, **kwargs):
airflow_context = kwargs['airflow_context']
relation = create_relation_from_task(task=task)
sql = airflow_context['templates_dict']['sql']
arguments = task.options.get('arguments', [])
runner.recreate_udf(
arguments=arguments,
query=sql,
relation=relation
)
def exec_replace_or_upsert(task: schema.Task, runner: QueryRunner, **kwargs):
airflow_context = kwargs['airflow_context']
relation = create_relation_from_task(task=task)
sql = airflow_context['templates_dict']['sql']
dag_run_conf = airflow_context['dag_run'].conf if airflow_context['dag_run'].conf else {}
full_refresh = dag_run_conf.get('full_refresh', False)
if type(full_refresh) != bool:
raise RuntimeError('Dag config `full_refresh` must be set to either `true` or `false` (as a boolean value).')
logging.info(f'### IS FULL REFRESH ENABLED: {full_refresh}')
runner.replace_or_upsert(
query=sql,
relation=relation,
options={
'partition_key': task.partitioning.field if task.partitioning else None,
'partition_data_type': task.partitioning.data_type if task.partitioning else None,
'full_refresh': full_refresh
}
)
def exec_create_schema(task, runner: QueryRunner, **kwargs):
runner.create_schema(project_id=task.database, dataset_id=task.schema)
def exec_assertion(task, runner: QueryRunner, **kwargs):
airflow_context = kwargs['airflow_context']
sql = airflow_context['templates_dict']['sql']
runner.assertion(query=sql)
def exec_assertion(task, runner: QueryRunner, **kwargs):
airflow_context = kwargs["airflow_context"]
sql = airflow_context["templates_dict"]["sql"]
runner.assertion(query=sql)
def exec_call_stored_procedure(task, runner: QueryRunner, **kwargs):
airflow_context = kwargs["airflow_context"]
sql = airflow_context["templates_dict"]["sql"]
runner.call_stored_procedure(query=sql)