def exec_call_stored_procedure(task, runner: QueryRunner, **kwargs): airflow_context = kwargs['airflow_context'] sql = airflow_context['templates_dict']['sql'] runner.call_stored_procedure( query=sql )
def exec_replace_or_upsert(task: schema.Task, runner: QueryRunner, **kwargs): airflow_context = kwargs["airflow_context"] relation = create_relation_from_task(task=task) sql = airflow_context["templates_dict"]["sql"] dag_run_conf = (airflow_context["dag_run"].conf if airflow_context["dag_run"].conf else {}) full_refresh = dag_run_conf.get("full_refresh", False) if type(full_refresh) != bool: raise RuntimeError( "Dag config `full_refresh` must be set to either `true` or `false` (as a boolean value)." ) logging.info(f"### IS FULL REFRESH ENABLED: {full_refresh}") runner.replace_or_upsert( query=sql, relation=relation, options={ "partition_key": task.partitioning.field if task.partitioning else None, "partition_data_type": task.partitioning.data_type if task.partitioning else None, "full_refresh": full_refresh, }, )
def exec_recreate_udf(task, runner: QueryRunner, **kwargs): airflow_context = kwargs["airflow_context"] relation = create_relation_from_task(task=task) sql = airflow_context["templates_dict"]["sql"] arguments = task.options.get("arguments", []) runner.recreate_udf(arguments=arguments, query=sql, relation=relation)
def exec_recreate_view(task, runner: QueryRunner, **kwargs): airflow_context = kwargs['airflow_context'] relation = create_relation_from_task(task=task) sql = airflow_context['templates_dict']['sql'] runner.recreate_view( query=sql, relation=relation )
def exec_recreate_udf(task, runner: QueryRunner, **kwargs): airflow_context = kwargs['airflow_context'] relation = create_relation_from_task(task=task) sql = airflow_context['templates_dict']['sql'] arguments = task.options.get('arguments', []) runner.recreate_udf( arguments=arguments, query=sql, relation=relation )
def exec_replace_or_upsert(task: schema.Task, runner: QueryRunner, **kwargs): airflow_context = kwargs['airflow_context'] relation = create_relation_from_task(task=task) sql = airflow_context['templates_dict']['sql'] dag_run_conf = airflow_context['dag_run'].conf if airflow_context['dag_run'].conf else {} full_refresh = dag_run_conf.get('full_refresh', False) if type(full_refresh) != bool: raise RuntimeError('Dag config `full_refresh` must be set to either `true` or `false` (as a boolean value).') logging.info(f'### IS FULL REFRESH ENABLED: {full_refresh}') runner.replace_or_upsert( query=sql, relation=relation, options={ 'partition_key': task.partitioning.field if task.partitioning else None, 'partition_data_type': task.partitioning.data_type if task.partitioning else None, 'full_refresh': full_refresh } )
def exec_create_schema(task, runner: QueryRunner, **kwargs): runner.create_schema(project_id=task.database, dataset_id=task.schema)
def exec_assertion(task, runner: QueryRunner, **kwargs): airflow_context = kwargs['airflow_context'] sql = airflow_context['templates_dict']['sql'] runner.assertion(query=sql)
def exec_assertion(task, runner: QueryRunner, **kwargs): airflow_context = kwargs["airflow_context"] sql = airflow_context["templates_dict"]["sql"] runner.assertion(query=sql)
def exec_call_stored_procedure(task, runner: QueryRunner, **kwargs): airflow_context = kwargs["airflow_context"] sql = airflow_context["templates_dict"]["sql"] runner.call_stored_procedure(query=sql)