def test_error_on_no_dag():
from flowetl.util import get_qa_checks
with pytest.raises(
TypeError,
match="Must set dag argument or be in a dag context manager."):
get_qa_checks()
def test_uses_context_dag():
from airflow import DAG
from flowetl.util import get_qa_checks
with DAG("DUMMY_DAG", start_date=datetime.now()) as dag:
get_qa_checks()
assert (str(
Path(__file__).parent.parent.parent / "flowetl" / "flowetl" /
"qa_checks") in dag.template_searchpath)
def test_module_path_added_to_dag_template_locations():
from airflow import DAG
from flowetl.util import get_qa_checks
dag = DAG("DUMMY_DAG", start_date=datetime.now())
get_qa_checks(dag=dag)
assert (str(
Path(__file__).parent.parent.parent / "flowetl" / "flowetl" /
"qa_checks") in dag.template_searchpath)
def test_default_qa_checks_found():
from airflow import DAG
from flowetl.util import get_qa_checks
dag = DAG("DUMMY_DAG", start_date=datetime.now())
check_operators = get_qa_checks(dag=dag)
assert {op.task_id: op.sql for op in check_operators} == qa_checks
def test_additional_checks_collected_in_subdirs(tmpdir):
from airflow import DAG
from flowetl.util import get_qa_checks
Path(tmpdir / "qa_checks" / "calls").mkdir(parents=True)
Path(tmpdir / "qa_checks" / "calls" / "DUMMY_CHECK.sql").touch()
check_operators = get_qa_checks(dag=DAG("DUMMY_DAG",
start_date=datetime.now(),
template_searchpath=str(tmpdir)))
assert len(check_operators) == len(qa_checks)
check_operators = get_qa_checks(dag=DAG(
"DUMMY_DAG",
start_date=datetime.now(),
template_searchpath=str(tmpdir),
params=dict(cdr_type="calls"),
), )
assert len(check_operators) > len(qa_checks)
def test_additional_checks_collected(tmpdir):
from airflow import DAG
from flowetl.util import get_qa_checks
Path(tmpdir / "qa_checks").mkdir()
Path(tmpdir / "qa_checks" / "DUMMY_CHECK.sql").touch()
check_operators = get_qa_checks(dag=DAG("DUMMY_DAG",
start_date=datetime.now(),
template_searchpath=str(tmpdir)))
assert len(check_operators) > len(qa_checks)
def test_additional_checks_collected_from_home():
from airflow import DAG, settings
from flowetl.util import get_qa_checks
checks_folder = Path(settings.DAGS_FOLDER) / "qa_checks"
checks_folder.mkdir(parents=True)
(checks_folder / "DUMMY_CHECK.sql").touch()
check_operators = get_qa_checks(
dag=DAG("DUMMY_DAG", start_date=datetime.now()))
assert len(check_operators) > len(qa_checks)
def test_name_suffix_added(tmpdir):
from airflow import DAG
from flowetl.util import get_qa_checks
Path(tmpdir / "qa_checks" / "calls").mkdir(parents=True)
Path(tmpdir / "qa_checks" / "calls" / "DUMMY_CHECK.sql").touch()
check_operators = get_qa_checks(dag=DAG(
"DUMMY_DAG",
start_date=datetime.now(),
template_searchpath=str(tmpdir),
params=dict(cdr_type="calls"),
))
assert any(op for op in check_operators
if op.task_id == "DUMMY_CHECK.calls")
def test_additional_checks_collected_if_specified(tmpdir):
from airflow import DAG
from flowetl.util import get_qa_checks
Path(tmpdir / "qa_checks").mkdir(parents=True)
Path(tmpdir / "qa_checks" / "DUMMY_CHECK.sql").touch()
check_operators = get_qa_checks(
dag=DAG(
"DUMMY_DAG",
start_date=datetime.now(),
),
additional_qa_check_paths=[str(tmpdir)],
)
assert len(check_operators) > len(qa_checks)
