def test_redis_version(webserver):
""" Redis pip module version 3.4.0 has an issue in the Astronomer platform
"""
try:
redis_module = webserver.pip_package.get_packages()['redis']
except KeyError:
raise Exception("redis pip module is not installed")
version = redis_module['version']
assert semantic_version(version) != semantic_version('3.4.0'), \
"redis module must not be 3.4.0"
def test_werkzeug_version(webserver):
""" Werkzeug pip module version >= 1.0.0 has an issue
"""
try:
werkzeug_module = webserver.pip_package.get_packages()['Werkzeug']
except KeyError:
raise Exception("Werkzeug pip module is not installed")
version = werkzeug_module['version']
assert semantic_version(version) < semantic_version('1.0.0'), \
"Werkzeug pip module version must be less than 1.0.0"
def test_elasticsearch_version(webserver):
""" Astronomer runs a version of ElasticSearch that requires
our users to run the client code of version 5.5.3 or greater
"""
try:
elasticsearch_module = webserver.pip_package.get_packages()['elasticsearch']
except KeyError:
raise Exception("elasticsearch pip module is not installed")
version = elasticsearch_module['version']
assert semantic_version(version) >= semantic_version('5.5.3'), \
"elasticsearch module must be version 5.5.3 or greater"
def test_astronomer_airflow_check_version(webserver):
""" astronomer-airflow-version-check 1.0.0 has an issue in the Astronomer platform
"""
try:
version_check_module = webserver.pip_package.get_packages()['astronomer-airflow-version-check']
except KeyError:
print("astronomer-airflow-version-check pip module is not installed")
return
version = version_check_module['version']
assert semantic_version(version) >= semantic_version('1.0.1'), \
"astronomer-airflow-version-check module must be greater than 1.0.0"
def test_airflow_configs(scheduler, docker_client):
"""Verify certain Airflow configurations"""
distro = get_label(docker_client, "io.astronomer.docker.distro")
relative_config_path = "site-packages/airflow/config_templates/default_airflow.cfg"
python_install_dir = str(
scheduler.check_output(
'python -c "import os; print(os.path.dirname(os.__file__))"')
).strip()
config_file_path = f"{python_install_dir}/{relative_config_path}"
assert scheduler.file(
config_file_path).exists, f"{relative_config_path} does not exist !"
if distro == "debian":
expected_run_as_user = "******"
elif distro == "alpine":
expected_run_as_user = "******"
elif distro == "rhel":
expected_run_as_user = "******"
else:
expected_run_as_user = ""
if airflow_2:
assert scheduler.check_output(
f"cat {config_file_path} | "
"grep '^lazy_load_plugins' | awk '{print $3}'"
) == "False", "[core] lazy_load_plugins needs to be False for astronomer-version-check plugin to work"
assert scheduler.check_output(
f"cat {config_file_path} | "
"grep '^auth_backend' | awk '{print $3}'"
) == "astronomer.flask_appbuilder.current_user_backend", \
"[api] auth_backend(s) needs to be set to 'astronomer.flask_appbuilder.current_user_backend' for Platform"
assert scheduler.check_output(
f"cat {config_file_path} | "
"grep '^operation_timeout' | awk '{print $3}'"
) == "10.0", "[celery] operation_timeout needs to be set for AC >= 2.0.0"
else:
# Confirm that run_as_user is the UID for astro user (and not root) for AC images
assert scheduler.check_output(f"cat {config_file_path} | "
"grep '^run_as_user' | awk '{print $3}'"
).strip() == expected_run_as_user
if semantic_version(airflow_version) >= semantic_version('1.10.7'):
assert scheduler.check_output(
f"cat {config_file_path} | "
"grep '^update_fab_perms' | awk '{print $3}'"
) == "False", "[webserver] update_fab_perms needs to be False for AC >= 1.10.10"
def test_airflow_configs(scheduler, docker_client):
"""Verify certain Airflow configurations"""
distro = get_label(docker_client, "io.astronomer.docker.distro")
if distro == "debian":
config_file_path = "/usr/local/lib/python3.7/site-packages/airflow/config_templates/default_airflow.cfg"
expected_run_as_user = "******"
elif distro == "alpine":
config_file_path = "/usr/lib/python3.7/site-packages/airflow/config_templates/default_airflow.cfg"
expected_run_as_user = "******"
elif distro == "rhel":
config_file_path = "/usr/local/lib/python3.6/site-packages/airflow/config_templates/default_airflow.cfg"
expected_run_as_user = "******"
else:
config_file_path = "/usr/lib/python3.7/site-packages/airflow/config_templates/default_airflow.cfg"
expected_run_as_user = ""
if airflow_2:
assert scheduler.check_output(
f"cat {config_file_path} | "
"grep '^lazy_load_plugins' | awk '{print $3}'"
) == "False", "[core] lazy_load_plugins needs to be False for astronomer-version-check plugin to work"
assert scheduler.check_output(
f"cat {config_file_path} | "
"grep '^auth_backend' | awk '{print $3}'"
) == "astronomer.flask_appbuilder.current_user_backend", \
"[api] auth_backend needs to be set to 'astronomer.flask_appbuilder.current_user_backend' for Platform"
else:
# Confirm that run_as_user is the UID for astro user (and not root) for AC images
assert scheduler.check_output(f"cat {config_file_path} | "
"grep '^run_as_user' | awk '{print $3}'"
).strip() == expected_run_as_user
if semantic_version(airflow_version) >= semantic_version('1.10.7'):
assert scheduler.check_output(
f"cat {config_file_path} | "
"grep '^update_fab_perms' | awk '{print $3}'"
) == "False", "[webserver] update_fab_perms needs to be False for AC >= 1.10.10"
instance of postgres, which is also running in Docker.
Testinfra is used to configuration test the image. In effect,
testinfra simplifies and provides syntactic sugar for doing
execs into a running container.
"""
import os
import docker
import pytest
import testinfra
from kubernetes import client, config
from packaging.version import parse as semantic_version
airflow_version = semantic_version(os.environ.get("AIRFLOW_VERSION"))
airflow_2 = airflow_version >= semantic_version("2.0.0")
def create_kube_client(in_cluster=False):
"""
Load and store authentication and cluster information from kube-config
file; if running inside a pod, use Kubernetes service account. Use that to
instantiate Kubernetes client.
"""
if in_cluster:
print("Using in cluster kubernetes configuration")
config.load_incluster_config()
else:
print("Using kubectl kubernetes configuration")
config.load_kube_config()
