def test_redis_version(webserver): """ Redis pip module version 3.4.0 has an issue in the Astronomer platform """ try: redis_module = webserver.pip_package.get_packages()['redis'] except KeyError: raise Exception("redis pip module is not installed") version = redis_module['version'] assert semantic_version(version) != semantic_version('3.4.0'), \ "redis module must not be 3.4.0"
def test_werkzeug_version(webserver): """ Werkzeug pip module version >= 1.0.0 has an issue """ try: werkzeug_module = webserver.pip_package.get_packages()['Werkzeug'] except KeyError: raise Exception("Werkzeug pip module is not installed") version = werkzeug_module['version'] assert semantic_version(version) < semantic_version('1.0.0'), \ "Werkzeug pip module version must be less than 1.0.0"
def test_elasticsearch_version(webserver): """ Astronomer runs a version of ElasticSearch that requires our users to run the client code of version 5.5.3 or greater """ try: elasticsearch_module = webserver.pip_package.get_packages()['elasticsearch'] except KeyError: raise Exception("elasticsearch pip module is not installed") version = elasticsearch_module['version'] assert semantic_version(version) >= semantic_version('5.5.3'), \ "elasticsearch module must be version 5.5.3 or greater"
def test_astronomer_airflow_check_version(webserver): """ astronomer-airflow-version-check 1.0.0 has an issue in the Astronomer platform """ try: version_check_module = webserver.pip_package.get_packages()['astronomer-airflow-version-check'] except KeyError: print("astronomer-airflow-version-check pip module is not installed") return version = version_check_module['version'] assert semantic_version(version) >= semantic_version('1.0.1'), \ "astronomer-airflow-version-check module must be greater than 1.0.0"
def test_airflow_configs(scheduler, docker_client): """Verify certain Airflow configurations""" distro = get_label(docker_client, "io.astronomer.docker.distro") relative_config_path = "site-packages/airflow/config_templates/default_airflow.cfg" python_install_dir = str( scheduler.check_output( 'python -c "import os; print(os.path.dirname(os.__file__))"') ).strip() config_file_path = f"{python_install_dir}/{relative_config_path}" assert scheduler.file( config_file_path).exists, f"{relative_config_path} does not exist !" if distro == "debian": expected_run_as_user = "******" elif distro == "alpine": expected_run_as_user = "******" elif distro == "rhel": expected_run_as_user = "******" else: expected_run_as_user = "" if airflow_2: assert scheduler.check_output( f"cat {config_file_path} | " "grep '^lazy_load_plugins' | awk '{print $3}'" ) == "False", "[core] lazy_load_plugins needs to be False for astronomer-version-check plugin to work" assert scheduler.check_output( f"cat {config_file_path} | " "grep '^auth_backend' | awk '{print $3}'" ) == "astronomer.flask_appbuilder.current_user_backend", \ "[api] auth_backend(s) needs to be set to 'astronomer.flask_appbuilder.current_user_backend' for Platform" assert scheduler.check_output( f"cat {config_file_path} | " "grep '^operation_timeout' | awk '{print $3}'" ) == "10.0", "[celery] operation_timeout needs to be set for AC >= 2.0.0" else: # Confirm that run_as_user is the UID for astro user (and not root) for AC images assert scheduler.check_output(f"cat {config_file_path} | " "grep '^run_as_user' | awk '{print $3}'" ).strip() == expected_run_as_user if semantic_version(airflow_version) >= semantic_version('1.10.7'): assert scheduler.check_output( f"cat {config_file_path} | " "grep '^update_fab_perms' | awk '{print $3}'" ) == "False", "[webserver] update_fab_perms needs to be False for AC >= 1.10.10"
def test_airflow_configs(scheduler, docker_client): """Verify certain Airflow configurations""" distro = get_label(docker_client, "io.astronomer.docker.distro") if distro == "debian": config_file_path = "/usr/local/lib/python3.7/site-packages/airflow/config_templates/default_airflow.cfg" expected_run_as_user = "******" elif distro == "alpine": config_file_path = "/usr/lib/python3.7/site-packages/airflow/config_templates/default_airflow.cfg" expected_run_as_user = "******" elif distro == "rhel": config_file_path = "/usr/local/lib/python3.6/site-packages/airflow/config_templates/default_airflow.cfg" expected_run_as_user = "******" else: config_file_path = "/usr/lib/python3.7/site-packages/airflow/config_templates/default_airflow.cfg" expected_run_as_user = "" if airflow_2: assert scheduler.check_output( f"cat {config_file_path} | " "grep '^lazy_load_plugins' | awk '{print $3}'" ) == "False", "[core] lazy_load_plugins needs to be False for astronomer-version-check plugin to work" assert scheduler.check_output( f"cat {config_file_path} | " "grep '^auth_backend' | awk '{print $3}'" ) == "astronomer.flask_appbuilder.current_user_backend", \ "[api] auth_backend needs to be set to 'astronomer.flask_appbuilder.current_user_backend' for Platform" else: # Confirm that run_as_user is the UID for astro user (and not root) for AC images assert scheduler.check_output(f"cat {config_file_path} | " "grep '^run_as_user' | awk '{print $3}'" ).strip() == expected_run_as_user if semantic_version(airflow_version) >= semantic_version('1.10.7'): assert scheduler.check_output( f"cat {config_file_path} | " "grep '^update_fab_perms' | awk '{print $3}'" ) == "False", "[webserver] update_fab_perms needs to be False for AC >= 1.10.10"
instance of postgres, which is also running in Docker. Testinfra is used to configuration test the image. In effect, testinfra simplifies and provides syntactic sugar for doing execs into a running container. """ import os import docker import pytest import testinfra from kubernetes import client, config from packaging.version import parse as semantic_version airflow_version = semantic_version(os.environ.get("AIRFLOW_VERSION")) airflow_2 = airflow_version >= semantic_version("2.0.0") def create_kube_client(in_cluster=False): """ Load and store authentication and cluster information from kube-config file; if running inside a pod, use Kubernetes service account. Use that to instantiate Kubernetes client. """ if in_cluster: print("Using in cluster kubernetes configuration") config.load_incluster_config() else: print("Using kubectl kubernetes configuration") config.load_kube_config()