def test_airflow_command(self):
"""Checking 'airflow' command It should return non-zero exit code."""
with pytest.raises(subprocess.CalledProcessError) as ctx:
run_command([
"docker", "run", "--rm", "-e", "COLUMNS=180", docker_image,
"airflow"
])
assert 2 == ctx.value.returncode
def test_pip_dependencies_conflict(self):
try:
run_command([
"docker", "run", "--rm", "--entrypoint", "/bin/bash",
docker_image, "-c", 'pip check'
])
except subprocess.CalledProcessError as ex:
display_dependency_conflict_message()
raise ex
def test_execute_airflow_as_root(self):
run_command([
"docker",
"run",
"--rm",
"--user",
"0",
"-e",
"PYTHONDONTWRITEBYTECODE=true",
docker_image,
"airflow",
"info",
])
def test_dockerfile_example(dockerfile):
rel_dockerfile_path = Path(dockerfile).relative_to(DOCKER_EXAMPLES_DIR)
image_name = str(rel_dockerfile_path).lower().replace("/", "-")
content = Path(dockerfile).read_text()
new_content = re.sub(
r'FROM apache/airflow:.*', fr'FROM apache/airflow:{get_latest_airflow_version_released()}', content
)
try:
run_command(
["docker", "build", ".", "--tag", image_name, '-f', '-'],
cwd=str(Path(dockerfile).parent),
input=new_content.encode(),
)
finally:
run_command(["docker", "rmi", "--force", image_name])
def test_bash_version(self):
"""Checking 'bash --version' command It should return zero exit code."""
output = run_command(
[
"docker", "run", "--rm", "-e", "COLUMNS=180", docker_image,
"bash", "--version"
],
return_output=True,
)
assert "GNU bash," in output
def test_python_version(self):
"""Checking 'python --version' command It should return zero exit code."""
output = run_command(
[
"docker", "run", "--rm", "-e", "COLUMNS=180", docker_image,
"python", "--version"
],
return_output=True,
)
assert "Python 3." in output
def test_airflow_version(self):
"""Checking 'airflow version' command It should return zero exit code."""
output = run_command(
[
"docker", "run", "--rm", "-e", "COLUMNS=180", docker_image,
"airflow", "version"
],
return_output=True,
)
assert "2." in output
def test_run_custom_python_packages_as_root(self):
with tempfile.TemporaryDirectory() as tmp_dir:
(Path(tmp_dir) / "__init__.py").write_text('')
(Path(tmp_dir) / "awesome.py").write_text('print("Awesome")')
run_command([
"docker",
"run",
"--rm",
"-e",
f"PYTHONPATH={tmp_dir}",
"-e",
"PYTHONDONTWRITEBYTECODE=true",
"-v",
f"{tmp_dir}:{tmp_dir}",
"--user",
"0",
docker_image,
"python",
"-c",
"import awesome",
])
def run_bash_in_docker(bash_script, **kwargs):
docker_command = [
"docker",
"run",
"--rm",
"-e",
"COLUMNS=180",
"--entrypoint",
"/bin/bash",
docker_image,
"-c",
bash_script,
]
return run_command(docker_command, **kwargs)
def run_python_in_docker(python_script, **kwargs):
docker_command = [
"docker",
"run",
"--rm",
"-e",
"COLUMNS=180",
"-e",
"PYTHONDONTWRITEBYTECODE=true",
docker_image,
"python",
"-c",
python_script,
]
return run_command(docker_command, **kwargs)
def test_trigger_dag_and_wait_for_result():
compose_file_path = SOURCE_ROOT / "docs" / "apache-airflow" / "start" / "docker-compose.yaml"
with tempfile.TemporaryDirectory() as tmp_dir, tmp_chdir(
tmp_dir), mock.patch.dict('os.environ',
AIRFLOW_IMAGE_NAME=docker_image):
copyfile(str(compose_file_path), f"{tmp_dir}/docker-compose.yaml")
os.mkdir(f"{tmp_dir}/dags")
os.mkdir(f"{tmp_dir}/logs")
os.mkdir(f"{tmp_dir}/plugins")
(Path(tmp_dir) / ".env").write_text(
f"AIRFLOW_UID={subprocess.check_output(['id', '-u']).decode()}\n")
print(".emv=", (Path(tmp_dir) / ".env").read_text())
copyfile(
str(SOURCE_ROOT / "airflow" / "example_dags" /
"example_bash_operator.py"),
f"{tmp_dir}/dags/example_bash_operator.py",
)
run_command(["docker-compose", "config"])
run_command(
["docker-compose", "down", "--volumes", "--remove-orphans"])
try:
run_command(["docker-compose", "up", "-d"])
# The --wait condition was released in docker-compose v2.1.1, but we want to support
# docker-compose v1 yet.
# See:
# https://github.com/docker/compose/releases/tag/v2.1.1
# https://github.com/docker/compose/pull/8777
for container_id in (subprocess.check_output(
["docker-compose", 'ps', '-q']).decode().strip().splitlines()):
wait_for_container(container_id)
api_request("PATCH",
path=f"dags/{DAG_ID}",
json={"is_paused": False})
api_request("POST",
path=f"dags/{DAG_ID}/dagRuns",
json={"dag_run_id": DAG_RUN_ID})
try:
wait_for_terminal_dag_state(dag_id=DAG_ID,
dag_run_id=DAG_RUN_ID)
dag_state = api_request(
"GET", f"dags/{DAG_ID}/dagRuns/{DAG_RUN_ID}").get("state")
assert dag_state == "success"
except Exception:
print(f"HTTP: GET dags/{DAG_ID}/dagRuns/{DAG_RUN_ID}")
pprint(
api_request("GET", f"dags/{DAG_ID}/dagRuns/{DAG_RUN_ID}"))
print(
f"HTTP: GET dags/{DAG_ID}/dagRuns/{DAG_RUN_ID}/taskInstances"
)
pprint(
api_request(
"GET",
f"dags/{DAG_ID}/dagRuns/{DAG_RUN_ID}/taskInstances"))
raise
except Exception:
run_command(["docker", "ps"])
run_command(["docker-compose", "logs"])
raise
finally:
run_command(["docker-compose", "down", "--volumes"])
def test_shell_script_example(script_file):
run_command(["bash", script_file])
