def run_shell_with_build_image_checks(
verbose: bool, dry_run: bool, shell_params: ShellParams
) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]:
"""
Executes a shell command built from params passed, checking if build is not needed.
* checks if there are enough resources to run shell
* checks if image was built at least once (if not - forces the build)
* if not forces, checks if build is needed and asks the user if so
* builds the image if needed
* prints information about the build
* constructs docker compose command to enter shell
* executes it
:param verbose: print commands when running
:param dry_run: do not execute "write" commands - just print what would happen
:param shell_params: parameters of the execution
"""
check_docker_resources(verbose,
shell_params.airflow_image_name,
dry_run=dry_run)
build_ci_image_check_cache = Path(BUILD_CACHE_DIR,
shell_params.airflow_branch,
f".built_{shell_params.python}")
ci_image_params = BuildCiParams(python=shell_params.python,
upgrade_to_newer_dependencies="false")
if build_ci_image_check_cache.exists():
console.print(
f'[bright_blue]{shell_params.the_image_type} image already built locally.[/]'
)
else:
console.print(
f'[bright_yellow]{shell_params.the_image_type} image not built locally. '
f'Forcing build.[/]')
ci_image_params.force_build = True
build_ci_image(verbose,
dry_run=dry_run,
with_ci_group=False,
ci_image_params=ci_image_params)
shell_params.print_badge_info()
cmd = [
'docker-compose', 'run', '--service-ports', "-e", "BREEZE", '--rm',
'airflow'
]
cmd_added = shell_params.command_passed
env_variables = construct_env_variables_docker_compose_command(
shell_params)
if cmd_added is not None:
cmd.extend(['-c', cmd_added])
return run_command(cmd,
verbose=verbose,
dry_run=dry_run,
env=env_variables,
text=True)
def pull_image(
verbose: bool,
dry_run: bool,
python: str,
github_repository: str,
run_in_parallel: bool,
python_versions: str,
parallelism: int,
image_tag: Optional[str],
wait_for_image: bool,
tag_as_latest: bool,
verify_image: bool,
extra_pytest_args: Tuple,
):
"""Pull and optionally verify CI images - possibly in parallel for all Python versions."""
if run_in_parallel:
python_version_list = get_python_version_list(python_versions)
ci_image_params_list = [
BuildCiParams(image_tag=image_tag,
python=python,
github_repository=github_repository)
for python in python_version_list
]
run_pull_in_parallel(
dry_run=dry_run,
parallelism=parallelism,
image_params_list=ci_image_params_list,
python_version_list=python_version_list,
verbose=verbose,
verify_image=verify_image,
wait_for_image=wait_for_image,
tag_as_latest=tag_as_latest,
extra_pytest_args=extra_pytest_args
if extra_pytest_args is not None else (),
)
else:
image_params = BuildCiParams(image_tag=image_tag,
python=python,
github_repository=github_repository)
return_code, info = run_pull_image(
image_params=image_params,
dry_run=dry_run,
verbose=verbose,
wait_for_image=wait_for_image,
tag_as_latest=tag_as_latest,
poll_time=10.0,
)
if return_code != 0:
console.print(
f"[red]There was an error when pulling CI image: {info}[/]")
sys.exit(return_code)
def build_docs(verbose: bool, dry_run: bool, docs_only: bool,
spellcheck_only: bool, package_filter: Tuple[str]):
"""
Builds documentation in the container.
* figures out CI image name
* checks if there are enough resources
* converts parameters into a DocParams class
"""
params = BuildCiParams()
ci_image_name = params.airflow_image_name
check_docker_resources(verbose, ci_image_name)
doc_builder = DocParams(
package_filter=package_filter,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
)
extra_docker_flags = get_extra_docker_flags(MOUNT_SELECTED)
cmd = []
cmd.extend(["docker", "run"])
cmd.extend(extra_docker_flags)
cmd.extend(["-t", "-e", "GITHUB_ACTIONS="])
cmd.extend(["--entrypoint", "/usr/local/bin/dumb-init", "--pull", "never"])
cmd.extend([
ci_image_name, "--",
"/opt/airflow/scripts/in_container/run_docs_build.sh"
])
cmd.extend(doc_builder.args_doc_builder)
run_command(cmd, verbose=verbose, dry_run=dry_run, text=True)
def build_image(
verbose: bool,
dry_run: bool,
with_ci_group: bool,
build_multiple_images: bool,
python_versions: str,
answer: str,
**kwargs,
):
"""Build CI image. Include building multiple images for all python versions (sequentially)."""
def run_build(ci_image_params: BuildCiParams) -> None:
return_code, info = build_ci_image(verbose=verbose,
dry_run=dry_run,
with_ci_group=with_ci_group,
ci_image_params=ci_image_params)
if return_code != 0:
console.print(f"[red]Error when building image! {info}")
sys.exit(return_code)
set_forced_answer(answer)
parameters_passed = filter_out_none(**kwargs)
if build_multiple_images:
with ci_group(f"Building images sequentially {python_versions}",
enabled=with_ci_group):
python_version_list = get_python_version_list(python_versions)
for python in python_version_list:
params = BuildCiParams(**parameters_passed)
params.python = python
params.answer = answer
run_build(ci_image_params=params)
else:
params = BuildCiParams(**parameters_passed)
run_build(ci_image_params=params)
def get_ci_image_build_params(parameters_passed: Dict) -> BuildCiParams:
"""
Converts parameters received as dict into BuildCiParams. In case cacheable
parameters are missing, it reads the last used value for that parameter
from the cache and if it is not found, it uses default value for that parameter.
This method updates cached based on parameters passed via Dict.
:param parameters_passed: parameters to use when constructing BuildCiParams
"""
ci_image_params = BuildCiParams(**parameters_passed)
synchronize_parameters_with_cache(ci_image_params, parameters_passed)
return ci_image_params
def build_docs(
verbose: bool,
dry_run: bool,
github_repository: str,
docs_only: bool,
spellcheck_only: bool,
package_filter: Tuple[str],
):
"""Build documentation in the container."""
params = BuildCiParams(github_repository=github_repository,
python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION)
ci_image_name = params.airflow_image_name
doc_builder = DocBuildParams(
package_filter=package_filter,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
)
extra_docker_flags = get_extra_docker_flags(MOUNT_SELECTED)
env = construct_env_variables_docker_compose_command(params)
cmd = [
"docker",
"run",
"-t",
*extra_docker_flags,
"-e",
"GITHUB_ACTIONS=",
"-e",
"SKIP_ENVIRONMENT_INITIALIZATION=true",
"--pull",
"never",
ci_image_name,
"/opt/airflow/scripts/in_container/run_docs_build.sh",
*doc_builder.args_doc_builder,
]
process = run_command(cmd,
verbose=verbose,
dry_run=dry_run,
text=True,
env=env,
check=False)
sys.exit(process.returncode)
def verify_image(
verbose: bool,
dry_run: bool,
python: str,
github_repository: str,
image_name: str,
image_tag: str,
extra_pytest_args: Tuple,
):
"""Verify CI image."""
if image_name is None:
build_params = BuildCiParams(python=python,
image_tag=image_tag,
github_repository=github_repository)
image_name = build_params.airflow_image_name_with_tag
console.print(f"[bright_blue]Verifying CI image: {image_name}[/]")
return_code, info = verify_an_image(
image_name=image_name,
verbose=verbose,
dry_run=dry_run,
image_type='CI',
extra_pytest_args=extra_pytest_args,
)
sys.exit(return_code)
def build_ci_image(verbose: bool, dry_run: bool, with_ci_group: bool,
ci_image_params: BuildCiParams) -> Tuple[int, str]:
"""
Builds CI image:
* fixes group permissions for files (to improve caching when umask is 002)
* converts all the parameters received via kwargs into BuildCIParams (including cache)
* prints info about the image to build
* logs int to docker registry on CI if build cache is being executed
* removes "tag" for previously build image so that inline cache uses only remote image
* constructs docker-compose command to run based on parameters passed
* run the build command
* update cached information that the build completed and saves checksums of all files
for quick future check if the build is needed
:param verbose: print commands when running
:param dry_run: do not execute "write" commands - just print what would happen
:param with_ci_group: whether to wrap the build in CI logging group
:param ci_image_params: CI image parameters
"""
fix_group_permissions()
if verbose or dry_run:
console.print(
f"\n[bright_blue]Building CI image of airflow from {AIRFLOW_SOURCES_ROOT} "
f"python version: {ci_image_params.python}[/]\n")
with ci_group(
f"Build CI image for Python {ci_image_params.python} "
f"with tag: {ci_image_params.image_tag}",
enabled=with_ci_group,
):
ci_image_params.print_info()
if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies:
if not should_we_run_the_build(build_ci_params=ci_image_params):
return 0, f"Image build: {ci_image_params.python}"
run_command(
[
"docker", "rmi", "--no-prune", "--force",
ci_image_params.airflow_image_name
],
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=False,
)
if ci_image_params.prepare_buildx_cache:
login_to_docker_registry(ci_image_params, dry_run=dry_run)
cmd = construct_docker_build_command(
image_params=ci_image_params,
verbose=verbose,
required_args=REQUIRED_CI_IMAGE_ARGS,
optional_args=OPTIONAL_CI_IMAGE_ARGS,
production_image=False,
)
if ci_image_params.empty_image:
console.print(
f"\n[blue]Building empty CI Image for Python {ci_image_params.python}\n"
)
cmd = construct_empty_docker_build_command(
image_params=ci_image_params)
build_result = run_command(
cmd,
input="FROM scratch\n",
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
)
else:
console.print(
f"\n[blue]Building CI Image for Python {ci_image_params.python}\n"
)
build_result = run_command(cmd,
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=False)
if not dry_run:
if build_result.returncode == 0:
ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch
ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
touch_cache_file(f"built_{ci_image_params.python}",
root_dir=ci_image_cache_dir)
calculate_md5_checksum_for_files(
ci_image_params.md5sum_cache_dir, update=True)
else:
console.print("[red]Error when building image![/]")
return (
build_result.returncode,
f"Image build: {ci_image_params.python}",
)
else:
console.print(
"[blue]Not updating build cache because we are in `dry_run` mode.[/]"
)
if ci_image_params.push_image:
return tag_and_push_image(image_params=ci_image_params,
dry_run=dry_run,
verbose=verbose)
return build_result.returncode, f"Image build: {ci_image_params.python}"