def build_image(verbose: bool, dry_run: bool, **kwargs) -> None:
"""
Builds CI image:
* fixes group permissions for files (to improve caching when umask is 002)
* converts all the parameters received via kwargs into BuildCIParams (including cache)
* prints info about the image to build
* logs int to docker registry on CI if build cache is being executed
* removes "tag" for previously build image so that inline cache uses only remote image
* constructs docker-compose command to run based on parameters passed
* run the build command
* update cached information that the build completed and saves checksums of all files
for quick future check if the build is needed
:param verbose: print commands when running
:param dry_run: do not execute "write" commands - just print what would happen
:param kwargs: arguments passed from the command
"""
fix_group_permissions()
parameters_passed = filter_out_none(**kwargs)
ci_image_params = get_ci_image_build_params(parameters_passed)
ci_image_params.print_info()
run_command(
[
"docker", "rmi", "--no-prune", "--force",
ci_image_params.airflow_image_name
],
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=False,
)
cmd = construct_build_docker_command(
image_params=ci_image_params,
verbose=verbose,
required_args=REQUIRED_CI_IMAGE_ARGS,
optional_args=OPTIONAL_CI_IMAGE_ARGS,
production_image=False,
)
if ci_image_params.prepare_buildx_cache:
login_to_docker_registry(ci_image_params)
console.print(
f"\n[blue]Building CI Image for Python {ci_image_params.python}\n")
run_command(cmd,
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True)
if not dry_run:
ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch
ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
touch_cache_file(f"built_{ci_image_params.python}",
root_dir=ci_image_cache_dir)
calculate_md5_checksum_for_files(ci_image_params.md5sum_cache_dir,
update=True)
else:
console.print(
"[blue]Not updating build cache because we are in `dry_run` mode.[/]"
)
def tag_and_push_image(image_params: Union[BuildProdParams, BuildCiParams],
dry_run: bool, verbose: bool):
"""
Tag and push the image according to parameters.
:param image_params: parameters of the image
:param dry_run: whether we are in dry-run mode
:param verbose: whethere we produce verbose output
:return:
"""
console.print(
f"[blue]Tagging and pushing the {image_params.airflow_image_name} as "
f"{image_params.airflow_image_name_with_tag}.[/]")
cmd = construct_docker_tag_command(image_params)
run_command(cmd,
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=True)
login_to_docker_registry(image_params)
cmd = construct_docker_push_command(image_params)
run_command(cmd,
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=True)
def build_production_image(verbose: bool, dry_run: bool, **kwargs):
"""
Builds PROD image:
* fixes group permissions for files (to improve caching when umask is 002)
* converts all the parameters received via kwargs into BuildProdParams (including cache)
* prints info about the image to build
* removes docker-context-files if requested
* performs sanity check if the files are present in docker-context-files if expected
* logs int to docker registry on CI if build cache is being executed
* removes "tag" for previously build image so that inline cache uses only remote image
* constructs docker-compose command to run based on parameters passed
* run the build command
* update cached information that the build completed and saves checksums of all files
for quick future check if the build is needed
:param verbose: print commands when running
:param dry_run: do not execute "write" commands - just print what would happen
:param kwargs: arguments passed from the command
"""
fix_group_permissions()
parameters_passed = filter_out_none(**kwargs)
prod_image_params = get_prod_image_build_params(parameters_passed)
prod_image_params.print_info()
if prod_image_params.cleanup_docker_context_files:
clean_docker_context_files()
check_docker_context_files(prod_image_params.install_docker_context_files)
if prod_image_params.prepare_buildx_cache:
login_to_docker_registry(prod_image_params)
run_command(
[
"docker", "rmi", "--no-prune", "--force",
prod_image_params.airflow_image_name
],
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=False,
)
console.print(
f"\n[blue]Building PROD Image for Python {prod_image_params.python}\n")
cmd = construct_build_docker_command(
image_params=prod_image_params,
verbose=verbose,
required_args=REQUIRED_PROD_IMAGE_ARGS,
optional_args=OPTIONAL_PROD_IMAGE_ARGS,
production_image=True,
)
run_command(cmd,
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True)
def tag_and_push_image(
image_params: Union[BuildProdParams, BuildCiParams], dry_run: bool, verbose: bool
) -> Tuple[int, str]:
"""
Tag and push the image according to parameters.
:param image_params: parameters of the image
:param dry_run: whether we are in dry-run mode
:param verbose: whethere we produce verbose output
:return:
"""
console.print(
f"[blue]Tagging and pushing the {image_params.airflow_image_name} as "
f"{image_params.airflow_image_name_with_tag}.[/]"
)
cmd = construct_docker_tag_command(image_params)
command_result = run_command(
cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False
)
if command_result.returncode == 0:
return_code, info = login_to_docker_registry(image_params, dry_run=dry_run)
if return_code != 0:
return return_code, f"Tag and pushing the image {image_params.python}: {info}"
cmd = construct_docker_push_command(image_params)
command_result = run_command(
cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False
)
if command_result.returncode == 0:
return 0, f"Tag and pushing the image {image_params.python}"
return command_result.returncode, f"Tag and pushing the image {image_params.python}"
def build_ci_image(verbose: bool, dry_run: bool, with_ci_group: bool,
ci_image_params: BuildCiParams) -> Tuple[int, str]:
"""
Builds CI image:
* fixes group permissions for files (to improve caching when umask is 002)
* converts all the parameters received via kwargs into BuildCIParams (including cache)
* prints info about the image to build
* logs int to docker registry on CI if build cache is being executed
* removes "tag" for previously build image so that inline cache uses only remote image
* constructs docker-compose command to run based on parameters passed
* run the build command
* update cached information that the build completed and saves checksums of all files
for quick future check if the build is needed
:param verbose: print commands when running
:param dry_run: do not execute "write" commands - just print what would happen
:param with_ci_group: whether to wrap the build in CI logging group
:param ci_image_params: CI image parameters
"""
fix_group_permissions()
if verbose or dry_run:
console.print(
f"\n[bright_blue]Building CI image of airflow from {AIRFLOW_SOURCES_ROOT} "
f"python version: {ci_image_params.python}[/]\n")
with ci_group(
f"Build CI image for Python {ci_image_params.python} "
f"with tag: {ci_image_params.image_tag}",
enabled=with_ci_group,
):
ci_image_params.print_info()
if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies:
if not should_we_run_the_build(build_ci_params=ci_image_params):
return 0, f"Image build: {ci_image_params.python}"
run_command(
[
"docker", "rmi", "--no-prune", "--force",
ci_image_params.airflow_image_name
],
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=False,
)
if ci_image_params.prepare_buildx_cache:
login_to_docker_registry(ci_image_params, dry_run=dry_run)
cmd = construct_docker_build_command(
image_params=ci_image_params,
verbose=verbose,
required_args=REQUIRED_CI_IMAGE_ARGS,
optional_args=OPTIONAL_CI_IMAGE_ARGS,
production_image=False,
)
if ci_image_params.empty_image:
console.print(
f"\n[blue]Building empty CI Image for Python {ci_image_params.python}\n"
)
cmd = construct_empty_docker_build_command(
image_params=ci_image_params)
build_result = run_command(
cmd,
input="FROM scratch\n",
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
)
else:
console.print(
f"\n[blue]Building CI Image for Python {ci_image_params.python}\n"
)
build_result = run_command(cmd,
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=False)
if not dry_run:
if build_result.returncode == 0:
ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch
ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
touch_cache_file(f"built_{ci_image_params.python}",
root_dir=ci_image_cache_dir)
calculate_md5_checksum_for_files(
ci_image_params.md5sum_cache_dir, update=True)
else:
console.print("[red]Error when building image![/]")
return (
build_result.returncode,
f"Image build: {ci_image_params.python}",
)
else:
console.print(
"[blue]Not updating build cache because we are in `dry_run` mode.[/]"
)
if ci_image_params.push_image:
return tag_and_push_image(image_params=ci_image_params,
dry_run=dry_run,
verbose=verbose)
return build_result.returncode, f"Image build: {ci_image_params.python}"