def build_image(verbose: bool, dry_run: bool, **kwargs) -> None:
"""
Builds CI image:
* fixes group permissions for files (to improve caching when umask is 002)
* converts all the parameters received via kwargs into BuildCIParams (including cache)
* prints info about the image to build
* logs int to docker registry on CI if build cache is being executed
* removes "tag" for previously build image so that inline cache uses only remote image
* constructs docker-compose command to run based on parameters passed
* run the build command
* update cached information that the build completed and saves checksums of all files
for quick future check if the build is needed
:param verbose: print commands when running
:param dry_run: do not execute "write" commands - just print what would happen
:param kwargs: arguments passed from the command
"""
fix_group_permissions()
parameters_passed = filter_out_none(**kwargs)
ci_image_params = get_ci_image_build_params(parameters_passed)
ci_image_params.print_info()
run_command(
[
"docker", "rmi", "--no-prune", "--force",
ci_image_params.airflow_image_name
],
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=False,
)
cmd = construct_build_docker_command(
image_params=ci_image_params,
verbose=verbose,
required_args=REQUIRED_CI_IMAGE_ARGS,
optional_args=OPTIONAL_CI_IMAGE_ARGS,
production_image=False,
)
if ci_image_params.prepare_buildx_cache:
login_to_docker_registry(ci_image_params)
console.print(
f"\n[blue]Building CI Image for Python {ci_image_params.python}\n")
run_command(cmd,
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True)
if not dry_run:
ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch
ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
touch_cache_file(f"built_{ci_image_params.python}",
root_dir=ci_image_cache_dir)
calculate_md5_checksum_for_files(ci_image_params.md5sum_cache_dir,
update=True)
else:
console.print(
"[blue]Not updating build cache because we are in `dry_run` mode.[/]"
)
def change_config(
python: str,
backend: str,
postgres_version: str,
mysql_version: str,
mssql_version: str,
cheatsheet: bool,
asciiart: bool,
):
"""
Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).
"""
asciiart_file = "suppress_asciiart"
cheatsheet_file = "suppress_cheatsheet"
if asciiart is not None:
if asciiart:
delete_cache(asciiart_file)
console.print('[bright_blue]Enable ASCIIART![/]')
else:
touch_cache_file(asciiart_file)
console.print('[bright_blue]Disable ASCIIART![/]')
if cheatsheet is not None:
if cheatsheet:
delete_cache(cheatsheet_file)
console.print('[bright_blue]Enable Cheatsheet[/]')
elif cheatsheet is not None:
touch_cache_file(cheatsheet_file)
console.print('[bright_blue]Disable Cheatsheet[/]')
def get_status(file: str):
return "disabled" if check_if_cache_exists(file) else "enabled"
console.print()
console.print("[bright_blue]Current configuration:[/]")
console.print()
console.print(f"[bright_blue]* Python: {python}[/]")
console.print(f"[bright_blue]* Backend: {backend}[/]")
console.print()
console.print(f"[bright_blue]* Postgres version: {postgres_version}[/]")
console.print(f"[bright_blue]* MySQL version: {mysql_version}[/]")
console.print(f"[bright_blue]* MsSQL version: {mssql_version}[/]")
console.print()
console.print(f"[bright_blue]* ASCIIART: {get_status(asciiart_file)}[/]")
console.print(
f"[bright_blue]* Cheatsheet: {get_status(cheatsheet_file)}[/]")
console.print()
def change_config(python, backend, cheatsheet, asciiart):
"""
Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).
"""
asciiart_file = "suppress_asciiart"
cheatsheet_file = "suppress_cheatsheet"
python_file = 'PYTHON_MAJOR_MINOR_VERSION'
backend_file = 'BACKEND'
if asciiart is not None:
if asciiart:
delete_cache(asciiart_file)
console.print('[bright_blue]Enable ASCIIART![/]')
else:
touch_cache_file(asciiart_file)
console.print('[bright_blue]Disable ASCIIART![/]')
if cheatsheet is not None:
if cheatsheet:
delete_cache(cheatsheet_file)
console.print('[bright_blue]Enable Cheatsheet[/]')
elif cheatsheet is not None:
touch_cache_file(cheatsheet_file)
console.print('[bright_blue]Disable Cheatsheet[/]')
if python is not None:
write_to_cache_file(python_file, python)
console.print(f'[bright_blue]Python default value set to: {python}[/]')
if backend is not None:
write_to_cache_file(backend_file, backend)
console.print(
f'[bright_blue]Backend default value set to: {backend}[/]')
def get_status(file: str):
return "disabled" if check_if_cache_exists(file) else "enabled"
console.print()
console.print("[bright_blue]Current configuration:[/]")
console.print()
console.print(
f"[bright_blue]* Python: {read_from_cache_file(python_file)}[/]")
console.print(
f"[bright_blue]* Backend: {read_from_cache_file(backend_file)}[/]")
console.print(f"[bright_blue]* ASCIIART: {get_status(asciiart_file)}[/]")
console.print(
f"[bright_blue]* Cheatsheet: {get_status(cheatsheet_file)}[/]")
console.print()
def change_config(python, backend, cheatsheet, asciiart):
"""
Toggles on/off cheatsheet, asciiart. Sets default Python and backend.
"""
if asciiart:
console.print('[bright_blue] ASCIIART enabled')
delete_cache('suppress_asciiart')
elif asciiart is not None:
touch_cache_file('suppress_asciiart')
else:
pass
if cheatsheet:
console.print('[bright_blue] Cheatsheet enabled')
delete_cache('suppress_cheatsheet')
elif cheatsheet is not None:
touch_cache_file('suppress_cheatsheet')
else:
pass
if python is not None:
write_to_cache_file('PYTHON_MAJOR_MINOR_VERSION', python)
console.print(f'[bright_blue]Python cached_value {python}')
if backend is not None:
write_to_cache_file('BACKEND', backend)
console.print(f'[bright_blue]Backend cached_value {backend}')
def build_ci_image(verbose: bool, dry_run: bool, with_ci_group: bool,
ci_image_params: BuildCiParams) -> Tuple[int, str]:
"""
Builds CI image:
* fixes group permissions for files (to improve caching when umask is 002)
* converts all the parameters received via kwargs into BuildCIParams (including cache)
* prints info about the image to build
* logs int to docker registry on CI if build cache is being executed
* removes "tag" for previously build image so that inline cache uses only remote image
* constructs docker-compose command to run based on parameters passed
* run the build command
* update cached information that the build completed and saves checksums of all files
for quick future check if the build is needed
:param verbose: print commands when running
:param dry_run: do not execute "write" commands - just print what would happen
:param with_ci_group: whether to wrap the build in CI logging group
:param ci_image_params: CI image parameters
"""
fix_group_permissions()
if verbose or dry_run:
console.print(
f"\n[bright_blue]Building CI image of airflow from {AIRFLOW_SOURCES_ROOT} "
f"python version: {ci_image_params.python}[/]\n")
with ci_group(
f"Build CI image for Python {ci_image_params.python} "
f"with tag: {ci_image_params.image_tag}",
enabled=with_ci_group,
):
ci_image_params.print_info()
if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies:
if not should_we_run_the_build(build_ci_params=ci_image_params):
return 0, f"Image build: {ci_image_params.python}"
run_command(
[
"docker", "rmi", "--no-prune", "--force",
ci_image_params.airflow_image_name
],
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=False,
)
if ci_image_params.prepare_buildx_cache:
login_to_docker_registry(ci_image_params, dry_run=dry_run)
cmd = construct_docker_build_command(
image_params=ci_image_params,
verbose=verbose,
required_args=REQUIRED_CI_IMAGE_ARGS,
optional_args=OPTIONAL_CI_IMAGE_ARGS,
production_image=False,
)
if ci_image_params.empty_image:
console.print(
f"\n[blue]Building empty CI Image for Python {ci_image_params.python}\n"
)
cmd = construct_empty_docker_build_command(
image_params=ci_image_params)
build_result = run_command(
cmd,
input="FROM scratch\n",
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
)
else:
console.print(
f"\n[blue]Building CI Image for Python {ci_image_params.python}\n"
)
build_result = run_command(cmd,
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=False)
if not dry_run:
if build_result.returncode == 0:
ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch
ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
touch_cache_file(f"built_{ci_image_params.python}",
root_dir=ci_image_cache_dir)
calculate_md5_checksum_for_files(
ci_image_params.md5sum_cache_dir, update=True)
else:
console.print("[red]Error when building image![/]")
return (
build_result.returncode,
f"Image build: {ci_image_params.python}",
)
else:
console.print(
"[blue]Not updating build cache because we are in `dry_run` mode.[/]"
)
if ci_image_params.push_image:
return tag_and_push_image(image_params=ci_image_params,
dry_run=dry_run,
verbose=verbose)
return build_result.returncode, f"Image build: {ci_image_params.python}"