def build_image(
verbose: bool,
dry_run: bool,
with_ci_group: bool,
build_multiple_images: bool,
python_versions: str,
answer: str,
**kwargs,
):
"""Build CI image. Include building multiple images for all python versions (sequentially)."""
def run_build(ci_image_params: BuildCiParams) -> None:
return_code, info = build_ci_image(verbose=verbose,
dry_run=dry_run,
with_ci_group=with_ci_group,
ci_image_params=ci_image_params)
if return_code != 0:
console.print(f"[red]Error when building image! {info}")
sys.exit(return_code)
set_forced_answer(answer)
parameters_passed = filter_out_none(**kwargs)
if build_multiple_images:
with ci_group(f"Building images sequentially {python_versions}",
enabled=with_ci_group):
python_version_list = get_python_version_list(python_versions)
for python in python_version_list:
params = BuildCiParams(**parameters_passed)
params.python = python
params.answer = answer
run_build(ci_image_params=params)
else:
params = BuildCiParams(**parameters_passed)
run_build(ci_image_params=params)
def enter_shell(
**kwargs
) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]:
"""
Executes entering shell using the parameters passed as kwargs:
* checks if docker version is good
* checks if docker-compose version is good
* updates kwargs with cached parameters
* displays ASCIIART and CHEATSHEET unless disabled
* build ShellParams from the updated kwargs
* executes the command to drop the user to Breeze shell
"""
verbose = kwargs['verbose']
dry_run = kwargs['dry_run']
if not check_docker_is_running(verbose):
console.print(
'[red]Docker is not running.[/]\n'
'[bright_yellow]Please make sure Docker is installed and running.[/]'
)
sys.exit(1)
check_docker_version(verbose)
check_docker_compose_version(verbose)
updated_kwargs = synchronize_cached_params(kwargs)
if read_from_cache_file('suppress_asciiart') is None:
console.print(ASCIIART, style=ASCIIART_STYLE)
if read_from_cache_file('suppress_cheatsheet') is None:
console.print(CHEATSHEET, style=CHEATSHEET_STYLE)
enter_shell_params = ShellParams(**filter_out_none(**updated_kwargs))
return run_shell_with_build_image_checks(verbose, dry_run,
enter_shell_params)
def build_production_image(verbose, **kwargs):
parameters_passed = filter_out_none(**kwargs)
prod_params = get_image_build_params(parameters_passed)
prod_params.print_info()
if prod_params.cleanup_docker_context_files:
clean_docker_context_files()
check_docker_context_files(prod_params.install_docker_context_files)
if prod_params.skip_building_prod_image:
console.print(
'[bright_yellow]\nSkip building production image. Assume the one we have is good!'
)
console.print(
'bright_yellow]\nYou must run Breeze2 build-prod-image before for all python versions!'
)
if prod_params.prepare_buildx_cache:
login_to_docker_registry(prod_params)
cmd = construct_docker_command(prod_params)
run_command(
[
"docker", "rmi", "--no-prune", "--force",
prod_params.airflow_prod_image_name
],
verbose=verbose,
cwd=AIRFLOW_SOURCE,
text=True,
suppress_raise_exception=True,
)
run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True)
if prod_params.prepare_buildx_cache:
run_command(['docker', 'push', prod_params.airflow_prod_image_name],
verbose=True,
text=True)
def build_image(verbose: bool, dry_run: bool, **kwargs) -> None:
"""
Builds CI image:
* fixes group permissions for files (to improve caching when umask is 002)
* converts all the parameters received via kwargs into BuildCIParams (including cache)
* prints info about the image to build
* logs int to docker registry on CI if build cache is being executed
* removes "tag" for previously build image so that inline cache uses only remote image
* constructs docker-compose command to run based on parameters passed
* run the build command
* update cached information that the build completed and saves checksums of all files
for quick future check if the build is needed
:param verbose: print commands when running
:param dry_run: do not execute "write" commands - just print what would happen
:param kwargs: arguments passed from the command
"""
fix_group_permissions()
parameters_passed = filter_out_none(**kwargs)
ci_image_params = get_ci_image_build_params(parameters_passed)
ci_image_params.print_info()
run_command(
[
"docker", "rmi", "--no-prune", "--force",
ci_image_params.airflow_image_name
],
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=False,
)
cmd = construct_build_docker_command(
image_params=ci_image_params,
verbose=verbose,
required_args=REQUIRED_CI_IMAGE_ARGS,
optional_args=OPTIONAL_CI_IMAGE_ARGS,
production_image=False,
)
if ci_image_params.prepare_buildx_cache:
login_to_docker_registry(ci_image_params)
console.print(
f"\n[blue]Building CI Image for Python {ci_image_params.python}\n")
run_command(cmd,
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True)
if not dry_run:
ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch
ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
touch_cache_file(f"built_{ci_image_params.python}",
root_dir=ci_image_cache_dir)
calculate_md5_checksum_for_files(ci_image_params.md5sum_cache_dir,
update=True)
else:
console.print(
"[blue]Not updating build cache because we are in `dry_run` mode.[/]"
)
def build_image(verbose, **kwargs):
ci_image_params = BuildParams(**filter_out_none(**kwargs))
is_cached, value = check_cache_and_write_if_not_cached(
"PYTHON_MAJOR_MINOR_VERSION", ci_image_params.python_version)
if is_cached:
ci_image_params.python_version = value
cmd = construct_docker_command(ci_image_params)
output = run_command(cmd, verbose=verbose, text=True)
console.print(f"[blue]{output}")
def build_shell(verbose, **kwargs):
check_docker_version(verbose)
check_docker_compose_version(verbose)
updated_kwargs = get_cached_params(kwargs)
if read_from_cache_file('suppress_asciiart') is None:
console.print(ASCIIART, style=ASCIIART_STYLE)
if read_from_cache_file('suppress_cheatsheet') is None:
console.print(CHEATSHEET, style=CHEATSHEET_STYLE)
enter_shell_params = ShellBuilder(**filter_out_none(**updated_kwargs))
build_image_checks(verbose, enter_shell_params)
def build_production_image(verbose: bool, dry_run: bool, **kwargs):
"""
Builds PROD image:
* fixes group permissions for files (to improve caching when umask is 002)
* converts all the parameters received via kwargs into BuildProdParams (including cache)
* prints info about the image to build
* removes docker-context-files if requested
* performs sanity check if the files are present in docker-context-files if expected
* logs int to docker registry on CI if build cache is being executed
* removes "tag" for previously build image so that inline cache uses only remote image
* constructs docker-compose command to run based on parameters passed
* run the build command
* update cached information that the build completed and saves checksums of all files
for quick future check if the build is needed
:param verbose: print commands when running
:param dry_run: do not execute "write" commands - just print what would happen
:param kwargs: arguments passed from the command
"""
fix_group_permissions()
parameters_passed = filter_out_none(**kwargs)
prod_image_params = get_prod_image_build_params(parameters_passed)
prod_image_params.print_info()
if prod_image_params.cleanup_docker_context_files:
clean_docker_context_files()
check_docker_context_files(prod_image_params.install_docker_context_files)
if prod_image_params.prepare_buildx_cache:
login_to_docker_registry(prod_image_params)
run_command(
[
"docker", "rmi", "--no-prune", "--force",
prod_image_params.airflow_image_name
],
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
check=False,
)
console.print(
f"\n[blue]Building PROD Image for Python {prod_image_params.python}\n")
cmd = construct_build_docker_command(
image_params=prod_image_params,
verbose=verbose,
required_args=REQUIRED_PROD_IMAGE_ARGS,
optional_args=OPTIONAL_PROD_IMAGE_ARGS,
production_image=True,
)
run_command(cmd,
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True)
def build_image(verbose, **kwargs):
parameters_passed = filter_out_none(**kwargs)
ci_image_params = get_image_build_params(parameters_passed)
ci_image_cache_dir = Path(BUILD_CACHE_DIR, ci_image_params.airflow_branch)
ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
touch_cache_file(
f"built_{ci_image_params.python_version}",
root_dir=ci_image_cache_dir,
)
cmd = construct_docker_command(ci_image_params)
output = run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True)
console.print(f"[blue]{output}")
def enter_shell(**kwargs):
"""
Executes entering shell using the parameters passed as kwargs:
* checks if docker version is good
* checks if docker-compose version is good
* updates kwargs with cached parameters
* displays ASCIIART and CHEATSHEET unless disabled
* build ShellParams from the updated kwargs
* executes the command to drop the user to Breeze shell
"""
verbose = kwargs['verbose']
dry_run = kwargs['dry_run']
check_docker_version(verbose)
check_docker_compose_version(verbose)
updated_kwargs = synchronize_cached_params(kwargs)
if read_from_cache_file('suppress_asciiart') is None:
console.print(ASCIIART, style=ASCIIART_STYLE)
if read_from_cache_file('suppress_cheatsheet') is None:
console.print(CHEATSHEET, style=CHEATSHEET_STYLE)
enter_shell_params = ShellParams(**filter_out_none(**updated_kwargs))
run_shell_with_build_image_checks(verbose, dry_run, enter_shell_params)
def build_image(verbose, **kwargs):
parameters_passed = filter_out_none(**kwargs)
ci_image_params = get_image_build_params(parameters_passed)
cmd = construct_docker_command(ci_image_params)
output = run_command(cmd, verbose=verbose, text=True)
console.print(f"[blue]{output}")