def _prepare_docker_image_env(conf: AiscalatorConfig):
"""
Assemble the list of volumes to mount specific to
building the docker image
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
Returns
-------
list
list of commands to bind those volumes
"""
commands = []
if conf.config_path() is not None:
commands += [
"--mount",
"type=bind,source=" + os.path.realpath(conf.config_path()) +
",target="
"/home/jovyan/work/" + os.path.basename(conf.config_path()),
]
if conf.has_step_field("docker_image.apt_repository_path"):
apt_repo = conf.step_file_path('docker_image.apt_repository_path')
if apt_repo and os.path.isfile(apt_repo):
commands += [
"--mount",
"type=bind,source=" + apt_repo +
",target=/home/jovyan/work/apt_repository.txt",
]
if conf.has_step_field("docker_image.apt_package_path"):
apt_packages = conf.step_file_path('docker_image.apt_package_path')
if apt_packages and os.path.isfile(apt_packages):
commands += [
"--mount",
"type=bind,source=" + apt_packages +
",target=/home/jovyan/work/apt_packages.txt",
]
if conf.has_step_field("docker_image.requirements_path"):
requirements = conf.step_file_path('docker_image.requirements_path')
if requirements and os.path.isfile(requirements):
commands += [
"--mount",
"type=bind,source=" + requirements +
",target=/home/jovyan/work/requirements.txt",
]
if conf.has_step_field("docker_image.lab_extension_path"):
lab_extensions = conf.step_file_path('docker_image.lab_extension_path')
if lab_extensions and os.path.isfile(lab_extensions):
commands += [
"--mount",
"type=bind,source=" + lab_extensions +
",target=/home/jovyan/work/lab_extensions.txt",
]
# allow to pass a list of extra options like ["--network", "bridge"]
if conf.has_step_field("docker_image.docker_extra_options"):
commands += conf.step_field("docker_image.docker_extra_options")
return commands
def _prepare_docker_env(conf: AiscalatorConfig, program, port):
"""
Assembles the list of commands to execute a docker run call
When calling "docker run ...", this function also adds a set of
additional parameters to mount the proper volumes and expose the
correct environment for the call in the docker image.
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
program : List
the rest of the commands to execute as part of
the docker run call
Returns
-------
List
The full Array of Strings representing the commands to execute
in the docker run call
"""
logger = logging.getLogger(__name__)
commands = [
"docker", "run", "--name", conf.dag_container_name() + "_edit",
"--rm",
# TODO improve port publishing
"-p", str(port) + ":8888",
"-p", "18080:8080",
]
for env in conf.user_env_file(conf.dag_field("definition.env")):
if isfile(env):
commands += ["--env-file", env]
commands += [
"--mount", "type=bind,source=/var/run/docker.sock,"
"target=/var/run/docker.sock",
]
code_path = conf.dag_file_path('definition.code_path')
notebook, _ = utils.notebook_file(code_path)
utils.check_notebook_dir(logger, notebook)
commands += [
"--mount", "type=bind,source=" + dirname(notebook) +
",target=/usr/local/airflow/work/",
]
if conf.config_path() is not None:
commands += [
"--mount",
"type=bind,source=" + abspath(conf.config_path()) +
",target="
"/usr/local/airflow/" + basename(conf.config_path()),
]
workspace = []
ws_path = "airflow.setup.workspace_paths"
if conf.app_config_has(ws_path):
ws_home = join(conf.app_config_home(),
"workspace")
makedirs(ws_home, exist_ok=True)
for folder in conf.app_config()[ws_path]:
src, dst = _split_workspace_string(conf, folder)
# bind the same path from host in the container (after creating
# a symbolic link at dst path)
workspace += [src + ":" + src]
commands += [
"--mount", "type=bind,source=" + src +
",target=" + src
]
commands += [
"--mount", "type=bind,source=" + ws_home +
",target=/usr/local/airflow/workspace/"
]
commands += program + workspace
return commands