def _docker_compose(conf: AiscalatorConfig,
extra_commands: list):
"""
Run the docker-compose command
Parameters
----------
conf : AiscalatorConfig
Configuration object for the application
extra_commands : list
list of sub-commands to run in docker-compose
"""
logger = logging.getLogger(__name__)
conf.validate_config()
dockerfile = join(conf.app_config_home(), "config",
conf.airflow_docker_compose_file())
commands = ["docker-compose"]
# Prepare a temp folder to run the command from
with TemporaryDirectory(prefix="aiscalator_") as tmp:
with open(join(tmp, ".env"), mode="w") as env_file:
# concatenate all the env files into one
for env in conf.user_env_file(conf.dag_field("definition.env")):
if isfile(env):
with open(env, mode="r") as file:
for line in file:
env_file.write(line)
utils.copy_replace(join(tmp, ".env"),
join(dirname(dockerfile), ".env"))
commands += ["-f", dockerfile] + extra_commands
logger.info("Running...: %s", " ".join(commands))
utils.subprocess_run(commands, no_redirect=True)
def _prepare_docker_env(conf: AiscalatorConfig, program, reason):
"""
Assembles the list of commands to execute a docker run call
When calling "docker run ...", this function also adds a set of
additional parameters to mount the proper volumes and expose the
correct environment for the call in the docker image mapped to the
host directories. This is done so only some specific data and code
folders are accessible within the docker image.
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
program : List
the rest of the commands to execute as part of
the docker run call
Returns
-------
List
The full Array of Strings representing the commands to execute
in the docker run call
"""
logger = logging.getLogger(__name__)
commands = [
"docker", "run", "--name",
conf.step_container_name() + "_" + reason, "--rm"
]
for env in conf.user_env_file(conf.step_field("task.env")):
if os.path.isfile(env):
commands += ["--env-file", env]
commands += _prepare_docker_image_env(conf)
code_path = conf.step_file_path('task.code_path')
if conf.has_step_field('task.code_format'):
from_format = conf.step_field('task.code_format')
else:
from_format = "py"
from_format += ':'
if conf.has_step_field('task.jupytext_format'):
from_format += conf.step_field('task.jupytext_format')
else:
from_format += "percent"
notebook, _ = notebook_file(code_path)
check_notebook_dir(logger, notebook, from_format)
commands += [
"--mount",
"type=bind,source=" + os.path.dirname(notebook) +
",target=/home/jovyan/work/notebook/",
]
commands += _prepare_task_env(conf)
if conf.has_step_field("task.execution_dir_path"):
execution_dir_path = conf.step_file_path('task.execution_dir_path')
if execution_dir_path:
makedirs(execution_dir_path, exist_ok=True)
commands += [
"--mount", "type=bind,source=" + execution_dir_path +
",target=/home/jovyan/work/notebook_run/"
]
commands += program
return commands
def _prepare_docker_env(conf: AiscalatorConfig, program, port):
"""
Assembles the list of commands to execute a docker run call
When calling "docker run ...", this function also adds a set of
additional parameters to mount the proper volumes and expose the
correct environment for the call in the docker image.
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
program : List
the rest of the commands to execute as part of
the docker run call
Returns
-------
List
The full Array of Strings representing the commands to execute
in the docker run call
"""
logger = logging.getLogger(__name__)
commands = [
"docker", "run", "--name", conf.dag_container_name() + "_edit",
"--rm",
# TODO improve port publishing
"-p", str(port) + ":8888",
"-p", "18080:8080",
]
for env in conf.user_env_file(conf.dag_field("definition.env")):
if isfile(env):
commands += ["--env-file", env]
commands += [
"--mount", "type=bind,source=/var/run/docker.sock,"
"target=/var/run/docker.sock",
]
code_path = conf.dag_file_path('definition.code_path')
notebook, _ = utils.notebook_file(code_path)
utils.check_notebook_dir(logger, notebook)
commands += [
"--mount", "type=bind,source=" + dirname(notebook) +
",target=/usr/local/airflow/work/",
]
if conf.config_path() is not None:
commands += [
"--mount",
"type=bind,source=" + abspath(conf.config_path()) +
",target="
"/usr/local/airflow/" + basename(conf.config_path()),
]
workspace = []
ws_path = "airflow.setup.workspace_paths"
if conf.app_config_has(ws_path):
ws_home = join(conf.app_config_home(),
"workspace")
makedirs(ws_home, exist_ok=True)
for folder in conf.app_config()[ws_path]:
src, dst = _split_workspace_string(conf, folder)
# bind the same path from host in the container (after creating
# a symbolic link at dst path)
workspace += [src + ":" + src]
commands += [
"--mount", "type=bind,source=" + src +
",target=" + src
]
commands += [
"--mount", "type=bind,source=" + ws_home +
",target=/usr/local/airflow/workspace/"
]
commands += program + workspace
return commands
