def _run_build(conf: AiscalatorConfig):
"""
Run the docker build command to produce the image and tag it.
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
Returns
-------
str
the docker image ID that was built
"""
logger = logging.getLogger(__name__)
commands = ["docker", "build", "--rm"]
output_docker_name = None
if conf.has_step_field("docker_image.output_docker_name"):
output_docker_name = conf.step_field("docker_image.output_docker_name")
commands += ["-t", output_docker_name + ":latest"]
commands += ["."]
log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n')
logger.info("Running...: %s", " ".join(commands))
utils.subprocess_run(commands, log_function=log.grep_logs)
result = log.artifact()
test = (result and output_docker_name is not None
and conf.has_step_field("docker_image.output_docker_tag"))
if test:
commands = ["docker", "tag"]
output_docker_tag = conf.step_field("docker_image.output_docker_tag")
commands += [result, output_docker_name + ":" + output_docker_tag]
# TODO implement docker tag output_docker_tag_commit_hash
logger.info("Running...: %s", " ".join(commands))
utils.subprocess_run(commands)
return result
def jupyter_edit(conf: AiscalatorConfig, param=None, param_raw=None):
"""
Starts a Jupyter Lab environment configured to edit the focused step
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
param : list
list of tuples of parameters
param_raw : list
list of tuples of raw parameters
Returns
-------
string
Url of the running jupyter lab
"""
logger = logging.getLogger(__name__)
conf.validate_config()
docker_image = build(conf)
if docker_image:
# TODO: shutdown other jupyter lab still running
notebook, _ = notebook_file(conf.step_field('task.code_path'))
notebook = os.path.basename(notebook)
if conf.step_extract_parameters():
jupyter_run(conf,
prepare_only=True,
param=param,
param_raw=param_raw)
commands = _prepare_docker_env(
conf, [docker_image, "start.sh", 'jupyter', 'lab'], "edit")
return wait_for_jupyter_lab(commands, logger, notebook, 10000,
"work/notebook")
raise Exception("Failed to build docker image")
def build(conf: AiscalatorConfig):
"""
Builds the docker image following the parameters specified in the
focused step's configuration file
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
Returns
-------
string
the docker artifact name of the image built
"""
input_docker_src = conf.step_field("docker_image.input_docker_src")
# TODO check if input_docker_src refers to an existing docker image
# in which case, if no customization is needed, no need to build
cwd = getcwd()
result = None
try:
# Prepare a temp folder to build docker image
with TemporaryDirectory(prefix="aiscalator_") as tmp:
_prepare_build_dir(conf, tmp, input_docker_src)
chdir(tmp)
result = _run_build(conf)
finally:
chdir(cwd)
return result
def _prepare_docker_image_env(conf: AiscalatorConfig):
"""
Assemble the list of volumes to mount specific to
building the docker image
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
Returns
-------
list
list of commands to bind those volumes
"""
commands = []
if conf.config_path() is not None:
commands += [
"--mount",
"type=bind,source=" + os.path.realpath(conf.config_path()) +
",target="
"/home/jovyan/work/" + os.path.basename(conf.config_path()),
]
if conf.has_step_field("docker_image.apt_repository_path"):
apt_repo = conf.step_file_path('docker_image.apt_repository_path')
if apt_repo and os.path.isfile(apt_repo):
commands += [
"--mount",
"type=bind,source=" + apt_repo +
",target=/home/jovyan/work/apt_repository.txt",
]
if conf.has_step_field("docker_image.apt_package_path"):
apt_packages = conf.step_file_path('docker_image.apt_package_path')
if apt_packages and os.path.isfile(apt_packages):
commands += [
"--mount",
"type=bind,source=" + apt_packages +
",target=/home/jovyan/work/apt_packages.txt",
]
if conf.has_step_field("docker_image.requirements_path"):
requirements = conf.step_file_path('docker_image.requirements_path')
if requirements and os.path.isfile(requirements):
commands += [
"--mount",
"type=bind,source=" + requirements +
",target=/home/jovyan/work/requirements.txt",
]
if conf.has_step_field("docker_image.lab_extension_path"):
lab_extensions = conf.step_file_path('docker_image.lab_extension_path')
if lab_extensions and os.path.isfile(lab_extensions):
commands += [
"--mount",
"type=bind,source=" + lab_extensions +
",target=/home/jovyan/work/lab_extensions.txt",
]
# allow to pass a list of extra options like ["--network", "bridge"]
if conf.has_step_field("docker_image.docker_extra_options"):
commands += conf.step_field("docker_image.docker_extra_options")
return commands
def _mount_path(conf: AiscalatorConfig,
field,
target_path,
readonly=False,
make_dirs=False):
"""
Returu commands to mount path from list field into the
docker image when running.
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
field : str
the field in the configuration step that contains the path
target_path : str
where to mount them inside the container
readonly : bool
flag to mount the path as read-only
make_dirs : bool
flag to create the folder on the host before mounting if
it doesn't exists.
Returns
-------
list
commands to mount all the paths from the field
"""
commands = []
if conf.has_step_field(field):
for value in conf.step_field(field):
# TODO handle URL
for i in value:
if make_dirs:
makedirs(os.path.realpath(conf.root_dir() + value[i]),
exist_ok=True)
if os.path.exists(conf.root_dir() + value[i]):
commands += [
"--mount", "type=bind,source=" +
os.path.realpath(conf.root_dir() + value[i]) +
",target=" + os.path.join(target_path, i) +
(",readonly" if readonly else "")
]
return commands
def _prepare_docker_env(conf: AiscalatorConfig, program, reason):
"""
Assembles the list of commands to execute a docker run call
When calling "docker run ...", this function also adds a set of
additional parameters to mount the proper volumes and expose the
correct environment for the call in the docker image mapped to the
host directories. This is done so only some specific data and code
folders are accessible within the docker image.
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
program : List
the rest of the commands to execute as part of
the docker run call
Returns
-------
List
The full Array of Strings representing the commands to execute
in the docker run call
"""
logger = logging.getLogger(__name__)
commands = [
"docker", "run", "--name",
conf.step_container_name() + "_" + reason, "--rm"
]
for env in conf.user_env_file(conf.step_field("task.env")):
if os.path.isfile(env):
commands += ["--env-file", env]
commands += _prepare_docker_image_env(conf)
code_path = conf.step_file_path('task.code_path')
if conf.has_step_field('task.code_format'):
from_format = conf.step_field('task.code_format')
else:
from_format = "py"
from_format += ':'
if conf.has_step_field('task.jupytext_format'):
from_format += conf.step_field('task.jupytext_format')
else:
from_format += "percent"
notebook, _ = notebook_file(code_path)
check_notebook_dir(logger, notebook, from_format)
commands += [
"--mount",
"type=bind,source=" + os.path.dirname(notebook) +
",target=/home/jovyan/work/notebook/",
]
commands += _prepare_task_env(conf)
if conf.has_step_field("task.execution_dir_path"):
execution_dir_path = conf.step_file_path('task.execution_dir_path')
if execution_dir_path:
makedirs(execution_dir_path, exist_ok=True)
commands += [
"--mount", "type=bind,source=" + execution_dir_path +
",target=/home/jovyan/work/notebook_run/"
]
commands += program
return commands