def _run_build(conf: AiscalatorConfig): """ Run the docker build command to produce the image and tag it. Parameters ---------- conf : AiscalatorConfig Configuration object for this step Returns ------- str the docker image ID that was built """ logger = logging.getLogger(__name__) commands = ["docker", "build", "--rm"] output_docker_name = None if conf.has_step_field("docker_image.output_docker_name"): output_docker_name = conf.step_field("docker_image.output_docker_name") commands += ["-t", output_docker_name + ":latest"] commands += ["."] log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n') logger.info("Running...: %s", " ".join(commands)) utils.subprocess_run(commands, log_function=log.grep_logs) result = log.artifact() test = (result and output_docker_name is not None and conf.has_step_field("docker_image.output_docker_tag")) if test: commands = ["docker", "tag"] output_docker_tag = conf.step_field("docker_image.output_docker_tag") commands += [result, output_docker_name + ":" + output_docker_tag] # TODO implement docker tag output_docker_tag_commit_hash logger.info("Running...: %s", " ".join(commands)) utils.subprocess_run(commands) return result
def _prepare_docker_image_env(conf: AiscalatorConfig): """ Assemble the list of volumes to mount specific to building the docker image Parameters ---------- conf : AiscalatorConfig Configuration object for the step Returns ------- list list of commands to bind those volumes """ commands = [] if conf.config_path() is not None: commands += [ "--mount", "type=bind,source=" + os.path.realpath(conf.config_path()) + ",target=" "/home/jovyan/work/" + os.path.basename(conf.config_path()), ] if conf.has_step_field("docker_image.apt_repository_path"): apt_repo = conf.step_file_path('docker_image.apt_repository_path') if apt_repo and os.path.isfile(apt_repo): commands += [ "--mount", "type=bind,source=" + apt_repo + ",target=/home/jovyan/work/apt_repository.txt", ] if conf.has_step_field("docker_image.apt_package_path"): apt_packages = conf.step_file_path('docker_image.apt_package_path') if apt_packages and os.path.isfile(apt_packages): commands += [ "--mount", "type=bind,source=" + apt_packages + ",target=/home/jovyan/work/apt_packages.txt", ] if conf.has_step_field("docker_image.requirements_path"): requirements = conf.step_file_path('docker_image.requirements_path') if requirements and os.path.isfile(requirements): commands += [ "--mount", "type=bind,source=" + requirements + ",target=/home/jovyan/work/requirements.txt", ] if conf.has_step_field("docker_image.lab_extension_path"): lab_extensions = conf.step_file_path('docker_image.lab_extension_path') if lab_extensions and os.path.isfile(lab_extensions): commands += [ "--mount", "type=bind,source=" + lab_extensions + ",target=/home/jovyan/work/lab_extensions.txt", ] # allow to pass a list of extra options like ["--network", "bridge"] if conf.has_step_field("docker_image.docker_extra_options"): commands += conf.step_field("docker_image.docker_extra_options") return commands
def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp): """ Include jupyter lab extensions packages into the dockerfile Parameters ---------- conf : AiscalatorConfig Configuration object for this step dockerfile : str path to the dockerfile to modify tmp : str path to the temporary dockerfile output Returns ------- path to the resulting dockerfile """ if conf.has_step_field("docker_image.lab_extension_path"): content = conf.step_file_path("docker_image.lab_extension_path") prefix = "&& jupyter labextension install " value = utils.format_file_content(content, prefix=prefix, suffix=" \\\n") if value: value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value utils.copy_replace(dockerfile, tmp, pattern="# lab_extensions.txt #", replace_value=value) return tmp return dockerfile
def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst): """ Include pip install packages into the dockerfile Parameters ---------- conf : AiscalatorConfig Configuration object for this step dockerfile : str path to the dockerfile to modify tmp : str path to the temporary dockerfile output dst : str path to the final temporary directory Returns ------- path to the resulting dockerfile """ if conf.has_step_field("docker_image.requirements_path"): content = conf.step_file_path("docker_image.requirements_path") copy(content, join(dst, 'requirements.txt')) utils.copy_replace(dockerfile, tmp, pattern="# requirements.txt #", replace_value=""" COPY requirements.txt requirements.txt RUN pip install -r requirements.txt RUN rm requirements.txt""") return tmp return dockerfile
def _include_apt_repo(conf: AiscalatorConfig, dockerfile, tmp): """ Include add-apt-repository packages into the dockerfile Parameters ---------- conf : AiscalatorConfig Configuration object for this step dockerfile : str path to the dockerfile to modify tmp : str path to the temporary dockerfile output Returns ------- path to the resulting dockerfile """ if conf.has_step_field("docker_image.apt_repository_path"): content = conf.step_file_path("docker_image.apt_repository_path") value = utils.format_file_content(content, prefix=" ", suffix="\\\n") if value: value = ("RUN apt-get update \\\n" + " && apt-get install -yqq \\\n" + " software-properties-common \\\n" + " && apt-add-repository \\\n" + value + " && apt-get update") utils.copy_replace(dockerfile, tmp, pattern="# apt_repository.txt #", replace_value=value) return tmp return dockerfile
def _mount_path(conf: AiscalatorConfig, field, target_path, readonly=False, make_dirs=False): """ Returu commands to mount path from list field into the docker image when running. Parameters ---------- conf : AiscalatorConfig Configuration object for the step field : str the field in the configuration step that contains the path target_path : str where to mount them inside the container readonly : bool flag to mount the path as read-only make_dirs : bool flag to create the folder on the host before mounting if it doesn't exists. Returns ------- list commands to mount all the paths from the field """ commands = [] if conf.has_step_field(field): for value in conf.step_field(field): # TODO handle URL for i in value: if make_dirs: makedirs(os.path.realpath(conf.root_dir() + value[i]), exist_ok=True) if os.path.exists(conf.root_dir() + value[i]): commands += [ "--mount", "type=bind,source=" + os.path.realpath(conf.root_dir() + value[i]) + ",target=" + os.path.join(target_path, i) + (",readonly" if readonly else "") ] return commands
def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp): """ Include apt-install packages into the dockerfile Parameters ---------- conf : AiscalatorConfig Configuration object for this step dockerfile : str path to the dockerfile to modify tmp : str path to the temporary dockerfile output Returns ------- path to the resulting dockerfile """ if conf.has_step_field("docker_image.apt_package_path"): content = conf.step_file_path("docker_image.apt_package_path") value = utils.format_file_content(content, prefix=" ", suffix="\\\n") if value: value = ("RUN apt-get update && apt-get install -yqq \\\n" + value + """ && apt-get purge --auto-remove -yqq $buildDeps \\ && apt-get autoremove -yqq --purge \\ && apt-get clean \\ && rm -rf \\ /var/lib/apt/lists/* \\ /tmp/* \\ /var/tmp/* \\ /usr/share/man \\ /usr/share/doc \\ /usr/share/doc-base """) utils.copy_replace(dockerfile, tmp, pattern="# apt_packages.txt #", replace_value=value) return tmp return dockerfile
def _prepare_docker_env(conf: AiscalatorConfig, program, reason): """ Assembles the list of commands to execute a docker run call When calling "docker run ...", this function also adds a set of additional parameters to mount the proper volumes and expose the correct environment for the call in the docker image mapped to the host directories. This is done so only some specific data and code folders are accessible within the docker image. Parameters ---------- conf : AiscalatorConfig Configuration object for the step program : List the rest of the commands to execute as part of the docker run call Returns ------- List The full Array of Strings representing the commands to execute in the docker run call """ logger = logging.getLogger(__name__) commands = [ "docker", "run", "--name", conf.step_container_name() + "_" + reason, "--rm" ] for env in conf.user_env_file(conf.step_field("task.env")): if os.path.isfile(env): commands += ["--env-file", env] commands += _prepare_docker_image_env(conf) code_path = conf.step_file_path('task.code_path') if conf.has_step_field('task.code_format'): from_format = conf.step_field('task.code_format') else: from_format = "py" from_format += ':' if conf.has_step_field('task.jupytext_format'): from_format += conf.step_field('task.jupytext_format') else: from_format += "percent" notebook, _ = notebook_file(code_path) check_notebook_dir(logger, notebook, from_format) commands += [ "--mount", "type=bind,source=" + os.path.dirname(notebook) + ",target=/home/jovyan/work/notebook/", ] commands += _prepare_task_env(conf) if conf.has_step_field("task.execution_dir_path"): execution_dir_path = conf.step_file_path('task.execution_dir_path') if execution_dir_path: makedirs(execution_dir_path, exist_ok=True) commands += [ "--mount", "type=bind,source=" + execution_dir_path + ",target=/home/jovyan/work/notebook_run/" ] commands += program return commands