def jupyter_run(conf: AiscalatorConfig,
prepare_only=False,
param=None,
param_raw=None):
"""
Executes the step in browserless mode using papermill
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
prepare_only : bool
Indicates if papermill should replace the parameters of the
notebook only or it should execute all the cells too
Returns
-------
string
the path to the output notebook resulting from the execution
of this step
"""
logger = logging.getLogger(__name__)
conf.validate_config()
docker_image = build(conf)
if not docker_image:
raise Exception("Failed to build docker image")
notebook, _ = notebook_file(conf.step_file_path('task.code_path'))
notebook = os.path.join("/home/jovyan/work/notebook/",
os.path.basename(notebook))
notebook_output = conf.step_notebook_output_path(notebook)
commands = _prepare_docker_env(
conf,
[
docker_image, "bash", "start-papermill.sh", "papermill", notebook,
notebook_output
],
"run_" + conf.step_name() + "_"
# add timestamp to name to handle multiple concurrent runs
+ datetime.datetime.now().strftime('%Y%m%d_%H%M%S'))
if prepare_only:
commands.append("--prepare-only")
parameters = conf.step_extract_parameters()
if parameters:
commands += parameters
if param:
for parameter in param:
commands += ["-p", parameter[0], parameter[1]]
if param_raw:
for raw_parameter in param_raw:
commands += ["-r", raw_parameter[0], raw_parameter[1]]
log = LogRegexAnalyzer()
logger.info("Running...: %s", " ".join(commands))
returncode = subprocess_run(commands, log_function=log.grep_logs)
if returncode:
logger.error("Run was not successful, returned status code is: " +
str(returncode))
sys.exit(returncode)
return os.path.join(conf.step_file_path('task.execution_dir_path'),
os.path.basename(notebook_output))
def _prepare_docker_image_env(conf: AiscalatorConfig):
"""
Assemble the list of volumes to mount specific to
building the docker image
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
Returns
-------
list
list of commands to bind those volumes
"""
commands = []
if conf.config_path() is not None:
commands += [
"--mount",
"type=bind,source=" + os.path.realpath(conf.config_path()) +
",target="
"/home/jovyan/work/" + os.path.basename(conf.config_path()),
]
if conf.has_step_field("docker_image.apt_repository_path"):
apt_repo = conf.step_file_path('docker_image.apt_repository_path')
if apt_repo and os.path.isfile(apt_repo):
commands += [
"--mount",
"type=bind,source=" + apt_repo +
",target=/home/jovyan/work/apt_repository.txt",
]
if conf.has_step_field("docker_image.apt_package_path"):
apt_packages = conf.step_file_path('docker_image.apt_package_path')
if apt_packages and os.path.isfile(apt_packages):
commands += [
"--mount",
"type=bind,source=" + apt_packages +
",target=/home/jovyan/work/apt_packages.txt",
]
if conf.has_step_field("docker_image.requirements_path"):
requirements = conf.step_file_path('docker_image.requirements_path')
if requirements and os.path.isfile(requirements):
commands += [
"--mount",
"type=bind,source=" + requirements +
",target=/home/jovyan/work/requirements.txt",
]
if conf.has_step_field("docker_image.lab_extension_path"):
lab_extensions = conf.step_file_path('docker_image.lab_extension_path')
if lab_extensions and os.path.isfile(lab_extensions):
commands += [
"--mount",
"type=bind,source=" + lab_extensions +
",target=/home/jovyan/work/lab_extensions.txt",
]
# allow to pass a list of extra options like ["--network", "bridge"]
if conf.has_step_field("docker_image.docker_extra_options"):
commands += conf.step_field("docker_image.docker_extra_options")
return commands
def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp):
"""
Include jupyter lab extensions packages into the dockerfile
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
dockerfile : str
path to the dockerfile to modify
tmp : str
path to the temporary dockerfile output
Returns
-------
path to the resulting dockerfile
"""
if conf.has_step_field("docker_image.lab_extension_path"):
content = conf.step_file_path("docker_image.lab_extension_path")
prefix = "&& jupyter labextension install "
value = utils.format_file_content(content,
prefix=prefix,
suffix=" \\\n")
if value:
value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value
utils.copy_replace(dockerfile,
tmp,
pattern="# lab_extensions.txt #",
replace_value=value)
return tmp
return dockerfile
def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst):
"""
Include pip install packages into the dockerfile
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
dockerfile : str
path to the dockerfile to modify
tmp : str
path to the temporary dockerfile output
dst : str
path to the final temporary directory
Returns
-------
path to the resulting dockerfile
"""
if conf.has_step_field("docker_image.requirements_path"):
content = conf.step_file_path("docker_image.requirements_path")
copy(content, join(dst, 'requirements.txt'))
utils.copy_replace(dockerfile,
tmp,
pattern="# requirements.txt #",
replace_value="""
COPY requirements.txt requirements.txt
RUN pip install -r requirements.txt
RUN rm requirements.txt""")
return tmp
return dockerfile
def _include_apt_repo(conf: AiscalatorConfig, dockerfile, tmp):
"""
Include add-apt-repository packages into the dockerfile
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
dockerfile : str
path to the dockerfile to modify
tmp : str
path to the temporary dockerfile output
Returns
-------
path to the resulting dockerfile
"""
if conf.has_step_field("docker_image.apt_repository_path"):
content = conf.step_file_path("docker_image.apt_repository_path")
value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
if value:
value = ("RUN apt-get update \\\n" +
" && apt-get install -yqq \\\n" +
" software-properties-common \\\n" +
" && apt-add-repository \\\n" + value +
" && apt-get update")
utils.copy_replace(dockerfile,
tmp,
pattern="# apt_repository.txt #",
replace_value=value)
return tmp
return dockerfile
def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp):
"""
Include apt-install packages into the dockerfile
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
dockerfile : str
path to the dockerfile to modify
tmp : str
path to the temporary dockerfile output
Returns
-------
path to the resulting dockerfile
"""
if conf.has_step_field("docker_image.apt_package_path"):
content = conf.step_file_path("docker_image.apt_package_path")
value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
if value:
value = ("RUN apt-get update && apt-get install -yqq \\\n" +
value +
""" && apt-get purge --auto-remove -yqq $buildDeps \\
&& apt-get autoremove -yqq --purge \\
&& apt-get clean \\
&& rm -rf \\
/var/lib/apt/lists/* \\
/tmp/* \\
/var/tmp/* \\
/usr/share/man \\
/usr/share/doc \\
/usr/share/doc-base
""")
utils.copy_replace(dockerfile,
tmp,
pattern="# apt_packages.txt #",
replace_value=value)
return tmp
return dockerfile
def _prepare_docker_env(conf: AiscalatorConfig, program, reason):
"""
Assembles the list of commands to execute a docker run call
When calling "docker run ...", this function also adds a set of
additional parameters to mount the proper volumes and expose the
correct environment for the call in the docker image mapped to the
host directories. This is done so only some specific data and code
folders are accessible within the docker image.
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
program : List
the rest of the commands to execute as part of
the docker run call
Returns
-------
List
The full Array of Strings representing the commands to execute
in the docker run call
"""
logger = logging.getLogger(__name__)
commands = [
"docker", "run", "--name",
conf.step_container_name() + "_" + reason, "--rm"
]
for env in conf.user_env_file(conf.step_field("task.env")):
if os.path.isfile(env):
commands += ["--env-file", env]
commands += _prepare_docker_image_env(conf)
code_path = conf.step_file_path('task.code_path')
if conf.has_step_field('task.code_format'):
from_format = conf.step_field('task.code_format')
else:
from_format = "py"
from_format += ':'
if conf.has_step_field('task.jupytext_format'):
from_format += conf.step_field('task.jupytext_format')
else:
from_format += "percent"
notebook, _ = notebook_file(code_path)
check_notebook_dir(logger, notebook, from_format)
commands += [
"--mount",
"type=bind,source=" + os.path.dirname(notebook) +
",target=/home/jovyan/work/notebook/",
]
commands += _prepare_task_env(conf)
if conf.has_step_field("task.execution_dir_path"):
execution_dir_path = conf.step_file_path('task.execution_dir_path')
if execution_dir_path:
makedirs(execution_dir_path, exist_ok=True)
commands += [
"--mount", "type=bind,source=" + execution_dir_path +
",target=/home/jovyan/work/notebook_run/"
]
commands += program
return commands
