def jupyter_edit(conf: AiscalatorConfig, param=None, param_raw=None):
"""
Starts a Jupyter Lab environment configured to edit the focused step
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
param : list
list of tuples of parameters
param_raw : list
list of tuples of raw parameters
Returns
-------
string
Url of the running jupyter lab
"""
logger = logging.getLogger(__name__)
conf.validate_config()
docker_image = build(conf)
if docker_image:
# TODO: shutdown other jupyter lab still running
notebook, _ = notebook_file(conf.step_field('task.code_path'))
notebook = os.path.basename(notebook)
if conf.step_extract_parameters():
jupyter_run(conf,
prepare_only=True,
param=param,
param_raw=param_raw)
commands = _prepare_docker_env(
conf, [docker_image, "start.sh", 'jupyter', 'lab'], "edit")
return wait_for_jupyter_lab(commands, logger, notebook, 10000,
"work/notebook")
raise Exception("Failed to build docker image")
def _docker_compose_grep(conf: AiscalatorConfig):
"""
Checks if the docker-compose file is using the
aiscalator/airflow docker image. In which case,
we need to make sure that image is properly built
and available.
Parameters
----------
conf : AiscalatorConfig
Configuration object for the application
Returns
-------
bool
Returns if aiscalator/airflow docker image is
needed and should be built.
"""
docker_compose_file = join(conf.app_config_home(), "config",
conf.airflow_docker_compose_file())
pattern = re.compile(r"\s+image:\s+aiscalator/airflow")
try:
with open(docker_compose_file, "r") as file:
for line in file:
if re.match(pattern, line):
# docker compose needs the image
return True
except FileNotFoundError:
# no docker compose, default will need the image
return True
return False
def _run_build(conf: AiscalatorConfig):
"""
Run the docker build command to produce the image and tag it.
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
Returns
-------
str
the docker image ID that was built
"""
logger = logging.getLogger(__name__)
commands = ["docker", "build", "--rm"]
output_docker_name = None
if conf.has_step_field("docker_image.output_docker_name"):
output_docker_name = conf.step_field("docker_image.output_docker_name")
commands += ["-t", output_docker_name + ":latest"]
commands += ["."]
log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n')
logger.info("Running...: %s", " ".join(commands))
utils.subprocess_run(commands, log_function=log.grep_logs)
result = log.artifact()
test = (result and output_docker_name is not None
and conf.has_step_field("docker_image.output_docker_tag"))
if test:
commands = ["docker", "tag"]
output_docker_tag = conf.step_field("docker_image.output_docker_tag")
commands += [result, output_docker_name + ":" + output_docker_tag]
# TODO implement docker tag output_docker_tag_commit_hash
logger.info("Running...: %s", " ".join(commands))
utils.subprocess_run(commands)
return result
def jupyter_run(config,
notebook=None,
prepare_only=False,
param=None,
param_raw=None):
"""
Executes the step in browserless mode using papermill
Parameters
----------
config : str
path to the configuration file
notebook : str
name of node to run, if None, then run the first one
parameters : list
List of parameters and their values
prepare_only : bool
Indicates if papermill should replace the parameters of the
notebook only or it should execute all the cells too
Returns
-------
string
the path to the output notebook resulting from the execution
of this step
"""
# TODO implements parameters passing
if notebook:
app_config = AiscalatorConfig(config=config, step_selection=notebook)
else:
app_config = AiscalatorConfig(config=config)
return command.jupyter_run(app_config,
prepare_only=prepare_only,
param=param,
param_raw=param_raw)
def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst):
"""
Include pip install packages into the dockerfile
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
dockerfile : str
path to the dockerfile to modify
tmp : str
path to the temporary dockerfile output
dst : str
path to the final temporary directory
Returns
-------
path to the resulting dockerfile
"""
if conf.has_step_field("docker_image.requirements_path"):
content = conf.step_file_path("docker_image.requirements_path")
copy(content, join(dst, 'requirements.txt'))
utils.copy_replace(dockerfile,
tmp,
pattern="# requirements.txt #",
replace_value="""
COPY requirements.txt requirements.txt
RUN pip install -r requirements.txt
RUN rm requirements.txt""")
return tmp
return dockerfile
def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp):
"""
Include jupyter lab extensions packages into the dockerfile
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
dockerfile : str
path to the dockerfile to modify
tmp : str
path to the temporary dockerfile output
Returns
-------
path to the resulting dockerfile
"""
if conf.has_step_field("docker_image.lab_extension_path"):
content = conf.step_file_path("docker_image.lab_extension_path")
prefix = "&& jupyter labextension install "
value = utils.format_file_content(content,
prefix=prefix,
suffix=" \\\n")
if value:
value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value
utils.copy_replace(dockerfile,
tmp,
pattern="# lab_extensions.txt #",
replace_value=value)
return tmp
return dockerfile
def _include_apt_repo(conf: AiscalatorConfig, dockerfile, tmp):
"""
Include add-apt-repository packages into the dockerfile
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
dockerfile : str
path to the dockerfile to modify
tmp : str
path to the temporary dockerfile output
Returns
-------
path to the resulting dockerfile
"""
if conf.has_step_field("docker_image.apt_repository_path"):
content = conf.step_file_path("docker_image.apt_repository_path")
value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
if value:
value = ("RUN apt-get update \\\n" +
" && apt-get install -yqq \\\n" +
" software-properties-common \\\n" +
" && apt-add-repository \\\n" + value +
" && apt-get update")
utils.copy_replace(dockerfile,
tmp,
pattern="# apt_repository.txt #",
replace_value=value)
return tmp
return dockerfile
def airflow_edit(conf: AiscalatorConfig):
"""
Starts an airflow environment
Parameters
----------
conf : AiscalatorConfig
Configuration object for the application
"""
logger = logging.getLogger(__name__)
conf.validate_config()
docker_image = _airflow_docker_build(conf)
if docker_image:
# TODO: shutdown other jupyter lab still running
port = 10001
notebook = basename(conf.dag_field('definition.code_path'))
notebook, notebook_py = utils.notebook_file(notebook)
commands = _prepare_docker_env(conf, [
"aiscalator/airflow:" + docker_image, "bash",
"/start-jupyter.sh",
"/usr/local/airflow/work/" + notebook_py +
":/usr/local/airflow/dags/" + notebook_py
], port)
return utils.wait_for_jupyter_lab(commands, logger, notebook,
port, "work")
raise Exception("Failed to build docker image")
def _airflow_docker_build(conf: AiscalatorConfig):
""" Build the aiscalator/airflow image and return its ID."""
logger = logging.getLogger(__name__)
# TODO get airflow dockerfile from conf?
conf.app_config_home()
dockerfile_dir = utils.data_file("../config/docker/airflow")
# TODO customize dockerfile with apt_packages, requirements etc
docker_gid, docker_group = _find_docker_gid_group()
commands = [
"docker", "build",
"--build-arg", "DOCKER_GID=" + str(docker_gid),
"--build-arg", "DOCKER_GROUP=" + str(docker_group),
"--rm", "-t", "aiscalator/airflow:latest",
dockerfile_dir
]
log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n')
logger.info("Running...: %s", " ".join(commands))
utils.subprocess_run(commands, log_function=log.grep_logs)
result = log.artifact()
if result:
# tag the image built with the sha256 of the dockerfile
tag = utils.sha256(join(dockerfile_dir, 'Dockerfile'))[:12]
commands = [
"docker", "tag", result, "aiscalator/airflow:" + tag
]
logger.info("Running...: %s", " ".join(commands))
utils.subprocess_run(commands)
return tag
return None
def push(conf, notebook):
"""Push a job into the DAGS folder to schedule in Airflow."""
if notebook:
for note in notebook:
app_config = AiscalatorConfig(config=conf,
dag_selection=note)
click.echo(command.airflow_push(app_config))
else:
app_config = AiscalatorConfig(config=conf)
click.echo(command.airflow_push(app_config))
def run(conf, notebook, param, param_raw):
"""Run the notebook from an aiscalate config without GUI."""
if notebook:
for note in notebook:
app_config = AiscalatorConfig(config=conf, step_selection=note)
click.echo(
command.jupyter_run(app_config,
param=param,
param_raw=param_raw))
else:
app_config = AiscalatorConfig(config=conf)
click.echo(
command.jupyter_run(app_config, param=param, param_raw=param_raw))
def start():
"""Start docker images to bring airflow services up."""
click.echo(command.airflow_up(AiscalatorConfig()))
click.echo("""
Airflow: http://localhost:8080
Flower: http://localhost:5555
""")
def build(conf: AiscalatorConfig):
"""
Builds the docker image following the parameters specified in the
focused step's configuration file
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
Returns
-------
string
the docker artifact name of the image built
"""
input_docker_src = conf.step_field("docker_image.input_docker_src")
# TODO check if input_docker_src refers to an existing docker image
# in which case, if no customization is needed, no need to build
cwd = getcwd()
result = None
try:
# Prepare a temp folder to build docker image
with TemporaryDirectory(prefix="aiscalator_") as tmp:
_prepare_build_dir(conf, tmp, input_docker_src)
chdir(tmp)
result = _run_build(conf)
finally:
chdir(cwd)
return result
def _prepare_task_env(conf: AiscalatorConfig):
"""
Assemble the list of volumes to mount specific to
the task execution
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
Returns
-------
list
list of commands to bind those volumes
"""
commands = []
if conf.root_dir():
commands += _mount_path(conf, "task.modules_src_path",
"/home/jovyan/work/modules/")
commands += _mount_path(conf,
"task.input_data_path",
"/home/jovyan/work/data/input/",
readonly=True)
commands += _mount_path(conf,
"task.output_data_path",
"/home/jovyan/work/data/output/",
make_dirs=True)
return commands
def _mount_path(conf: AiscalatorConfig,
field,
target_path,
readonly=False,
make_dirs=False):
"""
Returu commands to mount path from list field into the
docker image when running.
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
field : str
the field in the configuration step that contains the path
target_path : str
where to mount them inside the container
readonly : bool
flag to mount the path as read-only
make_dirs : bool
flag to create the folder on the host before mounting if
it doesn't exists.
Returns
-------
list
commands to mount all the paths from the field
"""
commands = []
if conf.has_step_field(field):
for value in conf.step_field(field):
# TODO handle URL
for i in value:
if make_dirs:
makedirs(os.path.realpath(conf.root_dir() + value[i]),
exist_ok=True)
if os.path.exists(conf.root_dir() + value[i]):
commands += [
"--mount", "type=bind,source=" +
os.path.realpath(conf.root_dir() + value[i]) +
",target=" + os.path.join(target_path, i) +
(",readonly" if readonly else "")
]
return commands
def edit(conf, notebook):
"""Edit DAG job"""
if len(notebook) < 2:
notebook = notebook[0] if notebook else None
app_config = AiscalatorConfig(config=conf,
dag_selection=notebook)
click.echo(command.airflow_edit(app_config))
else:
raise click.BadArgumentUsage("Expecting one or less notebook names")
def edit(conf, notebook, param, param_raw):
"""Edit the notebook from an aiscalate config with JupyterLab."""
if len(notebook) < 2:
notebook = notebook[0] if notebook else None
app_config = AiscalatorConfig(config=conf, step_selection=notebook)
click.echo(
command.jupyter_edit(app_config, param=param, param_raw=param_raw))
else:
raise click.BadArgumentUsage("Expecting one or less notebook names")
def jupyter_run(conf: AiscalatorConfig,
prepare_only=False,
param=None,
param_raw=None):
"""
Executes the step in browserless mode using papermill
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
prepare_only : bool
Indicates if papermill should replace the parameters of the
notebook only or it should execute all the cells too
Returns
-------
string
the path to the output notebook resulting from the execution
of this step
"""
logger = logging.getLogger(__name__)
conf.validate_config()
docker_image = build(conf)
if not docker_image:
raise Exception("Failed to build docker image")
notebook, _ = notebook_file(conf.step_file_path('task.code_path'))
notebook = os.path.join("/home/jovyan/work/notebook/",
os.path.basename(notebook))
notebook_output = conf.step_notebook_output_path(notebook)
commands = _prepare_docker_env(
conf,
[
docker_image, "bash", "start-papermill.sh", "papermill", notebook,
notebook_output
],
"run_" + conf.step_name() + "_"
# add timestamp to name to handle multiple concurrent runs
+ datetime.datetime.now().strftime('%Y%m%d_%H%M%S'))
if prepare_only:
commands.append("--prepare-only")
parameters = conf.step_extract_parameters()
if parameters:
commands += parameters
if param:
for parameter in param:
commands += ["-p", parameter[0], parameter[1]]
if param_raw:
for raw_parameter in param_raw:
commands += ["-r", raw_parameter[0], raw_parameter[1]]
log = LogRegexAnalyzer()
logger.info("Running...: %s", " ".join(commands))
returncode = subprocess_run(commands, log_function=log.grep_logs)
if returncode:
logger.error("Run was not successful, returned status code is: " +
str(returncode))
sys.exit(returncode)
return os.path.join(conf.step_file_path('task.execution_dir_path'),
os.path.basename(notebook_output))
def airflow_push(conf: AiscalatorConfig):
"""
Starts an airflow environment
Parameters
----------
conf : AiscalatorConfig
Configuration object for the application
"""
# TODO to implement
logging.error("Not implemented yet %s", conf.app_config_home())
def _split_workspace_string(conf: AiscalatorConfig, workspace):
"""
Interprets the workspace string and split into src and dst
paths:
- The src is a path on the host machine.
- The dst is a path on the container.
In case, the workspace string doesn't specify both paths
separated by a ':', this function will automatically mount it
in the $app_config_home_directory/work/ folder creating a
symbolic link with the same basename as the workspace.
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
workspace : str
the workspace string to interpret
Returns
-------
(str, str)
A tuple with both src and dst paths
"""
logger = logging.getLogger(__name__)
root_dir = conf.app_config_home()
if workspace.strip():
if ':' in workspace:
src = abspath(workspace.split(':')[0])
if not src.startswith('/'):
src = abspath(join(root_dir, src))
dst = workspace.split(':')[1]
if not dst.startswith('/'):
dst = abspath(join(root_dir, dst))
else:
src = abspath(workspace)
if not src.startswith('/'):
src = abspath(join(root_dir, src))
# in the workspace special folder, we'll create the same named
# folder (basename) that is actually a link back to the one
# we want to include in our workspace.
dst = join("workspace", basename(workspace.strip('/')))
link = join(root_dir, dst)
if realpath(src) != realpath(link):
if exists(link) and islink(link):
logger.warning("Removing an existing symbolic"
" link %s -> %s",
link, realpath(link))
remove(link)
if not exists(link):
logger.info("Creating a symbolic link %s -> %s", link, src)
symlink(src, link)
dst = "/usr/local/airflow/" + dst
return src, dst
return None, None
def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp):
"""
Include apt-install packages into the dockerfile
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
dockerfile : str
path to the dockerfile to modify
tmp : str
path to the temporary dockerfile output
Returns
-------
path to the resulting dockerfile
"""
if conf.has_step_field("docker_image.apt_package_path"):
content = conf.step_file_path("docker_image.apt_package_path")
value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
if value:
value = ("RUN apt-get update && apt-get install -yqq \\\n" +
value +
""" && apt-get purge --auto-remove -yqq $buildDeps \\
&& apt-get autoremove -yqq --purge \\
&& apt-get clean \\
&& rm -rf \\
/var/lib/apt/lists/* \\
/tmp/* \\
/var/tmp/* \\
/usr/share/man \\
/usr/share/doc \\
/usr/share/doc-base
""")
utils.copy_replace(dockerfile,
tmp,
pattern="# apt_packages.txt #",
replace_value=value)
return tmp
return dockerfile
def _docker_compose(conf: AiscalatorConfig,
extra_commands: list):
"""
Run the docker-compose command
Parameters
----------
conf : AiscalatorConfig
Configuration object for the application
extra_commands : list
list of sub-commands to run in docker-compose
"""
logger = logging.getLogger(__name__)
conf.validate_config()
dockerfile = join(conf.app_config_home(), "config",
conf.airflow_docker_compose_file())
commands = ["docker-compose"]
# Prepare a temp folder to run the command from
with TemporaryDirectory(prefix="aiscalator_") as tmp:
with open(join(tmp, ".env"), mode="w") as env_file:
# concatenate all the env files into one
for env in conf.user_env_file(conf.dag_field("definition.env")):
if isfile(env):
with open(env, mode="r") as file:
for line in file:
env_file.write(line)
utils.copy_replace(join(tmp, ".env"),
join(dirname(dockerfile), ".env"))
commands += ["-f", dockerfile] + extra_commands
logger.info("Running...: %s", " ".join(commands))
utils.subprocess_run(commands, no_redirect=True)
def prompt_edit(file):
"""
When creating a new step, if it is already defined,
ask to edit instead
Parameters
----------
file : str
existing configuration file
"""
msg = file + ' already exists. Did you mean to run:\n'
for i in sys.argv:
if i != "new":
msg += i + ' '
else:
break
msg += "edit " + file + " instead?"
if click.confirm(msg, abort=True):
conf = AiscalatorConfig(config=file)
click.echo(command.jupyter_edit(conf))
def jupyter_new(name, path, output_format="hocon"):
"""
Starts a Jupyter Lab environment configured to edit a brand new step
Parameters
----------
name : str
name of the new step
path : str
path to where the new step files should be created
output_format : str
the format of the new configuration file to produce
Returns
-------
string
Url of the running jupyter lab
"""
step_file = os.path.join(path, name, name) + '.conf'
if os.path.dirname(step_file):
makedirs(os.path.dirname(step_file), exist_ok=True)
copy_replace(data_file("../config/template/step.conf"),
step_file,
pattern="Untitled",
replace_value=name)
if output_format != 'hocon':
file = os.path.join(path, name, name) + '.' + output_format
step_file = convert_to_format(step_file,
output=file,
output_format=output_format)
notebook = os.path.join(path, name, 'notebook', name) + '.ipynb'
if os.path.dirname(notebook):
makedirs(os.path.dirname(notebook), exist_ok=True)
copy_replace(data_file("../config/template/notebook.json"), notebook)
open(os.path.join(path, name, "apt_repository.txt"), 'a').close()
open(os.path.join(path, name, "apt_packages.txt"), 'a').close()
open(os.path.join(path, name, "requirements.txt"), 'a').close()
open(os.path.join(path, name, "lab_extensions.txt"), 'a').close()
jupyter_edit(AiscalatorConfig(config=step_file, step_selection=name))
def _prepare_docker_env(conf: AiscalatorConfig, program, reason):
"""
Assembles the list of commands to execute a docker run call
When calling "docker run ...", this function also adds a set of
additional parameters to mount the proper volumes and expose the
correct environment for the call in the docker image mapped to the
host directories. This is done so only some specific data and code
folders are accessible within the docker image.
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
program : List
the rest of the commands to execute as part of
the docker run call
Returns
-------
List
The full Array of Strings representing the commands to execute
in the docker run call
"""
logger = logging.getLogger(__name__)
commands = [
"docker", "run", "--name",
conf.step_container_name() + "_" + reason, "--rm"
]
for env in conf.user_env_file(conf.step_field("task.env")):
if os.path.isfile(env):
commands += ["--env-file", env]
commands += _prepare_docker_image_env(conf)
code_path = conf.step_file_path('task.code_path')
if conf.has_step_field('task.code_format'):
from_format = conf.step_field('task.code_format')
else:
from_format = "py"
from_format += ':'
if conf.has_step_field('task.jupytext_format'):
from_format += conf.step_field('task.jupytext_format')
else:
from_format += "percent"
notebook, _ = notebook_file(code_path)
check_notebook_dir(logger, notebook, from_format)
commands += [
"--mount",
"type=bind,source=" + os.path.dirname(notebook) +
",target=/home/jovyan/work/notebook/",
]
commands += _prepare_task_env(conf)
if conf.has_step_field("task.execution_dir_path"):
execution_dir_path = conf.step_file_path('task.execution_dir_path')
if execution_dir_path:
makedirs(execution_dir_path, exist_ok=True)
commands += [
"--mount", "type=bind,source=" + execution_dir_path +
",target=/home/jovyan/work/notebook_run/"
]
commands += program
return commands
def _prepare_docker_image_env(conf: AiscalatorConfig):
"""
Assemble the list of volumes to mount specific to
building the docker image
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
Returns
-------
list
list of commands to bind those volumes
"""
commands = []
if conf.config_path() is not None:
commands += [
"--mount",
"type=bind,source=" + os.path.realpath(conf.config_path()) +
",target="
"/home/jovyan/work/" + os.path.basename(conf.config_path()),
]
if conf.has_step_field("docker_image.apt_repository_path"):
apt_repo = conf.step_file_path('docker_image.apt_repository_path')
if apt_repo and os.path.isfile(apt_repo):
commands += [
"--mount",
"type=bind,source=" + apt_repo +
",target=/home/jovyan/work/apt_repository.txt",
]
if conf.has_step_field("docker_image.apt_package_path"):
apt_packages = conf.step_file_path('docker_image.apt_package_path')
if apt_packages and os.path.isfile(apt_packages):
commands += [
"--mount",
"type=bind,source=" + apt_packages +
",target=/home/jovyan/work/apt_packages.txt",
]
if conf.has_step_field("docker_image.requirements_path"):
requirements = conf.step_file_path('docker_image.requirements_path')
if requirements and os.path.isfile(requirements):
commands += [
"--mount",
"type=bind,source=" + requirements +
",target=/home/jovyan/work/requirements.txt",
]
if conf.has_step_field("docker_image.lab_extension_path"):
lab_extensions = conf.step_file_path('docker_image.lab_extension_path')
if lab_extensions and os.path.isfile(lab_extensions):
commands += [
"--mount",
"type=bind,source=" + lab_extensions +
",target=/home/jovyan/work/lab_extensions.txt",
]
# allow to pass a list of extra options like ["--network", "bridge"]
if conf.has_step_field("docker_image.docker_extra_options"):
commands += conf.step_field("docker_image.docker_extra_options")
return commands
def run(service, subcommand):
"""Run sub-command in a running docker service."""
if not subcommand:
subcommand = None
click.echo(command.airflow_cmd(AiscalatorConfig(),
service=service, cmd=subcommand))
def test_prepare_docker_image_env_extra_options():
"""Test the _prepare_docker_image_env."""
options_list = command._prepare_docker_image_env(
AiscalatorConfig("tests/jupyter/sample_pyhocon.conf"))
'bridge' == options_list[-1]
'--network' == options_list[-2]
def airflow_setup(conf: AiscalatorConfig,
config_home: str,
workspace: list,
append: bool = True):
"""
Setup the airflow configuration files and environment
Parameters
----------
conf : AiscalatorConfig
Configuration object for the application
config_home : str
path to the configuration home directory
workspace : list
List of path to directories to mount as volumes
to airflow workers to use as workspaces
append : bool
flag to tell if workspace should be appended to
the list in the config or replace it.
"""
logger = logging.getLogger(__name__)
conf.validate_config()
if config_home:
makedirs(config_home, exist_ok=True)
conf.redefine_app_config_home(config_home)
ws_path = "airflow.setup.workspace_paths"
if conf.app_config_has(ws_path):
if append:
workspace += conf.app_config()[ws_path]
conf.redefine_airflow_workspaces(workspace)
image = 'latest'
if _docker_compose_grep(conf):
image = _airflow_docker_build(conf)
if not image:
raise Exception("Failed to build docker image")
src = utils.data_file("../config/docker/airflow/config/")
dst = join(conf.app_config_home(), "config")
logger.info("Generating a new configuration folder for aiscalator:\n\t%s",
dst)
makedirs(dst, exist_ok=True)
makedirs(join(conf.app_config_home(), "dags"), exist_ok=True)
makedirs(join(conf.app_config_home(), "pgdata"), exist_ok=True)
makedirs(join(conf.app_config_home(), "workspace"), exist_ok=True)
pattern = [
r"(\s+)# - workspace #",
"aiscalator/airflow:latest",
]
workspace = []
for line in conf.app_config()[ws_path]:
host_src, container_dst = _split_workspace_string(conf, line)
# bind the same path from host in the container (after creating a
# symbolic link at container_dst path)
workspace += [r"\1- " + host_src + ':' + host_src]
workspace += [r"\1# - workspace #"]
value = [
"\n".join(workspace),
"aiscalator/airflow:" + image,
]
for file in listdir(src):
utils.copy_replace(join(src, file),
join(dst, file),
pattern=pattern,
replace_value=value)
def _prepare_docker_env(conf: AiscalatorConfig, program, port):
"""
Assembles the list of commands to execute a docker run call
When calling "docker run ...", this function also adds a set of
additional parameters to mount the proper volumes and expose the
correct environment for the call in the docker image.
Parameters
----------
conf : AiscalatorConfig
Configuration object for the step
program : List
the rest of the commands to execute as part of
the docker run call
Returns
-------
List
The full Array of Strings representing the commands to execute
in the docker run call
"""
logger = logging.getLogger(__name__)
commands = [
"docker", "run", "--name", conf.dag_container_name() + "_edit",
"--rm",
# TODO improve port publishing
"-p", str(port) + ":8888",
"-p", "18080:8080",
]
for env in conf.user_env_file(conf.dag_field("definition.env")):
if isfile(env):
commands += ["--env-file", env]
commands += [
"--mount", "type=bind,source=/var/run/docker.sock,"
"target=/var/run/docker.sock",
]
code_path = conf.dag_file_path('definition.code_path')
notebook, _ = utils.notebook_file(code_path)
utils.check_notebook_dir(logger, notebook)
commands += [
"--mount", "type=bind,source=" + dirname(notebook) +
",target=/usr/local/airflow/work/",
]
if conf.config_path() is not None:
commands += [
"--mount",
"type=bind,source=" + abspath(conf.config_path()) +
",target="
"/usr/local/airflow/" + basename(conf.config_path()),
]
workspace = []
ws_path = "airflow.setup.workspace_paths"
if conf.app_config_has(ws_path):
ws_home = join(conf.app_config_home(),
"workspace")
makedirs(ws_home, exist_ok=True)
for folder in conf.app_config()[ws_path]:
src, dst = _split_workspace_string(conf, folder)
# bind the same path from host in the container (after creating
# a symbolic link at dst path)
workspace += [src + ":" + src]
commands += [
"--mount", "type=bind,source=" + src +
",target=" + src
]
commands += [
"--mount", "type=bind,source=" + ws_home +
",target=/usr/local/airflow/workspace/"
]
commands += program + workspace
return commands
