def jupyter_edit(conf: AiscalatorConfig, param=None, param_raw=None): """ Starts a Jupyter Lab environment configured to edit the focused step Parameters ---------- conf : AiscalatorConfig Configuration object for the step param : list list of tuples of parameters param_raw : list list of tuples of raw parameters Returns ------- string Url of the running jupyter lab """ logger = logging.getLogger(__name__) conf.validate_config() docker_image = build(conf) if docker_image: # TODO: shutdown other jupyter lab still running notebook, _ = notebook_file(conf.step_field('task.code_path')) notebook = os.path.basename(notebook) if conf.step_extract_parameters(): jupyter_run(conf, prepare_only=True, param=param, param_raw=param_raw) commands = _prepare_docker_env( conf, [docker_image, "start.sh", 'jupyter', 'lab'], "edit") return wait_for_jupyter_lab(commands, logger, notebook, 10000, "work/notebook") raise Exception("Failed to build docker image")
def _docker_compose_grep(conf: AiscalatorConfig): """ Checks if the docker-compose file is using the aiscalator/airflow docker image. In which case, we need to make sure that image is properly built and available. Parameters ---------- conf : AiscalatorConfig Configuration object for the application Returns ------- bool Returns if aiscalator/airflow docker image is needed and should be built. """ docker_compose_file = join(conf.app_config_home(), "config", conf.airflow_docker_compose_file()) pattern = re.compile(r"\s+image:\s+aiscalator/airflow") try: with open(docker_compose_file, "r") as file: for line in file: if re.match(pattern, line): # docker compose needs the image return True except FileNotFoundError: # no docker compose, default will need the image return True return False
def _run_build(conf: AiscalatorConfig): """ Run the docker build command to produce the image and tag it. Parameters ---------- conf : AiscalatorConfig Configuration object for this step Returns ------- str the docker image ID that was built """ logger = logging.getLogger(__name__) commands = ["docker", "build", "--rm"] output_docker_name = None if conf.has_step_field("docker_image.output_docker_name"): output_docker_name = conf.step_field("docker_image.output_docker_name") commands += ["-t", output_docker_name + ":latest"] commands += ["."] log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n') logger.info("Running...: %s", " ".join(commands)) utils.subprocess_run(commands, log_function=log.grep_logs) result = log.artifact() test = (result and output_docker_name is not None and conf.has_step_field("docker_image.output_docker_tag")) if test: commands = ["docker", "tag"] output_docker_tag = conf.step_field("docker_image.output_docker_tag") commands += [result, output_docker_name + ":" + output_docker_tag] # TODO implement docker tag output_docker_tag_commit_hash logger.info("Running...: %s", " ".join(commands)) utils.subprocess_run(commands) return result
def jupyter_run(config, notebook=None, prepare_only=False, param=None, param_raw=None): """ Executes the step in browserless mode using papermill Parameters ---------- config : str path to the configuration file notebook : str name of node to run, if None, then run the first one parameters : list List of parameters and their values prepare_only : bool Indicates if papermill should replace the parameters of the notebook only or it should execute all the cells too Returns ------- string the path to the output notebook resulting from the execution of this step """ # TODO implements parameters passing if notebook: app_config = AiscalatorConfig(config=config, step_selection=notebook) else: app_config = AiscalatorConfig(config=config) return command.jupyter_run(app_config, prepare_only=prepare_only, param=param, param_raw=param_raw)
def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst): """ Include pip install packages into the dockerfile Parameters ---------- conf : AiscalatorConfig Configuration object for this step dockerfile : str path to the dockerfile to modify tmp : str path to the temporary dockerfile output dst : str path to the final temporary directory Returns ------- path to the resulting dockerfile """ if conf.has_step_field("docker_image.requirements_path"): content = conf.step_file_path("docker_image.requirements_path") copy(content, join(dst, 'requirements.txt')) utils.copy_replace(dockerfile, tmp, pattern="# requirements.txt #", replace_value=""" COPY requirements.txt requirements.txt RUN pip install -r requirements.txt RUN rm requirements.txt""") return tmp return dockerfile
def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp): """ Include jupyter lab extensions packages into the dockerfile Parameters ---------- conf : AiscalatorConfig Configuration object for this step dockerfile : str path to the dockerfile to modify tmp : str path to the temporary dockerfile output Returns ------- path to the resulting dockerfile """ if conf.has_step_field("docker_image.lab_extension_path"): content = conf.step_file_path("docker_image.lab_extension_path") prefix = "&& jupyter labextension install " value = utils.format_file_content(content, prefix=prefix, suffix=" \\\n") if value: value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value utils.copy_replace(dockerfile, tmp, pattern="# lab_extensions.txt #", replace_value=value) return tmp return dockerfile
def _include_apt_repo(conf: AiscalatorConfig, dockerfile, tmp): """ Include add-apt-repository packages into the dockerfile Parameters ---------- conf : AiscalatorConfig Configuration object for this step dockerfile : str path to the dockerfile to modify tmp : str path to the temporary dockerfile output Returns ------- path to the resulting dockerfile """ if conf.has_step_field("docker_image.apt_repository_path"): content = conf.step_file_path("docker_image.apt_repository_path") value = utils.format_file_content(content, prefix=" ", suffix="\\\n") if value: value = ("RUN apt-get update \\\n" + " && apt-get install -yqq \\\n" + " software-properties-common \\\n" + " && apt-add-repository \\\n" + value + " && apt-get update") utils.copy_replace(dockerfile, tmp, pattern="# apt_repository.txt #", replace_value=value) return tmp return dockerfile
def airflow_edit(conf: AiscalatorConfig): """ Starts an airflow environment Parameters ---------- conf : AiscalatorConfig Configuration object for the application """ logger = logging.getLogger(__name__) conf.validate_config() docker_image = _airflow_docker_build(conf) if docker_image: # TODO: shutdown other jupyter lab still running port = 10001 notebook = basename(conf.dag_field('definition.code_path')) notebook, notebook_py = utils.notebook_file(notebook) commands = _prepare_docker_env(conf, [ "aiscalator/airflow:" + docker_image, "bash", "/start-jupyter.sh", "/usr/local/airflow/work/" + notebook_py + ":/usr/local/airflow/dags/" + notebook_py ], port) return utils.wait_for_jupyter_lab(commands, logger, notebook, port, "work") raise Exception("Failed to build docker image")
def _airflow_docker_build(conf: AiscalatorConfig): """ Build the aiscalator/airflow image and return its ID.""" logger = logging.getLogger(__name__) # TODO get airflow dockerfile from conf? conf.app_config_home() dockerfile_dir = utils.data_file("../config/docker/airflow") # TODO customize dockerfile with apt_packages, requirements etc docker_gid, docker_group = _find_docker_gid_group() commands = [ "docker", "build", "--build-arg", "DOCKER_GID=" + str(docker_gid), "--build-arg", "DOCKER_GROUP=" + str(docker_group), "--rm", "-t", "aiscalator/airflow:latest", dockerfile_dir ] log = LogRegexAnalyzer(b'Successfully built ([a-zA-Z0-9]+)\n') logger.info("Running...: %s", " ".join(commands)) utils.subprocess_run(commands, log_function=log.grep_logs) result = log.artifact() if result: # tag the image built with the sha256 of the dockerfile tag = utils.sha256(join(dockerfile_dir, 'Dockerfile'))[:12] commands = [ "docker", "tag", result, "aiscalator/airflow:" + tag ] logger.info("Running...: %s", " ".join(commands)) utils.subprocess_run(commands) return tag return None
def push(conf, notebook): """Push a job into the DAGS folder to schedule in Airflow.""" if notebook: for note in notebook: app_config = AiscalatorConfig(config=conf, dag_selection=note) click.echo(command.airflow_push(app_config)) else: app_config = AiscalatorConfig(config=conf) click.echo(command.airflow_push(app_config))
def run(conf, notebook, param, param_raw): """Run the notebook from an aiscalate config without GUI.""" if notebook: for note in notebook: app_config = AiscalatorConfig(config=conf, step_selection=note) click.echo( command.jupyter_run(app_config, param=param, param_raw=param_raw)) else: app_config = AiscalatorConfig(config=conf) click.echo( command.jupyter_run(app_config, param=param, param_raw=param_raw))
def start(): """Start docker images to bring airflow services up.""" click.echo(command.airflow_up(AiscalatorConfig())) click.echo(""" Airflow: http://localhost:8080 Flower: http://localhost:5555 """)
def build(conf: AiscalatorConfig): """ Builds the docker image following the parameters specified in the focused step's configuration file Parameters ---------- conf : AiscalatorConfig Configuration object for this step Returns ------- string the docker artifact name of the image built """ input_docker_src = conf.step_field("docker_image.input_docker_src") # TODO check if input_docker_src refers to an existing docker image # in which case, if no customization is needed, no need to build cwd = getcwd() result = None try: # Prepare a temp folder to build docker image with TemporaryDirectory(prefix="aiscalator_") as tmp: _prepare_build_dir(conf, tmp, input_docker_src) chdir(tmp) result = _run_build(conf) finally: chdir(cwd) return result
def _prepare_task_env(conf: AiscalatorConfig): """ Assemble the list of volumes to mount specific to the task execution Parameters ---------- conf : AiscalatorConfig Configuration object for the step Returns ------- list list of commands to bind those volumes """ commands = [] if conf.root_dir(): commands += _mount_path(conf, "task.modules_src_path", "/home/jovyan/work/modules/") commands += _mount_path(conf, "task.input_data_path", "/home/jovyan/work/data/input/", readonly=True) commands += _mount_path(conf, "task.output_data_path", "/home/jovyan/work/data/output/", make_dirs=True) return commands
def _mount_path(conf: AiscalatorConfig, field, target_path, readonly=False, make_dirs=False): """ Returu commands to mount path from list field into the docker image when running. Parameters ---------- conf : AiscalatorConfig Configuration object for the step field : str the field in the configuration step that contains the path target_path : str where to mount them inside the container readonly : bool flag to mount the path as read-only make_dirs : bool flag to create the folder on the host before mounting if it doesn't exists. Returns ------- list commands to mount all the paths from the field """ commands = [] if conf.has_step_field(field): for value in conf.step_field(field): # TODO handle URL for i in value: if make_dirs: makedirs(os.path.realpath(conf.root_dir() + value[i]), exist_ok=True) if os.path.exists(conf.root_dir() + value[i]): commands += [ "--mount", "type=bind,source=" + os.path.realpath(conf.root_dir() + value[i]) + ",target=" + os.path.join(target_path, i) + (",readonly" if readonly else "") ] return commands
def edit(conf, notebook): """Edit DAG job""" if len(notebook) < 2: notebook = notebook[0] if notebook else None app_config = AiscalatorConfig(config=conf, dag_selection=notebook) click.echo(command.airflow_edit(app_config)) else: raise click.BadArgumentUsage("Expecting one or less notebook names")
def edit(conf, notebook, param, param_raw): """Edit the notebook from an aiscalate config with JupyterLab.""" if len(notebook) < 2: notebook = notebook[0] if notebook else None app_config = AiscalatorConfig(config=conf, step_selection=notebook) click.echo( command.jupyter_edit(app_config, param=param, param_raw=param_raw)) else: raise click.BadArgumentUsage("Expecting one or less notebook names")
def jupyter_run(conf: AiscalatorConfig, prepare_only=False, param=None, param_raw=None): """ Executes the step in browserless mode using papermill Parameters ---------- conf : AiscalatorConfig Configuration object for the step prepare_only : bool Indicates if papermill should replace the parameters of the notebook only or it should execute all the cells too Returns ------- string the path to the output notebook resulting from the execution of this step """ logger = logging.getLogger(__name__) conf.validate_config() docker_image = build(conf) if not docker_image: raise Exception("Failed to build docker image") notebook, _ = notebook_file(conf.step_file_path('task.code_path')) notebook = os.path.join("/home/jovyan/work/notebook/", os.path.basename(notebook)) notebook_output = conf.step_notebook_output_path(notebook) commands = _prepare_docker_env( conf, [ docker_image, "bash", "start-papermill.sh", "papermill", notebook, notebook_output ], "run_" + conf.step_name() + "_" # add timestamp to name to handle multiple concurrent runs + datetime.datetime.now().strftime('%Y%m%d_%H%M%S')) if prepare_only: commands.append("--prepare-only") parameters = conf.step_extract_parameters() if parameters: commands += parameters if param: for parameter in param: commands += ["-p", parameter[0], parameter[1]] if param_raw: for raw_parameter in param_raw: commands += ["-r", raw_parameter[0], raw_parameter[1]] log = LogRegexAnalyzer() logger.info("Running...: %s", " ".join(commands)) returncode = subprocess_run(commands, log_function=log.grep_logs) if returncode: logger.error("Run was not successful, returned status code is: " + str(returncode)) sys.exit(returncode) return os.path.join(conf.step_file_path('task.execution_dir_path'), os.path.basename(notebook_output))
def airflow_push(conf: AiscalatorConfig): """ Starts an airflow environment Parameters ---------- conf : AiscalatorConfig Configuration object for the application """ # TODO to implement logging.error("Not implemented yet %s", conf.app_config_home())
def _split_workspace_string(conf: AiscalatorConfig, workspace): """ Interprets the workspace string and split into src and dst paths: - The src is a path on the host machine. - The dst is a path on the container. In case, the workspace string doesn't specify both paths separated by a ':', this function will automatically mount it in the $app_config_home_directory/work/ folder creating a symbolic link with the same basename as the workspace. Parameters ---------- conf : AiscalatorConfig Configuration object for the step workspace : str the workspace string to interpret Returns ------- (str, str) A tuple with both src and dst paths """ logger = logging.getLogger(__name__) root_dir = conf.app_config_home() if workspace.strip(): if ':' in workspace: src = abspath(workspace.split(':')[0]) if not src.startswith('/'): src = abspath(join(root_dir, src)) dst = workspace.split(':')[1] if not dst.startswith('/'): dst = abspath(join(root_dir, dst)) else: src = abspath(workspace) if not src.startswith('/'): src = abspath(join(root_dir, src)) # in the workspace special folder, we'll create the same named # folder (basename) that is actually a link back to the one # we want to include in our workspace. dst = join("workspace", basename(workspace.strip('/'))) link = join(root_dir, dst) if realpath(src) != realpath(link): if exists(link) and islink(link): logger.warning("Removing an existing symbolic" " link %s -> %s", link, realpath(link)) remove(link) if not exists(link): logger.info("Creating a symbolic link %s -> %s", link, src) symlink(src, link) dst = "/usr/local/airflow/" + dst return src, dst return None, None
def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp): """ Include apt-install packages into the dockerfile Parameters ---------- conf : AiscalatorConfig Configuration object for this step dockerfile : str path to the dockerfile to modify tmp : str path to the temporary dockerfile output Returns ------- path to the resulting dockerfile """ if conf.has_step_field("docker_image.apt_package_path"): content = conf.step_file_path("docker_image.apt_package_path") value = utils.format_file_content(content, prefix=" ", suffix="\\\n") if value: value = ("RUN apt-get update && apt-get install -yqq \\\n" + value + """ && apt-get purge --auto-remove -yqq $buildDeps \\ && apt-get autoremove -yqq --purge \\ && apt-get clean \\ && rm -rf \\ /var/lib/apt/lists/* \\ /tmp/* \\ /var/tmp/* \\ /usr/share/man \\ /usr/share/doc \\ /usr/share/doc-base """) utils.copy_replace(dockerfile, tmp, pattern="# apt_packages.txt #", replace_value=value) return tmp return dockerfile
def _docker_compose(conf: AiscalatorConfig, extra_commands: list): """ Run the docker-compose command Parameters ---------- conf : AiscalatorConfig Configuration object for the application extra_commands : list list of sub-commands to run in docker-compose """ logger = logging.getLogger(__name__) conf.validate_config() dockerfile = join(conf.app_config_home(), "config", conf.airflow_docker_compose_file()) commands = ["docker-compose"] # Prepare a temp folder to run the command from with TemporaryDirectory(prefix="aiscalator_") as tmp: with open(join(tmp, ".env"), mode="w") as env_file: # concatenate all the env files into one for env in conf.user_env_file(conf.dag_field("definition.env")): if isfile(env): with open(env, mode="r") as file: for line in file: env_file.write(line) utils.copy_replace(join(tmp, ".env"), join(dirname(dockerfile), ".env")) commands += ["-f", dockerfile] + extra_commands logger.info("Running...: %s", " ".join(commands)) utils.subprocess_run(commands, no_redirect=True)
def prompt_edit(file): """ When creating a new step, if it is already defined, ask to edit instead Parameters ---------- file : str existing configuration file """ msg = file + ' already exists. Did you mean to run:\n' for i in sys.argv: if i != "new": msg += i + ' ' else: break msg += "edit " + file + " instead?" if click.confirm(msg, abort=True): conf = AiscalatorConfig(config=file) click.echo(command.jupyter_edit(conf))
def jupyter_new(name, path, output_format="hocon"): """ Starts a Jupyter Lab environment configured to edit a brand new step Parameters ---------- name : str name of the new step path : str path to where the new step files should be created output_format : str the format of the new configuration file to produce Returns ------- string Url of the running jupyter lab """ step_file = os.path.join(path, name, name) + '.conf' if os.path.dirname(step_file): makedirs(os.path.dirname(step_file), exist_ok=True) copy_replace(data_file("../config/template/step.conf"), step_file, pattern="Untitled", replace_value=name) if output_format != 'hocon': file = os.path.join(path, name, name) + '.' + output_format step_file = convert_to_format(step_file, output=file, output_format=output_format) notebook = os.path.join(path, name, 'notebook', name) + '.ipynb' if os.path.dirname(notebook): makedirs(os.path.dirname(notebook), exist_ok=True) copy_replace(data_file("../config/template/notebook.json"), notebook) open(os.path.join(path, name, "apt_repository.txt"), 'a').close() open(os.path.join(path, name, "apt_packages.txt"), 'a').close() open(os.path.join(path, name, "requirements.txt"), 'a').close() open(os.path.join(path, name, "lab_extensions.txt"), 'a').close() jupyter_edit(AiscalatorConfig(config=step_file, step_selection=name))
def _prepare_docker_env(conf: AiscalatorConfig, program, reason): """ Assembles the list of commands to execute a docker run call When calling "docker run ...", this function also adds a set of additional parameters to mount the proper volumes and expose the correct environment for the call in the docker image mapped to the host directories. This is done so only some specific data and code folders are accessible within the docker image. Parameters ---------- conf : AiscalatorConfig Configuration object for the step program : List the rest of the commands to execute as part of the docker run call Returns ------- List The full Array of Strings representing the commands to execute in the docker run call """ logger = logging.getLogger(__name__) commands = [ "docker", "run", "--name", conf.step_container_name() + "_" + reason, "--rm" ] for env in conf.user_env_file(conf.step_field("task.env")): if os.path.isfile(env): commands += ["--env-file", env] commands += _prepare_docker_image_env(conf) code_path = conf.step_file_path('task.code_path') if conf.has_step_field('task.code_format'): from_format = conf.step_field('task.code_format') else: from_format = "py" from_format += ':' if conf.has_step_field('task.jupytext_format'): from_format += conf.step_field('task.jupytext_format') else: from_format += "percent" notebook, _ = notebook_file(code_path) check_notebook_dir(logger, notebook, from_format) commands += [ "--mount", "type=bind,source=" + os.path.dirname(notebook) + ",target=/home/jovyan/work/notebook/", ] commands += _prepare_task_env(conf) if conf.has_step_field("task.execution_dir_path"): execution_dir_path = conf.step_file_path('task.execution_dir_path') if execution_dir_path: makedirs(execution_dir_path, exist_ok=True) commands += [ "--mount", "type=bind,source=" + execution_dir_path + ",target=/home/jovyan/work/notebook_run/" ] commands += program return commands
def _prepare_docker_image_env(conf: AiscalatorConfig): """ Assemble the list of volumes to mount specific to building the docker image Parameters ---------- conf : AiscalatorConfig Configuration object for the step Returns ------- list list of commands to bind those volumes """ commands = [] if conf.config_path() is not None: commands += [ "--mount", "type=bind,source=" + os.path.realpath(conf.config_path()) + ",target=" "/home/jovyan/work/" + os.path.basename(conf.config_path()), ] if conf.has_step_field("docker_image.apt_repository_path"): apt_repo = conf.step_file_path('docker_image.apt_repository_path') if apt_repo and os.path.isfile(apt_repo): commands += [ "--mount", "type=bind,source=" + apt_repo + ",target=/home/jovyan/work/apt_repository.txt", ] if conf.has_step_field("docker_image.apt_package_path"): apt_packages = conf.step_file_path('docker_image.apt_package_path') if apt_packages and os.path.isfile(apt_packages): commands += [ "--mount", "type=bind,source=" + apt_packages + ",target=/home/jovyan/work/apt_packages.txt", ] if conf.has_step_field("docker_image.requirements_path"): requirements = conf.step_file_path('docker_image.requirements_path') if requirements and os.path.isfile(requirements): commands += [ "--mount", "type=bind,source=" + requirements + ",target=/home/jovyan/work/requirements.txt", ] if conf.has_step_field("docker_image.lab_extension_path"): lab_extensions = conf.step_file_path('docker_image.lab_extension_path') if lab_extensions and os.path.isfile(lab_extensions): commands += [ "--mount", "type=bind,source=" + lab_extensions + ",target=/home/jovyan/work/lab_extensions.txt", ] # allow to pass a list of extra options like ["--network", "bridge"] if conf.has_step_field("docker_image.docker_extra_options"): commands += conf.step_field("docker_image.docker_extra_options") return commands
def run(service, subcommand): """Run sub-command in a running docker service.""" if not subcommand: subcommand = None click.echo(command.airflow_cmd(AiscalatorConfig(), service=service, cmd=subcommand))
def test_prepare_docker_image_env_extra_options(): """Test the _prepare_docker_image_env.""" options_list = command._prepare_docker_image_env( AiscalatorConfig("tests/jupyter/sample_pyhocon.conf")) 'bridge' == options_list[-1] '--network' == options_list[-2]
def airflow_setup(conf: AiscalatorConfig, config_home: str, workspace: list, append: bool = True): """ Setup the airflow configuration files and environment Parameters ---------- conf : AiscalatorConfig Configuration object for the application config_home : str path to the configuration home directory workspace : list List of path to directories to mount as volumes to airflow workers to use as workspaces append : bool flag to tell if workspace should be appended to the list in the config or replace it. """ logger = logging.getLogger(__name__) conf.validate_config() if config_home: makedirs(config_home, exist_ok=True) conf.redefine_app_config_home(config_home) ws_path = "airflow.setup.workspace_paths" if conf.app_config_has(ws_path): if append: workspace += conf.app_config()[ws_path] conf.redefine_airflow_workspaces(workspace) image = 'latest' if _docker_compose_grep(conf): image = _airflow_docker_build(conf) if not image: raise Exception("Failed to build docker image") src = utils.data_file("../config/docker/airflow/config/") dst = join(conf.app_config_home(), "config") logger.info("Generating a new configuration folder for aiscalator:\n\t%s", dst) makedirs(dst, exist_ok=True) makedirs(join(conf.app_config_home(), "dags"), exist_ok=True) makedirs(join(conf.app_config_home(), "pgdata"), exist_ok=True) makedirs(join(conf.app_config_home(), "workspace"), exist_ok=True) pattern = [ r"(\s+)# - workspace #", "aiscalator/airflow:latest", ] workspace = [] for line in conf.app_config()[ws_path]: host_src, container_dst = _split_workspace_string(conf, line) # bind the same path from host in the container (after creating a # symbolic link at container_dst path) workspace += [r"\1- " + host_src + ':' + host_src] workspace += [r"\1# - workspace #"] value = [ "\n".join(workspace), "aiscalator/airflow:" + image, ] for file in listdir(src): utils.copy_replace(join(src, file), join(dst, file), pattern=pattern, replace_value=value)
def _prepare_docker_env(conf: AiscalatorConfig, program, port): """ Assembles the list of commands to execute a docker run call When calling "docker run ...", this function also adds a set of additional parameters to mount the proper volumes and expose the correct environment for the call in the docker image. Parameters ---------- conf : AiscalatorConfig Configuration object for the step program : List the rest of the commands to execute as part of the docker run call Returns ------- List The full Array of Strings representing the commands to execute in the docker run call """ logger = logging.getLogger(__name__) commands = [ "docker", "run", "--name", conf.dag_container_name() + "_edit", "--rm", # TODO improve port publishing "-p", str(port) + ":8888", "-p", "18080:8080", ] for env in conf.user_env_file(conf.dag_field("definition.env")): if isfile(env): commands += ["--env-file", env] commands += [ "--mount", "type=bind,source=/var/run/docker.sock," "target=/var/run/docker.sock", ] code_path = conf.dag_file_path('definition.code_path') notebook, _ = utils.notebook_file(code_path) utils.check_notebook_dir(logger, notebook) commands += [ "--mount", "type=bind,source=" + dirname(notebook) + ",target=/usr/local/airflow/work/", ] if conf.config_path() is not None: commands += [ "--mount", "type=bind,source=" + abspath(conf.config_path()) + ",target=" "/usr/local/airflow/" + basename(conf.config_path()), ] workspace = [] ws_path = "airflow.setup.workspace_paths" if conf.app_config_has(ws_path): ws_home = join(conf.app_config_home(), "workspace") makedirs(ws_home, exist_ok=True) for folder in conf.app_config()[ws_path]: src, dst = _split_workspace_string(conf, folder) # bind the same path from host in the container (after creating # a symbolic link at dst path) workspace += [src + ":" + src] commands += [ "--mount", "type=bind,source=" + src + ",target=" + src ] commands += [ "--mount", "type=bind,source=" + ws_home + ",target=/usr/local/airflow/workspace/" ] commands += program + workspace return commands