def _docker_compose(conf: AiscalatorConfig, extra_commands: list): """ Run the docker-compose command Parameters ---------- conf : AiscalatorConfig Configuration object for the application extra_commands : list list of sub-commands to run in docker-compose """ logger = logging.getLogger(__name__) conf.validate_config() dockerfile = join(conf.app_config_home(), "config", conf.airflow_docker_compose_file()) commands = ["docker-compose"] # Prepare a temp folder to run the command from with TemporaryDirectory(prefix="aiscalator_") as tmp: with open(join(tmp, ".env"), mode="w") as env_file: # concatenate all the env files into one for env in conf.user_env_file(conf.dag_field("definition.env")): if isfile(env): with open(env, mode="r") as file: for line in file: env_file.write(line) utils.copy_replace(join(tmp, ".env"), join(dirname(dockerfile), ".env")) commands += ["-f", dockerfile] + extra_commands logger.info("Running...: %s", " ".join(commands)) utils.subprocess_run(commands, no_redirect=True)
def airflow_edit(conf: AiscalatorConfig): """ Starts an airflow environment Parameters ---------- conf : AiscalatorConfig Configuration object for the application """ logger = logging.getLogger(__name__) conf.validate_config() docker_image = _airflow_docker_build(conf) if docker_image: # TODO: shutdown other jupyter lab still running port = 10001 notebook = basename(conf.dag_field('definition.code_path')) notebook, notebook_py = utils.notebook_file(notebook) commands = _prepare_docker_env(conf, [ "aiscalator/airflow:" + docker_image, "bash", "/start-jupyter.sh", "/usr/local/airflow/work/" + notebook_py + ":/usr/local/airflow/dags/" + notebook_py ], port) return utils.wait_for_jupyter_lab(commands, logger, notebook, port, "work") raise Exception("Failed to build docker image")
def _prepare_docker_env(conf: AiscalatorConfig, program, port): """ Assembles the list of commands to execute a docker run call When calling "docker run ...", this function also adds a set of additional parameters to mount the proper volumes and expose the correct environment for the call in the docker image. Parameters ---------- conf : AiscalatorConfig Configuration object for the step program : List the rest of the commands to execute as part of the docker run call Returns ------- List The full Array of Strings representing the commands to execute in the docker run call """ logger = logging.getLogger(__name__) commands = [ "docker", "run", "--name", conf.dag_container_name() + "_edit", "--rm", # TODO improve port publishing "-p", str(port) + ":8888", "-p", "18080:8080", ] for env in conf.user_env_file(conf.dag_field("definition.env")): if isfile(env): commands += ["--env-file", env] commands += [ "--mount", "type=bind,source=/var/run/docker.sock," "target=/var/run/docker.sock", ] code_path = conf.dag_file_path('definition.code_path') notebook, _ = utils.notebook_file(code_path) utils.check_notebook_dir(logger, notebook) commands += [ "--mount", "type=bind,source=" + dirname(notebook) + ",target=/usr/local/airflow/work/", ] if conf.config_path() is not None: commands += [ "--mount", "type=bind,source=" + abspath(conf.config_path()) + ",target=" "/usr/local/airflow/" + basename(conf.config_path()), ] workspace = [] ws_path = "airflow.setup.workspace_paths" if conf.app_config_has(ws_path): ws_home = join(conf.app_config_home(), "workspace") makedirs(ws_home, exist_ok=True) for folder in conf.app_config()[ws_path]: src, dst = _split_workspace_string(conf, folder) # bind the same path from host in the container (after creating # a symbolic link at dst path) workspace += [src + ":" + src] commands += [ "--mount", "type=bind,source=" + src + ",target=" + src ] commands += [ "--mount", "type=bind,source=" + ws_home + ",target=/usr/local/airflow/workspace/" ] commands += program + workspace return commands