def airflow_setup(conf: AiscalatorConfig, config_home: str, workspace: list, append: bool = True): """ Setup the airflow configuration files and environment Parameters ---------- conf : AiscalatorConfig Configuration object for the application config_home : str path to the configuration home directory workspace : list List of path to directories to mount as volumes to airflow workers to use as workspaces append : bool flag to tell if workspace should be appended to the list in the config or replace it. """ logger = logging.getLogger(__name__) conf.validate_config() if config_home: makedirs(config_home, exist_ok=True) conf.redefine_app_config_home(config_home) ws_path = "airflow.setup.workspace_paths" if conf.app_config_has(ws_path): if append: workspace += conf.app_config()[ws_path] conf.redefine_airflow_workspaces(workspace) image = 'latest' if _docker_compose_grep(conf): image = _airflow_docker_build(conf) if not image: raise Exception("Failed to build docker image") src = utils.data_file("../config/docker/airflow/config/") dst = join(conf.app_config_home(), "config") logger.info("Generating a new configuration folder for aiscalator:\n\t%s", dst) makedirs(dst, exist_ok=True) makedirs(join(conf.app_config_home(), "dags"), exist_ok=True) makedirs(join(conf.app_config_home(), "pgdata"), exist_ok=True) makedirs(join(conf.app_config_home(), "workspace"), exist_ok=True) pattern = [ r"(\s+)# - workspace #", "aiscalator/airflow:latest", ] workspace = [] for line in conf.app_config()[ws_path]: host_src, container_dst = _split_workspace_string(conf, line) # bind the same path from host in the container (after creating a # symbolic link at container_dst path) workspace += [r"\1- " + host_src + ':' + host_src] workspace += [r"\1# - workspace #"] value = [ "\n".join(workspace), "aiscalator/airflow:" + image, ] for file in listdir(src): utils.copy_replace(join(src, file), join(dst, file), pattern=pattern, replace_value=value)
def _prepare_docker_env(conf: AiscalatorConfig, program, port): """ Assembles the list of commands to execute a docker run call When calling "docker run ...", this function also adds a set of additional parameters to mount the proper volumes and expose the correct environment for the call in the docker image. Parameters ---------- conf : AiscalatorConfig Configuration object for the step program : List the rest of the commands to execute as part of the docker run call Returns ------- List The full Array of Strings representing the commands to execute in the docker run call """ logger = logging.getLogger(__name__) commands = [ "docker", "run", "--name", conf.dag_container_name() + "_edit", "--rm", # TODO improve port publishing "-p", str(port) + ":8888", "-p", "18080:8080", ] for env in conf.user_env_file(conf.dag_field("definition.env")): if isfile(env): commands += ["--env-file", env] commands += [ "--mount", "type=bind,source=/var/run/docker.sock," "target=/var/run/docker.sock", ] code_path = conf.dag_file_path('definition.code_path') notebook, _ = utils.notebook_file(code_path) utils.check_notebook_dir(logger, notebook) commands += [ "--mount", "type=bind,source=" + dirname(notebook) + ",target=/usr/local/airflow/work/", ] if conf.config_path() is not None: commands += [ "--mount", "type=bind,source=" + abspath(conf.config_path()) + ",target=" "/usr/local/airflow/" + basename(conf.config_path()), ] workspace = [] ws_path = "airflow.setup.workspace_paths" if conf.app_config_has(ws_path): ws_home = join(conf.app_config_home(), "workspace") makedirs(ws_home, exist_ok=True) for folder in conf.app_config()[ws_path]: src, dst = _split_workspace_string(conf, folder) # bind the same path from host in the container (after creating # a symbolic link at dst path) workspace += [src + ":" + src] commands += [ "--mount", "type=bind,source=" + src + ",target=" + src ] commands += [ "--mount", "type=bind,source=" + ws_home + ",target=/usr/local/airflow/workspace/" ] commands += program + workspace return commands