def _generate_global_config() -> str: """Generate a standard configuration file for the application in the user's home folder ~/.aiscalator/config/aiscalator.conf from the template file in aiscalator/config/template/aiscalator.conf """ logger = getLogger(__name__) dst = os.path.join(os.path.expanduser("~"), ".aiscalator/config/aiscalator.conf") logger.info("Generating a new configuration file for aiscalator:\n\t%s", dst) pattern = [ "testUserID", "generation_date", ] replace_value = [ generate_user_id(), '"' + str(datetime.utcnow().replace(tzinfo=timezone("UTC"))) + '" // in UTC timezone', ] dst_dir = os.path.dirname(dst) if dst_dir: os.makedirs(dst_dir, exist_ok=True) copy_replace(data_file("../config/template/aiscalator.conf"), dst, pattern=pattern, replace_value=replace_value) open(os.path.join(dst_dir, "apt_packages.txt"), 'a').close() open(os.path.join(dst_dir, "requirements.txt"), 'a').close() open(os.path.join(dst_dir, "lab_extensions.txt"), 'a').close() return dst
def _docker_compose(conf: AiscalatorConfig, extra_commands: list): """ Run the docker-compose command Parameters ---------- conf : AiscalatorConfig Configuration object for the application extra_commands : list list of sub-commands to run in docker-compose """ logger = logging.getLogger(__name__) conf.validate_config() dockerfile = join(conf.app_config_home(), "config", conf.airflow_docker_compose_file()) commands = ["docker-compose"] # Prepare a temp folder to run the command from with TemporaryDirectory(prefix="aiscalator_") as tmp: with open(join(tmp, ".env"), mode="w") as env_file: # concatenate all the env files into one for env in conf.user_env_file(conf.dag_field("definition.env")): if isfile(env): with open(env, mode="r") as file: for line in file: env_file.write(line) utils.copy_replace(join(tmp, ".env"), join(dirname(dockerfile), ".env")) commands += ["-f", dockerfile] + extra_commands logger.info("Running...: %s", " ".join(commands)) utils.subprocess_run(commands, no_redirect=True)
def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp): """ Include jupyter lab extensions packages into the dockerfile Parameters ---------- conf : AiscalatorConfig Configuration object for this step dockerfile : str path to the dockerfile to modify tmp : str path to the temporary dockerfile output Returns ------- path to the resulting dockerfile """ if conf.has_step_field("docker_image.lab_extension_path"): content = conf.step_file_path("docker_image.lab_extension_path") prefix = "&& jupyter labextension install " value = utils.format_file_content(content, prefix=prefix, suffix=" \\\n") if value: value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value utils.copy_replace(dockerfile, tmp, pattern="# lab_extensions.txt #", replace_value=value) return tmp return dockerfile
def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst): """ Include pip install packages into the dockerfile Parameters ---------- conf : AiscalatorConfig Configuration object for this step dockerfile : str path to the dockerfile to modify tmp : str path to the temporary dockerfile output dst : str path to the final temporary directory Returns ------- path to the resulting dockerfile """ if conf.has_step_field("docker_image.requirements_path"): content = conf.step_file_path("docker_image.requirements_path") copy(content, join(dst, 'requirements.txt')) utils.copy_replace(dockerfile, tmp, pattern="# requirements.txt #", replace_value=""" COPY requirements.txt requirements.txt RUN pip install -r requirements.txt RUN rm requirements.txt""") return tmp return dockerfile
def _include_apt_repo(conf: AiscalatorConfig, dockerfile, tmp): """ Include add-apt-repository packages into the dockerfile Parameters ---------- conf : AiscalatorConfig Configuration object for this step dockerfile : str path to the dockerfile to modify tmp : str path to the temporary dockerfile output Returns ------- path to the resulting dockerfile """ if conf.has_step_field("docker_image.apt_repository_path"): content = conf.step_file_path("docker_image.apt_repository_path") value = utils.format_file_content(content, prefix=" ", suffix="\\\n") if value: value = ("RUN apt-get update \\\n" + " && apt-get install -yqq \\\n" + " software-properties-common \\\n" + " && apt-add-repository \\\n" + value + " && apt-get update") utils.copy_replace(dockerfile, tmp, pattern="# apt_repository.txt #", replace_value=value) return tmp return dockerfile
def jupyter_new(name, path, output_format="hocon"): """ Starts a Jupyter Lab environment configured to edit a brand new step Parameters ---------- name : str name of the new step path : str path to where the new step files should be created output_format : str the format of the new configuration file to produce Returns ------- string Url of the running jupyter lab """ step_file = os.path.join(path, name, name) + '.conf' if os.path.dirname(step_file): makedirs(os.path.dirname(step_file), exist_ok=True) copy_replace(data_file("../config/template/step.conf"), step_file, pattern="Untitled", replace_value=name) if output_format != 'hocon': file = os.path.join(path, name, name) + '.' + output_format step_file = convert_to_format(step_file, output=file, output_format=output_format) notebook = os.path.join(path, name, 'notebook', name) + '.ipynb' if os.path.dirname(notebook): makedirs(os.path.dirname(notebook), exist_ok=True) copy_replace(data_file("../config/template/notebook.json"), notebook) open(os.path.join(path, name, "apt_repository.txt"), 'a').close() open(os.path.join(path, name, "apt_packages.txt"), 'a').close() open(os.path.join(path, name, "requirements.txt"), 'a').close() open(os.path.join(path, name, "lab_extensions.txt"), 'a').close() jupyter_edit(AiscalatorConfig(config=step_file, step_selection=name))
def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp): """ Include apt-install packages into the dockerfile Parameters ---------- conf : AiscalatorConfig Configuration object for this step dockerfile : str path to the dockerfile to modify tmp : str path to the temporary dockerfile output Returns ------- path to the resulting dockerfile """ if conf.has_step_field("docker_image.apt_package_path"): content = conf.step_file_path("docker_image.apt_package_path") value = utils.format_file_content(content, prefix=" ", suffix="\\\n") if value: value = ("RUN apt-get update && apt-get install -yqq \\\n" + value + """ && apt-get purge --auto-remove -yqq $buildDeps \\ && apt-get autoremove -yqq --purge \\ && apt-get clean \\ && rm -rf \\ /var/lib/apt/lists/* \\ /tmp/* \\ /var/tmp/* \\ /usr/share/man \\ /usr/share/doc \\ /usr/share/doc-base """) utils.copy_replace(dockerfile, tmp, pattern="# apt_packages.txt #", replace_value=value) return tmp return dockerfile
def airflow_setup(conf: AiscalatorConfig, config_home: str, workspace: list, append: bool = True): """ Setup the airflow configuration files and environment Parameters ---------- conf : AiscalatorConfig Configuration object for the application config_home : str path to the configuration home directory workspace : list List of path to directories to mount as volumes to airflow workers to use as workspaces append : bool flag to tell if workspace should be appended to the list in the config or replace it. """ logger = logging.getLogger(__name__) conf.validate_config() if config_home: makedirs(config_home, exist_ok=True) conf.redefine_app_config_home(config_home) ws_path = "airflow.setup.workspace_paths" if conf.app_config_has(ws_path): if append: workspace += conf.app_config()[ws_path] conf.redefine_airflow_workspaces(workspace) image = 'latest' if _docker_compose_grep(conf): image = _airflow_docker_build(conf) if not image: raise Exception("Failed to build docker image") src = utils.data_file("../config/docker/airflow/config/") dst = join(conf.app_config_home(), "config") logger.info("Generating a new configuration folder for aiscalator:\n\t%s", dst) makedirs(dst, exist_ok=True) makedirs(join(conf.app_config_home(), "dags"), exist_ok=True) makedirs(join(conf.app_config_home(), "pgdata"), exist_ok=True) makedirs(join(conf.app_config_home(), "workspace"), exist_ok=True) pattern = [ r"(\s+)# - workspace #", "aiscalator/airflow:latest", ] workspace = [] for line in conf.app_config()[ws_path]: host_src, container_dst = _split_workspace_string(conf, line) # bind the same path from host in the container (after creating a # symbolic link at container_dst path) workspace += [r"\1- " + host_src + ':' + host_src] workspace += [r"\1# - workspace #"] value = [ "\n".join(workspace), "aiscalator/airflow:" + image, ] for file in listdir(src): utils.copy_replace(join(src, file), join(dst, file), pattern=pattern, replace_value=value)