def _generate_global_config() -> str:
"""Generate a standard configuration file for the application in the
user's home folder ~/.aiscalator/config/aiscalator.conf from the
template file in aiscalator/config/template/aiscalator.conf
"""
logger = getLogger(__name__)
dst = os.path.join(os.path.expanduser("~"),
".aiscalator/config/aiscalator.conf")
logger.info("Generating a new configuration file for aiscalator:\n\t%s",
dst)
pattern = [
"testUserID",
"generation_date",
]
replace_value = [
generate_user_id(),
'"' + str(datetime.utcnow().replace(tzinfo=timezone("UTC"))) +
'" // in UTC timezone',
]
dst_dir = os.path.dirname(dst)
if dst_dir:
os.makedirs(dst_dir, exist_ok=True)
copy_replace(data_file("../config/template/aiscalator.conf"),
dst,
pattern=pattern,
replace_value=replace_value)
open(os.path.join(dst_dir, "apt_packages.txt"), 'a').close()
open(os.path.join(dst_dir, "requirements.txt"), 'a').close()
open(os.path.join(dst_dir, "lab_extensions.txt"), 'a').close()
return dst
def _docker_compose(conf: AiscalatorConfig,
extra_commands: list):
"""
Run the docker-compose command
Parameters
----------
conf : AiscalatorConfig
Configuration object for the application
extra_commands : list
list of sub-commands to run in docker-compose
"""
logger = logging.getLogger(__name__)
conf.validate_config()
dockerfile = join(conf.app_config_home(), "config",
conf.airflow_docker_compose_file())
commands = ["docker-compose"]
# Prepare a temp folder to run the command from
with TemporaryDirectory(prefix="aiscalator_") as tmp:
with open(join(tmp, ".env"), mode="w") as env_file:
# concatenate all the env files into one
for env in conf.user_env_file(conf.dag_field("definition.env")):
if isfile(env):
with open(env, mode="r") as file:
for line in file:
env_file.write(line)
utils.copy_replace(join(tmp, ".env"),
join(dirname(dockerfile), ".env"))
commands += ["-f", dockerfile] + extra_commands
logger.info("Running...: %s", " ".join(commands))
utils.subprocess_run(commands, no_redirect=True)
def _include_lab_extensions(conf: AiscalatorConfig, dockerfile, tmp):
"""
Include jupyter lab extensions packages into the dockerfile
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
dockerfile : str
path to the dockerfile to modify
tmp : str
path to the temporary dockerfile output
Returns
-------
path to the resulting dockerfile
"""
if conf.has_step_field("docker_image.lab_extension_path"):
content = conf.step_file_path("docker_image.lab_extension_path")
prefix = "&& jupyter labextension install "
value = utils.format_file_content(content,
prefix=prefix,
suffix=" \\\n")
if value:
value = "RUN echo 'Installing Jupyter Extensions' \\\n" + value
utils.copy_replace(dockerfile,
tmp,
pattern="# lab_extensions.txt #",
replace_value=value)
return tmp
return dockerfile
def _include_requirements(conf: AiscalatorConfig, dockerfile, tmp, dst):
"""
Include pip install packages into the dockerfile
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
dockerfile : str
path to the dockerfile to modify
tmp : str
path to the temporary dockerfile output
dst : str
path to the final temporary directory
Returns
-------
path to the resulting dockerfile
"""
if conf.has_step_field("docker_image.requirements_path"):
content = conf.step_file_path("docker_image.requirements_path")
copy(content, join(dst, 'requirements.txt'))
utils.copy_replace(dockerfile,
tmp,
pattern="# requirements.txt #",
replace_value="""
COPY requirements.txt requirements.txt
RUN pip install -r requirements.txt
RUN rm requirements.txt""")
return tmp
return dockerfile
def _include_apt_repo(conf: AiscalatorConfig, dockerfile, tmp):
"""
Include add-apt-repository packages into the dockerfile
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
dockerfile : str
path to the dockerfile to modify
tmp : str
path to the temporary dockerfile output
Returns
-------
path to the resulting dockerfile
"""
if conf.has_step_field("docker_image.apt_repository_path"):
content = conf.step_file_path("docker_image.apt_repository_path")
value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
if value:
value = ("RUN apt-get update \\\n" +
" && apt-get install -yqq \\\n" +
" software-properties-common \\\n" +
" && apt-add-repository \\\n" + value +
" && apt-get update")
utils.copy_replace(dockerfile,
tmp,
pattern="# apt_repository.txt #",
replace_value=value)
return tmp
return dockerfile
def jupyter_new(name, path, output_format="hocon"):
"""
Starts a Jupyter Lab environment configured to edit a brand new step
Parameters
----------
name : str
name of the new step
path : str
path to where the new step files should be created
output_format : str
the format of the new configuration file to produce
Returns
-------
string
Url of the running jupyter lab
"""
step_file = os.path.join(path, name, name) + '.conf'
if os.path.dirname(step_file):
makedirs(os.path.dirname(step_file), exist_ok=True)
copy_replace(data_file("../config/template/step.conf"),
step_file,
pattern="Untitled",
replace_value=name)
if output_format != 'hocon':
file = os.path.join(path, name, name) + '.' + output_format
step_file = convert_to_format(step_file,
output=file,
output_format=output_format)
notebook = os.path.join(path, name, 'notebook', name) + '.ipynb'
if os.path.dirname(notebook):
makedirs(os.path.dirname(notebook), exist_ok=True)
copy_replace(data_file("../config/template/notebook.json"), notebook)
open(os.path.join(path, name, "apt_repository.txt"), 'a').close()
open(os.path.join(path, name, "apt_packages.txt"), 'a').close()
open(os.path.join(path, name, "requirements.txt"), 'a').close()
open(os.path.join(path, name, "lab_extensions.txt"), 'a').close()
jupyter_edit(AiscalatorConfig(config=step_file, step_selection=name))
def _include_apt_package(conf: AiscalatorConfig, dockerfile, tmp):
"""
Include apt-install packages into the dockerfile
Parameters
----------
conf : AiscalatorConfig
Configuration object for this step
dockerfile : str
path to the dockerfile to modify
tmp : str
path to the temporary dockerfile output
Returns
-------
path to the resulting dockerfile
"""
if conf.has_step_field("docker_image.apt_package_path"):
content = conf.step_file_path("docker_image.apt_package_path")
value = utils.format_file_content(content, prefix=" ", suffix="\\\n")
if value:
value = ("RUN apt-get update && apt-get install -yqq \\\n" +
value +
""" && apt-get purge --auto-remove -yqq $buildDeps \\
&& apt-get autoremove -yqq --purge \\
&& apt-get clean \\
&& rm -rf \\
/var/lib/apt/lists/* \\
/tmp/* \\
/var/tmp/* \\
/usr/share/man \\
/usr/share/doc \\
/usr/share/doc-base
""")
utils.copy_replace(dockerfile,
tmp,
pattern="# apt_packages.txt #",
replace_value=value)
return tmp
return dockerfile
def airflow_setup(conf: AiscalatorConfig,
config_home: str,
workspace: list,
append: bool = True):
"""
Setup the airflow configuration files and environment
Parameters
----------
conf : AiscalatorConfig
Configuration object for the application
config_home : str
path to the configuration home directory
workspace : list
List of path to directories to mount as volumes
to airflow workers to use as workspaces
append : bool
flag to tell if workspace should be appended to
the list in the config or replace it.
"""
logger = logging.getLogger(__name__)
conf.validate_config()
if config_home:
makedirs(config_home, exist_ok=True)
conf.redefine_app_config_home(config_home)
ws_path = "airflow.setup.workspace_paths"
if conf.app_config_has(ws_path):
if append:
workspace += conf.app_config()[ws_path]
conf.redefine_airflow_workspaces(workspace)
image = 'latest'
if _docker_compose_grep(conf):
image = _airflow_docker_build(conf)
if not image:
raise Exception("Failed to build docker image")
src = utils.data_file("../config/docker/airflow/config/")
dst = join(conf.app_config_home(), "config")
logger.info("Generating a new configuration folder for aiscalator:\n\t%s",
dst)
makedirs(dst, exist_ok=True)
makedirs(join(conf.app_config_home(), "dags"), exist_ok=True)
makedirs(join(conf.app_config_home(), "pgdata"), exist_ok=True)
makedirs(join(conf.app_config_home(), "workspace"), exist_ok=True)
pattern = [
r"(\s+)# - workspace #",
"aiscalator/airflow:latest",
]
workspace = []
for line in conf.app_config()[ws_path]:
host_src, container_dst = _split_workspace_string(conf, line)
# bind the same path from host in the container (after creating a
# symbolic link at container_dst path)
workspace += [r"\1- " + host_src + ':' + host_src]
workspace += [r"\1# - workspace #"]
value = [
"\n".join(workspace),
"aiscalator/airflow:" + image,
]
for file in listdir(src):
utils.copy_replace(join(src, file),
join(dst, file),
pattern=pattern,
replace_value=value)
