def build_distribution(exit_on_error: bool = True) -> None:
"""Build python package distribution.
Args:
exit_on_error (bool, optional): If `True`, exit process as soon as error occures. Defaults to True.
"""
try:
# Ensure there are no old builds
rmtree("./dist")
except OSError:
pass
try:
# Ensure there are no old builds
rmtree("./build")
except OSError:
pass
# Build the distribution archives
build_utils.run("python setup.py sdist bdist_wheel clean --all",
exit_on_error=exit_on_error)
# Check twine command
build_utils.command_exists("twine", exit_on_error=exit_on_error)
# Check the archives with twine
build_utils.run("twine check dist/*", exit_on_error=exit_on_error)
def generate_api_docs(
github_url: str,
main_package: str,
exit_on_error: bool = True,
) -> None:
"""Generates API documentation via lazydocs.
Args:
github_url (str): Github URL
main_package (str): The main package name to use for docs generation.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
command_prefix = ""
if is_pipenv_environment():
command_prefix = "pipenv run"
else:
# Check lazydocs command
build_utils.command_exists("lazydocs", exit_on_error=exit_on_error)
build_utils.run(
f"{command_prefix} lazydocs --overview-file=README.md"
f" --src-base-url={github_url}/blob/main {main_package}",
exit_on_error=exit_on_error,
)
def build_mkdocs(command_prefix: str = PIPENV_RUN) -> None:
"""Build mkdocs markdown documentation.
Args:
command_prefix (str, optional): Prefix to use for all commands. Defaults to `pipenv run`.
"""
build_utils.run(f"{command_prefix} mkdocs build", exit_on_error=True)
def install_build_env() -> None:
"""Installs a new virtual environment via pipenv."""
build_utils.run("pipenv --rm")
build_utils.run(
f"pipenv install --dev --python={sys.executable} --skip-lock --site-packages",
exit_on_error=True,
)
def generate_and_copy_js_client() -> bool:
temp_dir = "./temp"
pathlib.Path(temp_dir).mkdir(exist_ok=True)
swagger_codegen_cli = f"{temp_dir}/swagger-codegen-cli.jar"
is_successful = check_and_download_swagger_cli(swagger_codegen_cli)
if not is_successful:
return False
swagger_path = "./backend/lab-service/src/main/resources/swagger/swagger.json"
output_path = f"{temp_dir}/client"
build_utils.run(
f"java -jar {swagger_codegen_cli} generate -i {swagger_path} -l javascript -o {output_path} --additional-properties useES6=true"
)
# shutil.move(f"{output_path}/src/", "./webapp/src/services/mllab-client")
try:
for file in pathlib.Path(f"{output_path}/src/").iterdir():
file_name = str(file.parts[-1])
new_file_name = file_name
if file_name == "index.js":
new_file_name = "lab-api.js"
target_file_path = f"./webapp/src/services/client/{new_file_name}"
# Delete existing client files to be replaced with the new ones
if pathlib.Path(target_file_path).is_file():
pathlib.Path(target_file_path).unlink()
elif pathlib.Path(target_file_path).is_dir():
shutil.rmtree(target_file_path)
shutil.move(str(file), target_file_path)
except FileNotFoundError as e:
build_utils.log(str(e))
return False
return True
def publish_pypi_distribution(
pypi_token: str,
pypi_user: str = "__token__",
pypi_repository: Optional[str] = None,
exit_on_error: bool = True,
) -> None:
"""Publish distribution to pypi.
Args:
pypi_token (str): Token of PyPi repository.
pypi_user (str, optional): User of PyPi repository. Defaults to "__token__".
pypi_repository (Optional[str], optional): PyPi repository. If `None` provided, use the production instance.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
if not pypi_token:
build_utils.log(
"PyPI token is required for release (--pypi-token=<TOKEN>)")
if exit_on_error:
build_utils.exit_process(1)
return
pypi_repository_args = ""
if pypi_repository:
pypi_repository_args = f'--repository-url "{pypi_repository}"'
# Check twine command
build_utils.command_exists("twine", exit_on_error=exit_on_error)
# Publish on pypi
build_utils.run(
f'twine upload --non-interactive -u "{pypi_user}" -p "{pypi_token}" {pypi_repository_args} dist/*',
exit_on_error=exit_on_error,
)
def build_docker_image(
name: str,
version: str,
build_args: str = "",
docker_image_prefix: str = "",
dockerfile: Optional[str] = None,
additional_build_args: str = "",
exit_on_error: bool = True,
) -> subprocess.CompletedProcess:
"""Build a docker image from a Dockerfile in the working directory.
Args:
name (str): Name of the docker image.
version (str): Version to use as tag.
build_args (str, optional): Add additional build arguments for docker build.
docker_image_prefix (str, optional): The prefix added to the name to indicate an organization on DockerHub or a completely different repository.
dockerfile (str, optional): Specify a specific Dockerfile. If not specified, the default `Dockerfile` wil be used.
exit_on_error (bool, optional): If `True`, exit process as soon as an error occurs.
Returns:
subprocess.CompletedProcess: Returns the CompletedProcess object of the
"""
# Check if docker exists on the system
build_utils.command_exists("docker", exit_on_error=exit_on_error)
versioned_tag = get_image_name(name=name, tag=version)
latest_tag = get_image_name(name=name, tag="latest")
dockerfile_command = ""
if dockerfile:
dockerfile_command = " -f " + dockerfile
completed_process = build_utils.run(
"docker build "
+ dockerfile_command
+ "-t "
+ versioned_tag
+ " -t "
+ latest_tag
+ " "
+ build_args
+ " ./",
exit_on_error=exit_on_error,
)
if completed_process.returncode > 0:
build_utils.log(f"Failed to build Docker image {versioned_tag}")
return completed_process
if docker_image_prefix:
remote_versioned_tag = get_image_name(
name=name, tag=version, image_prefix=docker_image_prefix
)
build_utils.run(
"docker tag " + versioned_tag + " " + remote_versioned_tag,
exit_on_error=exit_on_error,
)
return completed_process
def lint_dockerfile() -> None:
"""Run hadolint on the Dockerfile."""
build_utils.log("Run linters and style checks:")
config_file_arg = ""
if os.path.exists(".hadolint.yml"):
config_file_arg = "--config=.hadolint.yml"
build_utils.run(f"hadolint {config_file_arg} Dockerfile", exit_on_error=True)
def lint_markdown() -> None:
"""Run markdownlint on markdown documentation."""
build_utils.log("Run linters and style checks:")
config_file_arg = ""
if os.path.exists(".markdown-lint.yml"):
config_file_arg = "--config='.markdown-lint.yml'"
build_utils.run(f"markdownlint {config_file_arg} ./docs",
exit_on_error=True)
def run_dev_mode(port: int = 8001, command_prefix: str = PIPENV_RUN) -> None:
"""Run mkdocs development server.
Args:
port (int, optional): Port to use for mkdocs development server. Defaults to 8001.
command_prefix (str, optional): Prefix to use for all commands. Defaults to `pipenv run`.
"""
build_utils.log(f"Run docs in development mode (http://localhost:{port}):")
build_utils.run(f"{command_prefix} mkdocs serve --dev-addr 0.0.0.0:{port}",
exit_on_error=True)
def deploy_gh_pages(command_prefix: str = PIPENV_RUN) -> None:
"""Deploy mkdocs documentation to Github pages.
Args:
command_prefix (str, optional): Prefix to use for all commands. Defaults to `pipenv run`.
"""
build_utils.log("Deploy documentation to Github pages:")
build_utils.run(f"{command_prefix} mkdocs gh-deploy --clean",
exit_on_error=True,
timeout=120)
def build_docker_image(
name: str, version: str, build_args: str = "", exit_on_error: bool = False
) -> subprocess.CompletedProcess:
"""Build a docker image from a Dockerfile in the working directory.
Args:
name (str): Name of the docker image.
version (str): Version to use as tag.
build_args (str, optional): Add additional build arguments for docker build.
exit_on_error (bool, optional): If `True`, exit process as soon as an error occurs.
Returns:
subprocess.CompletedProcess: Returns the CompletedProcess object of the
"""
versioned_image = name + ":" + version
latest_image = name + ":latest"
completed_process = build_utils.run(
"docker build -t "
+ versioned_image
+ " -t "
+ latest_image
+ " "
+ build_args
+ " ./",
exit_on_error=exit_on_error,
)
# TODO tag prefixed image names
if completed_process.returncode > 0:
build_utils.log(f"Failed to build Docker image {name}:{version}")
return completed_process
def build_mkdocs(exit_on_error: bool = True) -> None:
"""Build mkdocs markdown documentation.
Args:
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
command_prefix = ""
if is_pipenv_environment():
command_prefix = _PIPENV_RUN
else:
# Check mkdocs command
build_utils.command_exists("mkdocs", exit_on_error=exit_on_error)
build_utils.run(f"{command_prefix} mkdocs build",
exit_on_error=exit_on_error)
def generate_openapi_client(
openapi_spec_file: str,
target_language: str,
work_dir: str = DEFAULT_TEMP_DIR,
client_generator: OpenApiGenerator = OpenApiGenerator.OPENAPI_CODEGEN,
additional_properties: str = "",
additional_flags: str = "",
) -> Union[str, None]:
"""Generate an open api client.
The passed OpenAPI specification file will be taken to generate a client using the passed openapi-generator for the given programming language and optional additional properties (see the respective openapi cli for more information).
The client will be generated at the passed `work_dir` directory.
Args:
openapi_spec_file (str): The OpenAPI specification for which the client will be generated.
target_language (str): The client's programming language (e.g. `"javascript"`).
work_dir (str, optional): The directory in which the generator cli will be looked for and also the generated client will be placed. If it does not exist, it will be created.
client_generator (OpenApiGenerator, optional): The OpenApiGenerator which will be used to generate the client. It will check whether the cli can be found within the `work_dir` directory and if not it will try to download it according to the `_check_and_download_generator_cli` function.
additional_properties (str, optional): Additional properties passed to the OpenAPI generator client client (e.g. `"useES6=true"`)
Returns:
Union[str, None]: Returns the output path if the client generation was successful and None otherwise.
"""
pathlib.Path(work_dir).mkdir(exist_ok=True)
codegen_cli_path = f"{work_dir}/{client_generator.cli_name}"
is_successful = _check_and_download_generator_cli(
codegen_cli_path, client_generator=client_generator)
if not is_successful:
return None
if not pathlib.Path(openapi_spec_file).is_file():
build_utils.log(
f"The OpenAPI spec file {openapi_spec_file} does not exist")
return None
build_utils.run(
client_generator.get_generate_command(
openapi_spec_file=openapi_spec_file,
target_language=target_language,
work_dir=work_dir,
additional_properties=additional_properties,
additional_flags=additional_flags,
))
return client_generator.get_output_path(work_dir=work_dir)
def test_arguments_passed_to_submodule_error(self, cli_args_string: str):
"""Tests whether the cli args passed to a module will be passed in the same way to a submodule. The reason is that the argparser converts all dashes to underscores."""
HERE = os.path.abspath(os.path.dirname(__file__))
build_file_path = os.path.join(HERE, "build.py")
completed_process = build_utils.run(
f"python -u {build_file_path} --my-token=111 --deployment-token=666 --my_bool",
exit_on_error=False,
)
assert completed_process.returncode != 0
def lint_markdown(markdownlint: bool = True,
exit_on_error: bool = True) -> None:
"""Run markdownlint on markdown documentation.
Args:
markdownlint (bool, optional): Activate markdown linting via `markdownlint`. Defaults to `True`.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
build_utils.log("Run linters and style checks:")
if markdownlint and build_utils.command_exists(
"markdownlint", exit_on_error=exit_on_error):
config_file_arg = ""
if os.path.exists(".markdown-lint.yml"):
config_file_arg = "--config='.markdown-lint.yml'"
build_utils.run(f"markdownlint {config_file_arg} ./docs",
exit_on_error=exit_on_error)
def test_arguments_passed_as_env_to_submodule(self):
"""Tests whether the args passed as env variables to a module correctly to submodules."""
HERE = os.path.abspath(os.path.dirname(__file__))
build_file_path = os.path.join(HERE, "build.py")
completed_process = build_utils.run(
f"MY_TOKEN=111 python -u {build_file_path} --deployment-token 666 --my_bool",
exit_on_error=False,
)
assert completed_process.returncode == 0
def generate_api_docs(
github_url: str,
main_package: str,
command_prefix: str = "pipenv run",
exit_on_error: bool = True,
) -> None:
"""Generates API documentation via lazydocs.
Args:
github_url (str): Github URL
main_package (str): The main package name to use for docs generation.
command_prefix (str, optional): Prefix to use for all commands. Defaults to `pipenv run`.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
build_utils.run(
f"{command_prefix} lazydocs --overview-file=README.md"
f" --src-base-url={github_url}/blob/main {main_package}",
exit_on_error=exit_on_error,
)
def run_dev_mode(port: int = 8001, exit_on_error: bool = True) -> None:
"""Run mkdocs development server.
Args:
port (int, optional): Port to use for mkdocs development server. Defaults to 8001.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
build_utils.log(f"Run docs in development mode (http://localhost:{port}):")
command_prefix = ""
if is_pipenv_environment():
command_prefix = _PIPENV_RUN
else:
# Check mkdocs command
build_utils.command_exists("mkdocs", exit_on_error=exit_on_error)
build_utils.run(
f"{command_prefix} mkdocs serve --dev-addr 0.0.0.0:{port}",
exit_on_error=exit_on_error,
)
def deploy_gh_pages(exit_on_error: bool = True) -> None:
"""Deploy mkdocs documentation to Github pages.
Args:
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
build_utils.log("Deploy documentation to Github pages:")
command_prefix = ""
if is_pipenv_environment():
command_prefix = _PIPENV_RUN
else:
# Check mkdocs command
build_utils.command_exists("mkdocs", exit_on_error=exit_on_error)
build_utils.run(
f"{command_prefix} mkdocs gh-deploy --clean",
exit_on_error=exit_on_error,
timeout=120,
)
def lint_dockerfile(
hadolint: bool = True, dockerfile: str = "Dockerfile", exit_on_error: bool = True
) -> None:
"""Run hadolint on the Dockerfile.
Args:
hadolint (bool, optional): Activate hadolint dockerfile linter. Defaults to `True`.
dockerfile (str, optional): Specify a specific Dockerfile. If not specified, the default `Dockerfile` wil be used.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
build_utils.log("Run linters and style checks:")
if hadolint and build_utils.command_exists("hadolint", exit_on_error=exit_on_error):
config_file_arg = ""
if os.path.exists(".hadolint.yml"):
config_file_arg = "--config=.hadolint.yml"
build_utils.run(
f"hadolint {config_file_arg} {dockerfile}", exit_on_error=exit_on_error
)
def install_build_env(exit_on_error: bool = True) -> None:
"""Installs a new virtual environment via pipenv.
Args:
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
# Check if pipenv exists
build_utils.command_exists("pipenv", exit_on_error=exit_on_error)
if not os.path.exists("Pipfile"):
build_utils.log(
"No Pipfile discovered, cannot install pipenv environemnt")
if exit_on_error:
build_utils.exit_process(1)
return
build_utils.run("pipenv --rm", exit_on_error=False)
build_utils.run(
f"pipenv install --dev --python={sys.executable} --skip-lock --site-packages",
exit_on_error=exit_on_error,
)
def release_docker_image(
name: str, version: str, docker_image_prefix: str, exit_on_error: bool = True
) -> subprocess.CompletedProcess:
"""Push a Docker image to a repository.
Args:
name (str): The name of the image. Must not be prefixed!
version (str): The tag used for the image.
docker_image_prefix (str): The prefix added to the name to indicate an organization on DockerHub or a completely different repository.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
Returns:
subprocess.CompletedProcess: Returns the CompletedProcess object of the `docker push ...` command.
"""
# Check if docker exists on the system
build_utils.command_exists("docker", exit_on_error=exit_on_error)
if not docker_image_prefix:
build_utils.log(
"The flag --docker-image-prefix cannot be blank when pushing a Docker image."
)
build_utils.exit_process(build_utils.EXIT_CODE_GENERAL)
versioned_tag = get_image_name(name=name, tag=version)
remote_versioned_tag = get_image_name(
name=name, tag=version, image_prefix=docker_image_prefix
)
build_utils.run(
"docker tag " + versioned_tag + " " + remote_versioned_tag,
exit_on_error=exit_on_error,
)
completed_process = build_utils.run(
"docker push " + remote_versioned_tag, exit_on_error=exit_on_error
)
if completed_process.returncode > 0:
build_utils.log(f"Failed to release Docker image {name}:{version}")
# Only push version with latest tag if no suffix is added (pre-release)
if "-" not in version:
remote_latest_tag = get_image_name(
name=name, tag="latest", image_prefix=docker_image_prefix
)
build_utils.log(
"Release Docker image with latest tag as well: " + remote_latest_tag
)
build_utils.run(
"docker tag " + versioned_tag + " " + remote_latest_tag,
exit_on_error=exit_on_error,
)
build_utils.run("docker push " + remote_latest_tag, exit_on_error=exit_on_error)
return completed_process
def test_with_py_version(python_version: str,
exit_on_error: bool = True) -> None:
"""Run pytest in a environment wiht the specified python version.
Args:
python_version (str): Python version to use inside the virutal environment.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
if not os.path.exists("Pipfile"):
build_utils.log(
"No Pipfile discovered. Testing with specific python version only works with pipenv."
)
return
# Check if pyenv command exists
build_utils.command_exists("pyenv", exit_on_error=exit_on_error)
# Check if pipenv command exists
build_utils.command_exists("pipenv", exit_on_error=exit_on_error)
# Install pipenv environment with specific versio
build_utils.run(
f"pyenv install --skip-existing {python_version} && pyenv local {python_version}",
exit_on_error=exit_on_error,
)
# Install pipenv environment with specific version
build_utils.run(
f"pipenv install --dev --python={python_version} --skip-lock",
exit_on_error=exit_on_error,
)
# Run pytest in pipenv environment
build_utils.run("pipenv run pytest", exit_on_error=exit_on_error)
# Remove enviornment
build_utils.run("pipenv --rm", exit_on_error=False)
# Uninstall pyenv version
build_utils.run(
f"pyenv local --unset && pyenv uninstall -f {python_version}",
exit_on_error=False,
)
def publish_pypi_distribution(
pypi_token: str, pypi_user: str = "__token__", pypi_repository: Optional[str] = None
) -> None:
"""Publish distribution to pypi.
Args:
pypi_token (str): Token of PyPi repository.
pypi_user (str, optional): User of PyPi repository. Defaults to "__token__".
pypi_repository (Optional[str], optional): PyPi repository. If `None` provided, use the production instance.
"""
if not pypi_token:
build_utils.log("PyPI token is required for release (--pypi-token=<TOKEN>)")
build_utils.exit_process(1)
pypi_repository_args = ""
if pypi_repository:
pypi_repository_args = f'--repository-url "{pypi_repository}"'
# Publish on pypi
build_utils.run(
f'twine upload --non-interactive -u "{pypi_user}" -p "{pypi_token}" {pypi_repository_args} dist/*',
exit_on_error=True,
)
def build_distribution() -> None:
"""Build python package distribution."""
try:
# Ensure there are no old builds
rmtree("./dist")
except OSError:
pass
try:
# Ensure there are no old builds
rmtree("./build")
except OSError:
pass
# Build the distribution archives
build_utils.run(
"python setup.py sdist bdist_wheel clean --all",
exit_on_error=True,
)
# Check the archives with twine
build_utils.run("twine check dist/*", exit_on_error=True)
def test_with_py_version(python_version: str, exit_on_error: bool = True) -> None:
"""Run pytest in a environment wiht the specified python version.
Args:
python_version (str): Python version to use inside the virutal environment.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
# Install pipenv environment with specific versio
build_utils.run(
f"pyenv install --skip-existing {python_version} && pyenv local {python_version}",
exit_on_error=exit_on_error,
)
# Install pipenv environment with specific version
build_utils.run(
f"pipenv install --dev --python={python_version} --skip-lock",
exit_on_error=exit_on_error,
)
# Run pytest in pipenv environment
build_utils.run("pipenv run pytest", exit_on_error=exit_on_error)
# Remove enviornment
build_utils.run("pipenv --rm")
# Uninstall pyenv version
build_utils.run(f"pyenv local --unset && pyenv uninstall -f {python_version}")
def is_pipenv_environment() -> bool:
"""Check if current working directory is a valid pipenv environment."""
if not os.path.exists("Pipfile"):
return False
if not build_utils.command_exists("pipenv"):
return False
return (build_utils.run(
"pipenv --venv",
disable_stderr_logging=True,
disable_stdout_logging=True,
exit_on_error=False,
).returncode == 0)
def main(args: dict) -> None:
# set current path as working dir
os.chdir(HERE)
if args.get(build_utils.FLAG_MAKE):
# Install pipenv dev requirements
build_mkdocs.install_build_env(exit_on_error=True)
# Build mkdocs documentation
build_mkdocs.build_mkdocs(exit_on_error=True)
if args.get(build_utils.FLAG_CHECK):
# TODO: Markdown linting
# build_mkdocs.lint_markdown(exit_on_error=False)
pass
if args.get(build_utils.FLAG_RELEASE):
# TODO: Do not publish project-template
# Deploy to Github pages
# build_mkdocs.deploy_gh_pages(exit_on_error=True)
# Lock pipenv requirements
build_utils.run("pipenv lock", exit_on_error=False)
if args.get(build_utils.FLAG_RUN):
build_mkdocs.run_dev_mode(exit_on_error=True)
def install_build_env() -> None:
"""Installs a new virtual environment via pipenv."""
build_utils.run("pipenv --rm")
build_utils.run(
f"pipenv install --dev --python={sys.executable} --skip-lock",
exit_on_error=True,
)
# Show current environment
build_utils.run("pipenv graph", exit_on_error=False)
