def generate_api_docs(
github_url: str,
main_package: str,
exit_on_error: bool = True,
) -> None:
"""Generates API documentation via lazydocs.
Args:
github_url (str): Github URL
main_package (str): The main package name to use for docs generation.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
command_prefix = ""
if is_pipenv_environment():
command_prefix = "pipenv run"
else:
# Check lazydocs command
build_utils.command_exists("lazydocs", exit_on_error=exit_on_error)
build_utils.run(
f"{command_prefix} lazydocs --overview-file=README.md"
f" --src-base-url={github_url}/blob/main {main_package}",
exit_on_error=exit_on_error,
)
def build_distribution(exit_on_error: bool = True) -> None:
"""Build python package distribution.
Args:
exit_on_error (bool, optional): If `True`, exit process as soon as error occures. Defaults to True.
"""
try:
# Ensure there are no old builds
rmtree("./dist")
except OSError:
pass
try:
# Ensure there are no old builds
rmtree("./build")
except OSError:
pass
# Build the distribution archives
build_utils.run("python setup.py sdist bdist_wheel clean --all",
exit_on_error=exit_on_error)
# Check twine command
build_utils.command_exists("twine", exit_on_error=exit_on_error)
# Check the archives with twine
build_utils.run("twine check dist/*", exit_on_error=exit_on_error)
def publish_pypi_distribution(
pypi_token: str,
pypi_user: str = "__token__",
pypi_repository: Optional[str] = None,
exit_on_error: bool = True,
) -> None:
"""Publish distribution to pypi.
Args:
pypi_token (str): Token of PyPi repository.
pypi_user (str, optional): User of PyPi repository. Defaults to "__token__".
pypi_repository (Optional[str], optional): PyPi repository. If `None` provided, use the production instance.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
if not pypi_token:
build_utils.log(
"PyPI token is required for release (--pypi-token=<TOKEN>)")
if exit_on_error:
build_utils.exit_process(1)
return
pypi_repository_args = ""
if pypi_repository:
pypi_repository_args = f'--repository-url "{pypi_repository}"'
# Check twine command
build_utils.command_exists("twine", exit_on_error=exit_on_error)
# Publish on pypi
build_utils.run(
f'twine upload --non-interactive -u "{pypi_user}" -p "{pypi_token}" {pypi_repository_args} dist/*',
exit_on_error=exit_on_error,
)
def build_docker_image(
name: str,
version: str,
build_args: str = "",
docker_image_prefix: str = "",
dockerfile: Optional[str] = None,
additional_build_args: str = "",
exit_on_error: bool = True,
) -> subprocess.CompletedProcess:
"""Build a docker image from a Dockerfile in the working directory.
Args:
name (str): Name of the docker image.
version (str): Version to use as tag.
build_args (str, optional): Add additional build arguments for docker build.
docker_image_prefix (str, optional): The prefix added to the name to indicate an organization on DockerHub or a completely different repository.
dockerfile (str, optional): Specify a specific Dockerfile. If not specified, the default `Dockerfile` wil be used.
exit_on_error (bool, optional): If `True`, exit process as soon as an error occurs.
Returns:
subprocess.CompletedProcess: Returns the CompletedProcess object of the
"""
# Check if docker exists on the system
build_utils.command_exists("docker", exit_on_error=exit_on_error)
versioned_tag = get_image_name(name=name, tag=version)
latest_tag = get_image_name(name=name, tag="latest")
dockerfile_command = ""
if dockerfile:
dockerfile_command = " -f " + dockerfile
completed_process = build_utils.run(
"docker build "
+ dockerfile_command
+ "-t "
+ versioned_tag
+ " -t "
+ latest_tag
+ " "
+ build_args
+ " ./",
exit_on_error=exit_on_error,
)
if completed_process.returncode > 0:
build_utils.log(f"Failed to build Docker image {versioned_tag}")
return completed_process
if docker_image_prefix:
remote_versioned_tag = get_image_name(
name=name, tag=version, image_prefix=docker_image_prefix
)
build_utils.run(
"docker tag " + versioned_tag + " " + remote_versioned_tag,
exit_on_error=exit_on_error,
)
return completed_process
def release_docker_image(
name: str, version: str, docker_image_prefix: str, exit_on_error: bool = True
) -> subprocess.CompletedProcess:
"""Push a Docker image to a repository.
Args:
name (str): The name of the image. Must not be prefixed!
version (str): The tag used for the image.
docker_image_prefix (str): The prefix added to the name to indicate an organization on DockerHub or a completely different repository.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
Returns:
subprocess.CompletedProcess: Returns the CompletedProcess object of the `docker push ...` command.
"""
# Check if docker exists on the system
build_utils.command_exists("docker", exit_on_error=exit_on_error)
if not docker_image_prefix:
build_utils.log(
"The flag --docker-image-prefix cannot be blank when pushing a Docker image."
)
build_utils.exit_process(build_utils.EXIT_CODE_GENERAL)
versioned_tag = get_image_name(name=name, tag=version)
remote_versioned_tag = get_image_name(
name=name, tag=version, image_prefix=docker_image_prefix
)
build_utils.run(
"docker tag " + versioned_tag + " " + remote_versioned_tag,
exit_on_error=exit_on_error,
)
completed_process = build_utils.run(
"docker push " + remote_versioned_tag, exit_on_error=exit_on_error
)
if completed_process.returncode > 0:
build_utils.log(f"Failed to release Docker image {name}:{version}")
# Only push version with latest tag if no suffix is added (pre-release)
if "-" not in version:
remote_latest_tag = get_image_name(
name=name, tag="latest", image_prefix=docker_image_prefix
)
build_utils.log(
"Release Docker image with latest tag as well: " + remote_latest_tag
)
build_utils.run(
"docker tag " + versioned_tag + " " + remote_latest_tag,
exit_on_error=exit_on_error,
)
build_utils.run("docker push " + remote_latest_tag, exit_on_error=exit_on_error)
return completed_process
def build_mkdocs(exit_on_error: bool = True) -> None:
"""Build mkdocs markdown documentation.
Args:
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
command_prefix = ""
if is_pipenv_environment():
command_prefix = _PIPENV_RUN
else:
# Check mkdocs command
build_utils.command_exists("mkdocs", exit_on_error=exit_on_error)
build_utils.run(f"{command_prefix} mkdocs build",
exit_on_error=exit_on_error)
def run_dev_mode(port: int = 8001, exit_on_error: bool = True) -> None:
"""Run mkdocs development server.
Args:
port (int, optional): Port to use for mkdocs development server. Defaults to 8001.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
build_utils.log(f"Run docs in development mode (http://localhost:{port}):")
command_prefix = ""
if is_pipenv_environment():
command_prefix = _PIPENV_RUN
else:
# Check mkdocs command
build_utils.command_exists("mkdocs", exit_on_error=exit_on_error)
build_utils.run(
f"{command_prefix} mkdocs serve --dev-addr 0.0.0.0:{port}",
exit_on_error=exit_on_error,
)
def deploy_gh_pages(exit_on_error: bool = True) -> None:
"""Deploy mkdocs documentation to Github pages.
Args:
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
build_utils.log("Deploy documentation to Github pages:")
command_prefix = ""
if is_pipenv_environment():
command_prefix = _PIPENV_RUN
else:
# Check mkdocs command
build_utils.command_exists("mkdocs", exit_on_error=exit_on_error)
build_utils.run(
f"{command_prefix} mkdocs gh-deploy --clean",
exit_on_error=exit_on_error,
timeout=120,
)
def install_build_env(exit_on_error: bool = True) -> None:
"""Installs a new virtual environment via pipenv.
Args:
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
# Check if pipenv exists
build_utils.command_exists("pipenv", exit_on_error=exit_on_error)
if not os.path.exists("Pipfile"):
build_utils.log(
"No Pipfile discovered, cannot install pipenv environemnt")
if exit_on_error:
build_utils.exit_process(1)
return
build_utils.run("pipenv --rm", exit_on_error=False)
build_utils.run(
f"pipenv install --dev --python={sys.executable} --skip-lock --site-packages",
exit_on_error=exit_on_error,
)
def test_with_py_version(python_version: str,
exit_on_error: bool = True) -> None:
"""Run pytest in a environment wiht the specified python version.
Args:
python_version (str): Python version to use inside the virutal environment.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
if not os.path.exists("Pipfile"):
build_utils.log(
"No Pipfile discovered. Testing with specific python version only works with pipenv."
)
return
# Check if pyenv command exists
build_utils.command_exists("pyenv", exit_on_error=exit_on_error)
# Check if pipenv command exists
build_utils.command_exists("pipenv", exit_on_error=exit_on_error)
# Install pipenv environment with specific versio
build_utils.run(
f"pyenv install --skip-existing {python_version} && pyenv local {python_version}",
exit_on_error=exit_on_error,
)
# Install pipenv environment with specific version
build_utils.run(
f"pipenv install --dev --python={python_version} --skip-lock",
exit_on_error=exit_on_error,
)
# Run pytest in pipenv environment
build_utils.run("pipenv run pytest", exit_on_error=exit_on_error)
# Remove enviornment
build_utils.run("pipenv --rm", exit_on_error=False)
# Uninstall pyenv version
build_utils.run(
f"pyenv local --unset && pyenv uninstall -f {python_version}",
exit_on_error=False,
)
def is_pipenv_environment() -> bool:
"""Check if current working directory is a valid pipenv environment."""
if not os.path.exists("Pipfile"):
return False
if not build_utils.command_exists("pipenv"):
return False
return (build_utils.run(
"pipenv --venv",
disable_stderr_logging=True,
disable_stdout_logging=True,
exit_on_error=False,
).returncode == 0)
def lint_markdown(markdownlint: bool = True,
exit_on_error: bool = True) -> None:
"""Run markdownlint on markdown documentation.
Args:
markdownlint (bool, optional): Activate markdown linting via `markdownlint`. Defaults to `True`.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
build_utils.log("Run linters and style checks:")
if markdownlint and build_utils.command_exists(
"markdownlint", exit_on_error=exit_on_error):
config_file_arg = ""
if os.path.exists(".markdown-lint.yml"):
config_file_arg = "--config='.markdown-lint.yml'"
build_utils.run(f"markdownlint {config_file_arg} ./docs",
exit_on_error=exit_on_error)
def check_image(
image: str, trivy: bool = True, exit_on_error: bool = True
) -> subprocess.CompletedProcess:
"""Run vulnerability checks on Dockerimage.
Args:
image (str): The name of the docker image to check.
trivy (bool, optional): Activate trivy vulnerability check. Defaults to `True`.
exit_on_error (bool, optional): If `True`, exit process as soon as an error occurs.
"""
build_utils.log("Run vulnerability checks on docker image:")
if trivy and build_utils.command_exists("trivy", exit_on_error=exit_on_error):
return build_utils.run(
f"trivy image --timeout=20m0s --exit-code 1 --severity HIGH,CRITICAL {image}",
exit_on_error=exit_on_error,
)
return subprocess.CompletedProcess(args="", returncode=-1, stdout="", stderr="")
def lint_dockerfile(
hadolint: bool = True, dockerfile: str = "Dockerfile", exit_on_error: bool = True
) -> None:
"""Run hadolint on the Dockerfile.
Args:
hadolint (bool, optional): Activate hadolint dockerfile linter. Defaults to `True`.
dockerfile (str, optional): Specify a specific Dockerfile. If not specified, the default `Dockerfile` wil be used.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
build_utils.log("Run linters and style checks:")
if hadolint and build_utils.command_exists("hadolint", exit_on_error=exit_on_error):
config_file_arg = ""
if os.path.exists(".hadolint.yml"):
config_file_arg = "--config=.hadolint.yml"
build_utils.run(
f"hadolint {config_file_arg} {dockerfile}", exit_on_error=exit_on_error
)
def code_checks(
black: bool = True,
isort: bool = True,
pydocstyle: bool = True,
mypy: bool = True,
flake8: bool = True,
safety: bool = False,
exit_on_error: bool = True,
) -> None:
"""Run linting and style checks.
Args:
black (bool, optional): Activate black formatting check. Defaults to True.
isort (bool, optional): Activate isort import sorting check. Defaults to True.
pydocstyle (bool, optional): Activate pydocstyle docstring check. Defaults to True.
mypy (bool, optional): Activate mypy typing check. Defaults to True.
flake8 (bool, optional): Activate flake8 linting check. Defaults to True.
safety (bool, optional): Activate saftey check via pipenv. Defaults to False.
exit_on_error (bool, optional): If `True`, exit process as soon as error occures. Defaults to True.
"""
command_prefix = ""
if is_pipenv_environment():
command_prefix = "pipenv run"
successful: bool = True
if black:
if not command_prefix:
# Check twine command
build_utils.command_exists("black", exit_on_error=exit_on_error)
if (build_utils.run(f"{command_prefix} black --check src",
exit_on_error=False).returncode > 0):
successful = False
if (build_utils.run(f"{command_prefix} black --check tests",
exit_on_error=False).returncode > 0):
successful = False
if isort:
if not command_prefix:
# Check twine command
build_utils.command_exists("isort", exit_on_error=exit_on_error)
if (build_utils.run(
f"{command_prefix} isort --profile black --check-only src",
exit_on_error=False,
).returncode > 0):
successful = False
if (build_utils.run(
f"{command_prefix} isort --profile black --check-only tests",
exit_on_error=False,
).returncode > 0):
successful = False
if pydocstyle:
if not command_prefix:
# Check twine command
build_utils.command_exists("pydocstyle",
exit_on_error=exit_on_error)
if (build_utils.run(f"{command_prefix} pydocstyle src",
exit_on_error=False).returncode > 0):
successful = False
# Run linters and checks
if mypy:
if not command_prefix:
# Check twine command
build_utils.command_exists("mypy", exit_on_error=exit_on_error)
if (build_utils.run(f"{command_prefix} mypy src",
exit_on_error=False).returncode > 0):
successful = False
if flake8:
if not command_prefix:
# Check twine command
build_utils.command_exists("flake8", exit_on_error=exit_on_error)
if (build_utils.run(
f"{command_prefix} flake8 --show-source --statistics src",
exit_on_error=False,
).returncode > 0):
successful = False
if (build_utils.run(
f"{command_prefix} flake8 --show-source --statistics tests",
exit_on_error=False,
).returncode > 0):
successful = False
if safety:
# Check pipenv command
build_utils.command_exists("pipenv", exit_on_error=exit_on_error)
# Check using pipenv (runs safety check)
if build_utils.run("pipenv check", exit_on_error=False).returncode > 0:
successful = False
if not successful:
build_utils.log(
"Code checks (style, linting, safety, ...) failed. Please check the logs and fix the issues."
)
build_utils.exit_process(1)
