def update_version(module_path: str,
version: str,
exit_on_error: bool = True) -> None:
"""Update version in specified module.
Args:
module_path (str): Python module with a `__version__` attribute.
version (str): New version number to write into `__version__` attribute.
exit_on_error (bool, optional): If `True`, exit process as soon as error occures. Defaults to True.
"""
if not version:
build_utils.log("Cannot update version, no version provided")
if exit_on_error:
build_utils.exit_process(1)
return
if not os.path.exists(module_path):
build_utils.log("Couldn't find file: " + module_path)
if exit_on_error:
build_utils.exit_process(1)
return
with open(module_path, "r+") as f:
data = f.read()
f.seek(0)
f.write(
re.sub(r"__version__ = \".+\"", f'__version__ = "{version}"',
data))
f.truncate()
def publish_pypi_distribution(
pypi_token: str,
pypi_user: str = "__token__",
pypi_repository: Optional[str] = None,
exit_on_error: bool = True,
) -> None:
"""Publish distribution to pypi.
Args:
pypi_token (str): Token of PyPi repository.
pypi_user (str, optional): User of PyPi repository. Defaults to "__token__".
pypi_repository (Optional[str], optional): PyPi repository. If `None` provided, use the production instance.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
if not pypi_token:
build_utils.log(
"PyPI token is required for release (--pypi-token=<TOKEN>)")
if exit_on_error:
build_utils.exit_process(1)
return
pypi_repository_args = ""
if pypi_repository:
pypi_repository_args = f'--repository-url "{pypi_repository}"'
# Check twine command
build_utils.command_exists("twine", exit_on_error=exit_on_error)
# Publish on pypi
build_utils.run(
f'twine upload --non-interactive -u "{pypi_user}" -p "{pypi_token}" {pypi_repository_args} dist/*',
exit_on_error=exit_on_error,
)
def release_docker_image(
name: str, version: str, docker_image_prefix: str, exit_on_error: bool = True
) -> subprocess.CompletedProcess:
"""Push a Docker image to a repository.
Args:
name (str): The name of the image. Must not be prefixed!
version (str): The tag used for the image.
docker_image_prefix (str): The prefix added to the name to indicate an organization on DockerHub or a completely different repository.
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
Returns:
subprocess.CompletedProcess: Returns the CompletedProcess object of the `docker push ...` command.
"""
# Check if docker exists on the system
build_utils.command_exists("docker", exit_on_error=exit_on_error)
if not docker_image_prefix:
build_utils.log(
"The flag --docker-image-prefix cannot be blank when pushing a Docker image."
)
build_utils.exit_process(build_utils.EXIT_CODE_GENERAL)
versioned_tag = get_image_name(name=name, tag=version)
remote_versioned_tag = get_image_name(
name=name, tag=version, image_prefix=docker_image_prefix
)
build_utils.run(
"docker tag " + versioned_tag + " " + remote_versioned_tag,
exit_on_error=exit_on_error,
)
completed_process = build_utils.run(
"docker push " + remote_versioned_tag, exit_on_error=exit_on_error
)
if completed_process.returncode > 0:
build_utils.log(f"Failed to release Docker image {name}:{version}")
# Only push version with latest tag if no suffix is added (pre-release)
if "-" not in version:
remote_latest_tag = get_image_name(
name=name, tag="latest", image_prefix=docker_image_prefix
)
build_utils.log(
"Release Docker image with latest tag as well: " + remote_latest_tag
)
build_utils.run(
"docker tag " + versioned_tag + " " + remote_latest_tag,
exit_on_error=exit_on_error,
)
build_utils.run("docker push " + remote_latest_tag, exit_on_error=exit_on_error)
return completed_process
def release_docker_image(
name: str, version: str, docker_image_prefix: str = "", exit_on_error: bool = False
) -> subprocess.CompletedProcess:
"""Push a Docker image to a repository.
Args:
name (str): The name of the image. Must not be prefixed!
version (str): The tag used for the image.
docker_image_prefix (str, optional): The prefix added to the name to indicate an organization on DockerHub or a completely different repository. Defaults to "".
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
Returns:
subprocess.CompletedProcess: Returns the CompletedProcess object of the `docker push ...` command.
"""
if not docker_image_prefix:
build_utils.log(
"The flag --docker-image-prefix cannot be blank when pushing a Docker image."
)
build_utils.exit_process(build_utils.EXIT_CODE_GENERAL)
docker_image_prefix = docker_image_prefix.rstrip("/") + "/"
versioned_image = name + ":" + version
remote_versioned_image = docker_image_prefix + versioned_image
build_utils.run(
"docker tag " + versioned_image + " " + remote_versioned_image,
exit_on_error=exit_on_error,
)
completed_process = build_utils.run(
"docker push " + remote_versioned_image, exit_on_error=exit_on_error
)
if completed_process.returncode > 0:
build_utils.log(f"Failed to release Docker image {name}:{version}")
if "-dev" not in version:
build_utils.log("Release Docker image with latest tag as well.")
latest_image = name + ":latest"
remote_latest_image = docker_image_prefix + latest_image
build_utils.run(
"docker tag " + latest_image + " " + remote_latest_image,
exit_on_error=exit_on_error,
)
build_utils.run(
"docker push " + remote_latest_image, exit_on_error=exit_on_error
)
return completed_process
def install_build_env(exit_on_error: bool = True) -> None:
"""Installs a new virtual environment via pipenv.
Args:
exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`.
"""
# Check if pipenv exists
build_utils.command_exists("pipenv", exit_on_error=exit_on_error)
if not os.path.exists("Pipfile"):
build_utils.log(
"No Pipfile discovered, cannot install pipenv environemnt")
if exit_on_error:
build_utils.exit_process(1)
return
build_utils.run("pipenv --rm", exit_on_error=False)
build_utils.run(
f"pipenv install --dev --python={sys.executable} --skip-lock --site-packages",
exit_on_error=exit_on_error,
)
def main(args: dict) -> None:
"""Execute all component builds."""
# set script path as working dir
os.chdir(HERE)
# Build react webapp
build_utils.build(REACT_WEBAPP_COMPONENT, args)
# Build python lib
build_utils.build(PYTHON_LIB_COMPONENT, args)
if args.get(build_utils.FLAG_MAKE):
# Duplicate api docs into the mkdocs documentation
build_utils.duplicate_folder(f"./{PYTHON_LIB_COMPONENT}/docs/",
f"./{DOCS_COMPONENT}/docs/api-docs/")
# Copy python lib distribution to docker container
try:
dest_path = os.path.join("./", DOCKER_COMPONENT, "resources",
PYTHON_LIB_COMPONENT + ".tar.gz")
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
shutil.copy(
glob.glob(
f"./{PYTHON_LIB_COMPONENT}/dist/{PYTHON_LIB_COMPONENT}-*.tar.gz"
)[0],
os.path.join(dest_path),
)
except Exception as ex:
build_utils.log(
f"Failed to copy {PYTHON_LIB_COMPONENT} distribution to {DOCKER_COMPONENT} component: "
+ str(ex))
build_utils.exit_process(1)
# Build docker container
build_utils.build(DOCKER_COMPONENT, args)
# Build mkdocs documentation
build_utils.build(DOCS_COMPONENT, args)
def publish_pypi_distribution(
pypi_token: str, pypi_user: str = "__token__", pypi_repository: Optional[str] = None
) -> None:
"""Publish distribution to pypi.
Args:
pypi_token (str): Token of PyPi repository.
pypi_user (str, optional): User of PyPi repository. Defaults to "__token__".
pypi_repository (Optional[str], optional): PyPi repository. If `None` provided, use the production instance.
"""
if not pypi_token:
build_utils.log("PyPI token is required for release (--pypi-token=<TOKEN>)")
build_utils.exit_process(1)
pypi_repository_args = ""
if pypi_repository:
pypi_repository_args = f'--repository-url "{pypi_repository}"'
# Publish on pypi
build_utils.run(
f'twine upload --non-interactive -u "{pypi_user}" -p "{pypi_token}" {pypi_repository_args} dist/*',
exit_on_error=True,
)
VERSION = str(args.get(build_utils.FLAG_VERSION))
docker_image_prefix = args.get(build_docker.FLAG_DOCKER_IMAGE_PREFIX)
if not docker_image_prefix:
docker_image_prefix = REMOTE_IMAGE_PREFIX
if not args.get(FLAG_FLAVOR):
args[FLAG_FLAVOR] = "all"
flavor = str(args[FLAG_FLAVOR]).lower().strip()
if flavor == "all":
args[FLAG_FLAVOR] = "gpu-11.3"
build_utils.build(".", args)
build_utils.exit_process(0)
# unknown flavor -> try to build from subdirectory
if flavor not in ["gpu-11.3"]:
# assume that flavor has its own directory with build.py
build_utils.build(flavor + "-flavor", args)
build_utils.exit_process(0)
docker_image_name = IMAGE_NAME + "-" + flavor
# docker build
git_rev = "unknown"
try:
git_rev = (subprocess.check_output(["git", "rev-parse", "--short",
"HEAD"]).decode("ascii").strip())
except Exception:
from universal_build import build_utils
from universal_build.helpers import build_docker
COMPONENT_NAME = "simple-demo-job"
args = build_utils.parse_arguments()
if args[build_utils.FLAG_MAKE]:
completed_process = build_docker.build_docker_image(
COMPONENT_NAME, args[build_utils.FLAG_VERSION])
if completed_process.returncode > 0:
build_utils.exit_process(completed_process.returncode)
if args[build_utils.FLAG_RELEASE]:
completed_process = build_docker.release_docker_image(
COMPONENT_NAME, args[build_utils.FLAG_VERSION],
args[build_docker.FLAG_DOCKER_IMAGE_PREFIX])
def main(args: dict) -> None:
# set current path as working dir
os.chdir(HERE)
version = args.get(build_utils.FLAG_VERSION)
if version:
# Update version in _about.py
build_python.update_version(
os.path.join(HERE, f"src/{MAIN_PACKAGE}/_about.py"),
build_utils._Version.get_pip_compatible_string(str(version)),
)
if args.get(build_utils.FLAG_MAKE):
# Install pipenv dev requirements
build_python.install_build_env()
# Create API documentation via lazydocs
build_python.generate_api_docs(github_url=GITHUB_URL,
main_package=MAIN_PACKAGE)
# Build distribution via setuptools
build_python.build_distribution()
try:
dist_name = MAIN_PACKAGE.replace("_", "-")
dist_file = glob.glob(f"./dist/{dist_name}-*.tar.gz")[0]
shutil.copy(
dist_file,
os.path.join(HERE, "build-environment", "resources",
dist_name + ".tar.gz"),
)
except Exception:
build_utils.log("Failed to copy distribution to build container.")
build_utils.exit_process(1)
if args.get(build_utils.FLAG_CHECK):
build_python.code_checks(exit_on_error=True, safety=False)
if args.get(build_utils.FLAG_TEST):
# Remove coverage files
build_utils.run("pipenv run coverage erase", exit_on_error=False)
test_markers = args.get(build_utils.FLAG_TEST_MARKER)
if build_utils.TEST_MARKER_SLOW in test_markers: # type: ignore
# Run if slow test marker is set: test in multiple environments
# Python 3.6
build_python.test_with_py_version(python_version="3.6.12")
# Python 3.7
build_python.test_with_py_version(python_version="3.7.9")
# Activated Python Environment (3.8)
build_python.install_build_env()
# Run pytest in pipenv environment
build_utils.run("pipenv run pytest", exit_on_error=True)
# Update pipfile.lock when all tests are successfull (lock environment)
build_utils.run("pipenv lock", exit_on_error=True)
else:
# Run fast tests
build_utils.run('pipenv run pytest -m "not slow"',
exit_on_error=True)
if args.get(build_utils.FLAG_RELEASE):
# Bump all versions in some filess
previous_version = build_utils.get_latest_version()
if previous_version:
build_utils.replace_in_files(
previous_version,
version,
file_paths=[
"./actions/build-environment/Dockerfile",
"./README.md",
"./workflows/build-pipeline.yml",
"./workflows/release-pipeline.yml",
],
regex=False,
exit_on_error=True,
)
# Publish distribution on pypi
build_python.publish_pypi_distribution(
pypi_token=args.get(build_python.FLAG_PYPI_TOKEN),
pypi_repository=args.get(build_python.FLAG_PYPI_REPOSITORY),
)
# TODO: Publish coverage report: if private repo set CODECOV_TOKEN="token" or use -t
# build_utils.run("curl -s https://codecov.io/bash | bash -s", exit_on_error=False)
# Build the build-environment component
build_utils.build("build-environment", args)
# Build all examples components
build_utils.build("examples", args)
def code_checks(
black: bool = True,
isort: bool = True,
pydocstyle: bool = True,
mypy: bool = True,
flake8: bool = True,
safety: bool = False,
exit_on_error: bool = True,
) -> None:
"""Run linting and style checks.
Args:
black (bool, optional): Activate black formatting check. Defaults to True.
isort (bool, optional): Activate isort import sorting check. Defaults to True.
pydocstyle (bool, optional): Activate pydocstyle docstring check. Defaults to True.
mypy (bool, optional): Activate mypy typing check. Defaults to True.
flake8 (bool, optional): Activate flake8 linting check. Defaults to True.
safety (bool, optional): Activate saftey check via pipenv. Defaults to False.
exit_on_error (bool, optional): If `True`, exit process as soon as error occures. Defaults to True.
"""
command_prefix = ""
if is_pipenv_environment():
command_prefix = "pipenv run"
successful: bool = True
if black:
if not command_prefix:
# Check twine command
build_utils.command_exists("black", exit_on_error=exit_on_error)
if (build_utils.run(f"{command_prefix} black --check src",
exit_on_error=False).returncode > 0):
successful = False
if (build_utils.run(f"{command_prefix} black --check tests",
exit_on_error=False).returncode > 0):
successful = False
if isort:
if not command_prefix:
# Check twine command
build_utils.command_exists("isort", exit_on_error=exit_on_error)
if (build_utils.run(
f"{command_prefix} isort --profile black --check-only src",
exit_on_error=False,
).returncode > 0):
successful = False
if (build_utils.run(
f"{command_prefix} isort --profile black --check-only tests",
exit_on_error=False,
).returncode > 0):
successful = False
if pydocstyle:
if not command_prefix:
# Check twine command
build_utils.command_exists("pydocstyle",
exit_on_error=exit_on_error)
if (build_utils.run(f"{command_prefix} pydocstyle src",
exit_on_error=False).returncode > 0):
successful = False
# Run linters and checks
if mypy:
if not command_prefix:
# Check twine command
build_utils.command_exists("mypy", exit_on_error=exit_on_error)
if (build_utils.run(f"{command_prefix} mypy src",
exit_on_error=False).returncode > 0):
successful = False
if flake8:
if not command_prefix:
# Check twine command
build_utils.command_exists("flake8", exit_on_error=exit_on_error)
if (build_utils.run(
f"{command_prefix} flake8 --show-source --statistics src",
exit_on_error=False,
).returncode > 0):
successful = False
if (build_utils.run(
f"{command_prefix} flake8 --show-source --statistics tests",
exit_on_error=False,
).returncode > 0):
successful = False
if safety:
# Check pipenv command
build_utils.command_exists("pipenv", exit_on_error=exit_on_error)
# Check using pipenv (runs safety check)
if build_utils.run("pipenv check", exit_on_error=False).returncode > 0:
successful = False
if not successful:
build_utils.log(
"Code checks (style, linting, safety, ...) failed. Please check the logs and fix the issues."
)
build_utils.exit_process(1)
def main(args: Dict[str, Union[bool, str]]):
# Move libraries to
build_utils.run(
"rm -r -f services/lab-workspace/docker-res/duplicated-resources/")
build_utils.run(
"mkdir services/lab-workspace/docker-res/duplicated-resources/")
build_utils.run(
"cp -R libraries/* services/lab-workspace/docker-res/duplicated-resources/"
)
# build base images
# For just testing, the lab-workspace does not have to be built as it is not covered by tests yet
# TODO: in GitHub actions add workspace to --skip-path to ignore it
build_utils.build("services/lab-workspace", args)
build_utils.build("services/simple-workspace-service", args)
build_utils.build("services/lab-model-service", args)
# build demo services/jobs
build_utils.build("services/simple-demo-job", args)
build_utils.build("services/simple-demo-service", args)
build_utils.build("services/simple-fastapi-service", args)
# build webapp and move build into backend service
# TODO: MOVE SWAGGER API TO WEB APP
# build main application first time to generate swagger config
backend_args = {**args}
backend_args[build_utils.FLAG_TEST] = False
build_utils.build("backend", backend_args)
if args[build_utils.FLAG_MAKE]:
is_successful = generate_and_copy_js_client()
if not is_successful:
build_utils.log(
"Error in generating the JavaScript client library")
build_utils.exit_process(1)
# format the just generated JavaScript client to make it conform with the project and prevent showing format-related changes in Git
build_utils.run(
"cd webapp; npm run prettier ./src/services/client/; cd ..")
build_utils.build("webapp", args)
if args[build_utils.FLAG_MAKE]:
# Move webapp build into resources
build_utils.run("rm -r -f backend/lab-service/src/main/resources/app/")
build_utils.run("mkdir backend/lab-service/src/main/resources/app/")
build_utils.run(
"cp -R webapp/build/* backend/lab-service/src/main/resources/app/",
exit_on_error=True,
)
# build documentation
build_utils.build("docs", args)
if args[build_utils.FLAG_MAKE]:
# Move documentation build into resources
build_utils.run(
"rm -r -f backend/lab-service/src/main/resources/docs/")
build_utils.run("mkdir backend/lab-service/src/main/resources/docs/")
build_utils.run(
"cp -R docs/site/* backend/lab-service/src/main/resources/docs/")
# build main application second time to bundle webapp
build_utils.build("backend", args)
def main(args: dict) -> None:
# set current path as working dir
os.chdir(HERE)
version = args.get(build_utils.FLAG_VERSION)
if version:
# Update version in _about.py
build_python.update_version(
os.path.join(HERE, f"src/{MAIN_PACKAGE}/_about.py"),
build_utils._Version.get_pip_compatible_string(str(version)),
exit_on_error=True,
)
if args.get(build_utils.FLAG_MAKE):
# Install pipenv dev requirements
build_python.install_build_env(exit_on_error=True)
# Create API documentation via lazydocs
build_python.generate_api_docs(github_url=GITHUB_URL,
main_package=MAIN_PACKAGE,
exit_on_error=True)
# Build distribution via setuptools
build_python.build_distribution(exit_on_error=True)
# Copy distribution to playground
try:
dist_name = MAIN_PACKAGE.replace("_", "-")
dist_file = glob.glob(f"./dist/{dist_name}-*.tar.gz")[0]
shutil.copy(
dist_file,
os.path.join(HERE, "playground", "resources",
dist_name + ".tar.gz"),
)
except Exception as ex:
build_utils.log(
"Failed to copy distribution to playground container " +
str(ex))
build_utils.exit_process(1)
# Copy all Demo artifacts (only py and txt files for now)
files = []
for ext in ("*.py", "*.txt"):
files.extend(
glob.glob(os.path.join(HERE, "examples", "**", ext),
recursive=True))
DEMO_PATH = os.path.join(HERE, "playground", "resources", "demos")
if os.path.exists(DEMO_PATH):
shutil.rmtree(DEMO_PATH)
for file in files:
new_path = os.path.join(
DEMO_PATH,
os.path.relpath(file, os.path.join(HERE, "examples")),
)
os.makedirs(os.path.dirname(new_path), exist_ok=True)
shutil.copy(file, new_path)
if args.get(build_utils.FLAG_CHECK):
build_python.code_checks(exit_on_error=True, safety=False)
if args.get(build_utils.FLAG_TEST):
# Remove coverage files
build_utils.run("pipenv run coverage erase", exit_on_error=False)
test_markers = args.get(build_utils.FLAG_TEST_MARKER)
if (isinstance(test_markers, list)
and build_utils.TEST_MARKER_SLOW in test_markers):
# Run if slow test marker is set: test in multiple environments
# Python 3.6
build_python.test_with_py_version(python_version="3.6.12",
exit_on_error=True)
# Python 3.7
# build_python.test_with_py_version(
# python_version="3.7.9", exit_on_error=True
# )
# Activated Python Environment (3.8)
build_python.install_build_env()
# Run pytest in pipenv environment
build_utils.run("pipenv run pytest", exit_on_error=True)
# Update pipfile.lock when all tests are successfull (lock environment)
build_utils.run("pipenv lock", exit_on_error=True)
else:
# Run fast tests
build_utils.run('pipenv run pytest -m "not slow"',
exit_on_error=True)
if args.get(build_utils.FLAG_RELEASE):
# Publish distribution on pypi
build_python.publish_pypi_distribution(
pypi_token=args.get(build_python.FLAG_PYPI_TOKEN),
pypi_repository=args.get(build_python.FLAG_PYPI_REPOSITORY),
)
# TODO: Publish coverage report: if private repo set CODECOV_TOKEN="token" or use -t
# build_utils.run("curl -s https://codecov.io/bash | bash -s", exit_on_error=False)
pass
# Build the opyrator playground component
build_utils.build("playground", args)
