def update_version(module_path: str, version: str, exit_on_error: bool = True) -> None: """Update version in specified module. Args: module_path (str): Python module with a `__version__` attribute. version (str): New version number to write into `__version__` attribute. exit_on_error (bool, optional): If `True`, exit process as soon as error occures. Defaults to True. """ if not version: build_utils.log("Cannot update version, no version provided") if exit_on_error: build_utils.exit_process(1) return if not os.path.exists(module_path): build_utils.log("Couldn't find file: " + module_path) if exit_on_error: build_utils.exit_process(1) return with open(module_path, "r+") as f: data = f.read() f.seek(0) f.write( re.sub(r"__version__ = \".+\"", f'__version__ = "{version}"', data)) f.truncate()
def publish_pypi_distribution( pypi_token: str, pypi_user: str = "__token__", pypi_repository: Optional[str] = None, exit_on_error: bool = True, ) -> None: """Publish distribution to pypi. Args: pypi_token (str): Token of PyPi repository. pypi_user (str, optional): User of PyPi repository. Defaults to "__token__". pypi_repository (Optional[str], optional): PyPi repository. If `None` provided, use the production instance. exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ if not pypi_token: build_utils.log( "PyPI token is required for release (--pypi-token=<TOKEN>)") if exit_on_error: build_utils.exit_process(1) return pypi_repository_args = "" if pypi_repository: pypi_repository_args = f'--repository-url "{pypi_repository}"' # Check twine command build_utils.command_exists("twine", exit_on_error=exit_on_error) # Publish on pypi build_utils.run( f'twine upload --non-interactive -u "{pypi_user}" -p "{pypi_token}" {pypi_repository_args} dist/*', exit_on_error=exit_on_error, )
def release_docker_image( name: str, version: str, docker_image_prefix: str, exit_on_error: bool = True ) -> subprocess.CompletedProcess: """Push a Docker image to a repository. Args: name (str): The name of the image. Must not be prefixed! version (str): The tag used for the image. docker_image_prefix (str): The prefix added to the name to indicate an organization on DockerHub or a completely different repository. exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. Returns: subprocess.CompletedProcess: Returns the CompletedProcess object of the `docker push ...` command. """ # Check if docker exists on the system build_utils.command_exists("docker", exit_on_error=exit_on_error) if not docker_image_prefix: build_utils.log( "The flag --docker-image-prefix cannot be blank when pushing a Docker image." ) build_utils.exit_process(build_utils.EXIT_CODE_GENERAL) versioned_tag = get_image_name(name=name, tag=version) remote_versioned_tag = get_image_name( name=name, tag=version, image_prefix=docker_image_prefix ) build_utils.run( "docker tag " + versioned_tag + " " + remote_versioned_tag, exit_on_error=exit_on_error, ) completed_process = build_utils.run( "docker push " + remote_versioned_tag, exit_on_error=exit_on_error ) if completed_process.returncode > 0: build_utils.log(f"Failed to release Docker image {name}:{version}") # Only push version with latest tag if no suffix is added (pre-release) if "-" not in version: remote_latest_tag = get_image_name( name=name, tag="latest", image_prefix=docker_image_prefix ) build_utils.log( "Release Docker image with latest tag as well: " + remote_latest_tag ) build_utils.run( "docker tag " + versioned_tag + " " + remote_latest_tag, exit_on_error=exit_on_error, ) build_utils.run("docker push " + remote_latest_tag, exit_on_error=exit_on_error) return completed_process
def release_docker_image( name: str, version: str, docker_image_prefix: str = "", exit_on_error: bool = False ) -> subprocess.CompletedProcess: """Push a Docker image to a repository. Args: name (str): The name of the image. Must not be prefixed! version (str): The tag used for the image. docker_image_prefix (str, optional): The prefix added to the name to indicate an organization on DockerHub or a completely different repository. Defaults to "". exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. Returns: subprocess.CompletedProcess: Returns the CompletedProcess object of the `docker push ...` command. """ if not docker_image_prefix: build_utils.log( "The flag --docker-image-prefix cannot be blank when pushing a Docker image." ) build_utils.exit_process(build_utils.EXIT_CODE_GENERAL) docker_image_prefix = docker_image_prefix.rstrip("/") + "/" versioned_image = name + ":" + version remote_versioned_image = docker_image_prefix + versioned_image build_utils.run( "docker tag " + versioned_image + " " + remote_versioned_image, exit_on_error=exit_on_error, ) completed_process = build_utils.run( "docker push " + remote_versioned_image, exit_on_error=exit_on_error ) if completed_process.returncode > 0: build_utils.log(f"Failed to release Docker image {name}:{version}") if "-dev" not in version: build_utils.log("Release Docker image with latest tag as well.") latest_image = name + ":latest" remote_latest_image = docker_image_prefix + latest_image build_utils.run( "docker tag " + latest_image + " " + remote_latest_image, exit_on_error=exit_on_error, ) build_utils.run( "docker push " + remote_latest_image, exit_on_error=exit_on_error ) return completed_process
def install_build_env(exit_on_error: bool = True) -> None: """Installs a new virtual environment via pipenv. Args: exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ # Check if pipenv exists build_utils.command_exists("pipenv", exit_on_error=exit_on_error) if not os.path.exists("Pipfile"): build_utils.log( "No Pipfile discovered, cannot install pipenv environemnt") if exit_on_error: build_utils.exit_process(1) return build_utils.run("pipenv --rm", exit_on_error=False) build_utils.run( f"pipenv install --dev --python={sys.executable} --skip-lock --site-packages", exit_on_error=exit_on_error, )
def main(args: dict) -> None: """Execute all component builds.""" # set script path as working dir os.chdir(HERE) # Build react webapp build_utils.build(REACT_WEBAPP_COMPONENT, args) # Build python lib build_utils.build(PYTHON_LIB_COMPONENT, args) if args.get(build_utils.FLAG_MAKE): # Duplicate api docs into the mkdocs documentation build_utils.duplicate_folder(f"./{PYTHON_LIB_COMPONENT}/docs/", f"./{DOCS_COMPONENT}/docs/api-docs/") # Copy python lib distribution to docker container try: dest_path = os.path.join("./", DOCKER_COMPONENT, "resources", PYTHON_LIB_COMPONENT + ".tar.gz") os.makedirs(os.path.dirname(dest_path), exist_ok=True) shutil.copy( glob.glob( f"./{PYTHON_LIB_COMPONENT}/dist/{PYTHON_LIB_COMPONENT}-*.tar.gz" )[0], os.path.join(dest_path), ) except Exception as ex: build_utils.log( f"Failed to copy {PYTHON_LIB_COMPONENT} distribution to {DOCKER_COMPONENT} component: " + str(ex)) build_utils.exit_process(1) # Build docker container build_utils.build(DOCKER_COMPONENT, args) # Build mkdocs documentation build_utils.build(DOCS_COMPONENT, args)
def publish_pypi_distribution( pypi_token: str, pypi_user: str = "__token__", pypi_repository: Optional[str] = None ) -> None: """Publish distribution to pypi. Args: pypi_token (str): Token of PyPi repository. pypi_user (str, optional): User of PyPi repository. Defaults to "__token__". pypi_repository (Optional[str], optional): PyPi repository. If `None` provided, use the production instance. """ if not pypi_token: build_utils.log("PyPI token is required for release (--pypi-token=<TOKEN>)") build_utils.exit_process(1) pypi_repository_args = "" if pypi_repository: pypi_repository_args = f'--repository-url "{pypi_repository}"' # Publish on pypi build_utils.run( f'twine upload --non-interactive -u "{pypi_user}" -p "{pypi_token}" {pypi_repository_args} dist/*', exit_on_error=True, )
VERSION = str(args.get(build_utils.FLAG_VERSION)) docker_image_prefix = args.get(build_docker.FLAG_DOCKER_IMAGE_PREFIX) if not docker_image_prefix: docker_image_prefix = REMOTE_IMAGE_PREFIX if not args.get(FLAG_FLAVOR): args[FLAG_FLAVOR] = "all" flavor = str(args[FLAG_FLAVOR]).lower().strip() if flavor == "all": args[FLAG_FLAVOR] = "gpu-11.3" build_utils.build(".", args) build_utils.exit_process(0) # unknown flavor -> try to build from subdirectory if flavor not in ["gpu-11.3"]: # assume that flavor has its own directory with build.py build_utils.build(flavor + "-flavor", args) build_utils.exit_process(0) docker_image_name = IMAGE_NAME + "-" + flavor # docker build git_rev = "unknown" try: git_rev = (subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode("ascii").strip()) except Exception:
from universal_build import build_utils from universal_build.helpers import build_docker COMPONENT_NAME = "simple-demo-job" args = build_utils.parse_arguments() if args[build_utils.FLAG_MAKE]: completed_process = build_docker.build_docker_image( COMPONENT_NAME, args[build_utils.FLAG_VERSION]) if completed_process.returncode > 0: build_utils.exit_process(completed_process.returncode) if args[build_utils.FLAG_RELEASE]: completed_process = build_docker.release_docker_image( COMPONENT_NAME, args[build_utils.FLAG_VERSION], args[build_docker.FLAG_DOCKER_IMAGE_PREFIX])
def main(args: dict) -> None: # set current path as working dir os.chdir(HERE) version = args.get(build_utils.FLAG_VERSION) if version: # Update version in _about.py build_python.update_version( os.path.join(HERE, f"src/{MAIN_PACKAGE}/_about.py"), build_utils._Version.get_pip_compatible_string(str(version)), ) if args.get(build_utils.FLAG_MAKE): # Install pipenv dev requirements build_python.install_build_env() # Create API documentation via lazydocs build_python.generate_api_docs(github_url=GITHUB_URL, main_package=MAIN_PACKAGE) # Build distribution via setuptools build_python.build_distribution() try: dist_name = MAIN_PACKAGE.replace("_", "-") dist_file = glob.glob(f"./dist/{dist_name}-*.tar.gz")[0] shutil.copy( dist_file, os.path.join(HERE, "build-environment", "resources", dist_name + ".tar.gz"), ) except Exception: build_utils.log("Failed to copy distribution to build container.") build_utils.exit_process(1) if args.get(build_utils.FLAG_CHECK): build_python.code_checks(exit_on_error=True, safety=False) if args.get(build_utils.FLAG_TEST): # Remove coverage files build_utils.run("pipenv run coverage erase", exit_on_error=False) test_markers = args.get(build_utils.FLAG_TEST_MARKER) if build_utils.TEST_MARKER_SLOW in test_markers: # type: ignore # Run if slow test marker is set: test in multiple environments # Python 3.6 build_python.test_with_py_version(python_version="3.6.12") # Python 3.7 build_python.test_with_py_version(python_version="3.7.9") # Activated Python Environment (3.8) build_python.install_build_env() # Run pytest in pipenv environment build_utils.run("pipenv run pytest", exit_on_error=True) # Update pipfile.lock when all tests are successfull (lock environment) build_utils.run("pipenv lock", exit_on_error=True) else: # Run fast tests build_utils.run('pipenv run pytest -m "not slow"', exit_on_error=True) if args.get(build_utils.FLAG_RELEASE): # Bump all versions in some filess previous_version = build_utils.get_latest_version() if previous_version: build_utils.replace_in_files( previous_version, version, file_paths=[ "./actions/build-environment/Dockerfile", "./README.md", "./workflows/build-pipeline.yml", "./workflows/release-pipeline.yml", ], regex=False, exit_on_error=True, ) # Publish distribution on pypi build_python.publish_pypi_distribution( pypi_token=args.get(build_python.FLAG_PYPI_TOKEN), pypi_repository=args.get(build_python.FLAG_PYPI_REPOSITORY), ) # TODO: Publish coverage report: if private repo set CODECOV_TOKEN="token" or use -t # build_utils.run("curl -s https://codecov.io/bash | bash -s", exit_on_error=False) # Build the build-environment component build_utils.build("build-environment", args) # Build all examples components build_utils.build("examples", args)
def code_checks( black: bool = True, isort: bool = True, pydocstyle: bool = True, mypy: bool = True, flake8: bool = True, safety: bool = False, exit_on_error: bool = True, ) -> None: """Run linting and style checks. Args: black (bool, optional): Activate black formatting check. Defaults to True. isort (bool, optional): Activate isort import sorting check. Defaults to True. pydocstyle (bool, optional): Activate pydocstyle docstring check. Defaults to True. mypy (bool, optional): Activate mypy typing check. Defaults to True. flake8 (bool, optional): Activate flake8 linting check. Defaults to True. safety (bool, optional): Activate saftey check via pipenv. Defaults to False. exit_on_error (bool, optional): If `True`, exit process as soon as error occures. Defaults to True. """ command_prefix = "" if is_pipenv_environment(): command_prefix = "pipenv run" successful: bool = True if black: if not command_prefix: # Check twine command build_utils.command_exists("black", exit_on_error=exit_on_error) if (build_utils.run(f"{command_prefix} black --check src", exit_on_error=False).returncode > 0): successful = False if (build_utils.run(f"{command_prefix} black --check tests", exit_on_error=False).returncode > 0): successful = False if isort: if not command_prefix: # Check twine command build_utils.command_exists("isort", exit_on_error=exit_on_error) if (build_utils.run( f"{command_prefix} isort --profile black --check-only src", exit_on_error=False, ).returncode > 0): successful = False if (build_utils.run( f"{command_prefix} isort --profile black --check-only tests", exit_on_error=False, ).returncode > 0): successful = False if pydocstyle: if not command_prefix: # Check twine command build_utils.command_exists("pydocstyle", exit_on_error=exit_on_error) if (build_utils.run(f"{command_prefix} pydocstyle src", exit_on_error=False).returncode > 0): successful = False # Run linters and checks if mypy: if not command_prefix: # Check twine command build_utils.command_exists("mypy", exit_on_error=exit_on_error) if (build_utils.run(f"{command_prefix} mypy src", exit_on_error=False).returncode > 0): successful = False if flake8: if not command_prefix: # Check twine command build_utils.command_exists("flake8", exit_on_error=exit_on_error) if (build_utils.run( f"{command_prefix} flake8 --show-source --statistics src", exit_on_error=False, ).returncode > 0): successful = False if (build_utils.run( f"{command_prefix} flake8 --show-source --statistics tests", exit_on_error=False, ).returncode > 0): successful = False if safety: # Check pipenv command build_utils.command_exists("pipenv", exit_on_error=exit_on_error) # Check using pipenv (runs safety check) if build_utils.run("pipenv check", exit_on_error=False).returncode > 0: successful = False if not successful: build_utils.log( "Code checks (style, linting, safety, ...) failed. Please check the logs and fix the issues." ) build_utils.exit_process(1)
def main(args: Dict[str, Union[bool, str]]): # Move libraries to build_utils.run( "rm -r -f services/lab-workspace/docker-res/duplicated-resources/") build_utils.run( "mkdir services/lab-workspace/docker-res/duplicated-resources/") build_utils.run( "cp -R libraries/* services/lab-workspace/docker-res/duplicated-resources/" ) # build base images # For just testing, the lab-workspace does not have to be built as it is not covered by tests yet # TODO: in GitHub actions add workspace to --skip-path to ignore it build_utils.build("services/lab-workspace", args) build_utils.build("services/simple-workspace-service", args) build_utils.build("services/lab-model-service", args) # build demo services/jobs build_utils.build("services/simple-demo-job", args) build_utils.build("services/simple-demo-service", args) build_utils.build("services/simple-fastapi-service", args) # build webapp and move build into backend service # TODO: MOVE SWAGGER API TO WEB APP # build main application first time to generate swagger config backend_args = {**args} backend_args[build_utils.FLAG_TEST] = False build_utils.build("backend", backend_args) if args[build_utils.FLAG_MAKE]: is_successful = generate_and_copy_js_client() if not is_successful: build_utils.log( "Error in generating the JavaScript client library") build_utils.exit_process(1) # format the just generated JavaScript client to make it conform with the project and prevent showing format-related changes in Git build_utils.run( "cd webapp; npm run prettier ./src/services/client/; cd ..") build_utils.build("webapp", args) if args[build_utils.FLAG_MAKE]: # Move webapp build into resources build_utils.run("rm -r -f backend/lab-service/src/main/resources/app/") build_utils.run("mkdir backend/lab-service/src/main/resources/app/") build_utils.run( "cp -R webapp/build/* backend/lab-service/src/main/resources/app/", exit_on_error=True, ) # build documentation build_utils.build("docs", args) if args[build_utils.FLAG_MAKE]: # Move documentation build into resources build_utils.run( "rm -r -f backend/lab-service/src/main/resources/docs/") build_utils.run("mkdir backend/lab-service/src/main/resources/docs/") build_utils.run( "cp -R docs/site/* backend/lab-service/src/main/resources/docs/") # build main application second time to bundle webapp build_utils.build("backend", args)
def main(args: dict) -> None: # set current path as working dir os.chdir(HERE) version = args.get(build_utils.FLAG_VERSION) if version: # Update version in _about.py build_python.update_version( os.path.join(HERE, f"src/{MAIN_PACKAGE}/_about.py"), build_utils._Version.get_pip_compatible_string(str(version)), exit_on_error=True, ) if args.get(build_utils.FLAG_MAKE): # Install pipenv dev requirements build_python.install_build_env(exit_on_error=True) # Create API documentation via lazydocs build_python.generate_api_docs(github_url=GITHUB_URL, main_package=MAIN_PACKAGE, exit_on_error=True) # Build distribution via setuptools build_python.build_distribution(exit_on_error=True) # Copy distribution to playground try: dist_name = MAIN_PACKAGE.replace("_", "-") dist_file = glob.glob(f"./dist/{dist_name}-*.tar.gz")[0] shutil.copy( dist_file, os.path.join(HERE, "playground", "resources", dist_name + ".tar.gz"), ) except Exception as ex: build_utils.log( "Failed to copy distribution to playground container " + str(ex)) build_utils.exit_process(1) # Copy all Demo artifacts (only py and txt files for now) files = [] for ext in ("*.py", "*.txt"): files.extend( glob.glob(os.path.join(HERE, "examples", "**", ext), recursive=True)) DEMO_PATH = os.path.join(HERE, "playground", "resources", "demos") if os.path.exists(DEMO_PATH): shutil.rmtree(DEMO_PATH) for file in files: new_path = os.path.join( DEMO_PATH, os.path.relpath(file, os.path.join(HERE, "examples")), ) os.makedirs(os.path.dirname(new_path), exist_ok=True) shutil.copy(file, new_path) if args.get(build_utils.FLAG_CHECK): build_python.code_checks(exit_on_error=True, safety=False) if args.get(build_utils.FLAG_TEST): # Remove coverage files build_utils.run("pipenv run coverage erase", exit_on_error=False) test_markers = args.get(build_utils.FLAG_TEST_MARKER) if (isinstance(test_markers, list) and build_utils.TEST_MARKER_SLOW in test_markers): # Run if slow test marker is set: test in multiple environments # Python 3.6 build_python.test_with_py_version(python_version="3.6.12", exit_on_error=True) # Python 3.7 # build_python.test_with_py_version( # python_version="3.7.9", exit_on_error=True # ) # Activated Python Environment (3.8) build_python.install_build_env() # Run pytest in pipenv environment build_utils.run("pipenv run pytest", exit_on_error=True) # Update pipfile.lock when all tests are successfull (lock environment) build_utils.run("pipenv lock", exit_on_error=True) else: # Run fast tests build_utils.run('pipenv run pytest -m "not slow"', exit_on_error=True) if args.get(build_utils.FLAG_RELEASE): # Publish distribution on pypi build_python.publish_pypi_distribution( pypi_token=args.get(build_python.FLAG_PYPI_TOKEN), pypi_repository=args.get(build_python.FLAG_PYPI_REPOSITORY), ) # TODO: Publish coverage report: if private repo set CODECOV_TOKEN="token" or use -t # build_utils.run("curl -s https://codecov.io/bash | bash -s", exit_on_error=False) pass # Build the opyrator playground component build_utils.build("playground", args)