Example #1
0
def main(args: dict) -> None:
    """Execute all component builds."""

    # set script path as working dir
    os.chdir(HERE)

    # Build react webapp
    build_utils.build(REACT_WEBAPP_COMPONENT, args)
    # Build python lib
    build_utils.build(PYTHON_LIB_COMPONENT, args)

    if args.get(build_utils.FLAG_MAKE):
        # Duplicate api docs into the mkdocs documentation
        build_utils.duplicate_folder(f"./{PYTHON_LIB_COMPONENT}/docs/",
                                     f"./{DOCS_COMPONENT}/docs/api-docs/")

        # Copy python lib distribution to docker container
        try:
            dest_path = os.path.join("./", DOCKER_COMPONENT, "resources",
                                     PYTHON_LIB_COMPONENT + ".tar.gz")
            os.makedirs(os.path.dirname(dest_path), exist_ok=True)
            shutil.copy(
                glob.glob(
                    f"./{PYTHON_LIB_COMPONENT}/dist/{PYTHON_LIB_COMPONENT}-*.tar.gz"
                )[0],
                os.path.join(dest_path),
            )
        except Exception as ex:
            build_utils.log(
                f"Failed to copy {PYTHON_LIB_COMPONENT} distribution to {DOCKER_COMPONENT} component: "
                + str(ex))
            build_utils.exit_process(1)

    # Build docker container
    build_utils.build(DOCKER_COMPONENT, args)

    # Build mkdocs documentation
    build_utils.build(DOCS_COMPONENT, args)
Example #2
0
args = build_utils.parse_arguments(argument_parser=parser)

VERSION = str(args.get(build_utils.FLAG_VERSION))
docker_image_prefix = args.get(build_docker.FLAG_DOCKER_IMAGE_PREFIX)

if not docker_image_prefix:
    docker_image_prefix = REMOTE_IMAGE_PREFIX

if not args.get(FLAG_FLAVOR):
    args[FLAG_FLAVOR] = "all"

flavor = str(args[FLAG_FLAVOR]).lower().strip()

if flavor == "all":
    args[FLAG_FLAVOR] = "gpu-11.3"
    build_utils.build(".", args)
    build_utils.exit_process(0)

# unknown flavor -> try to build from subdirectory
if flavor not in ["gpu-11.3"]:
    # assume that flavor has its own directory with build.py
    build_utils.build(flavor + "-flavor", args)
    build_utils.exit_process(0)

docker_image_name = IMAGE_NAME + "-" + flavor

# docker build
git_rev = "unknown"
try:
    git_rev = (subprocess.check_output(["git", "rev-parse", "--short",
                                        "HEAD"]).decode("ascii").strip())
Example #3
0
args = build_utils.parse_arguments(argument_parser=parser)

VERSION = str(args.get(build_utils.FLAG_VERSION))
docker_image_prefix = args.get(build_docker.FLAG_DOCKER_IMAGE_PREFIX)

if not docker_image_prefix:
    docker_image_prefix = REMOTE_IMAGE_PREFIX

if not args.get(FLAG_FLAVOR):
    args[FLAG_FLAVOR] = "all"

flavor = str(args[FLAG_FLAVOR]).lower().strip()

if flavor == "all":
    args[FLAG_FLAVOR] = "minimal"
    build_utils.build(".", args)

    args[FLAG_FLAVOR] = "light"
    build_utils.build(".", args)

    args[FLAG_FLAVOR] = "full"
    build_utils.build(".", args)

    args[FLAG_FLAVOR] = "r"
    build_utils.build("r-flavor", args)

    args[FLAG_FLAVOR] = "spark"
    build_utils.build("spark-flavor", args)

    args[FLAG_FLAVOR] = "gpu"
    build_utils.build("gpu-flavor", args)
from universal_build import build_utils

COMPONENT_NAME = "unified-model-lib"

args = build_utils.parse_arguments()

if args[build_utils.FLAG_MAKE]:
    completed_process = build_utils.run("python setup.py develop")
    if completed_process.returncode > 0:
        build_utils.log(f"Error in building component {COMPONENT_NAME}")

    completed_process = build_utils.run("python generate_docs.py")
    if completed_process.returncode > 0:
        build_utils.log(
            f"Error in generating docs for component {COMPONENT_NAME}")

    build_utils.build("docker", args)
Example #5
0
def main(args: dict) -> None:
    # set current path as working dir
    os.chdir(HERE)

    version = args.get(build_utils.FLAG_VERSION)

    if version:
        # Update version in _about.py
        build_python.update_version(
            os.path.join(HERE, f"src/{MAIN_PACKAGE}/_about.py"),
            build_utils._Version.get_pip_compatible_string(str(version)),
        )

    if args.get(build_utils.FLAG_MAKE):
        # Install pipenv dev requirements
        build_python.install_build_env()
        # Create API documentation via lazydocs
        build_python.generate_api_docs(github_url=GITHUB_URL,
                                       main_package=MAIN_PACKAGE)
        # Build distribution via setuptools
        build_python.build_distribution()

        try:
            dist_name = MAIN_PACKAGE.replace("_", "-")
            dist_file = glob.glob(f"./dist/{dist_name}-*.tar.gz")[0]
            shutil.copy(
                dist_file,
                os.path.join(HERE, "build-environment", "resources",
                             dist_name + ".tar.gz"),
            )
        except Exception:
            build_utils.log("Failed to copy distribution to build container.")
            build_utils.exit_process(1)

    if args.get(build_utils.FLAG_CHECK):
        build_python.code_checks(exit_on_error=True, safety=False)

    if args.get(build_utils.FLAG_TEST):
        # Remove coverage files
        build_utils.run("pipenv run coverage erase", exit_on_error=False)

        test_markers = args.get(build_utils.FLAG_TEST_MARKER)

        if build_utils.TEST_MARKER_SLOW in test_markers:  # type: ignore
            # Run if slow test marker is set: test in multiple environments
            # Python 3.6
            build_python.test_with_py_version(python_version="3.6.12")

            # Python 3.7
            build_python.test_with_py_version(python_version="3.7.9")

            # Activated Python Environment (3.8)
            build_python.install_build_env()
            # Run pytest in pipenv environment
            build_utils.run("pipenv run pytest", exit_on_error=True)

            # Update pipfile.lock when all tests are successfull (lock environment)
            build_utils.run("pipenv lock", exit_on_error=True)
        else:
            # Run fast tests
            build_utils.run('pipenv run pytest -m "not slow"',
                            exit_on_error=True)

    if args.get(build_utils.FLAG_RELEASE):
        # Bump all versions in some filess
        previous_version = build_utils.get_latest_version()
        if previous_version:
            build_utils.replace_in_files(
                previous_version,
                version,
                file_paths=[
                    "./actions/build-environment/Dockerfile",
                    "./README.md",
                    "./workflows/build-pipeline.yml",
                    "./workflows/release-pipeline.yml",
                ],
                regex=False,
                exit_on_error=True,
            )

        # Publish distribution on pypi
        build_python.publish_pypi_distribution(
            pypi_token=args.get(build_python.FLAG_PYPI_TOKEN),
            pypi_repository=args.get(build_python.FLAG_PYPI_REPOSITORY),
        )

        # TODO: Publish coverage report: if private repo set CODECOV_TOKEN="token" or use -t
        # build_utils.run("curl -s https://codecov.io/bash | bash -s", exit_on_error=False)

    # Build the build-environment component
    build_utils.build("build-environment", args)
    # Build all examples components
    build_utils.build("examples", args)
Example #6
0
from universal_build import build_utils

args = build_utils.parse_arguments()

if args[build_utils.FLAG_VERSION]:
    # The version is also needed when just tests are executed, so set it independent of the passed flags
    completed_process = build_utils.run("mvn versions:set -DnewVersion=" +
                                        args[build_utils.FLAG_VERSION])
    if completed_process.returncode > 0:
        build_utils.log("Failed to apply version " +
                        args[build_utils.FLAG_VERSION])
        build_utils.run("mvn versions:revert")
        build_utils.exit_process(1)
    build_utils.run("mvn versions:commit")

if args[build_utils.FLAG_MAKE]:
    # Check if all project can be build, otherwise exit build script
    completed_process = build_utils.run("mvn clean package")
    if completed_process.returncode > 0:
        build_utils.log("Failed to build project")
        build_utils.exit_process(1)

    build_utils.run("mvn -N clean install")

# libraries
build_utils.build("environment-lib", args)
build_utils.build("service-lib", args)

# services
build_utils.build("lab-service", args)
Example #7
0
def main(args: Dict[str, Union[bool, str]]):
    # Move libraries to
    build_utils.run(
        "rm -r -f services/lab-workspace/docker-res/duplicated-resources/")
    build_utils.run(
        "mkdir services/lab-workspace/docker-res/duplicated-resources/")
    build_utils.run(
        "cp -R libraries/* services/lab-workspace/docker-res/duplicated-resources/"
    )

    # build base images
    # For just testing, the lab-workspace does not have to be built as it is not covered by tests yet
    # TODO: in GitHub actions add workspace to --skip-path to ignore it
    build_utils.build("services/lab-workspace", args)
    build_utils.build("services/simple-workspace-service", args)

    build_utils.build("services/lab-model-service", args)

    # build demo services/jobs
    build_utils.build("services/simple-demo-job", args)
    build_utils.build("services/simple-demo-service", args)
    build_utils.build("services/simple-fastapi-service", args)

    # build webapp and move build into backend service
    # TODO: MOVE SWAGGER API TO WEB APP
    # build main application first time to generate swagger config
    backend_args = {**args}
    backend_args[build_utils.FLAG_TEST] = False
    build_utils.build("backend", backend_args)

    if args[build_utils.FLAG_MAKE]:
        is_successful = generate_and_copy_js_client()
        if not is_successful:
            build_utils.log(
                "Error in generating the JavaScript client library")
            build_utils.exit_process(1)
        # format the just generated JavaScript client to make it conform with the project and prevent showing format-related changes in Git
        build_utils.run(
            "cd webapp; npm run prettier ./src/services/client/; cd ..")

    build_utils.build("webapp", args)
    if args[build_utils.FLAG_MAKE]:
        # Move webapp build into resources
        build_utils.run("rm -r -f backend/lab-service/src/main/resources/app/")
        build_utils.run("mkdir backend/lab-service/src/main/resources/app/")
        build_utils.run(
            "cp -R webapp/build/* backend/lab-service/src/main/resources/app/",
            exit_on_error=True,
        )

    # build documentation
    build_utils.build("docs", args)
    if args[build_utils.FLAG_MAKE]:
        # Move documentation build into resources
        build_utils.run(
            "rm -r -f backend/lab-service/src/main/resources/docs/")
        build_utils.run("mkdir backend/lab-service/src/main/resources/docs/")
        build_utils.run(
            "cp -R docs/site/* backend/lab-service/src/main/resources/docs/")

    # build main application second time to bundle webapp
    build_utils.build("backend", args)
Example #8
0
def main(args: dict) -> None:
    # set current path as working dir
    os.chdir(HERE)

    version = args.get(build_utils.FLAG_VERSION)

    if version:
        # Update version in _about.py
        build_python.update_version(
            os.path.join(HERE, f"src/{MAIN_PACKAGE}/_about.py"),
            build_utils._Version.get_pip_compatible_string(str(version)),
            exit_on_error=True,
        )

    if args.get(build_utils.FLAG_MAKE):
        # Install pipenv dev requirements
        build_python.install_build_env(exit_on_error=True)
        # Create API documentation via lazydocs
        build_python.generate_api_docs(github_url=GITHUB_URL,
                                       main_package=MAIN_PACKAGE,
                                       exit_on_error=True)
        # Build distribution via setuptools
        build_python.build_distribution(exit_on_error=True)

        # Copy distribution to playground
        try:
            dist_name = MAIN_PACKAGE.replace("_", "-")
            dist_file = glob.glob(f"./dist/{dist_name}-*.tar.gz")[0]
            shutil.copy(
                dist_file,
                os.path.join(HERE, "playground", "resources",
                             dist_name + ".tar.gz"),
            )
        except Exception as ex:
            build_utils.log(
                "Failed to copy distribution to playground container " +
                str(ex))
            build_utils.exit_process(1)

        # Copy all Demo artifacts (only py and txt files for now)
        files = []
        for ext in ("*.py", "*.txt"):
            files.extend(
                glob.glob(os.path.join(HERE, "examples", "**", ext),
                          recursive=True))

        DEMO_PATH = os.path.join(HERE, "playground", "resources", "demos")
        if os.path.exists(DEMO_PATH):
            shutil.rmtree(DEMO_PATH)

        for file in files:
            new_path = os.path.join(
                DEMO_PATH,
                os.path.relpath(file, os.path.join(HERE, "examples")),
            )
            os.makedirs(os.path.dirname(new_path), exist_ok=True)

            shutil.copy(file, new_path)

    if args.get(build_utils.FLAG_CHECK):
        build_python.code_checks(exit_on_error=True, safety=False)

    if args.get(build_utils.FLAG_TEST):
        # Remove coverage files
        build_utils.run("pipenv run coverage erase", exit_on_error=False)

        test_markers = args.get(build_utils.FLAG_TEST_MARKER)

        if (isinstance(test_markers, list)
                and build_utils.TEST_MARKER_SLOW in test_markers):
            # Run if slow test marker is set: test in multiple environments
            # Python 3.6
            build_python.test_with_py_version(python_version="3.6.12",
                                              exit_on_error=True)

            # Python 3.7
            # build_python.test_with_py_version(
            #    python_version="3.7.9", exit_on_error=True
            # )

            # Activated Python Environment (3.8)
            build_python.install_build_env()
            # Run pytest in pipenv environment
            build_utils.run("pipenv run pytest", exit_on_error=True)

            # Update pipfile.lock when all tests are successfull (lock environment)
            build_utils.run("pipenv lock", exit_on_error=True)
        else:
            # Run fast tests
            build_utils.run('pipenv run pytest -m "not slow"',
                            exit_on_error=True)

    if args.get(build_utils.FLAG_RELEASE):
        # Publish distribution on pypi
        build_python.publish_pypi_distribution(
            pypi_token=args.get(build_python.FLAG_PYPI_TOKEN),
            pypi_repository=args.get(build_python.FLAG_PYPI_REPOSITORY),
        )

        # TODO: Publish coverage report: if private repo set CODECOV_TOKEN="token" or use -t
        # build_utils.run("curl -s https://codecov.io/bash | bash -s", exit_on_error=False)
        pass

    # Build the opyrator playground component
    build_utils.build("playground", args)
                    help="flavor (lab, lab-gpu) used for docker container",
                    default="lab")

COMPONENT_NAME = "ml-workspace"
FLAG_FLAVOR = "flavor"

args = build_utils.parse_arguments(argument_parser=parser)

if not args[FLAG_FLAVOR]:
    args[FLAG_FLAVOR] = "lab"

args[FLAG_FLAVOR] = str(args[FLAG_FLAVOR]).lower()

if args[FLAG_FLAVOR] == "all":
    args[FLAG_FLAVOR] = "lab"
    build_utils.build(".", args)
    args[FLAG_FLAVOR] = "lab-gpu"
    build_utils.build(".", args)
    build_utils.exit_process(0)

# unknown flavor -> try to build from subdirectory
if args[FLAG_FLAVOR] not in ["lab", "lab-gpu"]:
    # assume that flavor has its own directory with build.py
    build_utils.build(args[FLAG_FLAVOR], args)
    build_utils.exit_process(0)

# Add flavor suffix to image name
# service_name += "-" + args.flavor
COMPONENT_NAME = f"{COMPONENT_NAME}-{args[FLAG_FLAVOR]}"

# Set base workspace image
Example #10
0
import argparse
import os

from universal_build import build_utils

HERE = os.path.abspath(os.path.dirname(__file__))
os.chdir(HERE)

if __name__ == "__main__":

    # Custom CLI arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("--deployment-token",
                        help="This is a custom token.",
                        default="")
    parser.add_argument("--my_token",
                        help="This is a custom token.",
                        default="")
    parser.add_argument("--my_bool",
                        help="This is a custom bool.",
                        action="store_true")

    args = build_utils.parse_arguments(argument_parser=parser)

    build_utils.build("submodule", args)