def build_distribution(exit_on_error: bool = True) -> None: """Build python package distribution. Args: exit_on_error (bool, optional): If `True`, exit process as soon as error occures. Defaults to True. """ try: # Ensure there are no old builds rmtree("./dist") except OSError: pass try: # Ensure there are no old builds rmtree("./build") except OSError: pass # Build the distribution archives build_utils.run("python setup.py sdist bdist_wheel clean --all", exit_on_error=exit_on_error) # Check twine command build_utils.command_exists("twine", exit_on_error=exit_on_error) # Check the archives with twine build_utils.run("twine check dist/*", exit_on_error=exit_on_error)
def generate_api_docs( github_url: str, main_package: str, exit_on_error: bool = True, ) -> None: """Generates API documentation via lazydocs. Args: github_url (str): Github URL main_package (str): The main package name to use for docs generation. exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ command_prefix = "" if is_pipenv_environment(): command_prefix = "pipenv run" else: # Check lazydocs command build_utils.command_exists("lazydocs", exit_on_error=exit_on_error) build_utils.run( f"{command_prefix} lazydocs --overview-file=README.md" f" --src-base-url={github_url}/blob/main {main_package}", exit_on_error=exit_on_error, )
def build_mkdocs(command_prefix: str = PIPENV_RUN) -> None: """Build mkdocs markdown documentation. Args: command_prefix (str, optional): Prefix to use for all commands. Defaults to `pipenv run`. """ build_utils.run(f"{command_prefix} mkdocs build", exit_on_error=True)
def install_build_env() -> None: """Installs a new virtual environment via pipenv.""" build_utils.run("pipenv --rm") build_utils.run( f"pipenv install --dev --python={sys.executable} --skip-lock --site-packages", exit_on_error=True, )
def generate_and_copy_js_client() -> bool: temp_dir = "./temp" pathlib.Path(temp_dir).mkdir(exist_ok=True) swagger_codegen_cli = f"{temp_dir}/swagger-codegen-cli.jar" is_successful = check_and_download_swagger_cli(swagger_codegen_cli) if not is_successful: return False swagger_path = "./backend/lab-service/src/main/resources/swagger/swagger.json" output_path = f"{temp_dir}/client" build_utils.run( f"java -jar {swagger_codegen_cli} generate -i {swagger_path} -l javascript -o {output_path} --additional-properties useES6=true" ) # shutil.move(f"{output_path}/src/", "./webapp/src/services/mllab-client") try: for file in pathlib.Path(f"{output_path}/src/").iterdir(): file_name = str(file.parts[-1]) new_file_name = file_name if file_name == "index.js": new_file_name = "lab-api.js" target_file_path = f"./webapp/src/services/client/{new_file_name}" # Delete existing client files to be replaced with the new ones if pathlib.Path(target_file_path).is_file(): pathlib.Path(target_file_path).unlink() elif pathlib.Path(target_file_path).is_dir(): shutil.rmtree(target_file_path) shutil.move(str(file), target_file_path) except FileNotFoundError as e: build_utils.log(str(e)) return False return True
def publish_pypi_distribution( pypi_token: str, pypi_user: str = "__token__", pypi_repository: Optional[str] = None, exit_on_error: bool = True, ) -> None: """Publish distribution to pypi. Args: pypi_token (str): Token of PyPi repository. pypi_user (str, optional): User of PyPi repository. Defaults to "__token__". pypi_repository (Optional[str], optional): PyPi repository. If `None` provided, use the production instance. exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ if not pypi_token: build_utils.log( "PyPI token is required for release (--pypi-token=<TOKEN>)") if exit_on_error: build_utils.exit_process(1) return pypi_repository_args = "" if pypi_repository: pypi_repository_args = f'--repository-url "{pypi_repository}"' # Check twine command build_utils.command_exists("twine", exit_on_error=exit_on_error) # Publish on pypi build_utils.run( f'twine upload --non-interactive -u "{pypi_user}" -p "{pypi_token}" {pypi_repository_args} dist/*', exit_on_error=exit_on_error, )
def build_docker_image( name: str, version: str, build_args: str = "", docker_image_prefix: str = "", dockerfile: Optional[str] = None, additional_build_args: str = "", exit_on_error: bool = True, ) -> subprocess.CompletedProcess: """Build a docker image from a Dockerfile in the working directory. Args: name (str): Name of the docker image. version (str): Version to use as tag. build_args (str, optional): Add additional build arguments for docker build. docker_image_prefix (str, optional): The prefix added to the name to indicate an organization on DockerHub or a completely different repository. dockerfile (str, optional): Specify a specific Dockerfile. If not specified, the default `Dockerfile` wil be used. exit_on_error (bool, optional): If `True`, exit process as soon as an error occurs. Returns: subprocess.CompletedProcess: Returns the CompletedProcess object of the """ # Check if docker exists on the system build_utils.command_exists("docker", exit_on_error=exit_on_error) versioned_tag = get_image_name(name=name, tag=version) latest_tag = get_image_name(name=name, tag="latest") dockerfile_command = "" if dockerfile: dockerfile_command = " -f " + dockerfile completed_process = build_utils.run( "docker build " + dockerfile_command + "-t " + versioned_tag + " -t " + latest_tag + " " + build_args + " ./", exit_on_error=exit_on_error, ) if completed_process.returncode > 0: build_utils.log(f"Failed to build Docker image {versioned_tag}") return completed_process if docker_image_prefix: remote_versioned_tag = get_image_name( name=name, tag=version, image_prefix=docker_image_prefix ) build_utils.run( "docker tag " + versioned_tag + " " + remote_versioned_tag, exit_on_error=exit_on_error, ) return completed_process
def lint_dockerfile() -> None: """Run hadolint on the Dockerfile.""" build_utils.log("Run linters and style checks:") config_file_arg = "" if os.path.exists(".hadolint.yml"): config_file_arg = "--config=.hadolint.yml" build_utils.run(f"hadolint {config_file_arg} Dockerfile", exit_on_error=True)
def lint_markdown() -> None: """Run markdownlint on markdown documentation.""" build_utils.log("Run linters and style checks:") config_file_arg = "" if os.path.exists(".markdown-lint.yml"): config_file_arg = "--config='.markdown-lint.yml'" build_utils.run(f"markdownlint {config_file_arg} ./docs", exit_on_error=True)
def run_dev_mode(port: int = 8001, command_prefix: str = PIPENV_RUN) -> None: """Run mkdocs development server. Args: port (int, optional): Port to use for mkdocs development server. Defaults to 8001. command_prefix (str, optional): Prefix to use for all commands. Defaults to `pipenv run`. """ build_utils.log(f"Run docs in development mode (http://localhost:{port}):") build_utils.run(f"{command_prefix} mkdocs serve --dev-addr 0.0.0.0:{port}", exit_on_error=True)
def deploy_gh_pages(command_prefix: str = PIPENV_RUN) -> None: """Deploy mkdocs documentation to Github pages. Args: command_prefix (str, optional): Prefix to use for all commands. Defaults to `pipenv run`. """ build_utils.log("Deploy documentation to Github pages:") build_utils.run(f"{command_prefix} mkdocs gh-deploy --clean", exit_on_error=True, timeout=120)
def build_docker_image( name: str, version: str, build_args: str = "", exit_on_error: bool = False ) -> subprocess.CompletedProcess: """Build a docker image from a Dockerfile in the working directory. Args: name (str): Name of the docker image. version (str): Version to use as tag. build_args (str, optional): Add additional build arguments for docker build. exit_on_error (bool, optional): If `True`, exit process as soon as an error occurs. Returns: subprocess.CompletedProcess: Returns the CompletedProcess object of the """ versioned_image = name + ":" + version latest_image = name + ":latest" completed_process = build_utils.run( "docker build -t " + versioned_image + " -t " + latest_image + " " + build_args + " ./", exit_on_error=exit_on_error, ) # TODO tag prefixed image names if completed_process.returncode > 0: build_utils.log(f"Failed to build Docker image {name}:{version}") return completed_process
def build_mkdocs(exit_on_error: bool = True) -> None: """Build mkdocs markdown documentation. Args: exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ command_prefix = "" if is_pipenv_environment(): command_prefix = _PIPENV_RUN else: # Check mkdocs command build_utils.command_exists("mkdocs", exit_on_error=exit_on_error) build_utils.run(f"{command_prefix} mkdocs build", exit_on_error=exit_on_error)
def generate_openapi_client( openapi_spec_file: str, target_language: str, work_dir: str = DEFAULT_TEMP_DIR, client_generator: OpenApiGenerator = OpenApiGenerator.OPENAPI_CODEGEN, additional_properties: str = "", additional_flags: str = "", ) -> Union[str, None]: """Generate an open api client. The passed OpenAPI specification file will be taken to generate a client using the passed openapi-generator for the given programming language and optional additional properties (see the respective openapi cli for more information). The client will be generated at the passed `work_dir` directory. Args: openapi_spec_file (str): The OpenAPI specification for which the client will be generated. target_language (str): The client's programming language (e.g. `"javascript"`). work_dir (str, optional): The directory in which the generator cli will be looked for and also the generated client will be placed. If it does not exist, it will be created. client_generator (OpenApiGenerator, optional): The OpenApiGenerator which will be used to generate the client. It will check whether the cli can be found within the `work_dir` directory and if not it will try to download it according to the `_check_and_download_generator_cli` function. additional_properties (str, optional): Additional properties passed to the OpenAPI generator client client (e.g. `"useES6=true"`) Returns: Union[str, None]: Returns the output path if the client generation was successful and None otherwise. """ pathlib.Path(work_dir).mkdir(exist_ok=True) codegen_cli_path = f"{work_dir}/{client_generator.cli_name}" is_successful = _check_and_download_generator_cli( codegen_cli_path, client_generator=client_generator) if not is_successful: return None if not pathlib.Path(openapi_spec_file).is_file(): build_utils.log( f"The OpenAPI spec file {openapi_spec_file} does not exist") return None build_utils.run( client_generator.get_generate_command( openapi_spec_file=openapi_spec_file, target_language=target_language, work_dir=work_dir, additional_properties=additional_properties, additional_flags=additional_flags, )) return client_generator.get_output_path(work_dir=work_dir)
def test_arguments_passed_to_submodule_error(self, cli_args_string: str): """Tests whether the cli args passed to a module will be passed in the same way to a submodule. The reason is that the argparser converts all dashes to underscores.""" HERE = os.path.abspath(os.path.dirname(__file__)) build_file_path = os.path.join(HERE, "build.py") completed_process = build_utils.run( f"python -u {build_file_path} --my-token=111 --deployment-token=666 --my_bool", exit_on_error=False, ) assert completed_process.returncode != 0
def lint_markdown(markdownlint: bool = True, exit_on_error: bool = True) -> None: """Run markdownlint on markdown documentation. Args: markdownlint (bool, optional): Activate markdown linting via `markdownlint`. Defaults to `True`. exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ build_utils.log("Run linters and style checks:") if markdownlint and build_utils.command_exists( "markdownlint", exit_on_error=exit_on_error): config_file_arg = "" if os.path.exists(".markdown-lint.yml"): config_file_arg = "--config='.markdown-lint.yml'" build_utils.run(f"markdownlint {config_file_arg} ./docs", exit_on_error=exit_on_error)
def test_arguments_passed_as_env_to_submodule(self): """Tests whether the args passed as env variables to a module correctly to submodules.""" HERE = os.path.abspath(os.path.dirname(__file__)) build_file_path = os.path.join(HERE, "build.py") completed_process = build_utils.run( f"MY_TOKEN=111 python -u {build_file_path} --deployment-token 666 --my_bool", exit_on_error=False, ) assert completed_process.returncode == 0
def generate_api_docs( github_url: str, main_package: str, command_prefix: str = "pipenv run", exit_on_error: bool = True, ) -> None: """Generates API documentation via lazydocs. Args: github_url (str): Github URL main_package (str): The main package name to use for docs generation. command_prefix (str, optional): Prefix to use for all commands. Defaults to `pipenv run`. exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ build_utils.run( f"{command_prefix} lazydocs --overview-file=README.md" f" --src-base-url={github_url}/blob/main {main_package}", exit_on_error=exit_on_error, )
def run_dev_mode(port: int = 8001, exit_on_error: bool = True) -> None: """Run mkdocs development server. Args: port (int, optional): Port to use for mkdocs development server. Defaults to 8001. exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ build_utils.log(f"Run docs in development mode (http://localhost:{port}):") command_prefix = "" if is_pipenv_environment(): command_prefix = _PIPENV_RUN else: # Check mkdocs command build_utils.command_exists("mkdocs", exit_on_error=exit_on_error) build_utils.run( f"{command_prefix} mkdocs serve --dev-addr 0.0.0.0:{port}", exit_on_error=exit_on_error, )
def deploy_gh_pages(exit_on_error: bool = True) -> None: """Deploy mkdocs documentation to Github pages. Args: exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ build_utils.log("Deploy documentation to Github pages:") command_prefix = "" if is_pipenv_environment(): command_prefix = _PIPENV_RUN else: # Check mkdocs command build_utils.command_exists("mkdocs", exit_on_error=exit_on_error) build_utils.run( f"{command_prefix} mkdocs gh-deploy --clean", exit_on_error=exit_on_error, timeout=120, )
def lint_dockerfile( hadolint: bool = True, dockerfile: str = "Dockerfile", exit_on_error: bool = True ) -> None: """Run hadolint on the Dockerfile. Args: hadolint (bool, optional): Activate hadolint dockerfile linter. Defaults to `True`. dockerfile (str, optional): Specify a specific Dockerfile. If not specified, the default `Dockerfile` wil be used. exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ build_utils.log("Run linters and style checks:") if hadolint and build_utils.command_exists("hadolint", exit_on_error=exit_on_error): config_file_arg = "" if os.path.exists(".hadolint.yml"): config_file_arg = "--config=.hadolint.yml" build_utils.run( f"hadolint {config_file_arg} {dockerfile}", exit_on_error=exit_on_error )
def install_build_env(exit_on_error: bool = True) -> None: """Installs a new virtual environment via pipenv. Args: exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ # Check if pipenv exists build_utils.command_exists("pipenv", exit_on_error=exit_on_error) if not os.path.exists("Pipfile"): build_utils.log( "No Pipfile discovered, cannot install pipenv environemnt") if exit_on_error: build_utils.exit_process(1) return build_utils.run("pipenv --rm", exit_on_error=False) build_utils.run( f"pipenv install --dev --python={sys.executable} --skip-lock --site-packages", exit_on_error=exit_on_error, )
def release_docker_image( name: str, version: str, docker_image_prefix: str, exit_on_error: bool = True ) -> subprocess.CompletedProcess: """Push a Docker image to a repository. Args: name (str): The name of the image. Must not be prefixed! version (str): The tag used for the image. docker_image_prefix (str): The prefix added to the name to indicate an organization on DockerHub or a completely different repository. exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. Returns: subprocess.CompletedProcess: Returns the CompletedProcess object of the `docker push ...` command. """ # Check if docker exists on the system build_utils.command_exists("docker", exit_on_error=exit_on_error) if not docker_image_prefix: build_utils.log( "The flag --docker-image-prefix cannot be blank when pushing a Docker image." ) build_utils.exit_process(build_utils.EXIT_CODE_GENERAL) versioned_tag = get_image_name(name=name, tag=version) remote_versioned_tag = get_image_name( name=name, tag=version, image_prefix=docker_image_prefix ) build_utils.run( "docker tag " + versioned_tag + " " + remote_versioned_tag, exit_on_error=exit_on_error, ) completed_process = build_utils.run( "docker push " + remote_versioned_tag, exit_on_error=exit_on_error ) if completed_process.returncode > 0: build_utils.log(f"Failed to release Docker image {name}:{version}") # Only push version with latest tag if no suffix is added (pre-release) if "-" not in version: remote_latest_tag = get_image_name( name=name, tag="latest", image_prefix=docker_image_prefix ) build_utils.log( "Release Docker image with latest tag as well: " + remote_latest_tag ) build_utils.run( "docker tag " + versioned_tag + " " + remote_latest_tag, exit_on_error=exit_on_error, ) build_utils.run("docker push " + remote_latest_tag, exit_on_error=exit_on_error) return completed_process
def test_with_py_version(python_version: str, exit_on_error: bool = True) -> None: """Run pytest in a environment wiht the specified python version. Args: python_version (str): Python version to use inside the virutal environment. exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ if not os.path.exists("Pipfile"): build_utils.log( "No Pipfile discovered. Testing with specific python version only works with pipenv." ) return # Check if pyenv command exists build_utils.command_exists("pyenv", exit_on_error=exit_on_error) # Check if pipenv command exists build_utils.command_exists("pipenv", exit_on_error=exit_on_error) # Install pipenv environment with specific versio build_utils.run( f"pyenv install --skip-existing {python_version} && pyenv local {python_version}", exit_on_error=exit_on_error, ) # Install pipenv environment with specific version build_utils.run( f"pipenv install --dev --python={python_version} --skip-lock", exit_on_error=exit_on_error, ) # Run pytest in pipenv environment build_utils.run("pipenv run pytest", exit_on_error=exit_on_error) # Remove enviornment build_utils.run("pipenv --rm", exit_on_error=False) # Uninstall pyenv version build_utils.run( f"pyenv local --unset && pyenv uninstall -f {python_version}", exit_on_error=False, )
def publish_pypi_distribution( pypi_token: str, pypi_user: str = "__token__", pypi_repository: Optional[str] = None ) -> None: """Publish distribution to pypi. Args: pypi_token (str): Token of PyPi repository. pypi_user (str, optional): User of PyPi repository. Defaults to "__token__". pypi_repository (Optional[str], optional): PyPi repository. If `None` provided, use the production instance. """ if not pypi_token: build_utils.log("PyPI token is required for release (--pypi-token=<TOKEN>)") build_utils.exit_process(1) pypi_repository_args = "" if pypi_repository: pypi_repository_args = f'--repository-url "{pypi_repository}"' # Publish on pypi build_utils.run( f'twine upload --non-interactive -u "{pypi_user}" -p "{pypi_token}" {pypi_repository_args} dist/*', exit_on_error=True, )
def build_distribution() -> None: """Build python package distribution.""" try: # Ensure there are no old builds rmtree("./dist") except OSError: pass try: # Ensure there are no old builds rmtree("./build") except OSError: pass # Build the distribution archives build_utils.run( "python setup.py sdist bdist_wheel clean --all", exit_on_error=True, ) # Check the archives with twine build_utils.run("twine check dist/*", exit_on_error=True)
def test_with_py_version(python_version: str, exit_on_error: bool = True) -> None: """Run pytest in a environment wiht the specified python version. Args: python_version (str): Python version to use inside the virutal environment. exit_on_error (bool, optional): Exit process if an error occurs. Defaults to `True`. """ # Install pipenv environment with specific versio build_utils.run( f"pyenv install --skip-existing {python_version} && pyenv local {python_version}", exit_on_error=exit_on_error, ) # Install pipenv environment with specific version build_utils.run( f"pipenv install --dev --python={python_version} --skip-lock", exit_on_error=exit_on_error, ) # Run pytest in pipenv environment build_utils.run("pipenv run pytest", exit_on_error=exit_on_error) # Remove enviornment build_utils.run("pipenv --rm") # Uninstall pyenv version build_utils.run(f"pyenv local --unset && pyenv uninstall -f {python_version}")
def is_pipenv_environment() -> bool: """Check if current working directory is a valid pipenv environment.""" if not os.path.exists("Pipfile"): return False if not build_utils.command_exists("pipenv"): return False return (build_utils.run( "pipenv --venv", disable_stderr_logging=True, disable_stdout_logging=True, exit_on_error=False, ).returncode == 0)
def main(args: dict) -> None: # set current path as working dir os.chdir(HERE) if args.get(build_utils.FLAG_MAKE): # Install pipenv dev requirements build_mkdocs.install_build_env(exit_on_error=True) # Build mkdocs documentation build_mkdocs.build_mkdocs(exit_on_error=True) if args.get(build_utils.FLAG_CHECK): # TODO: Markdown linting # build_mkdocs.lint_markdown(exit_on_error=False) pass if args.get(build_utils.FLAG_RELEASE): # TODO: Do not publish project-template # Deploy to Github pages # build_mkdocs.deploy_gh_pages(exit_on_error=True) # Lock pipenv requirements build_utils.run("pipenv lock", exit_on_error=False) if args.get(build_utils.FLAG_RUN): build_mkdocs.run_dev_mode(exit_on_error=True)
def install_build_env() -> None: """Installs a new virtual environment via pipenv.""" build_utils.run("pipenv --rm") build_utils.run( f"pipenv install --dev --python={sys.executable} --skip-lock", exit_on_error=True, ) # Show current environment build_utils.run("pipenv graph", exit_on_error=False)