def destroy(self) -> None: run_process([ 'gcloud', 'compute', '--project', self.gcp_project_id, 'routers', 'nats', 'delete', self.nat_name, '--router', self.router.name, '--quiet' ]) self.router.secure_destroy()
def pip_compile( requiremenets: Path, *, dry_run=False, verbose=False, upgrade=False, upgrade_package="", prereleases=False, rebuild=False, extra_args=(), ): """Wraps 'pip-tools' command. Include hash of source file into the generated one.""" requirements_txt = requiremenets.with_suffix(".txt") requirements_in = requiremenets.with_suffix(".in") logger.info("Compile requirements file %s ...", requirements_in) with tempfile.NamedTemporaryFile('w+t', prefix=f"{requirements_in.stem}-", suffix=".txt", delete=False) as txt_file: txt_path = Path(txt_file.name) if requirements_txt.exists() and not rebuild: txt_path.write_bytes(requirements_txt.read_bytes()) bf_commons.run_process([ "pip-compile", "--no-header", *(["-o", txt_path] if not dry_run else []), *(["--dry-run"] if dry_run else []), *(["--rebuild"] if rebuild else []), *(["--upgrade"] if upgrade else []), *(["--pre"] if prereleases else []), *(["--upgrade-package", upgrade_package] if upgrade_package else []), *(["-v"] if verbose else ["-q"]), *extra_args, str(requirements_in), ], check=True) reqs_content = txt_path.read_text() if dry_run: return source_hash = compute_requirements_in_hash(requirements_in) with open(requirements_txt, 'w+t') as out: logger.info("Write pip requirements file: %s", requirements_txt) out.write( textwrap.dedent(f"""\ # *** autogenerated: don't edit *** # $source-hash: {source_hash} # $source-file: {requirements_in} # # run 'bigflow build-requirements {requirements_in}' to update this file """)) out.write(reqs_content)
def _export_docker_image_to_file(tag: str, target_dir: Path, version: str): image_target_path = target_dir / f"image-{version}.tar" logger.info("Exporting the image to %s ...", image_target_path) bf_commons.run_process([ "docker", "image", "save", "-o", image_target_path, bf_commons.get_docker_image_id(tag) ])
def destroy(self) -> None: run_process([ 'gcloud', 'composer', 'environments', 'delete', self.composer_name, '--location', self.region, '--project', self.gcp_project_id, '--quiet' ]) self.cloud_nat.secure_destroy()
def _read_setuppy_args(path_to_setup: Path) -> dict: logger.info("Read project options from %s", path_to_setup) with tempfile.NamedTemporaryFile("r+b") as f: bf_commons.run_process( ["python", path_to_setup, DUMP_PARAMS_SETUPPY_CMDARG, f.name], cwd=str(path_to_setup.parent)) return pickle.load(f)
def create(self) -> None: self.router.create() run_process([ 'gcloud', 'compute', '--project', self.gcp_project_id, 'routers', 'nats', 'create', self.nat_name, f'--router={self.router.name}', '--auto-allocate-nat-external-ips', '--nat-all-subnet-ip-ranges', '--enable-logging' ])
def _cli_build_package(): validate_project_setup() bf_commons.run_process([ "python", bigflow.build.dev.find_setuppy(), "build_project", "--build-package", ])
def _cli_build(args): validate_project_setup() cmd = [ "python", bigflow.build.dev.find_setuppy(), "build_project", ] if _is_workflow_selected(args): cmd.append('--workflow') cmd.append(args.workflow) if _is_starttime_selected(args): cmd.append('--start-time') cmd.append(args.start_time) bf_commons.run_process(cmd)
def load_image_from_tar(image_tar_path: str) -> str: logger.info("Load docker image from %s...", image_tar_path) for line in bf_commons.run_process( ['docker', 'load', '-i', image_tar_path]).split('\n'): if 'Loaded image ID:' in line: return line.split()[-1].split(':')[-1] raise ValueError(f"Can't load image: {image_tar_path}")
def _read_setuppy_args(path_to_setup: Path) -> dict: logger.info("Read project options from %s", path_to_setup) with tempfile.NamedTemporaryFile("r+b") as f: bf_commons.run_process( ["python", path_to_setup, DUMP_PARAMS_SETUPPY_CMDARG, f.name], cwd=str(path_to_setup.parent)) params = pickle.load(f) legacy_project_name = _read_project_name_from_setup_legacy( path_to_setup.parent) if legacy_project_name and params.get('name') != legacy_project_name: logging.error( "Project name mismatch: setup.PROJECT_NAME == %r, " "but setup(name=%r). It is recommended to remove 'PROJECT_NAME' variable from 'project_setup.py'", legacy_project_name, params.get('name')) return params
def validate_project_setup(): cmd = [ "python", bigflow.build.dev.find_setuppy(), "build_project", "--validate-project-setup", ] output = bf_commons.run_process(cmd) if bigflow.build.dist.SETUP_VALIDATION_MESSAGE not in output: raise ValueError('The `setup.py` is invalid. Check the documentation how to create a valid `setup.py`: https://github.com/allegro/bigflow/blob/master/docs/build.md')
def pip_compile( req: Path, verbose=False, extra_args=(), ): """Wraps 'pip-tools' command. Include hash of source file into the generated one.""" req_txt = req.with_suffix(".txt") req_in = req.with_suffix(".in") logger.info("Compile requirements file %s ...", req_in) with tempfile.NamedTemporaryFile('w+t', prefix=f"{req_in.stem}-", suffix=".txt", delete=False) as txt_file: bf_commons.run_process([ "pip-compile", "--no-header", "-o", txt_file.name, *(["-v"] if verbose else ["-q"]), *extra_args, str(req_in), ]) with open(txt_file.name) as ff: reqs_content = ff.readlines() source_hash = bf_commons.generate_file_hash(req_in) with open(req_txt, 'w+t') as out: logger.info("Write pip requirements file: %s", req_txt) out.write( textwrap.dedent(f"""\ # *** AUTO GENERATED: DON'T EDIT *** # $source-hash: {source_hash} # $source-file: {req_in} # # run 'bigflow build-requirements {req_in}' to update this file """)) out.writelines(reqs_content)
def run_tests(project_spec: BigflowProjectSpec): logger.info('Runing tests...') output_dir = "build/junit-reports" try: bf_commons.run_process([ "python", "-m", "xmlrunner", "discover", "-s", ".", "-t", project_spec.project_dir, "-o", output_dir, ]) except subprocess.CalledProcessError: logger.error("Test suite was FAILED") exit(1) logger.info("Test suite was PASSED")
def _deploy_image_loaded_to_local_registry( build_ver, docker_repository, image_id, auth_method, vault_endpoint, vault_secret, ): tag_image(image_id, docker_repository, build_ver) docker_image = docker_repository + ":" + build_ver docker_image_latest = docker_repository + ":latest" logger.info("Deploying docker image tag=%s auth_method=%s", docker_image, auth_method) if auth_method == 'local_account': bf_commons.run_process(['gcloud', 'auth', 'configure-docker']) elif auth_method == 'vault': oauthtoken = get_vault_token(vault_endpoint, vault_secret) bf_commons.run_process( [ 'docker', 'login', '-u', 'oauth2accesstoken', '--password-stdin', 'https://eu.gcr.io' ], input=oauthtoken, ) else: raise ValueError('unsupported auth_method: ' + auth_method) bf_commons.run_process( ['docker', 'push', docker_image, docker_image_latest]) return docker_image
def _build_dist_package( project_name: str, suffix: str, cmdname: str, exargs: List[str], ): """Locates and runs 'bdist_*' command on 'setup.py'""" if project_name is None: project_name = infer_project_name(stack=3) with tempfile.TemporaryDirectory() as workdir: setuppy = materialize_setuppy(project_name, workdir) distdir = Path(workdir) / "dist" logger.info("Run setup.py %s", cmdname) fd, result_path = tempfile.mkstemp(suffix=suffix) result_path = Path(result_path) os.close(fd) bfc.run_process( [ "python", setuppy, cmdname, "--dist-dir", distdir, *(exargs or []), ], cwd=str(setuppy.parent), ) result_tmp = _expect_single_file(distdir, "*" + suffix) logger.debug("Rename %s to %s", result_tmp, result_path) result_tmp.rename(result_path) logger.info("Built package located at %s", result_path) return result_path
def read_setuppy_args(path_to_setup: Union[Path, str, None] = None) -> dict: """Loads `setup.py`, returns all parameters of `bigflow.build.setup()` function. This function doesn't unpack 'embeeded sdist' archive when package is installed via pip. You could use `bigflow.build.materialize_setuppy` for such purposes, although it is not recommended""" path_to_setup = path_to_setup or find_setuppy() logger.info("Read project options from %s", path_to_setup) with tempfile.NamedTemporaryFile("r+b") as f: bf_commons.run_process( ["python", path_to_setup, DUMP_PARAMS_SETUPPY_CMDARG, f.name], cwd=str(path_to_setup.parent)) params = pickle.load(f) legacy_project_name = _read_project_name_from_setup_legacy( path_to_setup.parent) if legacy_project_name and params.get('name') != legacy_project_name: logging.error( "Project name mismatch: setup.PROJECT_NAME == %r, " "but setup(name=%r). It is recommended to remove 'PROJECT_NAME' variable from 'project_setup.py'", legacy_project_name, params.get('name')) return params
def run_tests(project_dir: Path, build_dir: Path, test_package: Path): output_dir = build_dir / 'junit-reports' try: return bf_commons.run_process([ "python", "-m", "xmlrunner", "discover", "-s", test_package, "-t", project_dir, "-o", output_dir, ]) except subprocess.CalledProcessError: raise ValueError("Test suite failed.")
def destroy(self) -> None: run_process([ 'gcloud', 'compute', '--project', self.gcp_project_id, 'routers', 'delete', self.router_name, '--region', self.region, '--quiet' ])
def tag_image(image_id, repository, tag): return bf_commons.run_process( ["docker", "tag", image_id, f"{repository}:{tag}"])
def build_docker_image(project_dir: Path, tag: str): print('Building a Docker image. It might take a while.') bf_commons.run_process(f'docker build {project_dir} --tag {tag}')
def create(self) -> None: self.cloud_nat.create() run_process( _composer_create_command(self.composer_name, self.gcp_project_id, self.region, self.zone, self.environment_name))
def _build_docker_image(project_dir: Path, tag: str): logger.debug("Run docker build...") bf_commons.run_process(f'docker build {project_dir} --tag {tag}')
def create(self) -> None: run_process([ 'gcloud', 'compute', '--project', self.gcp_project_id, 'routers', 'create', self.router_name, '--network', 'default', '--region', self.region ])