def compile_fragment_files( self, write_fs: FS, found_fragments: Iterable[FoundFragment]) -> List[str]: """ Compile fragment files into `parent_dir`. """ outputs = [] for version_fs, filename in found_fragments: try: fragment = self.load_fragment(version_fs.readtext(filename)) fragment_type = fragment.get('type') showcontent = self.config.fragment_types.get( fragment_type, {}).get('showcontent', True) section = fragment.get('section') or None rendered_content = render_fragment( fragment, showcontent, self.config.changelog_output_type) if rendered_content.strip(): filename_stem = splitext(basename(filename))[0] output_path = join(*filter(None, [ section, '{}.{}'.format(filename_stem, fragment_type) ])) log.info('Compiling {} -> {}'.format( version_fs.getsyspath(filename), write_fs.getsyspath(output_path))) parent_dir = dirname(output_path) if parent_dir: write_fs.makedirs(parent_dir, recreate=True) write_fs.writetext(output_path, rendered_content) outputs.append(output_path) except Exception: raise FragmentCompilationError(filename) return outputs
def render_changelog(self, fs: FS, version: VersionInfo, version_date: date) -> str: """ Find compiled fragments in `parent_dir` and render a changelog with them. """ parent_dir = fs.getsyspath('.') return render_changelog(parent_dir, self.config.changelog_output_type, self.config._towncrier_sections(parent_dir), self.config._towncrier_fragment_types(), self.config._towncrier_underlines(), project_version=str(version), project_date=version_date.isoformat())
def _preprocess_images(base_fs, output_fs: FS, split): fname = "{}-image-tmp".format(split) result_path = output_fs.getsyspath(fname) batch_size = 32 img_ds = RawImageDataset(base_fs, split) dataloader = data.DataLoader(img_ds, batch_size=batch_size, num_workers=2) result_size = (len(img_ds), 1024, 14, 14) result = np.memmap(result_path, np.float32, "w+", shape=result_size) resnet = get_resnet().to(config.torch_device()) with torch.no_grad(): progbar = tqdm(dataloader, desc="Preprocessing images -- {}".format(split)) for ix, img in enumerate(progbar): img = resnet(img.to(config.torch_device())).cpu().numpy() result[ix * batch_size:(ix + 1) * batch_size] = img output_fs.move(fname, "{}-image".format(split), True)
def write_to_bento(self, bento_fs: FS, build_ctx: str): py_folder = fs.path.join("env", "python") wheels_folder = fs.path.join(py_folder, "wheels") bento_fs.makedirs(py_folder, recreate=True) # Save the python version of current build environment with bento_fs.open(fs.path.join(py_folder, "version.txt"), "w") as f: f.write(PYTHON_VERSION) # Move over required wheel files # Note: although wheel files outside of build_ctx will also work, we should # discourage users from doing that if self.wheels is not None: for whl_file in self.wheels: # pylint: disable=not-an-iterable whl_file = resolve_user_filepath(whl_file, build_ctx) copy_file_to_fs_folder(whl_file, bento_fs, wheels_folder) # If BentoML is installed in editable mode, build bentoml whl and save to Bento build_bentoml_whl_to_target_if_in_editable_mode( bento_fs.getsyspath(wheels_folder)) if self.requirements_txt is not None: requirements_txt_file = resolve_user_filepath( self.requirements_txt, build_ctx) copy_file_to_fs_folder( requirements_txt_file, bento_fs, py_folder, dst_filename="requirements.txt", ) elif self.packages is not None: with bento_fs.open(fs.path.join(py_folder, "requirements.txt"), "w") as f: f.write("\n".join(self.packages)) else: # Return early if no python packages were specified return pip_args: t.List[str] = [] if self.no_index: pip_args.append("--no-index") if self.index_url: pip_args.append(f"--index-url={self.index_url}") if self.trusted_host: for item in self.trusted_host: # pylint: disable=not-an-iterable pip_args.append(f"--trusted-host={item}") if self.find_links: for item in self.find_links: # pylint: disable=not-an-iterable pip_args.append(f"--find-links={item}") if self.extra_index_url: for item in self.extra_index_url: # pylint: disable=not-an-iterable pip_args.append(f"--extra-index-url={item}") if self.pip_args: # Additional user provided pip_args pip_args.append(self.pip_args) # write pip install args to a text file if applicable if pip_args: with bento_fs.open(fs.path.join(py_folder, "pip_args.txt"), "w") as f: f.write(" ".join(pip_args)) if self.lock_packages: # Note: "--allow-unsafe" is required for including setuptools in the # generated requirements.lock.txt file, and setuptool is required by # pyfilesystem2. Once pyfilesystem2 drop setuptools as dependency, we can # remove the "--allow-unsafe" flag here. # Note: "--generate-hashes" is purposefully not used here because it will # break if user includes PyPI package from version control system pip_compile_in = bento_fs.getsyspath( fs.path.join(py_folder, "requirements.txt")) pip_compile_out = bento_fs.getsyspath( fs.path.join(py_folder, "requirements.lock.txt")) pip_compile_args = ([pip_compile_in] + pip_args + [ "--quiet", "--allow-unsafe", "--no-header", f"--output-file={pip_compile_out}", ]) logger.info("Locking PyPI package versions..") click_ctx = pip_compile_cli.make_context("pip-compile", pip_compile_args) try: pip_compile_cli.invoke(click_ctx) except Exception as e: logger.error(f"Failed locking PyPI packages: {e}") logger.error( "Falling back to using user-provided package requirement specifier, equivalent to `lock_packages=False`" )