コード例 #1
0
 def compile_fragment_files(
         self, write_fs: FS,
         found_fragments: Iterable[FoundFragment]) -> List[str]:
     """
     Compile fragment files into `parent_dir`.
     """
     outputs = []
     for version_fs, filename in found_fragments:
         try:
             fragment = self.load_fragment(version_fs.readtext(filename))
             fragment_type = fragment.get('type')
             showcontent = self.config.fragment_types.get(
                 fragment_type, {}).get('showcontent', True)
             section = fragment.get('section') or None
             rendered_content = render_fragment(
                 fragment, showcontent, self.config.changelog_output_type)
             if rendered_content.strip():
                 filename_stem = splitext(basename(filename))[0]
                 output_path = join(*filter(None, [
                     section, '{}.{}'.format(filename_stem, fragment_type)
                 ]))
                 log.info('Compiling {} -> {}'.format(
                     version_fs.getsyspath(filename),
                     write_fs.getsyspath(output_path)))
                 parent_dir = dirname(output_path)
                 if parent_dir:
                     write_fs.makedirs(parent_dir, recreate=True)
                 write_fs.writetext(output_path, rendered_content)
                 outputs.append(output_path)
         except Exception:
             raise FragmentCompilationError(filename)
     return outputs
コード例 #2
0
 def render_changelog(self, fs: FS, version: VersionInfo,
                      version_date: date) -> str:
     """
     Find compiled fragments in `parent_dir` and render a changelog with
     them.
     """
     parent_dir = fs.getsyspath('.')
     return render_changelog(parent_dir,
                             self.config.changelog_output_type,
                             self.config._towncrier_sections(parent_dir),
                             self.config._towncrier_fragment_types(),
                             self.config._towncrier_underlines(),
                             project_version=str(version),
                             project_date=version_date.isoformat())
コード例 #3
0
def _preprocess_images(base_fs, output_fs: FS, split):
    fname = "{}-image-tmp".format(split)
    result_path = output_fs.getsyspath(fname)

    batch_size = 32
    img_ds = RawImageDataset(base_fs, split)
    dataloader = data.DataLoader(img_ds, batch_size=batch_size, num_workers=2)
    result_size = (len(img_ds), 1024, 14, 14)
    result = np.memmap(result_path, np.float32, "w+", shape=result_size)

    resnet = get_resnet().to(config.torch_device())

    with torch.no_grad():
        progbar = tqdm(dataloader,
                       desc="Preprocessing images -- {}".format(split))
        for ix, img in enumerate(progbar):
            img = resnet(img.to(config.torch_device())).cpu().numpy()
            result[ix * batch_size:(ix + 1) * batch_size] = img

    output_fs.move(fname, "{}-image".format(split), True)
コード例 #4
0
    def write_to_bento(self, bento_fs: FS, build_ctx: str):
        py_folder = fs.path.join("env", "python")
        wheels_folder = fs.path.join(py_folder, "wheels")
        bento_fs.makedirs(py_folder, recreate=True)

        # Save the python version of current build environment
        with bento_fs.open(fs.path.join(py_folder, "version.txt"), "w") as f:
            f.write(PYTHON_VERSION)

        # Move over required wheel files
        # Note: although wheel files outside of build_ctx will also work, we should
        # discourage users from doing that
        if self.wheels is not None:
            for whl_file in self.wheels:  # pylint: disable=not-an-iterable
                whl_file = resolve_user_filepath(whl_file, build_ctx)
                copy_file_to_fs_folder(whl_file, bento_fs, wheels_folder)

        # If BentoML is installed in editable mode, build bentoml whl and save to Bento
        build_bentoml_whl_to_target_if_in_editable_mode(
            bento_fs.getsyspath(wheels_folder))

        if self.requirements_txt is not None:
            requirements_txt_file = resolve_user_filepath(
                self.requirements_txt, build_ctx)
            copy_file_to_fs_folder(
                requirements_txt_file,
                bento_fs,
                py_folder,
                dst_filename="requirements.txt",
            )
        elif self.packages is not None:
            with bento_fs.open(fs.path.join(py_folder, "requirements.txt"),
                               "w") as f:
                f.write("\n".join(self.packages))
        else:
            # Return early if no python packages were specified
            return

        pip_args: t.List[str] = []
        if self.no_index:
            pip_args.append("--no-index")
        if self.index_url:
            pip_args.append(f"--index-url={self.index_url}")
        if self.trusted_host:
            for item in self.trusted_host:  # pylint: disable=not-an-iterable
                pip_args.append(f"--trusted-host={item}")
        if self.find_links:
            for item in self.find_links:  # pylint: disable=not-an-iterable
                pip_args.append(f"--find-links={item}")
        if self.extra_index_url:
            for item in self.extra_index_url:  # pylint: disable=not-an-iterable
                pip_args.append(f"--extra-index-url={item}")
        if self.pip_args:
            # Additional user provided pip_args
            pip_args.append(self.pip_args)

        # write pip install args to a text file if applicable
        if pip_args:
            with bento_fs.open(fs.path.join(py_folder, "pip_args.txt"),
                               "w") as f:
                f.write(" ".join(pip_args))

        if self.lock_packages:
            # Note: "--allow-unsafe" is required for including setuptools in the
            # generated requirements.lock.txt file, and setuptool is required by
            # pyfilesystem2. Once pyfilesystem2 drop setuptools as dependency, we can
            # remove the "--allow-unsafe" flag here.

            # Note: "--generate-hashes" is purposefully not used here because it will
            # break if user includes PyPI package from version control system
            pip_compile_in = bento_fs.getsyspath(
                fs.path.join(py_folder, "requirements.txt"))
            pip_compile_out = bento_fs.getsyspath(
                fs.path.join(py_folder, "requirements.lock.txt"))
            pip_compile_args = ([pip_compile_in] + pip_args + [
                "--quiet",
                "--allow-unsafe",
                "--no-header",
                f"--output-file={pip_compile_out}",
            ])
            logger.info("Locking PyPI package versions..")
            click_ctx = pip_compile_cli.make_context("pip-compile",
                                                     pip_compile_args)
            try:
                pip_compile_cli.invoke(click_ctx)
            except Exception as e:
                logger.error(f"Failed locking PyPI packages: {e}")
                logger.error(
                    "Falling back to using user-provided package requirement specifier, equivalent to `lock_packages=False`"
                )