def file_reader(*, fs: FS, resource: str, mimetype: str, proxy: bool = False, mmap_mode: Union[str, None] = None, **kwargs) -> Any: if mimetype is None and resource.endswith(".npy"): mimetype = "application/octet-stream" elif mimetype is None: mimetype, _ = guess_type(resource) if isinstance(resource, Path): resource = str(resource) mapping = { "application/octet-stream": ( np.load, "file", { "file": fs.open(resource, mode="rb"), "mmap_mode": mmap_mode, "allow_pickle": False, }, ), "application/json": ( json.load, "fp", { "fp": fs.open(resource, encoding="utf-8") }, ), "text/csv": ( pd.read_csv, "filepath_or_buffer", { "filepath_or_buffer": fs.open(resource) }, ), } try: func, label, kwargs = mapping[mimetype] except KeyError: raise InvalidMimetype("Mimetype '{}' not understoof".format(mimetype)) if proxy: return Proxy(func, label, kwargs) else: return func(**kwargs)
def write_to_bento(self, bento_fs: FS, build_ctx: str): conda_folder = fs.path.join("env", "conda") bento_fs.makedirs(conda_folder, recreate=True) if self.environment_yml is not None: environment_yml_file = resolve_user_filepath( self.environment_yml, build_ctx) copy_file_to_fs_folder( environment_yml_file, bento_fs, conda_folder, dst_filename="environment_yml", ) return deps_list = [] if self.dependencies is None else self.dependencies if self.pip is not None: deps_list.append(dict(pip=self.pip)) # type: ignore if not deps_list: return yaml_content = dict(dependencies=deps_list) yaml_content["channels"] = (["defaults"] if self.channels is None else self.channels) with bento_fs.open(fs.path.join(conda_folder, "environment_yml"), "w") as f: yaml.dump(yaml_content, f)
def write_to_bento(self, bento_fs: FS, build_ctx: str): docker_folder = fs.path.join("env", "docker") bento_fs.makedirs(docker_folder, recreate=True) dockerfile = fs.path.join(docker_folder, "Dockerfile") template_file = os.path.join(os.path.dirname(__file__), "docker", "Dockerfile.template") with open(template_file, "r", encoding="utf-8") as f: dockerfile_template = f.read() with bento_fs.open(dockerfile, "w") as dockerfile: dockerfile.write( dockerfile_template.format( base_image=self.get_base_image_tag())) for filename in ["init.sh", "entrypoint.sh"]: copy_file_to_fs_folder( os.path.join(os.path.dirname(__file__), "docker", filename), bento_fs, docker_folder, ) if self.setup_script: try: setup_script = resolve_user_filepath(self.setup_script, build_ctx) except FileNotFoundError as e: raise InvalidArgument(f"Invalid setup_script file: {e}") copy_file_to_fs_folder(setup_script, bento_fs, docker_folder, "setup_script")
def file_writer(*, data: Any, fs: FS, resource: str, mimetype: str, **kwargs) -> None: if isinstance(resource, Path): resource = str(resource) if mimetype == "application/octet-stream": return np.save(fs.open(resource, mode="wb"), data, allow_pickle=False) elif mimetype == "application/json": return json.dump( data, fs.open(resource, mode="w", encoding="utf-8"), indent=2, ensure_ascii=False, ) elif mimetype == "text/csv": assert isinstance(data, pd.DataFrame) data.to_csv(fs.open(resource, mode="w", encoding="utf-8"))
def package_json(cwd_fs: FS): """ Try guess a version from ``package.json``. """ log.debug('Looking for package.json') if cwd_fs.exists('package.json'): log.debug('Guessing version with package.json') try: with cwd_fs.open('package.json', 'r') as fd: return json.load(fd).get('version') except json.JSONDecodeError: pass return None
def from_fs(cls, item_fs: FS) -> "Model": try: with item_fs.open(MODEL_YAML_FILENAME, "r") as model_yaml: info = ModelInfo.from_yaml_file(model_yaml) except fs.errors.ResourceNotFound: raise BentoMLException( f"Failed to load bento model because it does not contain a '{MODEL_YAML_FILENAME}'" ) res = cls(info.tag, item_fs, info) if not res.validate(): raise BentoMLException( f"Failed to load bento model because it contains an invalid '{MODEL_YAML_FILENAME}'" ) res._info_flushed = True res._custom_objects_flushed = True return res
def write_to_bento(self, bento_fs: FS, build_ctx: str): py_folder = fs.path.join("env", "python") wheels_folder = fs.path.join(py_folder, "wheels") bento_fs.makedirs(py_folder, recreate=True) # Save the python version of current build environment with bento_fs.open(fs.path.join(py_folder, "version.txt"), "w") as f: f.write(PYTHON_VERSION) # Move over required wheel files # Note: although wheel files outside of build_ctx will also work, we should # discourage users from doing that if self.wheels is not None: for whl_file in self.wheels: # pylint: disable=not-an-iterable whl_file = resolve_user_filepath(whl_file, build_ctx) copy_file_to_fs_folder(whl_file, bento_fs, wheels_folder) # If BentoML is installed in editable mode, build bentoml whl and save to Bento build_bentoml_whl_to_target_if_in_editable_mode( bento_fs.getsyspath(wheels_folder)) if self.requirements_txt is not None: requirements_txt_file = resolve_user_filepath( self.requirements_txt, build_ctx) copy_file_to_fs_folder( requirements_txt_file, bento_fs, py_folder, dst_filename="requirements.txt", ) elif self.packages is not None: with bento_fs.open(fs.path.join(py_folder, "requirements.txt"), "w") as f: f.write("\n".join(self.packages)) else: # Return early if no python packages were specified return pip_args: t.List[str] = [] if self.no_index: pip_args.append("--no-index") if self.index_url: pip_args.append(f"--index-url={self.index_url}") if self.trusted_host: for item in self.trusted_host: # pylint: disable=not-an-iterable pip_args.append(f"--trusted-host={item}") if self.find_links: for item in self.find_links: # pylint: disable=not-an-iterable pip_args.append(f"--find-links={item}") if self.extra_index_url: for item in self.extra_index_url: # pylint: disable=not-an-iterable pip_args.append(f"--extra-index-url={item}") if self.pip_args: # Additional user provided pip_args pip_args.append(self.pip_args) # write pip install args to a text file if applicable if pip_args: with bento_fs.open(fs.path.join(py_folder, "pip_args.txt"), "w") as f: f.write(" ".join(pip_args)) if self.lock_packages: # Note: "--allow-unsafe" is required for including setuptools in the # generated requirements.lock.txt file, and setuptool is required by # pyfilesystem2. Once pyfilesystem2 drop setuptools as dependency, we can # remove the "--allow-unsafe" flag here. # Note: "--generate-hashes" is purposefully not used here because it will # break if user includes PyPI package from version control system pip_compile_in = bento_fs.getsyspath( fs.path.join(py_folder, "requirements.txt")) pip_compile_out = bento_fs.getsyspath( fs.path.join(py_folder, "requirements.lock.txt")) pip_compile_args = ([pip_compile_in] + pip_args + [ "--quiet", "--allow-unsafe", "--no-header", f"--output-file={pip_compile_out}", ]) logger.info("Locking PyPI package versions..") click_ctx = pip_compile_cli.make_context("pip-compile", pip_compile_args) try: pip_compile_cli.invoke(click_ctx) except Exception as e: logger.error(f"Failed locking PyPI packages: {e}") logger.error( "Falling back to using user-provided package requirement specifier, equivalent to `lock_packages=False`" )
def load_checkpoint(self, fs: FSBase, model: keras.models.Model) -> None: with tempfile.NamedTemporaryFile(suffix=".h5") as tf: local_fs = FileSystem() with fs.open("model.h5", "rb") as fin: local_fs.writefile(tf.name, fin) model.load_weights(tf.name)