def format_resource_sizes(resources: Optional[Dict[str, int]]) -> str: if resources is None: return "" else: return util.sizeof_fmt(sum(resources.values()))
def test_sizeof_fmt() -> None: assert sizeof_fmt(1024) == "1.0KB" assert sizeof_fmt(36) == "36.0B"
def from_local( cls, local_path: pathlib.Path, limit: int = constants.MAX_CONTEXT_SIZE, ) -> "Context": """ Given the path to a local directory, return a Context object. A .detignore file in the directory, if specified, indicates the wildcard paths that should be ignored. File paths are represented as relative paths (relative to the root directory). """ context = Context() local_path = local_path.resolve() if not local_path.exists(): raise Exception("Path '{}' doesn't exist".format(local_path)) if local_path.is_file(): raise ValueError( "Path '{}' must be a directory".format(local_path)) root_path = local_path ignore = list(constants.DEFAULT_DETIGNORE) ignore_path = root_path.joinpath(".detignore") if ignore_path.is_file(): with ignore_path.open("r") as detignore_file: ignore.extend(detignore_file) ignore_spec = pathspec.PathSpec.from_lines( pathspec.patterns.GitWildMatchPattern, ignore) msg = "Preparing files (in {}) to send to master... {} and {} files".format( root_path, sizeof_fmt(0), 0) print(msg, end="\r", flush=True) # We could use pathlib.Path.rglob for scanning the directory; # however, the Python documentation claims a warning that rglob may be # inefficient on large directory trees, so we use the older os.walk(). for parent, dirs, files in os.walk(str(root_path)): for directory in dirs: dir_path = pathlib.Path(parent).joinpath(directory) dir_rel_path = dir_path.relative_to(root_path) # If the file matches any path specified in .detignore, then ignore it. if ignore_spec.match_file(str(dir_rel_path) + "/"): continue # Determined only supports POSIX-style file paths. Use as_posix() in case this code # is executed in a non-POSIX environment. entry_path = dir_rel_path.as_posix() context.add_item( ContextItem.from_local_dir(entry_path, dir_path)) for file in files: file_path = pathlib.Path(parent).joinpath(file) file_rel_path = file_path.relative_to(root_path) # If the file is the .detignore file or matches one of the # paths specified in .detignore, then ignore it. if file_rel_path.name == ".detignore": continue if ignore_spec.match_file(str(file_rel_path)): continue # Determined only supports POSIX-style file paths. Use as_posix() in case this code # is executed in a non-POSIX environment. entry_path = file_rel_path.as_posix() try: entry = ContextItem.from_local_file(entry_path, file_path) except OSError: print("Error reading '{}', skipping this file.".format( entry_path)) continue context.add_item(entry) if context.size > limit: print() raise ValueError( "Directory '{}' exceeds the maximum allowed size {}.\n" "Consider using a .detignore file to specify that certain files " "or directories should be omitted from the model.". format(root_path, sizeof_fmt(constants.MAX_CONTEXT_SIZE))) print(" " * len(msg), end="\r") msg = "Preparing files ({}) to send to master... {} and {} files".format( root_path, sizeof_fmt(context.size), len(context)) print(msg, end="\r", flush=True) print() return context
def format_resource_sizes(resources: Optional[Dict[str, str]]) -> str: if resources is None: return "" else: sizes = map(float, resources.values()) return util.sizeof_fmt(sum(sizes))