def format_resource_sizes(resources: Optional[Dict[str, int]]) -> str:
if resources is None:
return ""
else:
return util.sizeof_fmt(sum(resources.values()))
def test_sizeof_fmt() -> None:
assert sizeof_fmt(1024) == "1.0KB"
assert sizeof_fmt(36) == "36.0B"
def from_local(
cls,
local_path: pathlib.Path,
limit: int = constants.MAX_CONTEXT_SIZE,
) -> "Context":
"""
Given the path to a local directory, return a Context object.
A .detignore file in the directory, if specified, indicates the wildcard paths
that should be ignored. File paths are represented as relative paths (relative to
the root directory).
"""
context = Context()
local_path = local_path.resolve()
if not local_path.exists():
raise Exception("Path '{}' doesn't exist".format(local_path))
if local_path.is_file():
raise ValueError(
"Path '{}' must be a directory".format(local_path))
root_path = local_path
ignore = list(constants.DEFAULT_DETIGNORE)
ignore_path = root_path.joinpath(".detignore")
if ignore_path.is_file():
with ignore_path.open("r") as detignore_file:
ignore.extend(detignore_file)
ignore_spec = pathspec.PathSpec.from_lines(
pathspec.patterns.GitWildMatchPattern, ignore)
msg = "Preparing files (in {}) to send to master... {} and {} files".format(
root_path, sizeof_fmt(0), 0)
print(msg, end="\r", flush=True)
# We could use pathlib.Path.rglob for scanning the directory;
# however, the Python documentation claims a warning that rglob may be
# inefficient on large directory trees, so we use the older os.walk().
for parent, dirs, files in os.walk(str(root_path)):
for directory in dirs:
dir_path = pathlib.Path(parent).joinpath(directory)
dir_rel_path = dir_path.relative_to(root_path)
# If the file matches any path specified in .detignore, then ignore it.
if ignore_spec.match_file(str(dir_rel_path) + "/"):
continue
# Determined only supports POSIX-style file paths. Use as_posix() in case this code
# is executed in a non-POSIX environment.
entry_path = dir_rel_path.as_posix()
context.add_item(
ContextItem.from_local_dir(entry_path, dir_path))
for file in files:
file_path = pathlib.Path(parent).joinpath(file)
file_rel_path = file_path.relative_to(root_path)
# If the file is the .detignore file or matches one of the
# paths specified in .detignore, then ignore it.
if file_rel_path.name == ".detignore":
continue
if ignore_spec.match_file(str(file_rel_path)):
continue
# Determined only supports POSIX-style file paths. Use as_posix() in case this code
# is executed in a non-POSIX environment.
entry_path = file_rel_path.as_posix()
try:
entry = ContextItem.from_local_file(entry_path, file_path)
except OSError:
print("Error reading '{}', skipping this file.".format(
entry_path))
continue
context.add_item(entry)
if context.size > limit:
print()
raise ValueError(
"Directory '{}' exceeds the maximum allowed size {}.\n"
"Consider using a .detignore file to specify that certain files "
"or directories should be omitted from the model.".
format(root_path,
sizeof_fmt(constants.MAX_CONTEXT_SIZE)))
print(" " * len(msg), end="\r")
msg = "Preparing files ({}) to send to master... {} and {} files".format(
root_path, sizeof_fmt(context.size), len(context))
print(msg, end="\r", flush=True)
print()
return context
def format_resource_sizes(resources: Optional[Dict[str, str]]) -> str:
if resources is None:
return ""
else:
sizes = map(float, resources.values())
return util.sizeof_fmt(sum(sizes))