def check_proj_consistency(actual, expected): # Check equality of all project fields (projects themselves are # not comparable), with extra semantic consistency checking # for paths. assert actual.name == expected.name assert actual.path == expected.path if actual.topdir is None or expected.topdir is None: assert actual.topdir is None and expected.topdir is None assert actual.abspath is None and expected.abspath is None assert actual.posixpath is None and expected.posixpath is None else: assert actual.topdir and actual.abspath and actual.posixpath assert expected.topdir and expected.abspath and expected.posixpath a_top, e_top = PurePath(actual.topdir), PurePath(expected.topdir) a_abs, e_abs = PurePath(actual.abspath), PurePath(expected.abspath) a_psx, e_psx = PurePath(actual.posixpath), PurePath(expected.posixpath) assert a_top.is_absolute() assert e_top.is_absolute() assert a_abs.is_absolute() assert e_abs.is_absolute() assert a_psx.is_absolute() assert e_psx.is_absolute() assert a_top == e_top assert a_abs == e_abs assert a_psx == e_psx assert (actual.url == expected.url or (WINDOWS and Path(expected.url).is_dir() and (PurePath(actual.url) == PurePath(expected.url)))) assert actual.clone_depth == expected.clone_depth assert actual.revision == expected.revision assert actual.west_commands == expected.west_commands
def path_starts_with(path: PurePath, prefix: PurePath) -> bool: """Return whether the path starts with prefix. Both arguments must be absolute paths. If not, this function raises a ValueError. This function compares the path components, so it's not a simple string prefix test. """ if not path.is_absolute(): raise ValueError("{!r} is not absolute".format(path)) if not prefix.is_absolute(): raise ValueError("{!r} is not absolute".format(prefix)) return path.parts[:len(prefix.parts)] == prefix.parts
def mkrelative(cls, asset_path: pathlib.Path, bfile_path: pathlib.PurePath) -> 'BlendPath': """Construct a BlendPath to the asset relative to the blend file. Assumes that bfile_path is absolute. """ from collections import deque assert bfile_path.is_absolute(), \ 'BlendPath().mkrelative(bfile_path=%r) should get absolute bfile_path' % bfile_path bdir_parts = deque(bfile_path.parent.parts) asset_parts = deque(asset_path.absolute().parts) # Remove matching initial parts. What is left in bdir_parts represents # the number of '..' we need. What is left in asset_parts represents # what we need after the '../../../'. while bdir_parts: if bdir_parts[0] != asset_parts[0]: break bdir_parts.popleft() asset_parts.popleft() rel_asset = pathlib.PurePath(*asset_parts) # TODO(Sybren): should we use sys.getfilesystemencoding() instead? rel_bytes = str(rel_asset).encode('utf-8') as_bytes = b'//' + len(bdir_parts) * b'../' + rel_bytes return cls(as_bytes)
def process_path_specs(specs: Optional[IncludesList]) -> InternalIncludesList: """ Prepare paths specified as config. The input is a list of either strings, or 2-tuples (source, target). Where single strings are supplied, the basenames are used as targets. Where targets are given explicitly, they must not be absolute paths. Returns a list of 2-tuples, or throws ConfigError if something is wrong in the input. """ if specs is None: specs = [] processed_specs: InternalIncludesList = [] for spec in specs: if not isinstance(spec, (list, tuple)): source = spec target = None elif len(spec) != 2: error = "path spec must be a list or tuple of length two" raise ConfigError(error) else: source, target = spec source = Path(source) if not source.exists(): raise ConfigError(f"cannot find file/directory named {source!s}") target = PurePath(target or source.name) if target.is_absolute(): error = f"target path named {target!s} cannot be absolute" raise ConfigError(error) processed_specs.append((source, target)) return processed_specs
def fnmatch_ex(pattern: str, path) -> bool: """FNMatcher port from py.path.common which works with PurePath() instances. The difference between this algorithm and PurePath.match() is that the latter matches "**" glob expressions for each part of the path, while this algorithm uses the whole path instead. For example: "tests/foo/bar/doc/test_foo.py" matches pattern "tests/**/doc/test*.py" with this algorithm, but not with PurePath.match(). This algorithm was ported to keep backward-compatibility with existing settings which assume paths match according this logic. References: * https://bugs.python.org/issue29249 * https://bugs.python.org/issue34731 """ path = PurePath(path) iswin32 = sys.platform.startswith("win") if iswin32 and sep not in pattern and posix_sep in pattern: # Running on Windows, the pattern has no Windows path separators, # and the pattern has one or more Posix path separators. Replace # the Posix path separators with the Windows path separator. pattern = pattern.replace(posix_sep, sep) if sep not in pattern: name = path.name else: name = str(path) if path.is_absolute() and not os.path.isabs(pattern): pattern = "*{}{}".format(os.sep, pattern) return fnmatch.fnmatch(name, pattern)
def _validate_uri(self, uri: Union[str, None]) -> str: """Format the uri provided to match mlflow expectations. Arguments: uri {Union[None, str]} -- A valid filepath for mlflow uri Returns: str -- A valid mlflow_tracking_uri """ # if no tracking uri is provided, we register the runs locally at the root of the project uri = uri or "mlruns" pathlib_uri = PurePath(uri) from urllib.parse import urlparse if pathlib_uri.is_absolute(): valid_uri = pathlib_uri.as_uri() else: parsed = urlparse(uri) if parsed.scheme == "": # if it is a local relative path, make it absolute # .resolve() does not work well on windows # .absolute is undocumented and have known bugs # Path.cwd() / uri is the recommend way by core developpers. # See : https://discuss.python.org/t/pathlib-absolute-vs-resolve/2573/6 valid_uri = (self.project_path / uri).as_uri() else: # else assume it is an uri valid_uri = uri return valid_uri
def build_stale_images(config, config_filepath): for image in config['images']: image_name = image['name'] print('[[ Checking Image [%s] ... ]]' % image_name) if not podman_commands.need_to_rebuild_image(image_name): print('[[ Image [%s] up to date ]]' % image_name) continue else: print('[[ Image [%s] needs rebuild ]]' % image_name) build_dir = PurePath(image['directory']) if not build_dir.is_absolute(): build_dir = config_filepath.parent.joinpath(build_dir) print('') print('[[ Building Image [%s] ... ]]' % image_name) print('') podman_commands.build_image(build_dir, image_name) print('') print('[[ Building Image [%s] Complete ]]' % image_name) print('') print('[[ Pruning Untagged Images ... ]]') podman_commands.prune_untagged_images() print('[[ Pruning Untagged Images Complete ]]')
def to_uri(file_path): pure_path = PurePath(file_path) if pure_path.is_absolute(): return pure_path.as_uri() else: posix_path = pure_path.as_posix() # Replace backslashes with slashes. return urlparse.quote(posix_path) # %-encode special characters.
def sane_members(members, destination): resolve = lambda path: realpath(normpath(join(destination, path))) destination = PurePath(destination) for member in members: mpath = PurePath(resolve(member.path)) # Check if mpath is under destination if destination not in mpath.parents: raise BadPathError( "Bad path to outside destination directory: {}".format(mpath)) elif member.issym() or member.islnk(): # Check link to make sure it resolves under destination lnkpath = PurePath(member.linkpath) if lnkpath.is_absolute() or lnkpath.is_reserved(): raise BadLinkError("Bad link: {}".format(lnkpath)) # resolve the link to an absolute path lnkpath = PurePath(resolve(lnkpath)) if destination not in lnkpath.parents: raise BadLinkError( "Bad link to outside destination directory: {}".format( lnkpath)) yield member
def __call__(self, data: str): path = PurePath(data) if path.is_absolute(): raise ValidationError(_("Absolute paths not allowed"), code="no_absolute") if ".." in path.parts: raise ValidationError(_("No parent directory references allowed"), code="no_parent_references")
def reroot_path(filename: PurePath, docpath: PurePath, project_root: Path) -> Tuple[FileId, Path]: """Files within a project may refer to other files. Return a canonical path relative to the project root.""" if filename.is_absolute(): rel_fn = FileId(*filename.parts[1:]) else: rel_fn = FileId(os.path.normpath(docpath.parent.joinpath(filename))) return rel_fn, project_root.joinpath(rel_fn).resolve()
def generate_relative_mounts(pvc_param, files): """ Maps a list of files as mounts, relative to the base volume mount. For example, given the pvc mount: { 'name': 'my_pvc', 'mountPath': '/galaxy/database/jobs', 'subPath': 'data', 'readOnly': False } and files: ['/galaxy/database/jobs/01/input.txt', '/galaxy/database/jobs/01/working'] returns each file as a relative mount as follows: [ { 'name': 'my_pvc', 'mountPath': '/galaxy/database/jobs/01/input.txt', 'subPath': 'data/01/input.txt', 'readOnly': False }, { 'name': 'my_pvc', 'mountPath': '/galaxy/database/jobs/01/working', 'subPath': 'data/01/working', 'readOnly': False } ] :param pvc_param: the pvc claim dict :param files: a list of file or folder names :return: A list of volume mounts """ if not pvc_param: return param_claim = parse_pvc_param_line(pvc_param) claim_name = param_claim['name'] base_subpath = PurePath(param_claim.get('subPath', "")) base_mount = PurePath(param_claim["mountPath"]) read_only = param_claim["readOnly"] volume_mounts = [] for f in files: file_path = PurePath(str(f)) if base_mount not in file_path.parents: # force relative directory, needed for the job working directory in particular file_path = base_mount.joinpath( file_path.relative_to("/") if file_path.is_absolute( ) else file_path) relpath = file_path.relative_to(base_mount) subpath = base_subpath.joinpath(relpath) volume_mounts.append({ 'name': claim_name, 'mountPath': str(file_path), 'subPath': str(subpath), 'readOnly': read_only }) return volume_mounts
def test_path(path_string, type=''): path = PurePath(path_string) if type != '': type = ' ' + type #pad for formatting if path.is_absolute(): raise Exception('Please provide a relative' + type + ' location. Do not start path with /') if '..' in path.parts: raise Exception('Please provide a relative' + type + ' location. Do not backtrack directories.') return True
def write_path_template(self): rootp = self.reg_root ret = PurePath(self._write_path_template) if self._read_path_template is None and rootp not in ret.parents: if not ret.is_absolute(): ret = rootp / ret else: raise ValueError( ('root: {!r} in not consistent with ' 'read_path_template: {!r}').format(rootp, ret)) return _ensure_trailing_slash(str(ret))
def __getitem__(self, k: PurePath) -> Union[str, Index]: assert not k.is_absolute(), "Index keys must be relative paths" if len(k.parts) == 0: return self elif len(k.parts) == 1: if k in self.files: return self.files[k] elif k in self.dirs: return self.dirs[k] else: raise KeyError(f"{k} not found") else: return self.dirs[PurePath(k.parts[0])][PurePath(*k.parts[1:])]
def compile(self, *paths): compilation = self.make_compilation() for path in paths: path = PurePath(path) if path.is_absolute(): path = path.relative_to('/') filename, storage = get_file_and_storage(str(path)) with storage.open(filename) as f: source = SourceFile.from_file(f, origin=path.parent, relpath=PurePath(path.name)) compilation.add_source(source) return self.call_and_catch_errors(compilation.run)
def send(self, subpath, guild, content, **attributes): # Get full path subpath = PurePath(subpath) assert not subpath.is_absolute( ), "Cannot broadcast on absolute subpath" path = self.path.joinpath(subpath) # Queue up event event = JournalEvent(path=path, guild=guild, content=content, attributes=attributes) self.router.queue.put_nowait(event)
def mkrelative(cls, asset_path: pathlib.PurePath, bfile_path: pathlib.PurePath) -> 'BlendPath': """Construct a BlendPath to the asset relative to the blend file. Assumes that bfile_path is absolute. Note that this can return an absolute path on Windows when 'asset_path' and 'bfile_path' are on different drives. """ from collections import deque # Only compare absolute paths. assert bfile_path.is_absolute(), \ 'BlendPath().mkrelative(bfile_path=%r) should get absolute bfile_path' % bfile_path assert asset_path.is_absolute(), \ 'BlendPath().mkrelative(asset_path=%r) should get absolute asset_path' % asset_path # There is no way to construct a relative path between drives. if bfile_path.drive != asset_path.drive: return cls(asset_path) bdir_parts = deque(bfile_path.parent.parts) asset_path = make_absolute(asset_path) asset_parts = deque(asset_path.parts) # Remove matching initial parts. What is left in bdir_parts represents # the number of '..' we need. What is left in asset_parts represents # what we need after the '../../../'. while bdir_parts: if bdir_parts[0] != asset_parts[0]: break bdir_parts.popleft() asset_parts.popleft() rel_asset = pathlib.PurePath(*asset_parts) # TODO(Sybren): should we use sys.getfilesystemencoding() instead? rel_bytes = str(rel_asset).encode('utf-8') as_bytes = b'//' + len(bdir_parts) * b'../' + rel_bytes return cls(as_bytes)
def get_relative_path(base_path: PurePath, target_path: PurePath) -> PurePath: if base_path == target_path: return Path('.') if not target_path.is_absolute(): return target_path parent_count: int = 0 children: List[str] = [] for base_part, target_part in zip_longest(base_path.parts, target_path.parts): if base_part == target_part and not parent_count: continue if base_part or not target_part: parent_count += 1 if target_part: children.append(target_part) return Path(*['..' for _ in range(parent_count)], *children)
def __delitem__(self, k: PurePath) -> None: assert not k.is_absolute(), "Index keys must be relative paths" if len(k.parts) == 0: raise ValueError("can't del self (i think)") elif len(k.parts) == 1: if k in self.files: del self.files[k] elif k in self.dirs: del self.dirs[k] else: raise KeyError(f"{k} not found") else: del self.dirs[PurePath(k.parts[0])][PurePath(*k.parts[1:])] if not self.dirs[PurePath(k.parts[0])]: del self.dirs[PurePath(k.parts[0])]
def main() -> int: args, argv = _parse_args() path = PurePath(args.path) if path == PurePath("-"): addn = {} else: key_path = path if path.is_absolute() else Path.home() / ".ssh" / path ssh_cmd = join( ("ssh", "-o", "IdentitiesOnly=yes", "-i", normcase(key_path))) addn = {"GIT_SSH_COMMAND": ssh_cmd} env = {**environ, **addn} if git := which("git"): execle(git, normcase(git), *argv, env)
def uri_from_path(path: PurePath) -> str: """ Convert a python path object to an URI """ # TODO: needs way more tests... See note [URI:java-python] if not path.is_absolute(): raise ValueError("uri_from_path requires an absolute path") java_uri = str(_normalize_pathlib_uris(path.as_uri()).toString()) # fixme: this should be replaced with a rfc3896 compliant solution... if re.match("file://([^/]|$)", java_uri): uri = f"file:////{java_uri[7:]}" # network shares have redundant authority on the java side # vvv this would only be required if we wouldn't normalize the uri like above # elif re.match("file:///([^/]|$)", java_uri): # uri = f"file:/{java_uri[8:]}" # the local windows absolute paths don't else: uri = java_uri return uri
def check_valid_loc(s: str): if s in _SPECIAL_ALLOWED_LOCS: return if s == "": raise ValueError(f"empty loc '{s}' is not allowed (try '.' instead)") p = PurePath(s) if p.is_absolute(): raise ValueError(f"loc '{p}' is absolute") if p.is_reserved(): raise ValueError(f"loc '{p}' is reserved.") if ".." in p.parts: raise ValueError(f"loc '{p}' contains disallowed '..'")
def read_path_template(self): "Returns write_path_template if read_path_template is not set" rootp = self.reg_root if self._read_path_template is None: ret = PurePath(self.write_path_template) else: ret = PurePath(self._read_path_template) if rootp not in ret.parents: if not ret.is_absolute(): ret = rootp / ret else: raise ValueError( ('root: {!r} in not consistent with ' 'read_path_template: {!r}').format(rootp, ret)) ret = os.path.join(ret, "") return str(ret)
def _validate_mlflow_tracking_uri(project_path: str, uri: Optional[str]) -> str: """Format the uri provided to match mlflow expectations. Arguments: uri {Union[None, str]} -- A valid filepath for mlflow uri Returns: str -- A valid mlflow_tracking_uri """ # this is a special reserved keyword for mlflow which should not be converted to a path # se: https://mlflow.org/docs/latest/tracking.html#where-runs-are-recorded if uri is None: # do not use mlflow.get_tracking_uri() because if there is no env var, # it resolves to 'Path.cwd() / "mlruns"' # but we want 'project_path / "mlruns"' uri = os.environ.get("MLFLOW_TRACKING_URI", "mlruns") if uri == "databricks": return uri # if no tracking uri is provided, we register the runs locally at the root of the project pathlib_uri = PurePath(uri) if pathlib_uri.is_absolute(): valid_uri = pathlib_uri.as_uri() else: parsed = urlparse(uri) if parsed.scheme == "": # if it is a local relative path, make it absolute # .resolve() does not work well on windows # .absolute is undocumented and have known bugs # Path.cwd() / uri is the recommend way by core developpers. # See : https://discuss.python.org/t/pathlib-absolute-vs-resolve/2573/6 valid_uri = (Path(project_path) / uri).as_uri() LOGGER.info( f"The 'mlflow_tracking_uri' key in mlflow.yml is relative ('server.mlflow_tracking_uri = {uri}'). It is converted to a valid uri: '{valid_uri}'" ) else: # else assume it is an uri valid_uri = uri return valid_uri
def __setitem__(self, k: PurePath, v: Union[str, Index]) -> None: assert not k.is_absolute(), "Index keys must be relative paths" if len(k.parts) == 0: raise ValueError("can't set self (i think)") elif len(k.parts) == 1: if isinstance(v, str): assert k not in self.dirs self.files[k] = v elif isinstance(v, Index): assert k not in self.files self.dirs[k] = v else: raise TypeError(f"Unallowed value of type {type(v)}") else: assert PurePath( k.parts[0]) not in self.files, "file/directory name collision" if PurePath(k.parts[0]) not in self.dirs: self[PurePath(k.parts[0])] = Index() self.dirs[PurePath(k.parts[0])][PurePath(*k.parts[1:])] = v
def relpath(filename): """Get relative path to root from filename. Args: filename (str): Filename Raises: exceptions.PathError: When path is not under root directory. Returns: PurePath: Relative path. """ ret = PurePath(os.fsdecode(filename)) if ret.is_absolute(): try: return PurePath(ret.relative_to(setting.ROOT)) except ValueError: raise exceptions.PathError(path) return ret
def relative_path(value: PurePath, base_directory: PurePath, allow_paths_outside_base=False) -> PurePath: """ Make a single path relative to the base directory if it is inside it. By default, will throw a ValueError if not able to make it relative to the path. >>> val = PurePath('/tmp/minimal-pkg/loch_ness_sightings_2019-07-04_blue.tif') >>> base = PurePath('/tmp/minimal-pkg') >>> relative_path(val, base).as_posix() 'loch_ness_sightings_2019-07-04_blue.tif' """ if not value or not value.is_absolute(): return value if base_directory not in value.parents: if not allow_paths_outside_base: raise ValueError( f"Path {value.as_posix()!r} is outside path {base_directory.as_posix()!r} " f"(allow_paths_outside_base={allow_paths_outside_base})") return value return value.relative_to(base_directory)
def handle_import(self, name, compilation, rule): """ Re-implementation of the core Sass import mechanism, which looks for files using the staticfiles storage and staticfiles finders. """ original_path = PurePath(name) search_exts = list(compilation.compiler.dynamic_extensions) if original_path.suffix and original_path.suffix in search_exts: basename = original_path.stem else: basename = original_path.name if original_path.is_absolute() or name[ 0] == '/': # such that name like '/path/to/file.scss' works on Windows # Remove the beginning slash search_path = original_path.relative_to('/').parent elif rule.source_file.origin: search_path = rule.source_file.origin if original_path.parent: search_path = os.path.normpath( str(search_path / original_path.parent)) else: search_path = original_path.parent for prefix, suffix in product(('_', ''), search_exts): filename = PurePath(prefix + basename + suffix) full_filename, storage = get_file_and_storage( str(search_path / filename)) if full_filename: with storage.open(full_filename) as f: return SourceFile.from_file(f, origin=search_path, relpath=filename)
pprint(p / 'piyo') # 純粋パスを扱う(PurePath)------------------------------------------------- from pathlib import PurePath p = PurePath('/hoge/fuga/piyo.txt') pprint(p.drive) pprint(p.root) pprint(p.anchor) pprint(list(p.parents)) pprint(p.parent) pprint(p.name) pprint(p.suffix) pprint(p.stem) pprint(p.is_absolute()) pprint(p.joinpath('foo', 'bar', 'baz')) pprint(p.match('piyo.*')) # 具象パスを扱う(Path)-------------------------------------------------------- from pathlib import Path p = Path.cwd() / 'newfile.txt' pprint(p) pprint(p.exists()) f = p.open('w+') pprint(p.exists()) pprint(p.resolve())
def handle_import(self, name, compilation, rule): """ Re-implementation of the core Sass import mechanism, which looks for files using the staticfiles storage and staticfiles finders. """ original_path = PurePath(name) if original_path.suffix: search_exts = [original_path.suffix] else: search_exts = compilation.compiler.dynamic_extensions if original_path.is_absolute(): # Remove the beginning slash search_path = original_path.relative_to('/').parent elif rule.source_file.origin: search_path = rule.source_file.origin if original_path.parent: search_path = search_path / original_path.parent else: search_path = original_path.parent basename = original_path.stem for prefix, suffix in product(('_', ''), search_exts): filename = PurePath(prefix + basename + suffix) full_filename, storage = get_file_and_storage(str(search_path / filename)) if full_filename: with storage.open(full_filename) as f: return SourceFile.from_file(f, origin=search_path, relpath=filename)
def compile(self, *paths): compilation = self.make_compilation() for path in paths: path = PurePath(path) if path.is_absolute(): path = path.relative_to('/') filename, storage = get_file_and_storage(str(path)) with storage.open(filename) as f: source = SourceFile.from_file(f, origin=path.parent, relpath=PurePath(path.name)) compilation.add_source(source) return self.call_and_catch_errors(compilation.run)
def write_path_template(self): rootp = self.reg_root ret = PurePath(self._write_path_template) if self._read_path_template is None and rootp not in ret.parents: if not ret.is_absolute(): ret = rootp / ret else: raise ValueError( ('root: {!r} in not consistent with ' 'read_path_template: {!r}').format(rootp, ret)) return _ensure_trailing_slash(str(ret))
def get_absolute_path(self, path, with_base=False): path = PurePath(path) try: new_path = None if path.is_absolute(): new_path = self.base_path / path.relative_to('/') else: new_path = self.base_path / self.cwd.relative_to('/') / path if new_path.exists(): new_path = new_path.resolve() except ValueError: new_path = None if new_path: if with_base: return str(new_path) else: return str(PurePath('/') / new_path.relative_to(self.base_path)) return None
def sane_members(members, destination): resolve = lambda path: realpath(normpath(join(destination, path))) destination = PurePath(destination) for member in members: mpath = PurePath(resolve(member.path)) # Check if mpath is under destination if destination not in mpath.parents: raise BadPathError("Bad path to outside destination directory: {}".format(mpath)) elif member.issym() or member.islnk(): # Check link to make sure it resolves under destination lnkpath = PurePath(member.linkpath) if lnkpath.is_absolute() or lnkpath.is_reserved(): raise BadLinkError("Bad link: {}".format(lnkpath)) # resolve the link to an absolute path lnkpath = PurePath(resolve(lnkpath)) if destination not in lnkpath.parents: raise BadLinkError("Bad link to outside destination directory: {}".format(lnkpath)) yield member