def _readlink(self, path: AbstractPath, recursive: bool) -> Path:
self.__ensure_sftp()
lpath = cast(PurePosixPath, path)
if recursive:
# SFTP's normalize() raises if there's a link loop or a \
# non-existing target, which we don't want, so we use \
# our own algorithm.
max_iter = 32
cur_path = lpath
iter_count = 0
while self._is_symlink(cur_path) and iter_count < max_iter:
target = PurePosixPath(self.__sftp.readlink(str(cur_path)))
if not target.is_absolute():
target = cur_path.parent / target
cur_path = target
iter_count += 1
if iter_count == max_iter:
raise RuntimeError('Too many symbolic links detected')
target = PurePosixPath(self.__sftp.normalize(str(path)))
else:
target = PurePosixPath(self.__sftp.readlink(str(path)))
if not target.is_absolute():
target = lpath.parent / target
return Path(self, target)
def find_page(self, model, data):
"""
Find a page by its URL and import its data.
Data importing has to be done here because often the page can't
be saved until the data is imported (i.e. null fields)
"""
try:
url = PurePosixPath(data.pop('url'))
if not url.is_absolute():
raise CommandError("Path %s must be absolute" % url)
except KeyError:
raise CommandError("Need `url' for page")
try:
page = model.objects.get(url_path=normalise(url))
self.import_data(page, data)
page.save()
self.stdout.write("Updating existing page %s" % url)
except model.DoesNotExist:
try:
# pylint:disable=no-member
parent = Page.objects.get(url_path=normalise(url.parent))
except Page.DoesNotExist:
raise CommandError("Parent of %s doesn't exist" % url)
page = model(slug=url.name)
self.import_data(page, data)
parent.add_child(instance=page)
self.stdout.write("Creating new page %s" % url)
return page
def _possible_names(cls, package: PackageInfo) -> List[str]:
names = list()
for url_str in package.get_urls():
url = urlparse(url_str)
host = url.netloc
if not host.endswith('github.com'):
continue
path = PurePosixPath(url.path)
parts = path.parts
if path.is_absolute():
parts = parts[1:]
if len(parts) >= 2:
# Get the first 2 path components without extensions
# this should handle:
# - owner/project
# - owner/project.git
# - owner/project/releases
name = PurePosixPath(parts[0])
# strip off .git if the project name contains it
# don't just strip off any ext because "." is valid
name_project = parts[1]
if name_project.endswith('.git'):
name_project = name_project[:-len('.git')]
name = name.joinpath(name_project)
names.append(str(name))
return names
def addPrefix(self, s: pathlib.PurePosixPath, n) -> pathlib.PurePosixPath:
if s.is_absolute():
if len(s.parts) == 1:
raise Exception('Invalid path: %s' % s)
prefix = s.parts[1]
else:
prefix = s.parts[0]
assert len(prefix) != 0
if len(prefix) > n:
prefix = prefix[0:n]
if s.is_absolute():
return pathlib.PurePosixPath('/', prefix, *s.parts[1:])
else:
return pathlib.PurePosixPath(prefix, *s.parts)
def default_logger(name=None,
logger=None,
level=None,
logfile=None,
message=None):
"""
使用python默认的日志记录的装饰器
:param logger: 日志记录器
:param level: 日志的等级
:param name: 日志名称
:param logfile: 日志文件
:param message: 日志消息
:return: 返回一个装饰器
"""
if logfile:
if not PurePosixPath.is_absolute(logfile):
raise OSError(f"{logfile} is not a absolute path!")
def decorater(func):
"""构造内部的logger"""
if logger:
inner_logger = logger
else:
logname = name if name else "common"
inner_logger = logging.getLogger(logname)
inner_logger.setLevel("INFO")
formatter = logging.Formatter(
'%(asctime)s :%(levelname)s %(message)s')
fh = logging.FileHandler(logfile if logfile else PurePosixPath.
joinpath(LOG_DIR, f"{logname}.log"))
fh.setFormatter(formatter)
streamh = logging.StreamHandler()
streamh.setFormatter(formatter)
logmsg = message if message else func.__name__
loglevel = level if level else logging.INFO
@wraps(func)
def wrapper(*args, **kwargs):
try:
inner_logger.addHandler(fh)
inner_logger.addHandler(streamh)
inner_logger.log(loglevel, f"executing function {logmsg}")
inner_logger.handlers.clear()
return func(*args, **kwargs)
except ValueError:
raise
except Exception:
inner_logger.addHandler(fh)
inner_logger.addHandler(streamh)
inner_logger.info(
"#####################################################")
inner_logger.exception(f"something wrong in function {logmsg}")
inner_logger.info(
"#####################################################")
inner_logger.handlers.clear()
return wrapper
return decorater
def update_modules(module_name_prefix: str, modules_path: Optional[str],
**kwargs):
LOG.debug("Generating update_modules proposal")
# Validate module name prefix
module_name_prefix_ = PurePosixPath(module_name_prefix)
if not module_name_prefix_.is_absolute():
raise ValueError("module name prefix must be an absolute path")
if any(folder in [".", ".."] for folder in module_name_prefix_.parents):
raise ValueError(
"module name prefix must not contain . or .. components")
if not module_name_prefix.endswith("/"):
raise ValueError("module name prefix must end with /")
# Read module files and build relative module names
modules = []
if modules_path:
for path in glob.glob(f"{modules_path}/**/*.js", recursive=True):
rel_module_name = os.path.relpath(path, modules_path)
rel_module_name = rel_module_name.replace("\\",
"/") # Windows support
with open(path) as f:
js = f.read()
modules.append({
"rel_name": rel_module_name,
"module": {
"js": js
}
})
proposal_args = {"prefix": module_name_prefix, "modules": modules}
return build_proposal("update_modules", proposal_args, **kwargs)
def normalise_path(path: H5Path,
current_gpath: H5Path = None,
relative=False) -> H5Path:
path = H5Path(path)
if current_gpath is not None:
current_gpath = H5Path(current_gpath)
path = current_gpath / path
if not path.is_absolute():
raise ValueError("Path must be absolute")
parts: List[str] = []
for part in path.parts:
if set(part) == {"."}:
for _ in part[1:]:
try:
parts.pop()
if len(parts) == 0:
raise IndexError()
except IndexError:
raise ValueError("Tried to traverse beyond root")
else:
parts.append(part)
out = H5Path(*parts)
if relative and current_gpath:
out = out.relative_to(current_gpath)
return out
def write_path_template(self):
rootp = self.reg_root
if self.path_semantics == 'posix':
ret = PurePosixPath(self._write_path_template)
elif self.path_semantics == 'windows':
ret = PureWindowsPath(self._write_path_template)
elif self.path_semantics is None:
# We are forced to guess which path semantics to use.
# Guess that the AD driver is running on the same OS as this client.
ret = PurePath(self._write_path_template)
else:
# This should never happen, but just for the sake of future-proofing...
raise ValueError(
f"Cannot handle path_semantics={self.path_semantics}")
if self._read_path_template is None and rootp not in ret.parents:
if not ret.is_absolute():
ret = rootp / ret
else:
raise ValueError(
('root: {!r} in not consistent with '
'read_path_template: {!r}').format(rootp, ret))
return _ensure_trailing_slash(str(ret),
path_semantics=self.path_semantics)
def _decode_name(name):
if name is None:
return None
path = PurePosixPath(name)
if path.is_absolute():
path = path.relative_to(path.root)
parts = [i for i in path.parts if i not in ('.', '..')]
return PurePosixPath(*parts)
def _check_path_relative(
self,
attribute: attr.Attribute[PurePosixPath],
value: PurePosixPath,
) -> None:
if value.is_absolute():
raise ConfigSemanticException(
f'Directory path "{value}" in addon config must be relative (not '
"absolute)")
def mkdir(self, cmd):
'''Create an empty directory. Usage: mkdir <directory>'''
directory = PurePosixPath(cmd[1])
if not directory.is_absolute():
directory = self.cwd / directory
r = requests.put(f'{self.url}/mkdir', data={'path': str(directory)})
if not r.ok:
print(r.text)
return
def main(args):
import logging
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
parser = argparse.ArgumentParser(description='Generate a distribution archive.', prog=args[0])
parser.add_argument('build_dir',
help='the build directory (defaults to CWD)',
default=Path(os.getcwd()),
type=Path,
nargs='?',
)
parser.add_argument('output',
help='the output file path',
type=Path,
)
parser.add_argument('--format',
help='the archive format',
default='zip',
choices=sorted(map(lambda x: x[0], shutil.get_archive_formats())),
)
parser.add_argument('--prefix',
help='add a common prefix to all files and directories in the archive',
default=None,
)
add_common_args(parser)
args = parser.parse_args(args[1:])
if args.prefix is not None:
p = PurePosixPath(args.prefix)
if p.is_absolute() or p.is_reserved() or '..' in p.parts:
raise ValueError('Bad prefix: {}'.format(args.prefix))
args.prefix = str(p)
if args.prefix == '.':
args.prefix = None
with temp_install(args.build_dir) as install_dir:
if args.prefix is not None:
os.chdir(str(install_dir.parent))
install_dir.rename(install_dir.parent / args.prefix)
archive = shutil.make_archive(str(args.output), args.format, '.', str(args.prefix))
else:
archive = shutil.make_archive(str(args.output), args.format, str(install_dir))
archive = Path(archive)
archive.rename(args.output)
print("Generated distribution archive {}".format(str(args.output)))
def convert_posix_to_win32(rootfs, cwd, path):
# rootfs is a concrete path.
rootfs = Path(rootfs)
# cwd and path are pure paths
cwd = PurePosixPath(cwd[1:])
path = PurePosixPath(path)
if path.is_absolute():
return rootfs / path.relative_to(path.anchor)
else:
return rootfs / cwd / path
def convert_posix_to_win32(rootfs, cwd, path):
# rootfs is a concrete path.
rootfs = Path(rootfs)
# cwd and path are pure paths
cwd = PurePosixPath(cwd[1:])
path = PurePosixPath(path)
if path.is_absolute():
return rootfs / PathUtils.normalize(path)
else:
return rootfs / PathUtils.normalize(cwd / path)
def convert_posix_to_win32(rootfs: Union[str, Path], cwd: str,
path: str) -> Path:
_rootfs = Path(rootfs)
_cwd = PurePosixPath(cwd[1:])
_path = PurePosixPath(path)
if _path.is_absolute():
return _rootfs / QlPathManager.normalize(_path)
else:
return _rootfs / QlPathManager.normalize(_cwd / _path)
def reroot_path(filename: PurePosixPath, docpath: PurePath,
project_root: Path) -> Tuple[FileId, Path]:
"""Files within a project may refer to other files. Return a canonical path
relative to the project root."""
if filename.is_absolute():
rel_fn = FileId(*filename.parts[1:])
else:
rel_fn = FileId(
*docpath.parent.joinpath(filename).parts).collapse_dots()
return rel_fn, project_root.joinpath(rel_fn).resolve()
def resolve_path(self, path):
"""
Resolves strings with variables relating to the job id.
"""
path = path.replace('$JOB_ID', self.job_id)
path = path.replace('$JOB_NUMBER', self.job_number)
path = PurePosixPath(path)
if path.is_absolute():
return path
return self.working_dir / path
def __init__(
self,
*,
datascheme: Optional[DataScheme] = None,
protocol: Protocol,
hostname: str,
port: Optional[int] = None,
path: PurePosixPath,
search: Optional[Mapping[str, str]] = None,
hash_: Optional[str] = None,
search_quoting_method: SearchQuotingMethod = SearchQuotingMethod.
QUOTE_PLUS,
):
if not path.is_absolute():
raise ValueError("Path '{path}' is not absolute")
path_parts: List[str] = []
for part in path.as_posix().split("/"):
if part == "." or part == "":
continue
if part == "..":
if len(path_parts) > 0:
_ = path_parts.pop()
else:
path_parts.append(part)
self.datascheme = datascheme
self.protocol = protocol
self.hostname = hostname
self.host = hostname + ("" if port is None else f":{port}")
self.port = port
self.path = PurePosixPath("/") / "/".join(path_parts)
self.search = search or {}
self.hash_ = hash_
self.schemeless_raw = f"{protocol}://{self.host}"
self.schemeless_raw += str(path)
if self.search:
if search_quoting_method == SearchQuotingMethod.QUOTE_PLUS:
quote_via = quote_plus
else:
quote_via = quote
self.schemeless_raw += "?" + urlencode(
self.search, doseq=True, quote_via=quote_via)
if self.hash_:
self.schemeless_raw += "#" + self.hash_
if self.datascheme:
self.raw = f"{self.datascheme}+{self.schemeless_raw}"
self.double_protocol_raw = f"{self.datascheme}://{self.schemeless_raw}"
else:
self.raw = self.schemeless_raw
self.double_protocol_raw = self.raw
if hostname == "" and protocol not in (Protocol.FILE, Protocol.MEMORY):
raise ValueError(f"Missing hostname in {self.raw}")
super().__init__()
def ls(self, cmd):
'''Lists directory contents. Usage: ls [directory]'''
directory = self.cwd
if len(cmd) > 1:
directory = PurePosixPath(cmd[1])
if not directory.is_absolute():
directory = self.cwd / directory
r = requests.get(f'{self.url}/ls', data={'path': str(directory)})
print(*r.json(), sep='\n')
def workfile_for_path(self, file):
# Transform work file path to some canonical form
posix_path = PurePosixPath(file)
if posix_path.is_absolute():
# TODO: what should we do with absolute workpath here?
raise Exception("Absolute work path not allowed: %s" % posix_path)
if posix_path not in self.workfiles:
wf = WorkFile(posix_path)
self.workfiles[posix_path] = wf
return wf
else:
return self.workfiles[posix_path]
def set_module(module_name: str, module_path: str, **kwargs):
module_name_ = PurePosixPath(module_name)
if not module_name_.is_absolute():
raise ValueError("module name must be an absolute path")
if any(folder in [".", ".."] for folder in module_name_.parents):
raise ValueError("module name must not contain . or .. components")
if module_name_.suffix == ".js":
with open(module_path) as f:
js = f.read()
proposal_args = {"name": module_name, "module": {"js": js}}
else:
raise ValueError("module name must end with .js")
return build_proposal("set_module", proposal_args, **kwargs)
def validate_pre_sds_if_applicable(self,
hds: HomeDs) -> Optional[TextRenderer]:
if self.path_str == '':
return text_docs.single_line(_EMPTY_FILE_NAME)
path = PurePosixPath(self.path_str)
if path.is_absolute():
return text_docs.single_line(
str_constructor.FormatMap(
'A {FILE_NAME} must not be absolute: {path}', {
'FILE_NAME': syntax.FILE_NAME.name,
'path': str(path),
}))
return None
def systemd_escape_path(path: pathlib.PurePosixPath) -> str:
"""Escape a path for inclusion in a systemd unit name.
See the 'systemd-escape --path' command for details.
"""
if not path.is_absolute():
raise ValueError("systemd_escape_path can only escape absolute paths")
if ".." in path.parts:
raise ValueError(
"systemd_escape_path can only escape paths without '..' components"
)
stdout: bytes = subprocess.check_output(
["systemd-escape", "--path", "--", str(path)]
)
return stdout.decode("utf-8").rstrip("\n")
def fetch_resource(self, resource_name):
resource_table = self._resource_tables[resource_name]
for destination, source_name in resource_table.items():
source_subpath = PurePosixPath(source_name)
if (
source_subpath.is_absolute() or
len(source_subpath.parts) > 1 or '..' in source_subpath.parts
):
raise RuntimeError(source_name)
source_path = self._resource_dir_path / source_name
destination_subpath = PurePosixPath(destination)
if (
destination_subpath.is_absolute() or
'..' in destination_subpath.parts
):
raise RuntimeError(destination)
destination_path: PosixPath = self.root / destination_subpath
if not destination_path.parent.exists():
destination_path.parent.mkdir(parents=True)
if destination_path.is_dir():
raise IsADirectoryError(str(destination))
if destination_path.is_symlink():
destination_path.unlink()
shutil.copyfile(str(source_path), str(destination_path))
def systemd_escape_path(path: pathlib.PurePosixPath) -> str:
"""Escape a path for inclusion in a systemd unit name.
See the 'systemd-escape --path' command for details.
"""
if not path.is_absolute():
raise ValueError("systemd_escape_path can only escape absolute paths")
if ".." in path.parts:
raise ValueError(
"systemd_escape_path can only escape paths without '..' components"
)
stdout: bytes = subprocess.check_output(
["systemd-escape", "--path", "--", str(path)]
)
return stdout.decode("utf-8").rstrip("\n")
def find_resource(root: Site, path: PurePosixPath) -> Resource:
"""Given a path-like string, walk the tree and return object.
Resources in a resource tree can be found with path-like lookups.
This implementation uses ``PurePosixPath`` as the path language.
Paths must start with a leading ``/``, but a trailing slash is optional.
Folder paths can be with or without a trailing ``index`` part.
If the provided path ends with ``index``, it will be removed for the
purposes of walking the resource tree.
Args:
root: The top of the resource tree.
path: A specification of how to walk down to the target resource.
Returns:
The resource at that path.
Raises:
ValueError: Paths must start with a slash.
KeyError: Asking for a path part that isn't in the dict at that
part of the tree.
"""
if not path.is_absolute():
# ResourceLike paths must start with a slash, so this is
# probably a static resource path
m = f'ResourceLike path "{path}" must start with a slash'
raise ValueError(m)
# "Normalize" the path if it ends with ``index``.
pp = path.parts
# noinspection PyTypeChecker
parts = iter(pp[1:-1] if path.name == "index" else pp[1:])
# Now walk the tree
# TODO: Fix the algorithm here to not need all the cast()
result = root
while True:
try:
part = next(parts)
try:
result = result[part] # type: ignore
except KeyError: # noqa: B904
m = f'No resource at path "{path}"'
raise KeyError(m) # noqa: B904
except StopIteration:
break
return cast(Resource, result)
def validate_pre_sds_if_applicable(self, hds: HomeDs) -> Optional[TextRenderer]:
path_str = self.path_str
if path_str == '':
return text_docs.single_line(_ERR__FILE_NAME__EMPTY)
for path_separator in _PATH_SEPARATORS:
if path_separator in path_str:
return self._err_msg(path_str, _ERR__FILE_NAME__PATH_SEPARATOR_IN_FILE_NAME)
path = PurePosixPath(path_str)
if path.is_absolute():
return self._err_msg(path_str, _ERR__FILE_NAME__ABSOLUTE)
if '..' in path.parts:
return self._err_msg(path_str, _ERR__FILE_NAME__RELATIVE_COMPONENTS)
return None
def rmdir(self, cmd):
'''Remove a directory. Usage: rmdir <directory>'''
directory = PurePosixPath(cmd[1])
if not directory.is_absolute():
directory = self.cwd / directory
r = requests.delete(f'{self.url}/rmdir', data={'path': str(directory)})
if not r.ok:
print(r.text)
if r.status_code == 400:
answer = ''
while answer not in ['y', 'n']:
print("Do you still want to delete? (y/n)")
answer = input()
if answer == 'y':
payload = {'path': str(directory), 'force': 'force'}
r = requests.delete(f'{self.url}/rmdir', data=payload)
def full_split(_path):
"""
Return a list with all the intermediate paths.
The input path must be a POSIX path string (i.e., Linux or OSX).
"""
intermediate_paths = list()
_path = PurePosixPath(_path)
if _path.is_absolute():
_path = _path.relative_to("/")
parts = _path.parts
for i in range(1, len(parts)):
intermediate_paths.append(PurePosixPath(*parts[0:i]).as_posix())
return intermediate_paths
def update_modules(module_name_prefix: str, modules_path: Optional[str],
**kwargs):
LOG.debug("Generating update_modules proposal")
# Validate module name prefix
module_name_prefix_ = PurePosixPath(module_name_prefix)
if not module_name_prefix_.is_absolute():
raise ValueError("module name prefix must be an absolute path")
if any(folder in [".", ".."] for folder in module_name_prefix_.parents):
raise ValueError(
"module name prefix must not contain . or .. components")
if not module_name_prefix.endswith("/"):
raise ValueError("module name prefix must end with /")
# Read module files and build relative module names
modules = []
if modules_path:
modules = read_modules(modules_path)
proposal_args = {"prefix": module_name_prefix, "modules": modules}
return build_proposal("update_modules", proposal_args, **kwargs)
def cd(self, cmd):
'''Changes the current directory. Usage: cd <directory>'''
directory = cmd[1]
parts = directory.split('/')
while parts[0] == '..':
self.cwd = self.cwd.parent
parts = parts[1:]
if len(parts) == 0:
return
directory = PurePosixPath('/'.join(parts))
if not directory.is_absolute():
directory = self.cwd / directory
payload = {'path': str(directory)}
r = requests.get(f'{self.url}/ls', data=payload)
if not r.ok:
print(r.text)
return
# If request passed, the folder exists
self.cwd = directory
def get_path_from_url(prefix, url, url_to_path):
if is_external(url, prefix):
raise ValueError(f'external url {url}')
path = url.path
if path.startswith(prefix.path):
path = path[len(prefix.path):]
result = url_to_path(path)
result = PurePosixPath(result)
if result.is_absolute():
raise ValueError(
f"Path may not be absolute: {result}(from {url.to_url()})")
assert '.' not in result.parts
if '..' in result.parts:
raise ValueError(
f"Path may not contain /../ segment: {result}(from {url.to_url()})"
)
return result
def get_by_path(user, path):
'''
Checks whether directory specified by given `path` exists for
given `user`.
Return value is a 2-tuple (n, directory) where directory
is the deepest ancestor found or the found directory, and n is the
number of directories that were not found.
So if given path = "/a/b/c/d/e", and "/a/b/c" exists but "d" doesn't,
This will return `(2, <directory object for /a/b/c>)` since "d" and
"e" were not found.
Also note that this function ignores anything that isn't a directory.
Using the above example, if "d" is a file, the return value will still
be the same since "d" isn't a directory and thus is ignored.
Arguments:
user -- user to be checked; Should match settings.AUTH_USER_MODEL.
path -- a path-like object specifying a POSIX path to check.
If relative, considered as relative-from-root.
'''
path = PurePosixPath(path)
parts = iter(path.parts)
if path.is_absolute():
next(parts) # discard first item
current_dir = UserStorage.objects.filter(
user=user).select_related('root_dir').get().root_dir
for (i, part) in enumerate(parts):
try:
next_dir = current_dir.children.get(name__exact=part).directory
except (DirectoryEntry.DoesNotExist, Directory.DoesNotExist):
return (len(parts) - i, current_dir)
current_dir = next_dir
return (0, current_dir)
def from_source_path( cls, source_path: PurePosixPath
) -> 'RecordPath':
if source_path.is_absolute():
raise ValueError(source_path)
return cls.from_parts(source_path.parts)
