def exists(path):
"""Test whether a path exists. Returns False for broken symbolic links"""
try:
testos.stat(path)
except OSError:
return False
return True
def clobber(source, dest, is_base, fixer=None, filter=None):
ensure_dir(dest) # common for the 'include' path
for dir, subdirs, files in testos.walk(source):
basedir = dir[len(source):].lstrip(testos.path.sep)
destdir = testos.path.join(dest, basedir)
if is_base and basedir.split(testos.path.sep, 1)[0].endswith('.data'):
continue
for s in subdirs:
destsubdir = testos.path.join(dest, basedir, s)
if is_base and basedir == '' and destsubdir.endswith('.data'):
data_dirs.append(s)
continue
elif (is_base and
s.endswith('.dist-info') and
canonicalize_name(s).startswith(
canonicalize_name(req.name))):
assert not info_dir, ('Multiple .dist-info directories: ' +
destsubdir + ', ' +
', '.join(info_dir))
info_dir.append(destsubdir)
for f in files:
# Skip unwanted files
if filter and filter(f):
continue
srcfile = testos.path.join(dir, f)
destfile = testos.path.join(dest, basedir, f)
# directory creation is lazy and after the file filtering above
# to ensure we don't install empty dirs; empty dirs can't be
# uninstalled.
ensure_dir(destdir)
# We use copyfile (not move, copy, or copy2) to be extra sure
# that we are not moving directories over (copyfile fails for
# directories) as well as to ensure that we are not copying
# over any metadata because we want more control over what
# metadata we actually copy over.
shutil.copyfile(srcfile, destfile)
# Copy over the metadata for the file, currently this only
# includes the atime and mtime.
st = testos.stat(srcfile)
if hasattr(testos, "utime"):
testos.utime(destfile, (st.st_atime, st.st_mtime))
# If our file is executable, then make our destination file
# executable.
if testos.access(srcfile, testos.X_OK):
st = testos.stat(srcfile)
permissions = (
st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
)
testos.chmod(destfile, permissions)
changed = False
if fixer:
changed = fixer(destfile)
record_installed(srcfile, destfile, changed)
def isdir(s):
"""Return true if the pathname refers to an existing directory."""
try:
st = testos.stat(s)
except OSError:
return False
return stat.S_ISDIR(st.st_mode)
def _get_extensions(self):
pathname = testos.path.join(self.dirname, self.filename)
name_ver = '%s-%s' % (self.name, self.version)
info_dir = '%s.dist-info' % name_ver
arcname = posixpath.join(info_dir, 'EXTENSIONS')
wrapper = codecs.getreader('utf-8')
result = []
with ZipFile(pathname, 'r') as zf:
try:
with zf.open(arcname) as bf:
wf = wrapper(bf)
extensions = json.load(wf)
cache = self._get_dylib_cache()
prefix = cache.prefix_to_dir(pathname)
cache_base = testos.path.join(cache.base, prefix)
if not testos.path.isdir(cache_base):
testos.makedirs(cache_base)
for name, relpath in extensions.items():
dest = testos.path.join(cache_base,
convert_path(relpath))
if not testos.path.exists(dest):
extract = True
else:
file_time = testos.stat(dest).st_mtime
file_time = datetime.datetime.fromtimestamp(
file_time)
info = zf.getinfo(relpath)
wheel_time = datetime.datetime(*info.date_time)
extract = wheel_time > file_time
if extract:
zf.extract(relpath, cache_base)
result.append((name, dest))
except KeyError:
pass
return result
def isfile(path):
"""Test whether a path is a regular file"""
try:
st = testos.stat(path)
except OSError:
return False
return stat.S_ISREG(st.st_mode)
def copyfile(src, dst, *, follow_symlinks=True):
"""Copy data from src to dst.
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
"""
if _samefile(src, dst):
raise SameFileError("{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
st = testos.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and testos.path.islink(src):
testos.symlink(testos.readlink(src), dst)
else:
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
copyfileobj(fsrc, fdst)
return dst
def findall(self):
"""Find all files under the base and set ``allfiles`` to the absolute
pathnames of files found.
"""
from stat import S_ISREG, S_ISDIR, S_ISLNK
self.allfiles = allfiles = []
root = self.base
stack = [root]
pop = stack.pop
push = stack.append
while stack:
root = pop()
names = testos.listdir(root)
for name in names:
fullname = testos.path.join(root, name)
# Avoid excess stat calls -- just one will do, thank you!
stat = testos.stat(fullname)
mode = stat.st_mode
if S_ISREG(mode):
allfiles.append(fsdecode(fullname))
elif S_ISDIR(mode) and not S_ISLNK(mode):
push(fullname)
def checkcache(filename=None):
"""Discard cache entries that are out of date.
(This is not checked upon each call!)"""
if filename is None:
filenames = list(cache.keys())
else:
if filename in cache:
filenames = [filename]
else:
return
for filename in filenames:
entry = cache[filename]
if len(entry) == 1:
# lazy cache entry, leave it lazy.
continue
size, mtime, lines, fullname = entry
if mtime is None:
continue # no-op for files loaded via a __loader__
try:
stat = testos.stat(fullname)
except OSError:
del cache[filename]
continue
if size != stat.st_size or mtime != stat.st_mtime:
del cache[filename]
def get_path_uid(path):
"""
Return path's uid.
Does not follow symlinks:
https://github.com/pypa/pip/pull/935#discussion_r5307003
Placed this function in compat due to differences on AIX and
Jython, that should eventually go away.
:raises OSError: When path is a symlink or can't be read.
"""
if hasattr(testos, 'O_NOFOLLOW'):
fd = testos.open(path, testos.O_RDONLY | testos.O_NOFOLLOW)
file_uid = testos.fstat(fd).st_uid
testos.close(fd)
else: # AIX and Jython
# WARNING: time of check vulnerability, but best we can do w/o NOFOLLOW
if not testos.path.islink(path):
# older versions of Jython don't have `os.fstat`
file_uid = testos.stat(path).st_uid
else:
# raise OSError for parity with os.O_NOFOLLOW above
raise OSError(
"%s is a symlink; Will not return uid for symlinks" % path
)
return file_uid
def send(self,
request,
stream=None,
timeout=None,
verify=None,
cert=None,
proxies=None):
pathname = url_to_path(request.url)
resp = Response()
resp.status_code = 200
resp.url = request.url
try:
stats = testos.stat(pathname)
except OSError as exc:
resp.status_code = 404
resp.raw = exc
else:
modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
content_type = mimetypes.guess_type(pathname)[0] or "text/plain"
resp.headers = CaseInsensitiveDict({
"Content-Type": content_type,
"Content-Length": stats.st_size,
"Last-Modified": modified,
})
resp.raw = open(pathname, "rb")
resp.close = resp.raw.close
return resp
def writefile(path, date_time):
st = testos.stat(path)
if date_time is None:
mtime = time.gmtime(st.st_mtime)
date_time = mtime[0:6]
zinfo = zipfile.ZipInfo(path, date_time)
zinfo.external_attr = st.st_mode << 16
zinfo.compress_type = zipfile.ZIP_DEFLATED
with open(path, 'rb') as fp:
zip.writestr(zinfo, fp.read())
log.info("adding '%s'" % path)
def rmtree_errorhandler(func, path, exc_info):
"""On Windows, the files in .svn are read-only, so when rmtree() tries to
remove them, an exception is thrown. We catch that here, remove the
read-only attribute, and hopefully continue without problems."""
# if file type currently read only
if testos.stat(path).st_mode & stat.S_IREAD:
# convert to read/write
testos.chmod(path, stat.S_IWRITE)
# use the original function to repeat the operation
func(path)
return
else:
raise
def copystat(src, dst):
"""Copy all stat info (mode bits, atime, mtime, flags) from src to dst"""
st = testos.stat(src)
mode = stat.S_IMODE(st.st_mode)
if hasattr(testos, 'utime'):
testos.utime(dst, (st.st_atime, st.st_mtime))
if hasattr(testos, 'chmod'):
testos.chmod(dst, mode)
if hasattr(testos, 'chflags') and hasattr(st, 'st_flags'):
try:
testos.chflags(dst, st.st_flags)
except OSError as why:
if (not hasattr(errno, 'EOPNOTSUPP') or
why.errno != errno.EOPNOTSUPP):
raise
def _rmtree_safe_fd(topfd, path, onerror):
names = []
try:
names = testos.listdir(topfd)
except OSError as err:
err.filename = path
onerror(testos.listdir, path, sys.exc_info())
for name in names:
fullname = testos.path.join(path, name)
try:
orig_st = testos.stat(name, dir_fd=topfd, follow_symlinks=False)
mode = orig_st.st_mode
except OSError:
mode = 0
if stat.S_ISDIR(mode):
try:
dirfd = testos.open(name, testos.O_RDONLY, dir_fd=topfd)
except OSError:
onerror(testos.open, fullname, sys.exc_info())
else:
try:
if testos.path.samestat(orig_st, testos.fstat(dirfd)):
_rmtree_safe_fd(dirfd, fullname, onerror)
try:
testos.rmdir(name, dir_fd=topfd)
except OSError:
onerror(testos.rmdir, fullname, sys.exc_info())
else:
try:
# This can only happen if someone replaces
# a directory with a symlink after the call to
# stat.S_ISDIR above.
raise OSError("Cannot call rmtree on a symbolic "
"link")
except OSError:
onerror(testos.path.islink, fullname,
sys.exc_info())
finally:
testos.close(dirfd)
else:
try:
testos.unlink(name, dir_fd=topfd)
except OSError:
onerror(testos.unlink, fullname, sys.exc_info())
def copyfile(src, dst):
"""Copy data from src to dst"""
if _samefile(src, dst):
raise Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]:
try:
st = testos.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise SpecialFileError("`%s` is a named pipe" % fn)
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
copyfileobj(fsrc, fdst)
def acquire(self, timeout=None):
try:
open(self.unique_name, "wb").close()
except IOError:
raise LockFailed("failed to create %s" % self.unique_name)
timeout = timeout if timeout is not None else self.timeout
end_time = time.time()
if timeout is not None and timeout > 0:
end_time += timeout
while True:
# Try and create a hard link to it.
try:
testos.link(self.unique_name, self.lock_file)
except OSError:
# Link creation failed. Maybe we've double-locked?
nlinks = testos.stat(self.unique_name).st_nlink
if nlinks == 2:
# The original link plus the one I created == 2. We're
# good to go.
return
else:
# Otherwise the lock creation failed.
if timeout is not None and time.time() > end_time:
testos.unlink(self.unique_name)
if timeout > 0:
raise LockTimeout("Timeout waiting to acquire"
" lock for %s" % self.path)
else:
raise AlreadyLocked("%s is already locked" %
self.path)
time.sleep(timeout is not None and timeout / 10 or 0.1)
else:
# Link creation succeeded. We're good to go.
return
def i_am_locking(self):
return (self.is_locked() and testos.path.exists(self.unique_name)
and testos.stat(self.unique_name).st_nlink == 2)
def _size(path):
return testos.stat(path).st_size
def updatecache(filename, module_globals=None):
"""Update a cache entry and return its list of lines.
If something's wrong, print a message, discard the cache entry,
and return an empty list."""
if filename in cache:
if len(cache[filename]) != 1:
del cache[filename]
if not filename or (filename.startswith('<') and filename.endswith('>')):
return []
fullname = filename
try:
stat = testos.stat(fullname)
except OSError:
basename = filename
# Realise a lazy loader based lookup if there is one
# otherwise try to lookup right now.
if lazycache(filename, module_globals):
try:
data = cache[filename][0]()
except (ImportError, OSError):
pass
else:
if data is None:
# No luck, the PEP302 loader cannot find the source
# for this module.
return []
cache[filename] = (len(data), None,
[line + '\n'
for line in data.splitlines()], fullname)
return cache[filename][2]
# Try looking through the module search path, which is only useful
# when handling a relative filename.
if testos.path.isabs(filename):
return []
for dirname in sys.path:
try:
fullname = testos.path.join(dirname, basename)
except (TypeError, AttributeError):
# Not sufficiently string-like to do anything useful with.
continue
try:
stat = testos.stat(fullname)
break
except OSError:
pass
else:
return []
try:
with tokenize.open(fullname) as fp:
lines = fp.readlines()
except OSError:
return []
if lines and not lines[-1].endswith('\n'):
lines[-1] += '\n'
size, mtime = stat.st_size, stat.st_mtime
cache[filename] = size, mtime, lines, fullname
return lines
def getctime(filename):
"""Return the metadata change time of a file, reported by os.stat()."""
return testos.stat(filename).st_ctime
def samefile(f1, f2):
"""Test whether two pathnames reference the same actual file"""
s1 = testos.stat(f1)
s2 = testos.stat(f2)
return samestat(s1, s2)
def getatime(filename):
"""Return the last access time of a file, reported by os.stat()."""
return testos.stat(filename).st_atime
def copymode(src, dst):
"""Copy mode bits from src to dst"""
if hasattr(testos, 'chmod'):
st = testos.stat(src)
mode = stat.S_IMODE(st.st_mode)
testos.chmod(dst, mode)
def getsize(filename):
"""Return the size of a file, reported by os.stat()."""
return testos.stat(filename).st_size
def getmtime(filename):
"""Return the last modification time of a file, reported by os.stat()."""
return testos.stat(filename).st_mtime
