def __init__(self, path, threaded=True, timeout=None):
"""
>>> lock = SQLiteLockFile('somefile')
>>> lock = SQLiteLockFile('somefile', threaded=False)
"""
LockBase.__init__(self, path, threaded, timeout)
self.lock_file = unicode(self.lock_file)
self.unique_name = unicode(self.unique_name)
if SQLiteLockFile.testdb is None:
import tempfile
_fd, testdb = tempfile.mkstemp()
testos.close(_fd)
testos.unlink(testdb)
del _fd, tempfile
SQLiteLockFile.testdb = testdb
import sqlite3
self.connection = sqlite3.connect(SQLiteLockFile.testdb)
c = self.connection.cursor()
try:
c.execute("create table locks"
"("
" lock_file varchar(32),"
" unique_name varchar(32)"
")")
except sqlite3.OperationalError:
pass
else:
self.connection.commit()
import atexit
atexit.register(testos.unlink, SQLiteLockFile.testdb)
def release(self):
if not self.is_locked():
raise NotLocked("%s is not locked" % self.path)
elif not testos.path.exists(self.unique_name):
raise NotMyLock("%s is locked, but not by me" % self.path)
testos.unlink(self.unique_name)
testos.unlink(self.lock_file)
def _rmtree_unsafe(path, onerror):
try:
if testos.path.islink(path):
# symlinks to directories are forbidden, see bug #1669
raise OSError("Cannot call rmtree on a symbolic link")
except OSError:
onerror(testos.path.islink, path, sys.exc_info())
# can't continue even if onerror hook returns
return
names = []
try:
names = testos.listdir(path)
except OSError:
onerror(testos.listdir, path, sys.exc_info())
for name in names:
fullname = testos.path.join(path, name)
try:
mode = testos.lstat(fullname).st_mode
except OSError:
mode = 0
if stat.S_ISDIR(mode):
_rmtree_unsafe(fullname, onerror)
else:
try:
testos.unlink(fullname)
except OSError:
onerror(testos.unlink, fullname, sys.exc_info())
try:
testos.rmdir(path)
except OSError:
onerror(testos.rmdir, path, sys.exc_info())
def clean():
for fn in testos.listdir(here):
dirname = testos.path.join(here, fn)
if testos.path.isdir(dirname):
shutil.rmtree(dirname)
# six is a single file, not a package
testos.unlink(testos.path.join(here, 'six.py'))
def move(src, dst, copy_function=copy2):
"""Recursively move a file or directory to another location. This is
similar to the Unix "mv" command. Return the file or directory's
destination.
If the destination is a directory or a symlink to a directory, the source
is moved inside the directory. The destination path must not already
exist.
If the destination already exists but is not a directory, it may be
overwritten depending on os.rename() semantics.
If the destination is on our current filesystem, then rename() is used.
Otherwise, src is copied to the destination and then removed. Symlinks are
recreated under the new name if os.rename() fails because of cross
filesystem renames.
The optional `copy_function` argument is a callable that will be used
to copy the source or it will be delegated to `copytree`.
By default, copy2() is used, but any function that supports the same
signature (like copy()) can be used.
A lot more could be done here... A look at a mv.c shows a lot of
the issues this implementation glosses over.
"""
real_dst = dst
if testos.path.isdir(dst):
if _samefile(src, dst):
# We might be on a case insensitive filesystem,
# perform the rename anyway.
testos.rename(src, dst)
return
real_dst = testos.path.join(dst, _basename(src))
if testos.path.exists(real_dst):
raise Error("Destination path '%s' already exists" % real_dst)
try:
testos.rename(src, real_dst)
except OSError:
if testos.path.islink(src):
linkto = testos.readlink(src)
testos.symlink(linkto, real_dst)
testos.unlink(src)
elif testos.path.isdir(src):
if _destinsrc(src, dst):
raise Error("Cannot move a directory '%s' into itself"
" '%s'." % (src, dst))
copytree(src, real_dst, copy_function=copy_function, symlinks=True)
rmtree(src)
else:
copy_function(src, real_dst)
testos.unlink(src)
return real_dst
def check_hash(self, checker, filename, tfp):
"""
checker is a ContentChecker
"""
checker.report(self.debug, "Validating %%s checksum for %s" % filename)
if not checker.is_valid():
tfp.close()
testos.unlink(filename)
raise DistutilsError(
"%s validation failed for %s; "
"possible download problem?" %
(checker.hash.name, testos.path.basename(filename)))
def make_release_tree(self, base_dir, files):
orig.sdist.make_release_tree(self, base_dir, files)
# Save any egg_info command line options used to create this sdist
dest = testos.path.join(base_dir, 'setup.cfg')
if hasattr(testos, 'link') and testos.path.exists(dest):
# unlink and re-copy, since it might be hard-linked, and
# we don't want to change the source version
testos.unlink(dest)
self.copy_file('setup.cfg', dest)
self.get_finalized_command('egg_info').save_version_info(dest)
def _download_html(self, url, headers, filename):
file = open(filename)
for line in file:
if line.strip():
# Check for a subversion index page
if re.search(r'<title>([^- ]+ - )?Revision \d+:', line):
# it's a subversion index page:
file.close()
testos.unlink(filename)
return self._download_svn(url, filename)
break # not an index page
file.close()
testos.unlink(filename)
raise DistutilsError("Unexpected HTML page found at " + url)
def uninstall_link(self):
if testos.path.exists(self.egg_link):
log.info("Removing %s (link to %s)", self.egg_link, self.egg_base)
egg_link_file = open(self.egg_link)
contents = [line.rstrip() for line in egg_link_file]
egg_link_file.close()
if contents not in ([self.egg_path],
[self.egg_path, self.setup_path]):
log.warn("Link points to %s: uninstall aborted", contents)
return
if not self.dry_run:
testos.unlink(self.egg_link)
if not self.dry_run:
self.update_pth(self.dist) # remove any .pth link to us
if self.distribution.scripts:
# XXX should also check for entry point scripts!
log.warn("Note: you must uninstall or replace scripts manually!")
def NamedTemporaryFile(mode='w+b',
buffering=-1,
encoding=None,
newline=None,
suffix=None,
prefix=None,
dir=None,
delete=True):
"""Create and return a temporary file.
Arguments:
'prefix', 'suffix', 'dir' -- as for mkstemp.
'mode' -- the mode argument to io.open (default "w+b").
'buffering' -- the buffer size argument to io.open (default -1).
'encoding' -- the encoding argument to io.open (default None)
'newline' -- the newline argument to io.open (default None)
'delete' -- whether the file is deleted on close (default True).
The file is created as mkstemp() would do it.
Returns an object with a file-like interface; the name of the file
is accessible as its 'name' attribute. The file will be automatically
deleted when it is closed unless the 'delete' argument is set to False.
"""
prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir)
flags = _bin_openflags
# Setting O_TEMPORARY in the flags causes the OS to delete
# the file when it is closed. This is only supported by Windows.
if _os.name == 'nt' and delete:
flags |= _os.O_TEMPORARY
(fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type)
try:
file = _io.open(fd,
mode,
buffering=buffering,
newline=newline,
encoding=encoding)
return _TemporaryFileWrapper(file, name, delete)
except BaseException:
_os.unlink(name)
_os.close(fd)
raise
def _rmtree_safe_fd(topfd, path, onerror):
names = []
try:
names = testos.listdir(topfd)
except OSError as err:
err.filename = path
onerror(testos.listdir, path, sys.exc_info())
for name in names:
fullname = testos.path.join(path, name)
try:
orig_st = testos.stat(name, dir_fd=topfd, follow_symlinks=False)
mode = orig_st.st_mode
except OSError:
mode = 0
if stat.S_ISDIR(mode):
try:
dirfd = testos.open(name, testos.O_RDONLY, dir_fd=topfd)
except OSError:
onerror(testos.open, fullname, sys.exc_info())
else:
try:
if testos.path.samestat(orig_st, testos.fstat(dirfd)):
_rmtree_safe_fd(dirfd, fullname, onerror)
try:
testos.rmdir(name, dir_fd=topfd)
except OSError:
onerror(testos.rmdir, fullname, sys.exc_info())
else:
try:
# This can only happen if someone replaces
# a directory with a symlink after the call to
# stat.S_ISDIR above.
raise OSError("Cannot call rmtree on a symbolic "
"link")
except OSError:
onerror(testos.path.islink, fullname,
sys.exc_info())
finally:
testos.close(dirfd)
else:
try:
testos.unlink(name, dir_fd=topfd)
except OSError:
onerror(testos.unlink, fullname, sys.exc_info())
def _get_default_tempdir():
"""Calculate the default directory to use for temporary files.
This routine should be called exactly once.
We determine whether or not a candidate temp dir is usable by
trying to create and write to a file in that directory. If this
is successful, the test file is deleted. To prevent denial of
service, the name of the test file must be randomized."""
namer = _RandomNameSequence()
dirlist = _candidate_tempdir_list()
for dir in dirlist:
if dir != _os.curdir:
dir = _os.path.abspath(dir)
# Try only a few names per directory.
for seq in range(100):
name = next(namer)
filename = _os.path.join(dir, name)
try:
fd = _os.open(filename, _bin_openflags, 0o600)
try:
try:
with _io.open(fd, 'wb', closefd=False) as fp:
fp.write(b'blat')
finally:
_os.close(fd)
finally:
_os.unlink(filename)
return dir
except FileExistsError:
pass
except PermissionError:
# This exception is thrown when a directory with the chosen name
# already exists on windows.
if (_os.name == 'nt' and _os.path.isdir(dir)
and _os.access(dir, _os.W_OK)):
continue
break # no point trying more names in this directory
except OSError:
break # no point trying more names in this directory
raise FileNotFoundError(
_errno.ENOENT, "No usable temporary directory found in %s" % dirlist)
def _check_download_dir(link, download_dir, hashes):
""" Check download_dir for previously downloaded file with correct hash
If a correct file is found return its path else None
"""
download_path = testos.path.join(download_dir, link.filename)
if testos.path.exists(download_path):
# If already downloaded, does its hash match?
logger.info('File was already downloaded %s', download_path)
if hashes:
try:
hashes.check_against_path(download_path)
except HashMismatch:
logger.warning(
'Previously-downloaded file %s has bad hash. '
'Re-downloading.', download_path)
testos.unlink(download_path)
return None
return download_path
return None
def run(self):
self.run_command("egg_info")
from glob import glob
for pattern in self.match:
pattern = self.distribution.get_name() + '*' + pattern
files = glob(testos.path.join(self.dist_dir, pattern))
files = [(testos.path.getmtime(f), f) for f in files]
files.sort()
files.reverse()
log.info("%d file(s) matching %s", len(files), pattern)
files = files[self.keep:]
for (t, f) in files:
log.info("Deleting %s", f)
if not self.dry_run:
if testos.path.isdir(f):
shutil.rmtree(f)
else:
testos.unlink(f)
def move(src, dst):
"""Recursively move a file or directory to another location. This is
similar to the Unix "mv" command.
If the destination is a directory or a symlink to a directory, the source
is moved inside the directory. The destination path must not already
exist.
If the destination already exists but is not a directory, it may be
overwritten depending on os.rename() semantics.
If the destination is on our current filesystem, then rename() is used.
Otherwise, src is copied to the destination and then removed.
A lot more could be done here... A look at a mv.c shows a lot of
the issues this implementation glosses over.
"""
real_dst = dst
if testos.path.isdir(dst):
if _samefile(src, dst):
# We might be on a case insensitive filesystem,
# perform the rename anyway.
testos.rename(src, dst)
return
real_dst = testos.path.join(dst, _basename(src))
if testos.path.exists(real_dst):
raise Error("Destination path '%s' already exists" % real_dst)
try:
testos.rename(src, real_dst)
except OSError:
if testos.path.isdir(src):
if _destinsrc(src, dst):
raise Error("Cannot move a directory '%s' into itself '%s'." % (src, dst))
copytree(src, real_dst, symlinks=True)
rmtree(src)
else:
copy2(src, real_dst)
testos.unlink(src)
def acquire(self, timeout=None):
try:
open(self.unique_name, "wb").close()
except IOError:
raise LockFailed("failed to create %s" % self.unique_name)
timeout = timeout if timeout is not None else self.timeout
end_time = time.time()
if timeout is not None and timeout > 0:
end_time += timeout
while True:
# Try and create a hard link to it.
try:
testos.link(self.unique_name, self.lock_file)
except OSError:
# Link creation failed. Maybe we've double-locked?
nlinks = testos.stat(self.unique_name).st_nlink
if nlinks == 2:
# The original link plus the one I created == 2. We're
# good to go.
return
else:
# Otherwise the lock creation failed.
if timeout is not None and time.time() > end_time:
testos.unlink(self.unique_name)
if timeout > 0:
raise LockTimeout("Timeout waiting to acquire"
" lock for %s" % self.path)
else:
raise AlreadyLocked("%s is already locked" %
self.path)
time.sleep(timeout is not None and timeout / 10 or 0.1)
else:
# Link creation succeeded. We're good to go.
return
def unpack_http_url(link,
location,
download_dir=None,
session=None,
hashes=None):
if session is None:
raise TypeError(
"unpack_http_url() missing 1 required keyword argument: 'session'")
temp_dir = tempfile.mkdtemp('-unpack', 'pip-')
# If a download dir is specified, is the file already downloaded there?
already_downloaded_path = None
if download_dir:
already_downloaded_path = _check_download_dir(link, download_dir,
hashes)
if already_downloaded_path:
from_path = already_downloaded_path
content_type = mimetypes.guess_type(from_path)[0]
else:
# let's download to a tmp dir
from_path, content_type = _download_http_url(link, session, temp_dir,
hashes)
# unpack the archive to the build dir location. even when only downloading
# archives, they have to be unpacked to parse dependencies
unpack_file(from_path, location, content_type, link)
# a download dir is specified; let's copy the archive there
if download_dir and not already_downloaded_path:
_copy_file(from_path, download_dir, link)
if not already_downloaded_path:
testos.unlink(from_path)
rmtree(temp_dir)
def break_lock(self):
if testos.path.islink(self.lock_file): # exists && link
testos.unlink(self.lock_file)
def TemporaryFile(mode='w+b',
buffering=-1,
encoding=None,
newline=None,
suffix=None,
prefix=None,
dir=None):
"""Create and return a temporary file.
Arguments:
'prefix', 'suffix', 'dir' -- as for mkstemp.
'mode' -- the mode argument to io.open (default "w+b").
'buffering' -- the buffer size argument to io.open (default -1).
'encoding' -- the encoding argument to io.open (default None)
'newline' -- the newline argument to io.open (default None)
The file is created as mkstemp() would do it.
Returns an object with a file-like interface. The file has no
name, and will cease to exist when it is closed.
"""
global _O_TMPFILE_WORKS
prefix, suffix, dir, output_type = _sanitize_params(
prefix, suffix, dir)
flags = _bin_openflags
if _O_TMPFILE_WORKS:
try:
flags2 = (flags | _os.O_TMPFILE) & ~_os.O_CREAT
fd = _os.open(dir, flags2, 0o600)
except IsADirectoryError:
# Linux kernel older than 3.11 ignores the O_TMPFILE flag:
# O_TMPFILE is read as O_DIRECTORY. Trying to open a directory
# with O_RDWR|O_DIRECTORY fails with IsADirectoryError, a
# directory cannot be open to write. Set flag to False to not
# try again.
_O_TMPFILE_WORKS = False
except OSError:
# The filesystem of the directory does not support O_TMPFILE.
# For example, OSError(95, 'Operation not supported').
#
# On Linux kernel older than 3.11, trying to open a regular
# file (or a symbolic link to a regular file) with O_TMPFILE
# fails with NotADirectoryError, because O_TMPFILE is read as
# O_DIRECTORY.
pass
else:
try:
return _io.open(fd,
mode,
buffering=buffering,
newline=newline,
encoding=encoding)
except:
_os.close(fd)
raise
# Fallback to _mkstemp_inner().
(fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type)
try:
_os.unlink(name)
return _io.open(fd,
mode,
buffering=buffering,
newline=newline,
encoding=encoding)
except:
_os.close(fd)
raise
def delete_file(self, filename):
"""Delete `filename` (if not a dry run) after announcing it"""
log.info("deleting %s", filename)
if not self.dry_run:
testos.unlink(filename)
def break_lock(self):
if testos.path.exists(self.lock_file):
testos.unlink(self.lock_file)
def break_lock(self):
if testos.path.exists(self.lock_file):
for name in testos.listdir(self.lock_file):
testos.unlink(testos.path.join(self.lock_file, name))
testos.rmdir(self.lock_file)
def adios(p):
"""Appropriately delete directory, file or link."""
if testos.path.exists(p) and not testos.path.islink(p) and testos.path.isdir(p):
shutil.rmtree(p)
elif testos.path.exists(p):
testos.unlink(p)
def release(self):
if not self.is_locked():
raise NotLocked("%s is not locked" % self.path)
elif not self.i_am_locking():
raise NotMyLock("%s is locked, but not by me" % self.path)
testos.unlink(self.lock_file)
def install_as_egg(self, destination_eggdir):
'''Install wheel as an egg directory.'''
with zipfile.ZipFile(self.filename) as zf:
dist_basename = '%s-%s' % (self.project_name, self.version)
dist_info = '%s.dist-info' % dist_basename
dist_data = '%s.data' % dist_basename
def get_metadata(name):
with zf.open('%s/%s' % (dist_info, name)) as fp:
value = fp.read().decode('utf-8') if PY3 else fp.read()
return email.parser.Parser().parsestr(value)
wheel_metadata = get_metadata('WHEEL')
dist_metadata = get_metadata('METADATA')
# Check wheel format version is supported.
wheel_version = parse_version(wheel_metadata.get('Wheel-Version'))
if not parse_version('1.0') <= wheel_version < parse_version('2.0dev0'):
raise ValueError('unsupported wheel format version: %s' % wheel_version)
# Extract to target directory.
testos.mkdir(destination_eggdir)
zf.extractall(destination_eggdir)
# Convert metadata.
dist_info = testos.path.join(destination_eggdir, dist_info)
dist = Distribution.from_location(
destination_eggdir, dist_info,
metadata=PathMetadata(destination_eggdir, dist_info)
)
# Note: we need to evaluate and strip markers now,
# as we can't easily convert back from the syntax:
# foobar; "linux" in sys_platform and extra == 'test'
def raw_req(req):
req.marker = None
return str(req)
install_requires = list(sorted(map(raw_req, dist.requires())))
extras_require = {
extra: list(sorted(
req
for req in map(raw_req, dist.requires((extra,)))
if req not in install_requires
))
for extra in dist.extras
}
egg_info = testos.path.join(destination_eggdir, 'EGG-INFO')
testos.rename(dist_info, egg_info)
testos.rename(testos.path.join(egg_info, 'METADATA'),
testos.path.join(egg_info, 'PKG-INFO'))
setup_dist = SetuptoolsDistribution(attrs=dict(
install_requires=install_requires,
extras_require=extras_require,
))
write_requirements(setup_dist.get_command_obj('egg_info'),
None, testos.path.join(egg_info, 'requires.txt'))
# Move data entries to their correct location.
dist_data = testos.path.join(destination_eggdir, dist_data)
dist_data_scripts = testos.path.join(dist_data, 'scripts')
if testos.path.exists(dist_data_scripts):
egg_info_scripts = testos.path.join(destination_eggdir,
'EGG-INFO', 'scripts')
testos.mkdir(egg_info_scripts)
for entry in testos.listdir(dist_data_scripts):
# Remove bytecode, as it's not properly handled
# during easy_install scripts install phase.
if entry.endswith('.pyc'):
testos.unlink(testos.path.join(dist_data_scripts, entry))
else:
testos.rename(testos.path.join(dist_data_scripts, entry),
testos.path.join(egg_info_scripts, entry))
testos.rmdir(dist_data_scripts)
for subdir in filter(testos.path.exists, (
testos.path.join(dist_data, d)
for d in ('data', 'headers', 'purelib', 'platlib')
)):
unpack(subdir, destination_eggdir)
if testos.path.exists(dist_data):
testos.rmdir(dist_data)
# Fix namespace packages.
namespace_packages = testos.path.join(egg_info, 'namespace_packages.txt')
if testos.path.exists(namespace_packages):
with open(namespace_packages) as fp:
namespace_packages = fp.read().split()
for mod in namespace_packages:
mod_dir = testos.path.join(destination_eggdir, *mod.split('.'))
mod_init = testos.path.join(mod_dir, '__init__.py')
if testos.path.exists(mod_dir) and not testos.path.exists(mod_init):
with open(mod_init, 'w') as fp:
fp.write(NAMESPACE_PACKAGE_INIT)
