def __init__(self, path, threaded=True, timeout=None): """ >>> lock = SQLiteLockFile('somefile') >>> lock = SQLiteLockFile('somefile', threaded=False) """ LockBase.__init__(self, path, threaded, timeout) self.lock_file = unicode(self.lock_file) self.unique_name = unicode(self.unique_name) if SQLiteLockFile.testdb is None: import tempfile _fd, testdb = tempfile.mkstemp() testos.close(_fd) testos.unlink(testdb) del _fd, tempfile SQLiteLockFile.testdb = testdb import sqlite3 self.connection = sqlite3.connect(SQLiteLockFile.testdb) c = self.connection.cursor() try: c.execute("create table locks" "(" " lock_file varchar(32)," " unique_name varchar(32)" ")") except sqlite3.OperationalError: pass else: self.connection.commit() import atexit atexit.register(testos.unlink, SQLiteLockFile.testdb)
def release(self): if not self.is_locked(): raise NotLocked("%s is not locked" % self.path) elif not testos.path.exists(self.unique_name): raise NotMyLock("%s is locked, but not by me" % self.path) testos.unlink(self.unique_name) testos.unlink(self.lock_file)
def _rmtree_unsafe(path, onerror): try: if testos.path.islink(path): # symlinks to directories are forbidden, see bug #1669 raise OSError("Cannot call rmtree on a symbolic link") except OSError: onerror(testos.path.islink, path, sys.exc_info()) # can't continue even if onerror hook returns return names = [] try: names = testos.listdir(path) except OSError: onerror(testos.listdir, path, sys.exc_info()) for name in names: fullname = testos.path.join(path, name) try: mode = testos.lstat(fullname).st_mode except OSError: mode = 0 if stat.S_ISDIR(mode): _rmtree_unsafe(fullname, onerror) else: try: testos.unlink(fullname) except OSError: onerror(testos.unlink, fullname, sys.exc_info()) try: testos.rmdir(path) except OSError: onerror(testos.rmdir, path, sys.exc_info())
def clean(): for fn in testos.listdir(here): dirname = testos.path.join(here, fn) if testos.path.isdir(dirname): shutil.rmtree(dirname) # six is a single file, not a package testos.unlink(testos.path.join(here, 'six.py'))
def move(src, dst, copy_function=copy2): """Recursively move a file or directory to another location. This is similar to the Unix "mv" command. Return the file or directory's destination. If the destination is a directory or a symlink to a directory, the source is moved inside the directory. The destination path must not already exist. If the destination already exists but is not a directory, it may be overwritten depending on os.rename() semantics. If the destination is on our current filesystem, then rename() is used. Otherwise, src is copied to the destination and then removed. Symlinks are recreated under the new name if os.rename() fails because of cross filesystem renames. The optional `copy_function` argument is a callable that will be used to copy the source or it will be delegated to `copytree`. By default, copy2() is used, but any function that supports the same signature (like copy()) can be used. A lot more could be done here... A look at a mv.c shows a lot of the issues this implementation glosses over. """ real_dst = dst if testos.path.isdir(dst): if _samefile(src, dst): # We might be on a case insensitive filesystem, # perform the rename anyway. testos.rename(src, dst) return real_dst = testos.path.join(dst, _basename(src)) if testos.path.exists(real_dst): raise Error("Destination path '%s' already exists" % real_dst) try: testos.rename(src, real_dst) except OSError: if testos.path.islink(src): linkto = testos.readlink(src) testos.symlink(linkto, real_dst) testos.unlink(src) elif testos.path.isdir(src): if _destinsrc(src, dst): raise Error("Cannot move a directory '%s' into itself" " '%s'." % (src, dst)) copytree(src, real_dst, copy_function=copy_function, symlinks=True) rmtree(src) else: copy_function(src, real_dst) testos.unlink(src) return real_dst
def check_hash(self, checker, filename, tfp): """ checker is a ContentChecker """ checker.report(self.debug, "Validating %%s checksum for %s" % filename) if not checker.is_valid(): tfp.close() testos.unlink(filename) raise DistutilsError( "%s validation failed for %s; " "possible download problem?" % (checker.hash.name, testos.path.basename(filename)))
def make_release_tree(self, base_dir, files): orig.sdist.make_release_tree(self, base_dir, files) # Save any egg_info command line options used to create this sdist dest = testos.path.join(base_dir, 'setup.cfg') if hasattr(testos, 'link') and testos.path.exists(dest): # unlink and re-copy, since it might be hard-linked, and # we don't want to change the source version testos.unlink(dest) self.copy_file('setup.cfg', dest) self.get_finalized_command('egg_info').save_version_info(dest)
def _download_html(self, url, headers, filename): file = open(filename) for line in file: if line.strip(): # Check for a subversion index page if re.search(r'<title>([^- ]+ - )?Revision \d+:', line): # it's a subversion index page: file.close() testos.unlink(filename) return self._download_svn(url, filename) break # not an index page file.close() testos.unlink(filename) raise DistutilsError("Unexpected HTML page found at " + url)
def uninstall_link(self): if testos.path.exists(self.egg_link): log.info("Removing %s (link to %s)", self.egg_link, self.egg_base) egg_link_file = open(self.egg_link) contents = [line.rstrip() for line in egg_link_file] egg_link_file.close() if contents not in ([self.egg_path], [self.egg_path, self.setup_path]): log.warn("Link points to %s: uninstall aborted", contents) return if not self.dry_run: testos.unlink(self.egg_link) if not self.dry_run: self.update_pth(self.dist) # remove any .pth link to us if self.distribution.scripts: # XXX should also check for entry point scripts! log.warn("Note: you must uninstall or replace scripts manually!")
def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True): """Create and return a temporary file. Arguments: 'prefix', 'suffix', 'dir' -- as for mkstemp. 'mode' -- the mode argument to io.open (default "w+b"). 'buffering' -- the buffer size argument to io.open (default -1). 'encoding' -- the encoding argument to io.open (default None) 'newline' -- the newline argument to io.open (default None) 'delete' -- whether the file is deleted on close (default True). The file is created as mkstemp() would do it. Returns an object with a file-like interface; the name of the file is accessible as its 'name' attribute. The file will be automatically deleted when it is closed unless the 'delete' argument is set to False. """ prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) flags = _bin_openflags # Setting O_TEMPORARY in the flags causes the OS to delete # the file when it is closed. This is only supported by Windows. if _os.name == 'nt' and delete: flags |= _os.O_TEMPORARY (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: file = _io.open(fd, mode, buffering=buffering, newline=newline, encoding=encoding) return _TemporaryFileWrapper(file, name, delete) except BaseException: _os.unlink(name) _os.close(fd) raise
def _rmtree_safe_fd(topfd, path, onerror): names = [] try: names = testos.listdir(topfd) except OSError as err: err.filename = path onerror(testos.listdir, path, sys.exc_info()) for name in names: fullname = testos.path.join(path, name) try: orig_st = testos.stat(name, dir_fd=topfd, follow_symlinks=False) mode = orig_st.st_mode except OSError: mode = 0 if stat.S_ISDIR(mode): try: dirfd = testos.open(name, testos.O_RDONLY, dir_fd=topfd) except OSError: onerror(testos.open, fullname, sys.exc_info()) else: try: if testos.path.samestat(orig_st, testos.fstat(dirfd)): _rmtree_safe_fd(dirfd, fullname, onerror) try: testos.rmdir(name, dir_fd=topfd) except OSError: onerror(testos.rmdir, fullname, sys.exc_info()) else: try: # This can only happen if someone replaces # a directory with a symlink after the call to # stat.S_ISDIR above. raise OSError("Cannot call rmtree on a symbolic " "link") except OSError: onerror(testos.path.islink, fullname, sys.exc_info()) finally: testos.close(dirfd) else: try: testos.unlink(name, dir_fd=topfd) except OSError: onerror(testos.unlink, fullname, sys.exc_info())
def _get_default_tempdir(): """Calculate the default directory to use for temporary files. This routine should be called exactly once. We determine whether or not a candidate temp dir is usable by trying to create and write to a file in that directory. If this is successful, the test file is deleted. To prevent denial of service, the name of the test file must be randomized.""" namer = _RandomNameSequence() dirlist = _candidate_tempdir_list() for dir in dirlist: if dir != _os.curdir: dir = _os.path.abspath(dir) # Try only a few names per directory. for seq in range(100): name = next(namer) filename = _os.path.join(dir, name) try: fd = _os.open(filename, _bin_openflags, 0o600) try: try: with _io.open(fd, 'wb', closefd=False) as fp: fp.write(b'blat') finally: _os.close(fd) finally: _os.unlink(filename) return dir except FileExistsError: pass except PermissionError: # This exception is thrown when a directory with the chosen name # already exists on windows. if (_os.name == 'nt' and _os.path.isdir(dir) and _os.access(dir, _os.W_OK)): continue break # no point trying more names in this directory except OSError: break # no point trying more names in this directory raise FileNotFoundError( _errno.ENOENT, "No usable temporary directory found in %s" % dirlist)
def _check_download_dir(link, download_dir, hashes): """ Check download_dir for previously downloaded file with correct hash If a correct file is found return its path else None """ download_path = testos.path.join(download_dir, link.filename) if testos.path.exists(download_path): # If already downloaded, does its hash match? logger.info('File was already downloaded %s', download_path) if hashes: try: hashes.check_against_path(download_path) except HashMismatch: logger.warning( 'Previously-downloaded file %s has bad hash. ' 'Re-downloading.', download_path) testos.unlink(download_path) return None return download_path return None
def run(self): self.run_command("egg_info") from glob import glob for pattern in self.match: pattern = self.distribution.get_name() + '*' + pattern files = glob(testos.path.join(self.dist_dir, pattern)) files = [(testos.path.getmtime(f), f) for f in files] files.sort() files.reverse() log.info("%d file(s) matching %s", len(files), pattern) files = files[self.keep:] for (t, f) in files: log.info("Deleting %s", f) if not self.dry_run: if testos.path.isdir(f): shutil.rmtree(f) else: testos.unlink(f)
def move(src, dst): """Recursively move a file or directory to another location. This is similar to the Unix "mv" command. If the destination is a directory or a symlink to a directory, the source is moved inside the directory. The destination path must not already exist. If the destination already exists but is not a directory, it may be overwritten depending on os.rename() semantics. If the destination is on our current filesystem, then rename() is used. Otherwise, src is copied to the destination and then removed. A lot more could be done here... A look at a mv.c shows a lot of the issues this implementation glosses over. """ real_dst = dst if testos.path.isdir(dst): if _samefile(src, dst): # We might be on a case insensitive filesystem, # perform the rename anyway. testos.rename(src, dst) return real_dst = testos.path.join(dst, _basename(src)) if testos.path.exists(real_dst): raise Error("Destination path '%s' already exists" % real_dst) try: testos.rename(src, real_dst) except OSError: if testos.path.isdir(src): if _destinsrc(src, dst): raise Error("Cannot move a directory '%s' into itself '%s'." % (src, dst)) copytree(src, real_dst, symlinks=True) rmtree(src) else: copy2(src, real_dst) testos.unlink(src)
def acquire(self, timeout=None): try: open(self.unique_name, "wb").close() except IOError: raise LockFailed("failed to create %s" % self.unique_name) timeout = timeout if timeout is not None else self.timeout end_time = time.time() if timeout is not None and timeout > 0: end_time += timeout while True: # Try and create a hard link to it. try: testos.link(self.unique_name, self.lock_file) except OSError: # Link creation failed. Maybe we've double-locked? nlinks = testos.stat(self.unique_name).st_nlink if nlinks == 2: # The original link plus the one I created == 2. We're # good to go. return else: # Otherwise the lock creation failed. if timeout is not None and time.time() > end_time: testos.unlink(self.unique_name) if timeout > 0: raise LockTimeout("Timeout waiting to acquire" " lock for %s" % self.path) else: raise AlreadyLocked("%s is already locked" % self.path) time.sleep(timeout is not None and timeout / 10 or 0.1) else: # Link creation succeeded. We're good to go. return
def unpack_http_url(link, location, download_dir=None, session=None, hashes=None): if session is None: raise TypeError( "unpack_http_url() missing 1 required keyword argument: 'session'") temp_dir = tempfile.mkdtemp('-unpack', 'pip-') # If a download dir is specified, is the file already downloaded there? already_downloaded_path = None if download_dir: already_downloaded_path = _check_download_dir(link, download_dir, hashes) if already_downloaded_path: from_path = already_downloaded_path content_type = mimetypes.guess_type(from_path)[0] else: # let's download to a tmp dir from_path, content_type = _download_http_url(link, session, temp_dir, hashes) # unpack the archive to the build dir location. even when only downloading # archives, they have to be unpacked to parse dependencies unpack_file(from_path, location, content_type, link) # a download dir is specified; let's copy the archive there if download_dir and not already_downloaded_path: _copy_file(from_path, download_dir, link) if not already_downloaded_path: testos.unlink(from_path) rmtree(temp_dir)
def break_lock(self): if testos.path.islink(self.lock_file): # exists && link testos.unlink(self.lock_file)
def TemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None): """Create and return a temporary file. Arguments: 'prefix', 'suffix', 'dir' -- as for mkstemp. 'mode' -- the mode argument to io.open (default "w+b"). 'buffering' -- the buffer size argument to io.open (default -1). 'encoding' -- the encoding argument to io.open (default None) 'newline' -- the newline argument to io.open (default None) The file is created as mkstemp() would do it. Returns an object with a file-like interface. The file has no name, and will cease to exist when it is closed. """ global _O_TMPFILE_WORKS prefix, suffix, dir, output_type = _sanitize_params( prefix, suffix, dir) flags = _bin_openflags if _O_TMPFILE_WORKS: try: flags2 = (flags | _os.O_TMPFILE) & ~_os.O_CREAT fd = _os.open(dir, flags2, 0o600) except IsADirectoryError: # Linux kernel older than 3.11 ignores the O_TMPFILE flag: # O_TMPFILE is read as O_DIRECTORY. Trying to open a directory # with O_RDWR|O_DIRECTORY fails with IsADirectoryError, a # directory cannot be open to write. Set flag to False to not # try again. _O_TMPFILE_WORKS = False except OSError: # The filesystem of the directory does not support O_TMPFILE. # For example, OSError(95, 'Operation not supported'). # # On Linux kernel older than 3.11, trying to open a regular # file (or a symbolic link to a regular file) with O_TMPFILE # fails with NotADirectoryError, because O_TMPFILE is read as # O_DIRECTORY. pass else: try: return _io.open(fd, mode, buffering=buffering, newline=newline, encoding=encoding) except: _os.close(fd) raise # Fallback to _mkstemp_inner(). (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: _os.unlink(name) return _io.open(fd, mode, buffering=buffering, newline=newline, encoding=encoding) except: _os.close(fd) raise
def delete_file(self, filename): """Delete `filename` (if not a dry run) after announcing it""" log.info("deleting %s", filename) if not self.dry_run: testos.unlink(filename)
def break_lock(self): if testos.path.exists(self.lock_file): testos.unlink(self.lock_file)
def break_lock(self): if testos.path.exists(self.lock_file): for name in testos.listdir(self.lock_file): testos.unlink(testos.path.join(self.lock_file, name)) testos.rmdir(self.lock_file)
def adios(p): """Appropriately delete directory, file or link.""" if testos.path.exists(p) and not testos.path.islink(p) and testos.path.isdir(p): shutil.rmtree(p) elif testos.path.exists(p): testos.unlink(p)
def release(self): if not self.is_locked(): raise NotLocked("%s is not locked" % self.path) elif not self.i_am_locking(): raise NotMyLock("%s is locked, but not by me" % self.path) testos.unlink(self.lock_file)
def install_as_egg(self, destination_eggdir): '''Install wheel as an egg directory.''' with zipfile.ZipFile(self.filename) as zf: dist_basename = '%s-%s' % (self.project_name, self.version) dist_info = '%s.dist-info' % dist_basename dist_data = '%s.data' % dist_basename def get_metadata(name): with zf.open('%s/%s' % (dist_info, name)) as fp: value = fp.read().decode('utf-8') if PY3 else fp.read() return email.parser.Parser().parsestr(value) wheel_metadata = get_metadata('WHEEL') dist_metadata = get_metadata('METADATA') # Check wheel format version is supported. wheel_version = parse_version(wheel_metadata.get('Wheel-Version')) if not parse_version('1.0') <= wheel_version < parse_version('2.0dev0'): raise ValueError('unsupported wheel format version: %s' % wheel_version) # Extract to target directory. testos.mkdir(destination_eggdir) zf.extractall(destination_eggdir) # Convert metadata. dist_info = testos.path.join(destination_eggdir, dist_info) dist = Distribution.from_location( destination_eggdir, dist_info, metadata=PathMetadata(destination_eggdir, dist_info) ) # Note: we need to evaluate and strip markers now, # as we can't easily convert back from the syntax: # foobar; "linux" in sys_platform and extra == 'test' def raw_req(req): req.marker = None return str(req) install_requires = list(sorted(map(raw_req, dist.requires()))) extras_require = { extra: list(sorted( req for req in map(raw_req, dist.requires((extra,))) if req not in install_requires )) for extra in dist.extras } egg_info = testos.path.join(destination_eggdir, 'EGG-INFO') testos.rename(dist_info, egg_info) testos.rename(testos.path.join(egg_info, 'METADATA'), testos.path.join(egg_info, 'PKG-INFO')) setup_dist = SetuptoolsDistribution(attrs=dict( install_requires=install_requires, extras_require=extras_require, )) write_requirements(setup_dist.get_command_obj('egg_info'), None, testos.path.join(egg_info, 'requires.txt')) # Move data entries to their correct location. dist_data = testos.path.join(destination_eggdir, dist_data) dist_data_scripts = testos.path.join(dist_data, 'scripts') if testos.path.exists(dist_data_scripts): egg_info_scripts = testos.path.join(destination_eggdir, 'EGG-INFO', 'scripts') testos.mkdir(egg_info_scripts) for entry in testos.listdir(dist_data_scripts): # Remove bytecode, as it's not properly handled # during easy_install scripts install phase. if entry.endswith('.pyc'): testos.unlink(testos.path.join(dist_data_scripts, entry)) else: testos.rename(testos.path.join(dist_data_scripts, entry), testos.path.join(egg_info_scripts, entry)) testos.rmdir(dist_data_scripts) for subdir in filter(testos.path.exists, ( testos.path.join(dist_data, d) for d in ('data', 'headers', 'purelib', 'platlib') )): unpack(subdir, destination_eggdir) if testos.path.exists(dist_data): testos.rmdir(dist_data) # Fix namespace packages. namespace_packages = testos.path.join(egg_info, 'namespace_packages.txt') if testos.path.exists(namespace_packages): with open(namespace_packages) as fp: namespace_packages = fp.read().split() for mod in namespace_packages: mod_dir = testos.path.join(destination_eggdir, *mod.split('.')) mod_init = testos.path.join(mod_dir, '__init__.py') if testos.path.exists(mod_dir) and not testos.path.exists(mod_init): with open(mod_init, 'w') as fp: fp.write(NAMESPACE_PACKAGE_INIT)