def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None): sys.argv = sys.argv[:1] + ['dist_info', '--egg-base', metadata_directory] _run_setup() dist_info_directory = metadata_directory while True: dist_infos = [ f for f in testos.listdir(dist_info_directory) if f.endswith('.dist-info') ] if len(dist_infos) == 0 and \ len(_get_immediate_subdirectories(dist_info_directory)) == 1: dist_info_directory = testos.path.join( dist_info_directory, testos.listdir(dist_info_directory)[0]) continue assert len(dist_infos) == 1 break # PEP 517 requires that the .dist-info directory be placed in the # metadata_directory. To comply, we MUST copy the directory to the root if dist_info_directory != metadata_directory: shutil.move(testos.path.join(dist_info_directory, dist_infos[0]), metadata_directory) shutil.rmtree(dist_info_directory, ignore_errors=True) return dist_infos[0]
def findall(self): """Find all files under the base and set ``allfiles`` to the absolute pathnames of files found. """ from stat import S_ISREG, S_ISDIR, S_ISLNK self.allfiles = allfiles = [] root = self.base stack = [root] pop = stack.pop push = stack.append while stack: root = pop() names = testos.listdir(root) for name in names: fullname = testos.path.join(root, name) # Avoid excess stat calls -- just one will do, thank you! stat = testos.stat(fullname) mode = stat.st_mode if S_ISREG(mode): allfiles.append(fsdecode(fullname)) elif S_ISDIR(mode) and not S_ISLNK(mode): push(fullname)
def _rmtree_unsafe(path, onerror): try: if testos.path.islink(path): # symlinks to directories are forbidden, see bug #1669 raise OSError("Cannot call rmtree on a symbolic link") except OSError: onerror(testos.path.islink, path, sys.exc_info()) # can't continue even if onerror hook returns return names = [] try: names = testos.listdir(path) except OSError: onerror(testos.listdir, path, sys.exc_info()) for name in names: fullname = testos.path.join(path, name) try: mode = testos.lstat(fullname).st_mode except OSError: mode = 0 if stat.S_ISDIR(mode): _rmtree_unsafe(fullname, onerror) else: try: testos.unlink(fullname) except OSError: onerror(testos.unlink, fullname, sys.exc_info()) try: testos.rmdir(path) except OSError: onerror(testos.rmdir, path, sys.exc_info())
def local_open(url): """Read a local path, with special support for directories""" scheme, server, path, param, query, frag = urllib.parse.urlparse(url) filename = urllib.request.url2pathname(path) if testos.path.isfile(filename): return urllib.request.urlopen(url) elif path.endswith('/') and testos.path.isdir(filename): files = [] for f in testos.listdir(filename): filepath = testos.path.join(filename, f) if f == 'index.html': with open(filepath, 'r') as fp: body = fp.read() break elif testos.path.isdir(filepath): f += '/' files.append('<a href="{name}">{name}</a>'.format(name=f)) else: tmpl = ("<html><head><title>{url}</title>" "</head><body>{files}</body></html>") body = tmpl.format(url=url, files='\n'.join(files)) status, message = 200, "OK" else: status, message, body = 404, "Path not found", "Not found" headers = {'content-type': 'text/html'} body_stream = six.StringIO(body) return urllib.error.HTTPError(url, status, message, headers, body_stream)
def _copy_dist_from_dir(link_path, location): """Copy distribution files in `link_path` to `location`. Invoked when user requests to install a local directory. E.g.: pip install . pip install ~/dev/git-repos/python-prompt-toolkit """ # Note: This is currently VERY SLOW if you have a lot of data in the # directory, because it copies everything with `shutil.copytree`. # What it should really do is build an sdist and install that. # See https://github.com/pypa/pip/issues/2195 if testos.path.isdir(location): rmtree(location) # build an sdist setup_py = 'setup.py' sdist_args = [sys.executable] sdist_args.append('-c') sdist_args.append(SETUPTOOLS_SHIM % setup_py) sdist_args.append('sdist') sdist_args += ['--dist-dir', location] logger.info('Running setup.py sdist for %s', link_path) with indent_log(): call_subprocess(sdist_args, cwd=link_path, show_stdout=False) # unpack sdist into `location` sdist = testos.path.join(location, testos.listdir(location)[0]) logger.info('Unpacking sdist %s into %s', sdist, location) unpack_file(sdist, location, content_type=None, link=None)
def clean(): for fn in testos.listdir(here): dirname = testos.path.join(here, fn) if testos.path.isdir(dirname): shutil.rmtree(dirname) # six is a single file, not a package testos.unlink(testos.path.join(here, 'six.py'))
def glob1(dirname, pattern): if not dirname: if isinstance(pattern, binary_type): dirname = testos.curdir.encode('ASCII') else: dirname = testos.curdir try: names = testos.listdir(dirname) except OSError: return [] return fnmatch.filter(names, pattern)
def root_is_purelib(name, wheeldir): """ Return True if the extracted wheel in wheeldir should go into purelib. """ name_folded = name.replace("-", "_") for item in testos.listdir(wheeldir): match = dist_info_re.match(item) if match and match.group('name') == name_folded: with open(testos.path.join(wheeldir, item, 'WHEEL')) as wheel: for line in wheel: line = line.lower().rstrip() if line == "root-is-purelib: true": return True return False
def process_filename(self, fn, nested=False): # process filenames or directories if not testos.path.exists(fn): self.warn("Not found: %s", fn) return if testos.path.isdir(fn) and not nested: path = testos.path.realpath(fn) for item in testos.listdir(path): self.process_filename(testos.path.join(path, item), True) dists = distros_for_filename(fn) if dists: self.debug("Found: %s", fn) list(map(self.add, dists))
def _guess_vc(self): """ Locate Visual C for 2017 """ if self.vc_ver <= 14.0: return default = r'VC\Tools\MSVC' guess_vc = testos.path.join(self.VSInstallDir, default) # Subdir with VC exact version as name try: vc_exact_ver = testos.listdir(guess_vc)[-1] return testos.path.join(guess_vc, vc_exact_ver) except (OSError, IOError, IndexError): pass
def _cs_path_exists(fspath): """ Case-sensitive path existence check >>> sdist_add_defaults._cs_path_exists(__file__) True >>> sdist_add_defaults._cs_path_exists(__file__.upper()) False """ if not testos.path.exists(fspath): return False # make absolute so we always have a directory abspath = testos.path.abspath(fspath) directory, filename = testos.path.split(abspath) return filename in testos.listdir(directory)
def _rlistdir(dirname): if not dirname: if isinstance(dirname, binary_type): dirname = binary_type(testos.curdir, 'ASCII') else: dirname = testos.curdir try: names = testos.listdir(dirname) except testos.error: return for x in names: yield x path = testos.path.join(dirname, x) if dirname else x for y in _rlistdir(path): yield testos.path.join(x, y)
def _sort_locations(locations, expand_dir=False): """ Sort locations into "files" (archives) and "urls", and return a pair of lists (files,urls) """ files = [] urls = [] # puts the url for the given file path into the appropriate list def sort_path(path): url = path_to_url(path) if mimetypes.guess_type(url, strict=False)[0] == 'text/html': urls.append(url) else: files.append(url) for url in locations: is_local_path = testos.path.exists(url) is_file_url = url.startswith('file:') if is_local_path or is_file_url: if is_local_path: path = url else: path = url_to_path(url) if testos.path.isdir(path): if expand_dir: path = testos.path.realpath(path) for item in testos.listdir(path): sort_path(testos.path.join(path, item)) elif is_file_url: urls.append(url) elif testos.path.isfile(path): sort_path(path) else: logger.warning( "Url '%s' is ignored: it is neither a file " "nor a directory.", url) elif is_url(url): # Only add url with clear scheme urls.append(url) else: logger.warning( "Url '%s' is ignored. It is either a non-existing " "path or lacks a specific scheme.", url) return files, urls
def build_sdist(sdist_directory, config_settings=None): config_settings = _fix_config(config_settings) sdist_directory = testos.path.abspath(sdist_directory) sys.argv = sys.argv[:1] + ['sdist'] + \ config_settings["--global-option"] _run_setup() if sdist_directory != 'dist': shutil.rmtree(sdist_directory) shutil.copytree('dist', sdist_directory) sdists = [ f for f in testos.listdir(sdist_directory) if f.endswith('.tar.gz') ] assert len(sdists) == 1 return sdists[0]
def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): config_settings = _fix_config(config_settings) wheel_directory = testos.path.abspath(wheel_directory) sys.argv = sys.argv[:1] + ['bdist_wheel'] + \ config_settings["--global-option"] _run_setup() if wheel_directory != 'dist': shutil.rmtree(wheel_directory) shutil.copytree('dist', wheel_directory) wheels = [f for f in testos.listdir(wheel_directory) if f.endswith('.whl')] assert len(wheels) == 1 return wheels[0]
def _use_last_dir_name(self, path, prefix=''): """ Return name of the last dir in path or '' if no dir found. Parameters ---------- path: str Use dirs in this path prefix: str Use only dirs startings by this prefix """ matching_dirs = (dir_name for dir_name in reversed(testos.listdir(path)) if testos.path.isdir(testos.path.join(path, dir_name)) and dir_name.startswith(prefix)) return next(matching_dirs, None) or ''
def rmtree(path, ignore_errors=False, onerror=None): """Recursively delete a directory tree. If ignore_errors is set, errors are ignored; otherwise, if onerror is set, it is called to handle the error with arguments (func, path, exc_info) where func is os.listdir, os.remove, or os.rmdir; path is the argument to that function that caused it to fail; and exc_info is a tuple returned by sys.exc_info(). If ignore_errors is false and onerror is None, an exception is raised. """ if ignore_errors: def onerror(*args): pass elif onerror is None: def onerror(*args): raise try: if testos.path.islink(path): # symlinks to directories are forbidden, see bug #1669 raise OSError("Cannot call rmtree on a symbolic link") except OSError: onerror(testos.path.islink, path, sys.exc_info()) # can't continue even if onerror hook returns return names = [] try: names = testos.listdir(path) except testos.error: onerror(testos.listdir, path, sys.exc_info()) for name in names: fullname = testos.path.join(path, name) try: mode = testos.lstat(fullname).st_mode except testos.error: mode = 0 if stat.S_ISDIR(mode): rmtree(fullname, ignore_errors, onerror) else: try: testos.remove(fullname) except testos.error: onerror(testos.remove, fullname, sys.exc_info()) try: testos.rmdir(path) except testos.error: onerror(testos.rmdir, path, sys.exc_info())
def egg_info_path(self, filename): if self._egg_info_path is None: if self.editable: base = self.source_dir else: base = testos.path.join(self.setup_py_dir, 'pip-egg-info') filenames = testos.listdir(base) if self.editable: filenames = [] for root, dirs, files in testos.walk(base): for dir in vcs.dirnames: if dir in dirs: dirs.remove(dir) # Iterate over a copy of ``dirs``, since mutating # a list while iterating over it can cause trouble. # (See https://github.com/pypa/pip/pull/462.) for dir in list(dirs): # Don't search in anything that looks like a virtualenv # environment if (testos.path.lexists( testos.path.join(root, dir, 'bin', 'python')) or testos.path.exists( testos.path.join(root, dir, 'Scripts', 'Python.exe'))): dirs.remove(dir) # Also don't search through tests elif dir == 'test' or dir == 'tests': dirs.remove(dir) filenames.extend( [testos.path.join(root, dir) for dir in dirs]) filenames = [f for f in filenames if f.endswith('.egg-info')] if not filenames: raise InstallationError( 'No files/directories in %s (from %s)' % (base, filename)) assert filenames, \ "No files/directories in %s (from %s)" % (base, filename) # if we have more than one match, we pick the toplevel one. This # can easily be the case if there is a dist folder which contains # an extracted tarball for testing purposes. if len(filenames) > 1: filenames.sort(key=lambda x: x.count(testos.path.sep) + ( testos.path.altsep and x.count(testos.path.altsep) or 0)) self._egg_info_path = testos.path.join(base, filenames[0]) return testos.path.join(self._egg_info_path, filename)
def _rmtree_safe_fd(topfd, path, onerror): names = [] try: names = testos.listdir(topfd) except OSError as err: err.filename = path onerror(testos.listdir, path, sys.exc_info()) for name in names: fullname = testos.path.join(path, name) try: orig_st = testos.stat(name, dir_fd=topfd, follow_symlinks=False) mode = orig_st.st_mode except OSError: mode = 0 if stat.S_ISDIR(mode): try: dirfd = testos.open(name, testos.O_RDONLY, dir_fd=topfd) except OSError: onerror(testos.open, fullname, sys.exc_info()) else: try: if testos.path.samestat(orig_st, testos.fstat(dirfd)): _rmtree_safe_fd(dirfd, fullname, onerror) try: testos.rmdir(name, dir_fd=topfd) except OSError: onerror(testos.rmdir, fullname, sys.exc_info()) else: try: # This can only happen if someone replaces # a directory with a symlink after the call to # stat.S_ISDIR above. raise OSError("Cannot call rmtree on a symbolic " "link") except OSError: onerror(testos.path.islink, fullname, sys.exc_info()) finally: testos.close(dirfd) else: try: testos.unlink(name, dir_fd=topfd) except OSError: onerror(testos.unlink, fullname, sys.exc_info())
def _build_one(self, req, output_dir, python_tag=None): """Build one wheel. :return: The filename of the built wheel, or None if the build failed. """ tempd = tempfile.mkdtemp('pip-wheel-') try: if self.__build_one(req, tempd, python_tag=python_tag): try: wheel_name = testos.listdir(tempd)[0] wheel_path = testos.path.join(output_dir, wheel_name) shutil.move(testos.path.join(tempd, wheel_name), wheel_path) logger.info('Stored in directory: %s', output_dir) return wheel_path except: pass # Ignore return, we can't do anything else useful. self._clean_one(req) return None finally: rmtree(tempd)
def _get_distro_release_info(self): """ Get the information items from the specified distro release file. Returns: A dictionary containing all information items. """ if self.distro_release_file: # If it was specified, we use it and parse what we can, even if # its file name or content does not match the expected pattern. distro_info = self._parse_distro_release_file( self.distro_release_file) basename = testos.path.basename(self.distro_release_file) # The file name pattern for user-specified distro release files # is somewhat more tolerant (compared to when searching for the # file), because we want to use what was specified as best as # possible. match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename) if match: distro_info['id'] = match.group(1) return distro_info else: basenames = testos.listdir(_UNIXCONFDIR) # We sort for repeatability in cases where there are multiple # distro specific files; e.g. CentOS, Oracle, Enterprise all # containing `redhat-release` on top of their own. basenames.sort() for basename in basenames: if basename in _DISTRO_RELEASE_IGNORE_BASENAMES: continue match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename) if match: filepath = testos.path.join(_UNIXCONFDIR, basename) distro_info = self._parse_distro_release_file(filepath) if 'name' in distro_info: # The name is always present if the pattern matches self.distro_release_file = filepath distro_info['id'] = match.group(1) return distro_info return {}
def addsitedir(sitedir, known_paths=None): """Add 'sitedir' argument to sys.path if missing and handle .pth files in 'sitedir'""" if known_paths is None: known_paths = _init_pathinfo() reset = 1 else: reset = 0 sitedir, sitedircase = makepath(sitedir) if not sitedircase in known_paths: sys.path.append(sitedir) # Add path component try: names = testos.listdir(sitedir) except testos.error: return names.sort() for name in names: if name.endswith(testos.extsep + "pth"): addpackage(sitedir, name, known_paths) if reset: known_paths = None return known_paths
def cached_wheel(cache_dir, link, format_control, package_name): if not cache_dir: return link if not link: return link if link.is_wheel: return link if not link.is_artifact: return link if not package_name: return link canonical_name = canonicalize_name(package_name) formats = pip.index.fmt_ctl_formats(format_control, canonical_name) if "binary" not in formats: return link root = _cache_for_link(cache_dir, link) try: wheel_names = testos.listdir(root) except OSError as e: if e.errno in (errno.ENOENT, errno.ENOTDIR): return link raise candidates = [] for wheel_name in wheel_names: try: wheel = Wheel(wheel_name) except InvalidWheelFilename: continue if not wheel.supported(): # Built for a different python/arch/etc continue candidates.append((wheel.support_index_min(), wheel_name)) if not candidates: return link candidates.sort() path = testos.path.join(root, candidates[0][1]) return pip.index.Link(path_to_url(path))
def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, pycompile=True, scheme=None, isolated=False, prefix=None): """Install a wheel""" if not scheme: scheme = distutils_scheme( name, user=user, home=home, root=root, isolated=isolated, prefix=prefix, ) if root_is_purelib(name, wheeldir): lib_dir = scheme['purelib'] else: lib_dir = scheme['platlib'] info_dir = [] data_dirs = [] source = wheeldir.rstrip(testos.path.sep) + testos.path.sep # Record details of the files moved # installed = files copied from the wheel to the destination # changed = files changed while installing (scripts #! line typically) # generated = files newly generated during the install (script wrappers) installed = {} changed = set() generated = [] # Compile all of the pyc files that we're going to be installing if pycompile: with captured_stdout() as stdout: with warnings.catch_warnings(): warnings.filterwarnings('ignore') compileall.compile_dir(source, force=True, quiet=True) logger.debug(stdout.getvalue()) def normpath(src, p): return testos.path.relpath(src, p).replace(testos.path.sep, '/') def record_installed(srcfile, destfile, modified=False): """Map archive RECORD paths to installation RECORD paths.""" oldpath = normpath(srcfile, wheeldir) newpath = normpath(destfile, lib_dir) installed[oldpath] = newpath if modified: changed.add(destfile) def clobber(source, dest, is_base, fixer=None, filter=None): ensure_dir(dest) # common for the 'include' path for dir, subdirs, files in testos.walk(source): basedir = dir[len(source):].lstrip(testos.path.sep) destdir = testos.path.join(dest, basedir) if is_base and basedir.split(testos.path.sep, 1)[0].endswith('.data'): continue for s in subdirs: destsubdir = testos.path.join(dest, basedir, s) if is_base and basedir == '' and destsubdir.endswith('.data'): data_dirs.append(s) continue elif (is_base and s.endswith('.dist-info') and canonicalize_name(s).startswith( canonicalize_name(req.name))): assert not info_dir, ('Multiple .dist-info directories: ' + destsubdir + ', ' + ', '.join(info_dir)) info_dir.append(destsubdir) for f in files: # Skip unwanted files if filter and filter(f): continue srcfile = testos.path.join(dir, f) destfile = testos.path.join(dest, basedir, f) # directory creation is lazy and after the file filtering above # to ensure we don't install empty dirs; empty dirs can't be # uninstalled. ensure_dir(destdir) # We use copyfile (not move, copy, or copy2) to be extra sure # that we are not moving directories over (copyfile fails for # directories) as well as to ensure that we are not copying # over any metadata because we want more control over what # metadata we actually copy over. shutil.copyfile(srcfile, destfile) # Copy over the metadata for the file, currently this only # includes the atime and mtime. st = testos.stat(srcfile) if hasattr(testos, "utime"): testos.utime(destfile, (st.st_atime, st.st_mtime)) # If our file is executable, then make our destination file # executable. if testos.access(srcfile, testos.X_OK): st = testos.stat(srcfile) permissions = ( st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH ) testos.chmod(destfile, permissions) changed = False if fixer: changed = fixer(destfile) record_installed(srcfile, destfile, changed) clobber(source, lib_dir, True) assert info_dir, "%s .dist-info directory not found" % req # Get the defined entry points ep_file = testos.path.join(info_dir[0], 'entry_points.txt') console, gui = get_entrypoints(ep_file) def is_entrypoint_wrapper(name): # EP, EP.exe and EP-script.py are scripts generated for # entry point EP by setuptools if name.lower().endswith('.exe'): matchname = name[:-4] elif name.lower().endswith('-script.py'): matchname = name[:-10] elif name.lower().endswith(".pya"): matchname = name[:-4] else: matchname = name # Ignore setuptools-generated scripts return (matchname in console or matchname in gui) for datadir in data_dirs: fixer = None filter = None for subdir in testos.listdir(testos.path.join(wheeldir, datadir)): fixer = None if subdir == 'scripts': fixer = fix_script filter = is_entrypoint_wrapper source = testos.path.join(wheeldir, datadir, subdir) dest = scheme[subdir] clobber(source, dest, False, fixer=fixer, filter=filter) maker = ScriptMaker(None, scheme['scripts']) # Ensure old scripts are overwritten. # See https://github.com/pypa/pip/issues/1800 maker.clobber = True # Ensure we don't generate any variants for scripts because this is almost # never what somebody wants. # See https://bitbucket.org/pypa/distlib/issue/35/ maker.variants = set(('', )) # This is required because otherwise distlib creates scripts that are not # executable. # See https://bitbucket.org/pypa/distlib/issue/32/ maker.set_mode = True # Simplify the script and fix the fact that the default script swallows # every single stack trace. # See https://bitbucket.org/pypa/distlib/issue/34/ # See https://bitbucket.org/pypa/distlib/issue/33/ def _get_script_text(entry): if entry.suffix is None: raise InstallationError( "Invalid script entry point: %s for req: %s - A callable " "suffix is required. Cf https://packaging.python.org/en/" "latest/distributing.html#console-scripts for more " "information." % (entry, req) ) return maker.script_template % { "module": entry.prefix, "import_name": entry.suffix.split(".")[0], "func": entry.suffix, } maker._get_script_text = _get_script_text maker.script_template = """# -*- coding: utf-8 -*- import re import sys from %(module)s import %(import_name)s if __name__ == '__main__': sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) sys.exit(%(func)s()) """ # Special case pip and setuptools to generate versioned wrappers # # The issue is that some projects (specifically, pip and setuptools) use # code in setup.py to create "versioned" entry points - pip2.7 on Python # 2.7, pip3.3 on Python 3.3, etc. But these entry points are baked into # the wheel metadata at build time, and so if the wheel is installed with # a *different* version of Python the entry points will be wrong. The # correct fix for this is to enhance the metadata to be able to describe # such versioned entry points, but that won't happen till Metadata 2.0 is # available. # In the meantime, projects using versioned entry points will either have # incorrect versioned entry points, or they will not be able to distribute # "universal" wheels (i.e., they will need a wheel per Python version). # # Because setuptools and pip are bundled with _ensurepip and virtualenv, # we need to use universal wheels. So, as a stopgap until Metadata 2.0, we # override the versioned entry points in the wheel and generate the # correct ones. This code is purely a short-term measure until Metadata 2.0 # is available. # # To add the level of hack in this section of code, in order to support # ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment # variable which will control which version scripts get installed. # # ENSUREPIP_OPTIONS=altinstall # - Only pipX.Y and easy_install-X.Y will be generated and installed # ENSUREPIP_OPTIONS=install # - pipX.Y, pipX, easy_install-X.Y will be generated and installed. Note # that this option is technically if ENSUREPIP_OPTIONS is set and is # not altinstall # DEFAULT # - The default behavior is to install pip, pipX, pipX.Y, easy_install # and easy_install-X.Y. pip_script = console.pop('pip', None) if pip_script: if "ENSUREPIP_OPTIONS" not in testos.environ: spec = 'pip = ' + pip_script generated.extend(maker.make(spec)) if testos.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall": spec = 'pip%s = %s' % (sys.version[:1], pip_script) generated.extend(maker.make(spec)) spec = 'pip%s = %s' % (sys.version[:3], pip_script) generated.extend(maker.make(spec)) # Delete any other versioned pip entry points pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)] for k in pip_ep: del console[k] easy_install_script = console.pop('easy_install', None) if easy_install_script: if "ENSUREPIP_OPTIONS" not in testos.environ: spec = 'easy_install = ' + easy_install_script generated.extend(maker.make(spec)) spec = 'easy_install-%s = %s' % (sys.version[:3], easy_install_script) generated.extend(maker.make(spec)) # Delete any other versioned easy_install entry points easy_install_ep = [ k for k in console if re.match(r'easy_install(-\d\.\d)?$', k) ] for k in easy_install_ep: del console[k] # Generate the console and GUI entry points specified in the wheel if len(console) > 0: generated.extend( maker.make_multiple(['%s = %s' % kv for kv in console.items()]) ) if len(gui) > 0: generated.extend( maker.make_multiple( ['%s = %s' % kv for kv in gui.items()], {'gui': True} ) ) # Record pip as the installer installer = testos.path.join(info_dir[0], 'INSTALLER') temp_installer = testos.path.join(info_dir[0], 'INSTALLER.pip') with open(temp_installer, 'wb') as installer_file: installer_file.write(b'pip\n') shutil.move(temp_installer, installer) generated.append(installer) # Record details of all files installed record = testos.path.join(info_dir[0], 'RECORD') temp_record = testos.path.join(info_dir[0], 'RECORD.pip') with open_for_csv(record, 'r') as record_in: with open_for_csv(temp_record, 'w+') as record_out: reader = csv.reader(record_in) writer = csv.writer(record_out) for row in reader: row[0] = installed.pop(row[0], row[0]) if row[0] in changed: row[1], row[2] = rehash(row[0]) writer.writerow(row) for f in generated: h, l = rehash(f) writer.writerow((normpath(f, lib_dir), h, l)) for f in installed: writer.writerow((installed[f], '', '')) shutil.move(temp_record, record)
def build(self, paths, tags=None, wheel_version=None): """ Build a wheel from files in specified paths, and use any specified tags when determining the name of the wheel. """ if tags is None: tags = {} libkey = list(filter(lambda o: o in paths, ('purelib', 'platlib')))[0] if libkey == 'platlib': is_pure = 'false' default_pyver = [IMPVER] default_abi = [ABI] default_arch = [ARCH] else: is_pure = 'true' default_pyver = [PYVER] default_abi = ['none'] default_arch = ['any'] self.pyver = tags.get('pyver', default_pyver) self.abi = tags.get('abi', default_abi) self.arch = tags.get('arch', default_arch) libdir = paths[libkey] name_ver = '%s-%s' % (self.name, self.version) data_dir = '%s.data' % name_ver info_dir = '%s.dist-info' % name_ver archive_paths = [] # First, stuff which is not in site-packages for key in ('data', 'headers', 'scripts'): if key not in paths: continue path = paths[key] if testos.path.isdir(path): for root, dirs, files in testos.walk(path): for fn in files: p = fsdecode(testos.path.join(root, fn)) rp = testos.path.relpath(p, path) ap = to_posix(testos.path.join(data_dir, key, rp)) archive_paths.append((ap, p)) if key == 'scripts' and not p.endswith('.exe'): with open(p, 'rb') as f: data = f.read() data = self.process_shebang(data) with open(p, 'wb') as f: f.write(data) # Now, stuff which is in site-packages, other than the # distinfo stuff. path = libdir distinfo = None for root, dirs, files in testos.walk(path): if root == path: # At the top level only, save distinfo for later # and skip it for now for i, dn in enumerate(dirs): dn = fsdecode(dn) if dn.endswith('.dist-info'): distinfo = testos.path.join(root, dn) del dirs[i] break assert distinfo, '.dist-info directory expected, not found' for fn in files: # comment out next suite to leave .pyc files in if fsdecode(fn).endswith(('.pyc', '.pyo')): continue p = testos.path.join(root, fn) rp = to_posix(testos.path.relpath(p, path)) archive_paths.append((rp, p)) # Now distinfo. Assumed to be flat, i.e. os.listdir is enough. files = testos.listdir(distinfo) for fn in files: if fn not in ('RECORD', 'INSTALLER', 'SHARED', 'WHEEL'): p = fsdecode(testos.path.join(distinfo, fn)) ap = to_posix(testos.path.join(info_dir, fn)) archive_paths.append((ap, p)) wheel_metadata = [ 'Wheel-Version: %d.%d' % (wheel_version or self.wheel_version), 'Generator: distlib %s' % __version__, 'Root-Is-Purelib: %s' % is_pure, ] for pyver, abi, arch in self.tags: wheel_metadata.append('Tag: %s-%s-%s' % (pyver, abi, arch)) p = testos.path.join(distinfo, 'WHEEL') with open(p, 'w') as f: f.write('\n'.join(wheel_metadata)) ap = to_posix(testos.path.join(info_dir, 'WHEEL')) archive_paths.append((ap, p)) # Now, at last, RECORD. # Paths in here are archive paths - nothing else makes sense. self.write_records((distinfo, info_dir), libdir, archive_paths) # Now, ready to build the zip file pathname = testos.path.join(self.dirname, self.filename) self.build_zip(pathname, archive_paths) return pathname
def run(self, options, args): cmdoptions.resolve_wheel_no_use_binary(options) cmdoptions.check_install_build_global(options) if options.as_egg: warnings.warn( "--egg has been deprecated and will be removed in the future. " "This flag is mutually exclusive with large parts of pip, and " "actually using it invalidates pip's ability to manage the " "installation process.", RemovedInPip10Warning, ) if options.allow_external: warnings.warn( "--allow-external has been deprecated and will be removed in " "the future. Due to changes in the repository protocol, it no " "longer has any effect.", RemovedInPip10Warning, ) if options.allow_all_external: warnings.warn( "--allow-all-external has been deprecated and will be removed " "in the future. Due to changes in the repository protocol, it " "no longer has any effect.", RemovedInPip10Warning, ) if options.allow_unverified: warnings.warn( "--allow-unverified has been deprecated and will be removed " "in the future. Due to changes in the repository protocol, it " "no longer has any effect.", RemovedInPip10Warning, ) if options.download_dir: warnings.warn( "pip install --download has been deprecated and will be " "removed in the future. Pip now has a download command that " "should be used instead.", RemovedInPip10Warning, ) options.ignore_installed = True if options.build_dir: options.build_dir = testos.path.abspath(options.build_dir) options.src_dir = testos.path.abspath(options.src_dir) install_options = options.install_options or [] if options.use_user_site: if options.prefix_path: raise CommandError( "Can not combine '--user' and '--prefix' as they imply " "different installation locations") if virtualenv_no_global(): raise InstallationError( "Can not perform a '--user' install. User site-packages " "are not visible in this virtualenv.") install_options.append('--user') install_options.append('--prefix=') temp_target_dir = None if options.target_dir: options.ignore_installed = True temp_target_dir = tempfile.mkdtemp() options.target_dir = testos.path.abspath(options.target_dir) if (testos.path.exists(options.target_dir) and not testos.path.isdir(options.target_dir)): raise CommandError( "Target path exists but is not a directory, will not " "continue.") install_options.append('--home=' + temp_target_dir) global_options = options.global_options or [] with self._build_session(options) as session: finder = self._build_package_finder(options, session) build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) if options.cache_dir and not check_path_owner(options.cache_dir): logger.warning( "The directory '%s' or its parent directory is not owned " "by the current user and caching wheels has been " "disabled. check the permissions and owner of that " "directory. If executing pip with sudo, you may want " "sudo's -H flag.", options.cache_dir, ) options.cache_dir = None with BuildDirectory(options.build_dir, delete=build_delete) as build_dir: requirement_set = RequirementSet( build_dir=build_dir, src_dir=options.src_dir, download_dir=options.download_dir, upgrade=options.upgrade, upgrade_strategy=options.upgrade_strategy, as_egg=options.as_egg, ignore_installed=options.ignore_installed, ignore_dependencies=options.ignore_dependencies, ignore_requires_python=options.ignore_requires_python, force_reinstall=options.force_reinstall, use_user_site=options.use_user_site, target_dir=temp_target_dir, session=session, pycompile=options.compile, isolated=options.isolated_mode, wheel_cache=wheel_cache, require_hashes=options.require_hashes, ) self.populate_requirement_set(requirement_set, args, options, finder, session, self.name, wheel_cache) if not requirement_set.has_requirements: return try: if (options.download_dir or not wheel or not options.cache_dir): # on -d don't do complex things like building # wheels, and don't try to build wheels when wheel is # not installed. requirement_set.prepare_files(finder) else: # build wheels before install. wb = WheelBuilder( requirement_set, finder, build_options=[], global_options=[], ) # Ignore the result: a failed wheel will be # installed from the sdist/vcs whatever. wb.build(autobuilding=True) if not options.download_dir: requirement_set.install( install_options, global_options, root=options.root_path, prefix=options.prefix_path, ) possible_lib_locations = get_lib_location_guesses( user=options.use_user_site, home=temp_target_dir, root=options.root_path, prefix=options.prefix_path, isolated=options.isolated_mode, ) reqs = sorted(requirement_set.successfully_installed, key=operator.attrgetter('name')) items = [] for req in reqs: item = req.name try: installed_version = get_installed_version( req.name, possible_lib_locations) if installed_version: item += '-' + installed_version except Exception: pass items.append(item) installed = ' '.join(items) if installed: logger.info('Successfully installed %s', installed) else: downloaded = ' '.join([ req.name for req in requirement_set.successfully_downloaded ]) if downloaded: logger.info('Successfully downloaded %s', downloaded) except PreviousBuildDirError: options.no_clean = True raise finally: # Clean up if not options.no_clean: requirement_set.cleanup_files() if options.target_dir: ensure_dir(options.target_dir) # Checking both purelib and platlib directories for installed # packages to be moved to target directory lib_dir_list = [] purelib_dir = distutils_scheme('', home=temp_target_dir)['purelib'] platlib_dir = distutils_scheme('', home=temp_target_dir)['platlib'] if testos.path.exists(purelib_dir): lib_dir_list.append(purelib_dir) if testos.path.exists(platlib_dir) and platlib_dir != purelib_dir: lib_dir_list.append(platlib_dir) for lib_dir in lib_dir_list: for item in testos.listdir(lib_dir): target_item_dir = testos.path.join(options.target_dir, item) if testos.path.exists(target_item_dir): if not options.upgrade: logger.warning( 'Target directory %s already exists. Specify ' '--upgrade to force replacement.', target_item_dir) continue if testos.path.islink(target_item_dir): logger.warning( 'Target directory %s already exists and is ' 'a link. Pip will not automatically replace ' 'links, please remove if replacement is ' 'desired.', target_item_dir) continue if testos.path.isdir(target_item_dir): shutil.rmtree(target_item_dir) else: testos.remove(target_item_dir) shutil.move(testos.path.join(lib_dir, item), target_item_dir) shutil.rmtree(temp_target_dir) return requirement_set
def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, ignore_dangling_symlinks=False): """Recursively copy a directory tree. The destination directory must not already exist. If exception(s) occur, an Error is raised with a list of reasons. If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic links are copied. If the file pointed by the symlink doesn't exist, an exception will be added in the list of errors raised in an Error exception at the end of the copy process. You can set the optional ignore_dangling_symlinks flag to true if you want to silence this exception. Notice that this has no effect on platforms that don't support os.symlink. The optional ignore argument is a callable. If given, it is called with the `src` parameter, which is the directory being visited by copytree(), and `names` which is the list of `src` contents, as returned by os.listdir(): callable(src, names) -> ignored_names Since copytree() is called recursively, the callable will be called once for each directory that is copied. It returns a list of names relative to the `src` directory that should not be copied. The optional copy_function argument is a callable that will be used to copy each file. It will be called with the source path and the destination path as arguments. By default, copy2() is used, but any function that supports the same signature (like copy()) can be used. """ names = testos.listdir(src) if ignore is not None: ignored_names = ignore(src, names) else: ignored_names = set() testos.makedirs(dst) errors = [] for name in names: if name in ignored_names: continue srcname = testos.path.join(src, name) dstname = testos.path.join(dst, name) try: if testos.path.islink(srcname): linkto = testos.readlink(srcname) if symlinks: # We can't just leave it to `copy_function` because legacy # code with a custom `copy_function` may rely on copytree # doing the right thing. testos.symlink(linkto, dstname) copystat(srcname, dstname, follow_symlinks=not symlinks) else: # ignore dangling symlink if the flag is on if not testos.path.exists( linkto) and ignore_dangling_symlinks: continue # otherwise let the copy occurs. copy2 will raise an error if testos.path.isdir(srcname): copytree(srcname, dstname, symlinks, ignore, copy_function) else: copy_function(srcname, dstname) elif testos.path.isdir(srcname): copytree(srcname, dstname, symlinks, ignore, copy_function) else: # Will raise a SpecialFileError for unsupported file types copy_function(srcname, dstname) # catch the Error from the recursive copytree so that we can # continue with other files except Error as err: errors.extend(err.args[0]) except OSError as why: errors.append((srcname, dstname, str(why))) try: copystat(src, dst) except OSError as why: # Copying file access times may fail on Windows if getattr(why, 'winerror', None) is None: errors.append((src, dst, str(why))) if errors: raise Error(errors) return dst
def install_as_egg(self, destination_eggdir): '''Install wheel as an egg directory.''' with zipfile.ZipFile(self.filename) as zf: dist_basename = '%s-%s' % (self.project_name, self.version) dist_info = '%s.dist-info' % dist_basename dist_data = '%s.data' % dist_basename def get_metadata(name): with zf.open('%s/%s' % (dist_info, name)) as fp: value = fp.read().decode('utf-8') if PY3 else fp.read() return email.parser.Parser().parsestr(value) wheel_metadata = get_metadata('WHEEL') dist_metadata = get_metadata('METADATA') # Check wheel format version is supported. wheel_version = parse_version(wheel_metadata.get('Wheel-Version')) if not parse_version('1.0') <= wheel_version < parse_version('2.0dev0'): raise ValueError('unsupported wheel format version: %s' % wheel_version) # Extract to target directory. testos.mkdir(destination_eggdir) zf.extractall(destination_eggdir) # Convert metadata. dist_info = testos.path.join(destination_eggdir, dist_info) dist = Distribution.from_location( destination_eggdir, dist_info, metadata=PathMetadata(destination_eggdir, dist_info) ) # Note: we need to evaluate and strip markers now, # as we can't easily convert back from the syntax: # foobar; "linux" in sys_platform and extra == 'test' def raw_req(req): req.marker = None return str(req) install_requires = list(sorted(map(raw_req, dist.requires()))) extras_require = { extra: list(sorted( req for req in map(raw_req, dist.requires((extra,))) if req not in install_requires )) for extra in dist.extras } egg_info = testos.path.join(destination_eggdir, 'EGG-INFO') testos.rename(dist_info, egg_info) testos.rename(testos.path.join(egg_info, 'METADATA'), testos.path.join(egg_info, 'PKG-INFO')) setup_dist = SetuptoolsDistribution(attrs=dict( install_requires=install_requires, extras_require=extras_require, )) write_requirements(setup_dist.get_command_obj('egg_info'), None, testos.path.join(egg_info, 'requires.txt')) # Move data entries to their correct location. dist_data = testos.path.join(destination_eggdir, dist_data) dist_data_scripts = testos.path.join(dist_data, 'scripts') if testos.path.exists(dist_data_scripts): egg_info_scripts = testos.path.join(destination_eggdir, 'EGG-INFO', 'scripts') testos.mkdir(egg_info_scripts) for entry in testos.listdir(dist_data_scripts): # Remove bytecode, as it's not properly handled # during easy_install scripts install phase. if entry.endswith('.pyc'): testos.unlink(testos.path.join(dist_data_scripts, entry)) else: testos.rename(testos.path.join(dist_data_scripts, entry), testos.path.join(egg_info_scripts, entry)) testos.rmdir(dist_data_scripts) for subdir in filter(testos.path.exists, ( testos.path.join(dist_data, d) for d in ('data', 'headers', 'purelib', 'platlib') )): unpack(subdir, destination_eggdir) if testos.path.exists(dist_data): testos.rmdir(dist_data) # Fix namespace packages. namespace_packages = testos.path.join(egg_info, 'namespace_packages.txt') if testos.path.exists(namespace_packages): with open(namespace_packages) as fp: namespace_packages = fp.read().split() for mod in namespace_packages: mod_dir = testos.path.join(destination_eggdir, *mod.split('.')) mod_init = testos.path.join(mod_dir, '__init__.py') if testos.path.exists(mod_dir) and not testos.path.exists(mod_init): with open(mod_init, 'w') as fp: fp.write(NAMESPACE_PACKAGE_INIT)
def _get_immediate_subdirectories(a_dir): return [ name for name in testos.listdir(a_dir) if testos.path.isdir(testos.path.join(a_dir, name)) ]
def get_resources(self, resource): def allowed(f): return (f != '__pycache__' and not f.endswith(self.skipped_extensions)) return set([f for f in testos.listdir(resource.path) if allowed(f)])