def test_root_modifies_appropriately(self, monkeypatch): # This deals with nt/posix path differences # root is c:\somewhere\else or /somewhere/else root = os.path.normcase(os.path.abspath( os.path.join(os.path.sep, 'somewhere', 'else'))) norm_scheme = distutils_scheme("example") root_scheme = distutils_scheme("example", root=root) for key, value in norm_scheme.items(): drive, path = os.path.splitdrive(os.path.abspath(value)) expected = os.path.join(root, path[1:]) assert os.path.abspath(root_scheme[key]) == expected
def test_root_modifies_appropriately(self, monkeypatch): # This deals with nt/posix path differences # root is c:\somewhere\else or /somewhere/else root = os.path.normcase( os.path.abspath(os.path.join(os.path.sep, 'somewhere', 'else'))) norm_scheme = distutils_scheme("example") root_scheme = distutils_scheme("example", root=root) for key, value in norm_scheme.items(): drive, path = os.path.splitdrive(os.path.abspath(value)) expected = os.path.join(root, path[1:]) assert os.path.abspath(root_scheme[key]) == expected
def test_prefix_modifies_appropriately(self): prefix = os.path.abspath(os.path.join('somewhere', 'else')) normal_scheme = distutils_scheme("example") prefix_scheme = distutils_scheme("example", prefix=prefix) def _calculate_expected(value): path = os.path.join(prefix, os.path.relpath(value, sys.prefix)) return os.path.normpath(path) expected = { k: _calculate_expected(v) for k, v in normal_scheme.items() } assert prefix_scheme == expected
def is_local(path): # type: (str) -> bool """ Return True if this is a path pip is allowed to modify. If we're in a virtualenv, sys.prefix points to the virtualenv's prefix; only sys.prefix is considered local. If we're not in a virtualenv, in general we can modify anything. However, if the OS vendor has configured distutils to install somewhere other than sys.prefix (which could be a subdirectory of sys.prefix, e.g. /usr/local), we consider sys.prefix itself nonlocal and the domain of the OS vendor. (In other words, everything _other than_ sys.prefix is considered local.) Caution: this function assumes the head of path has been normalized with normalize_path. """ path = normalize_path(path) prefix = normalize_path(sys.prefix) if running_under_virtualenv(): return path.startswith(normalize_path(sys.prefix)) else: from pip._internal.locations import distutils_scheme if path.startswith(prefix): for local_path in distutils_scheme("").values(): if path.startswith(normalize_path(local_path)): return True return False else: return True
def dist_in_install_path(dist): """ Return True if given Distribution is installed in path matching distutils_scheme layout. """ norm_path = normalize_path(dist_location(dist)) return norm_path.startswith( normalize_path(distutils_scheme("")['purelib'].split('python')[0]))
def get_directories(location, auto_upgrade=True): """ Returns a dictionary that contains information on the install location of Node.py packages. The dictionary contains the following keys: - packages - bin - pip_bin Only when *location* is `'local'` or `'global'`, the following keys are available: - pip_prefix - pip_lib If *auto_upgrade* is #True (which it is by default), `'global'` will automatically be upgraded into `'root'` inside a virtual environment. """ assert location in ('local', 'global', 'root') if auto_upgrade and location == 'global' and is_virtualenv(): location = 'root' local = pip_locations.distutils_scheme('', prefix=PIP_DIRECTORY) if location == 'local': return { 'packages': MODULES_DIRECTORY, 'bin': PROGRAM_DIRECTORY, 'pip_prefix': local['data'], 'pip_bin': local['scripts'], 'pip_lib': local['purelib'] } user = (location == 'global') scheme = pip_locations.distutils_scheme('', user=user) return { # Install Node.py modules near the site-packages. 'packages': os.path.join(os.path.dirname(scheme['purelib']), 'nodepy-modules'), 'bin': scheme['scripts'], 'pip_prefix': scheme['data'], # Re-use Pip's script and site-packages directory. 'pip_bin': scheme['scripts'], 'pip_lib': scheme['purelib'] }
def get_lib_location_guesses( user=False, # type: bool home=None, # type: Optional[str] root=None, # type: Optional[str] isolated=False, # type: bool prefix=None # type: Optional[str] ): # type:(...) -> List[str] scheme = distutils_scheme('', user=user, home=home, root=root, isolated=isolated, prefix=prefix)
def _handle_target_dir(self, target_dir, target_temp_dir, upgrade): ensure_dir(target_dir) # Checking both purelib and platlib directories for installed # packages to be moved to target directory lib_dir_list = [] with target_temp_dir: # Checking both purelib and platlib directories for installed # packages to be moved to target directory scheme = distutils_scheme('', home=target_temp_dir.path) purelib_dir = scheme['purelib'] platlib_dir = scheme['platlib'] data_dir = scheme['data'] if os.path.exists(purelib_dir): lib_dir_list.append(purelib_dir) if os.path.exists(platlib_dir) and platlib_dir != purelib_dir: lib_dir_list.append(platlib_dir) if os.path.exists(data_dir): lib_dir_list.append(data_dir) for lib_dir in lib_dir_list: for item in os.listdir(lib_dir): if lib_dir == data_dir: ddir = os.path.join(data_dir, item) if any(s.startswith(ddir) for s in lib_dir_list[:-1]): continue target_item_dir = os.path.join(target_dir, item) if os.path.exists(target_item_dir): if not upgrade: logger.warning( 'Target directory %s already exists. Specify ' '--upgrade to force replacement.', target_item_dir ) continue if os.path.islink(target_item_dir): logger.warning( 'Target directory %s already exists and is ' 'a link. Pip will not automatically replace ' 'links, please remove if replacement is ' 'desired.', target_item_dir ) continue if os.path.isdir(target_item_dir): shutil.rmtree(target_item_dir) else: os.remove(target_item_dir) shutil.move( os.path.join(lib_dir, item), target_item_dir )
def lib_dir(name, wheeldir, user=False, home=None, root=None, isolated=False, prefix=None): from pip._internal.locations import distutils_scheme from pip._internal.wheel import root_is_purelib scheme = distutils_scheme( "", user=user, home=home, root=root, isolated=isolated, prefix=prefix) if root_is_purelib(name, wheeldir): return scheme['purelib'] else: return scheme['platlib']
def _handle_target_dir(self, target_dir, target_temp_dir, upgrade): # type: (str, TempDirectory, bool) -> None ensure_dir(target_dir) # Checking both purelib and platlib directories for installed # packages to be moved to target directory lib_dir_list = [] # Checking both purelib and platlib directories for installed # packages to be moved to target directory scheme = distutils_scheme("", home=target_temp_dir.path) purelib_dir = scheme["purelib"] platlib_dir = scheme["platlib"] data_dir = scheme["data"] if os.path.exists(purelib_dir): lib_dir_list.append(purelib_dir) if os.path.exists(platlib_dir) and platlib_dir != purelib_dir: lib_dir_list.append(platlib_dir) if os.path.exists(data_dir): lib_dir_list.append(data_dir) for lib_dir in lib_dir_list: for item in os.listdir(lib_dir): if lib_dir == data_dir: ddir = os.path.join(data_dir, item) if any(s.startswith(ddir) for s in lib_dir_list[:-1]): continue target_item_dir = os.path.join(target_dir, item) if os.path.exists(target_item_dir): if not upgrade: logger.warning( "Target directory %s already exists. Specify " "--upgrade to force replacement.", target_item_dir, ) continue if os.path.islink(target_item_dir): logger.warning( "Target directory %s already exists and is " "a link. pip will not automatically replace " "links, please remove if replacement is " "desired.", target_item_dir, ) continue if os.path.isdir(target_item_dir): shutil.rmtree(target_item_dir) else: os.remove(target_item_dir) shutil.move(os.path.join(lib_dir, item), target_item_dir)
def pip_locations_for(directory): """ Returns a dictionary that contains the `pip_prefix`, `pip_bin` and `pip_lib` members as returned by #get_directories() but for a directory that contains the pip data (i.e. `myproject/.nodepy`). """ scheme = pip_locations.distutils_scheme('', prefix=PIP_DIRECTORY) return { 'pip_prefix': os.path.join(directory, scheme['data']), 'pip_bin': os.path.join(directory, scheme['scripts']), 'pip_lib': os.path.join(directory, scheme['purelib']) }
def test_distutils_config_file_read(self, tmpdir, monkeypatch): # This deals with nt/posix path differences install_scripts = os.path.normcase( os.path.abspath(os.path.join(os.path.sep, 'somewhere', 'else'))) f = tmpdir.mkdir("config").joinpath("setup.cfg") f.write_text("[install]\ninstall-scripts=" + install_scripts) from distutils.dist import Distribution # patch the function that returns what config files are present monkeypatch.setattr( Distribution, 'find_config_files', lambda self: [f], ) scheme = distutils_scheme('example') assert scheme['scripts'] == install_scripts
def test_distutils_config_file_read(self, tmpdir, monkeypatch): # This deals with nt/posix path differences install_scripts = os.path.normcase(os.path.abspath( os.path.join(os.path.sep, 'somewhere', 'else'))) f = tmpdir.mkdir("config").join("setup.cfg") f.write("[install]\ninstall-scripts=" + install_scripts) from distutils.dist import Distribution # patch the function that returns what config files are present monkeypatch.setattr( Distribution, 'find_config_files', lambda self: [f], ) scheme = distutils_scheme('example') assert scheme['scripts'] == install_scripts
def get_dist_to_uninstall(self, candidate): # type: (Candidate) -> Optional[Distribution] # TODO: Are there more cases this needs to return True? Editable? dist = self._installed_dists.get(candidate.name) if dist is None: # Not installed, no uninstallation required. return None # Prevent uninstalling packages from /usr if dist_location(dist) in ( distutils_scheme('', prefix=sys.base_prefix)['purelib'], distutils_scheme('', prefix=sys.base_prefix)['platlib'], ): return None # We're installing into global site. The current installation must # be uninstalled, no matter it's in global or user site, because the # user site installation has precedence over global. if not self._use_user_site: return dist # We're installing into user site. Remove the user site installation. if dist_in_usersite(dist): return dist # We're installing into user site, but the installed incompatible # package is in global site. We can't uninstall that, and would let # the new user installation to "shadow" it. But shadowing won't work # in virtual environments, so we error out. if running_under_virtualenv() and dist_in_site_packages(dist): raise InstallationError( "Will not install to the user site because it will " "lack sys.path precedence to {} in {}".format( dist.project_name, dist.location, )) return None
def test_install_lib_takes_precedence(self, tmpdir, monkeypatch): # This deals with nt/posix path differences install_lib = os.path.normcase( os.path.abspath(os.path.join(os.path.sep, 'somewhere', 'else'))) f = tmpdir.mkdir("config").join("setup.cfg") f.write("[install]\ninstall-lib=" + install_lib) from distutils.dist import Distribution # patch the function that returns what config files are present monkeypatch.setattr( Distribution, 'find_config_files', lambda self: [f], ) scheme = distutils_scheme('example') assert scheme['platlib'] == install_lib + os.path.sep assert scheme['purelib'] == install_lib + os.path.sep
def lib_dir(name, wheeldir, user=False, home=None, root=None, isolated=False, prefix=None): scheme = distutils_scheme("", user=user, home=home, root=root, isolated=isolated, prefix=prefix) if root_is_purelib(name, wheeldir): return scheme['purelib'] else: return scheme['platlib']
def test_install_prefix(self, data, tmpdir): prefix = os.path.join(os.path.sep, 'some', 'path') self.prep(data, tmpdir) scheme = distutils_scheme( self.name, user=False, home=None, root=tmpdir, isolated=False, prefix=prefix, ) wheel.install_unpacked_wheel( self.name, self.src, scheme=scheme, req_description=str(self.req), ) bin_dir = 'Scripts' if WINDOWS else 'bin' assert os.path.exists(os.path.join(tmpdir, 'some', 'path', bin_dir)) assert os.path.exists(os.path.join(tmpdir, 'some', 'path', 'my_data'))
def get_lib_location_guesses(*args, **kwargs): scheme = distutils_scheme('', *args, **kwargs) return [scheme['purelib'], scheme['platlib']]
def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, pycompile=True, scheme=None, isolated=False, prefix=None, warn_script_location=True): """Install a wheel""" if not scheme: scheme = distutils_scheme( name, user=user, home=home, root=root, isolated=isolated, prefix=prefix, ) if root_is_purelib(name, wheeldir): lib_dir = scheme['purelib'] else: lib_dir = scheme['platlib'] info_dir = [] data_dirs = [] source = wheeldir.rstrip(os.path.sep) + os.path.sep # Record details of the files moved # installed = files copied from the wheel to the destination # changed = files changed while installing (scripts #! line typically) # generated = files newly generated during the install (script wrappers) installed = {} changed = set() generated = [] # Compile all of the pyc files that we're going to be installing if pycompile: with captured_stdout() as stdout: with warnings.catch_warnings(): warnings.filterwarnings('ignore') compileall.compile_dir(source, force=True, quiet=True) logger.debug(stdout.getvalue()) def normpath(src, p): return os.path.relpath(src, p).replace(os.path.sep, '/') def record_installed(srcfile, destfile, modified=False): """Map archive RECORD paths to installation RECORD paths.""" oldpath = normpath(srcfile, wheeldir) newpath = normpath(destfile, lib_dir) installed[oldpath] = newpath if modified: changed.add(destfile) def clobber(source, dest, is_base, fixer=None, filter=None): ensure_dir(dest) # common for the 'include' path for dir, subdirs, files in os.walk(source): basedir = dir[len(source):].lstrip(os.path.sep) destdir = os.path.join(dest, basedir) if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'): continue for s in subdirs: destsubdir = os.path.join(dest, basedir, s) if is_base and basedir == '' and destsubdir.endswith('.data'): data_dirs.append(s) continue elif (is_base and s.endswith('.dist-info') and canonicalize_name(s).startswith( canonicalize_name(req.name))): assert not info_dir, ('Multiple .dist-info directories: ' + destsubdir + ', ' + ', '.join(info_dir)) info_dir.append(destsubdir) for f in files: # Skip unwanted files if filter and filter(f): continue srcfile = os.path.join(dir, f) destfile = os.path.join(dest, basedir, f) # directory creation is lazy and after the file filtering above # to ensure we don't install empty dirs; empty dirs can't be # uninstalled. ensure_dir(destdir) # We use copyfile (not move, copy, or copy2) to be extra sure # that we are not moving directories over (copyfile fails for # directories) as well as to ensure that we are not copying # over any metadata because we want more control over what # metadata we actually copy over. shutil.copyfile(srcfile, destfile) # Copy over the metadata for the file, currently this only # includes the atime and mtime. st = os.stat(srcfile) if hasattr(os, "utime"): os.utime(destfile, (st.st_atime, st.st_mtime)) # If our file is executable, then make our destination file # executable. if os.access(srcfile, os.X_OK): st = os.stat(srcfile) permissions = ( st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH ) os.chmod(destfile, permissions) changed = False if fixer: changed = fixer(destfile) record_installed(srcfile, destfile, changed) clobber(source, lib_dir, True) assert info_dir, "%s .dist-info directory not found" % req # Get the defined entry points ep_file = os.path.join(info_dir[0], 'entry_points.txt') console, gui = get_entrypoints(ep_file) def is_entrypoint_wrapper(name): # EP, EP.exe and EP-script.py are scripts generated for # entry point EP by setuptools if name.lower().endswith('.exe'): matchname = name[:-4] elif name.lower().endswith('-script.py'): matchname = name[:-10] elif name.lower().endswith(".pya"): matchname = name[:-4] else: matchname = name # Ignore setuptools-generated scripts return (matchname in console or matchname in gui) for datadir in data_dirs: fixer = None filter = None for subdir in os.listdir(os.path.join(wheeldir, datadir)): fixer = None if subdir == 'scripts': fixer = fix_script filter = is_entrypoint_wrapper source = os.path.join(wheeldir, datadir, subdir) dest = scheme[subdir] clobber(source, dest, False, fixer=fixer, filter=filter) maker = ScriptMaker(None, scheme['scripts']) # Ensure old scripts are overwritten. # See https://github.com/pypa/pip/issues/1800 maker.clobber = True # Ensure we don't generate any variants for scripts because this is almost # never what somebody wants. # See https://bitbucket.org/pypa/distlib/issue/35/ maker.variants = {''} # This is required because otherwise distlib creates scripts that are not # executable. # See https://bitbucket.org/pypa/distlib/issue/32/ maker.set_mode = True # Simplify the script and fix the fact that the default script swallows # every single stack trace. # See https://bitbucket.org/pypa/distlib/issue/34/ # See https://bitbucket.org/pypa/distlib/issue/33/ def _get_script_text(entry): if entry.suffix is None: raise InstallationError( "Invalid script entry point: %s for req: %s - A callable " "suffix is required. Cf https://packaging.python.org/en/" "latest/distributing.html#console-scripts for more " "information." % (entry, req) ) return maker.script_template % { "module": entry.prefix, "import_name": entry.suffix.split(".")[0], "func": entry.suffix, } maker._get_script_text = _get_script_text maker.script_template = r"""# -*- coding: utf-8 -*- import re import sys from %(module)s import %(import_name)s if __name__ == '__main__': sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) sys.exit(%(func)s()) """ # Special case pip and setuptools to generate versioned wrappers # # The issue is that some projects (specifically, pip and setuptools) use # code in setup.py to create "versioned" entry points - pip2.7 on Python # 2.7, pip3.3 on Python 3.3, etc. But these entry points are baked into # the wheel metadata at build time, and so if the wheel is installed with # a *different* version of Python the entry points will be wrong. The # correct fix for this is to enhance the metadata to be able to describe # such versioned entry points, but that won't happen till Metadata 2.0 is # available. # In the meantime, projects using versioned entry points will either have # incorrect versioned entry points, or they will not be able to distribute # "universal" wheels (i.e., they will need a wheel per Python version). # # Because setuptools and pip are bundled with _ensurepip and virtualenv, # we need to use universal wheels. So, as a stopgap until Metadata 2.0, we # override the versioned entry points in the wheel and generate the # correct ones. This code is purely a short-term measure until Metadata 2.0 # is available. # # To add the level of hack in this section of code, in order to support # ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment # variable which will control which version scripts get installed. # # ENSUREPIP_OPTIONS=altinstall # - Only pipX.Y and easy_install-X.Y will be generated and installed # ENSUREPIP_OPTIONS=install # - pipX.Y, pipX, easy_install-X.Y will be generated and installed. Note # that this option is technically if ENSUREPIP_OPTIONS is set and is # not altinstall # DEFAULT # - The default behavior is to install pip, pipX, pipX.Y, easy_install # and easy_install-X.Y. pip_script = console.pop('pip', None) if pip_script: if "ENSUREPIP_OPTIONS" not in os.environ: spec = 'pip = ' + pip_script generated.extend(maker.make(spec)) if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall": spec = 'pip%s = %s' % (sys.version[:1], pip_script) generated.extend(maker.make(spec)) spec = 'pip%s = %s' % (sys.version[:3], pip_script) generated.extend(maker.make(spec)) # Delete any other versioned pip entry points pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)] for k in pip_ep: del console[k] easy_install_script = console.pop('easy_install', None) if easy_install_script: if "ENSUREPIP_OPTIONS" not in os.environ: spec = 'easy_install = ' + easy_install_script generated.extend(maker.make(spec)) spec = 'easy_install-%s = %s' % (sys.version[:3], easy_install_script) generated.extend(maker.make(spec)) # Delete any other versioned easy_install entry points easy_install_ep = [ k for k in console if re.match(r'easy_install(-\d\.\d)?$', k) ] for k in easy_install_ep: del console[k] # Generate the console and GUI entry points specified in the wheel if len(console) > 0: generated_console_scripts = maker.make_multiple( ['%s = %s' % kv for kv in console.items()] ) generated.extend(generated_console_scripts) if warn_script_location: msg = message_about_scripts_not_on_PATH(generated_console_scripts) if msg is not None: logger.warn(msg) if len(gui) > 0: generated.extend( maker.make_multiple( ['%s = %s' % kv for kv in gui.items()], {'gui': True} ) ) # Record pip as the installer installer = os.path.join(info_dir[0], 'INSTALLER') temp_installer = os.path.join(info_dir[0], 'INSTALLER.pip') with open(temp_installer, 'wb') as installer_file: installer_file.write(b'pip\n') shutil.move(temp_installer, installer) generated.append(installer) # Record details of all files installed record = os.path.join(info_dir[0], 'RECORD') temp_record = os.path.join(info_dir[0], 'RECORD.pip') with open_for_csv(record, 'r') as record_in: with open_for_csv(temp_record, 'w+') as record_out: reader = csv.reader(record_in) writer = csv.writer(record_out) for row in reader: row[0] = installed.pop(row[0], row[0]) if row[0] in changed: row[1], row[2] = rehash(row[0]) writer.writerow(row) for f in generated: h, l = rehash(f) writer.writerow((normpath(f, lib_dir), h, l)) for f in installed: writer.writerow((installed[f], '', '')) shutil.move(temp_record, record)
def install( self, install_options, # type: List[str] global_options=None, # type: Optional[Sequence[str]] root=None, # type: Optional[str] home=None, # type: Optional[str] prefix=None, # type: Optional[str] warn_script_location=True, # type: bool use_user_site=False, # type: bool pycompile=True # type: bool ): # type: (...) -> None global_options = global_options if global_options is not None else [] if self.editable: self.install_editable( install_options, global_options, prefix=prefix, ) return if self.is_wheel: version = wheel.wheel_version(self.source_dir) wheel.check_compatibility(version, self.name) scheme = distutils_scheme( self.name, user=use_user_site, home=home, root=root, isolated=self.isolated, prefix=prefix, ) self.move_wheel_files( self.source_dir, scheme=scheme, warn_script_location=warn_script_location, pycompile=pycompile, ) self.install_succeeded = True return # Extend the list of global and install options passed on to # the setup.py call with the ones from the requirements file. # Options specified in requirements file override those # specified on the command line, since the last option given # to setup.py is the one that is used. global_options = list(global_options) + \ self.options.get('global_options', []) install_options = list(install_options) + \ self.options.get('install_options', []) header_dir = None # type: Optional[str] if running_under_virtualenv(): py_ver_str = 'python' + sysconfig.get_python_version() header_dir = os.path.join( sys.prefix, 'include', 'site', py_ver_str, self.name ) with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') install_args = make_setuptools_install_args( self.setup_py_path, global_options=global_options, install_options=install_options, record_filename=record_filename, root=root, prefix=prefix, header_dir=header_dir, no_user_config=self.isolated, pycompile=pycompile, ) runner = runner_with_spinner_message( "Running setup.py install for {}".format(self.name) ) with indent_log(), self.build_env: runner( cmd=install_args, cwd=self.unpacked_source_directory, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True def prepend_root(path): # type: (str) -> str if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: logger.warning( 'Could not find .egg-info directory in install record' ' for %s', self, ) # FIXME: put the record somewhere return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( os.path.relpath(prepend_root(filename), egg_info_dir) ) new_lines.sort() ensure_dir(egg_info_dir) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n')
def get_lib_location_guesses(*args, **kwargs): scheme = distutils_scheme("", *args, **kwargs) return [scheme["purelib"], scheme["platlib"]]
def move_wheel_files( name, # type: str req, # type: Requirement wheeldir, # type: str user=False, # type: bool home=None, # type: Optional[str] root=None, # type: Optional[str] pycompile=True, # type: bool scheme=None, # type: Optional[Mapping[str, str]] isolated=False, # type: bool prefix=None, # type: Optional[str] warn_script_location=True # type: bool ): # type: (...) -> None """Install a wheel""" # TODO: Investigate and break this up. # TODO: Look into moving this into a dedicated class for representing an # installation. if not scheme: scheme = distutils_scheme( name, user=user, home=home, root=root, isolated=isolated, prefix=prefix, ) if root_is_purelib(name, wheeldir): lib_dir = scheme['purelib'] else: lib_dir = scheme['platlib'] info_dir = [] # type: List[str] data_dirs = [] source = wheeldir.rstrip(os.path.sep) + os.path.sep # Record details of the files moved # installed = files copied from the wheel to the destination # changed = files changed while installing (scripts #! line typically) # generated = files newly generated during the install (script wrappers) installed = {} # type: Dict[str, str] changed = set() generated = [] # type: List[str] # Compile all of the pyc files that we're going to be installing if pycompile: with captured_stdout() as stdout: with warnings.catch_warnings(): warnings.filterwarnings('ignore') compileall.compile_dir(source, force=True, quiet=True) logger.debug(stdout.getvalue()) def record_installed(srcfile, destfile, modified=False): """Map archive RECORD paths to installation RECORD paths.""" oldpath = normpath(srcfile, wheeldir) newpath = normpath(destfile, lib_dir) installed[oldpath] = newpath if modified: changed.add(destfile) def clobber(source, dest, is_base, fixer=None, filter=None): ensure_dir(dest) # common for the 'include' path for dir, subdirs, files in os.walk(source): basedir = dir[len(source):].lstrip(os.path.sep) destdir = os.path.join(dest, basedir) if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'): continue for s in subdirs: destsubdir = os.path.join(dest, basedir, s) if is_base and basedir == '' and destsubdir.endswith('.data'): data_dirs.append(s) continue elif (is_base and s.endswith('.dist-info') and canonicalize_name(s).startswith( canonicalize_name(req.name))): assert not info_dir, ('Multiple .dist-info directories: ' + destsubdir + ', ' + ', '.join(info_dir)) info_dir.append(destsubdir) for f in files: # Skip unwanted files if filter and filter(f): continue srcfile = os.path.join(dir, f) destfile = os.path.join(dest, basedir, f) # directory creation is lazy and after the file filtering above # to ensure we don't install empty dirs; empty dirs can't be # uninstalled. ensure_dir(destdir) # copyfile (called below) truncates the destination if it # exists and then writes the new contents. This is fine in most # cases, but can cause a segfault if pip has loaded a shared # object (e.g. from pyopenssl through its vendored urllib3) # Since the shared object is mmap'd an attempt to call a # symbol in it will then cause a segfault. Unlinking the file # allows writing of new contents while allowing the process to # continue to use the old copy. if os.path.exists(destfile): os.unlink(destfile) # We use copyfile (not move, copy, or copy2) to be extra sure # that we are not moving directories over (copyfile fails for # directories) as well as to ensure that we are not copying # over any metadata because we want more control over what # metadata we actually copy over. shutil.copyfile(srcfile, destfile) # Copy over the metadata for the file, currently this only # includes the atime and mtime. st = os.stat(srcfile) if hasattr(os, "utime"): os.utime(destfile, (st.st_atime, st.st_mtime)) # If our file is executable, then make our destination file # executable. if os.access(srcfile, os.X_OK): st = os.stat(srcfile) permissions = ( st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH ) os.chmod(destfile, permissions) changed = False if fixer: changed = fixer(destfile) record_installed(srcfile, destfile, changed) clobber(source, lib_dir, True) assert info_dir, "%s .dist-info directory not found" % req # Get the defined entry points ep_file = os.path.join(info_dir[0], 'entry_points.txt') console, gui = get_entrypoints(ep_file) def is_entrypoint_wrapper(name): # EP, EP.exe and EP-script.py are scripts generated for # entry point EP by setuptools if name.lower().endswith('.exe'): matchname = name[:-4] elif name.lower().endswith('-script.py'): matchname = name[:-10] elif name.lower().endswith(".pya"): matchname = name[:-4] else: matchname = name # Ignore setuptools-generated scripts return (matchname in console or matchname in gui) for datadir in data_dirs: fixer = None filter = None for subdir in os.listdir(os.path.join(wheeldir, datadir)): fixer = None if subdir == 'scripts': fixer = fix_script filter = is_entrypoint_wrapper source = os.path.join(wheeldir, datadir, subdir) dest = scheme[subdir] clobber(source, dest, False, fixer=fixer, filter=filter) maker = ScriptMaker(None, scheme['scripts']) # Ensure old scripts are overwritten. # See https://github.com/pypa/pip/issues/1800 maker.clobber = True # Ensure we don't generate any variants for scripts because this is almost # never what somebody wants. # See https://bitbucket.org/pypa/distlib/issue/35/ maker.variants = {''} # This is required because otherwise distlib creates scripts that are not # executable. # See https://bitbucket.org/pypa/distlib/issue/32/ maker.set_mode = True # Simplify the script and fix the fact that the default script swallows # every single stack trace. # See https://bitbucket.org/pypa/distlib/issue/34/ # See https://bitbucket.org/pypa/distlib/issue/33/ def _get_script_text(entry): if entry.suffix is None: raise InstallationError( "Invalid script entry point: %s for req: %s - A callable " "suffix is required. Cf https://packaging.python.org/en/" "latest/distributing.html#console-scripts for more " "information." % (entry, req) ) return maker.script_template % { "module": entry.prefix, "import_name": entry.suffix.split(".")[0], "func": entry.suffix, } # ignore type, because mypy disallows assigning to a method, # see https://github.com/python/mypy/issues/2427 maker._get_script_text = _get_script_text # type: ignore maker.script_template = r"""# -*- coding: utf-8 -*- import re import sys from %(module)s import %(import_name)s if __name__ == '__main__': sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) sys.exit(%(func)s()) """ # Special case pip and setuptools to generate versioned wrappers # # The issue is that some projects (specifically, pip and setuptools) use # code in setup.py to create "versioned" entry points - pip2.7 on Python # 2.7, pip3.3 on Python 3.3, etc. But these entry points are baked into # the wheel metadata at build time, and so if the wheel is installed with # a *different* version of Python the entry points will be wrong. The # correct fix for this is to enhance the metadata to be able to describe # such versioned entry points, but that won't happen till Metadata 2.0 is # available. # In the meantime, projects using versioned entry points will either have # incorrect versioned entry points, or they will not be able to distribute # "universal" wheels (i.e., they will need a wheel per Python version). # # Because setuptools and pip are bundled with _ensurepip and virtualenv, # we need to use universal wheels. So, as a stopgap until Metadata 2.0, we # override the versioned entry points in the wheel and generate the # correct ones. This code is purely a short-term measure until Metadata 2.0 # is available. # # To add the level of hack in this section of code, in order to support # ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment # variable which will control which version scripts get installed. # # ENSUREPIP_OPTIONS=altinstall # - Only pipX.Y and easy_install-X.Y will be generated and installed # ENSUREPIP_OPTIONS=install # - pipX.Y, pipX, easy_install-X.Y will be generated and installed. Note # that this option is technically if ENSUREPIP_OPTIONS is set and is # not altinstall # DEFAULT # - The default behavior is to install pip, pipX, pipX.Y, easy_install # and easy_install-X.Y. pip_script = console.pop('pip', None) if pip_script: if "ENSUREPIP_OPTIONS" not in os.environ: spec = 'pip = ' + pip_script generated.extend(maker.make(spec)) if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall": spec = 'pip%s = %s' % (sys.version[:1], pip_script) generated.extend(maker.make(spec)) spec = 'pip%s = %s' % (sys.version[:3], pip_script) generated.extend(maker.make(spec)) # Delete any other versioned pip entry points pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)] for k in pip_ep: del console[k] easy_install_script = console.pop('easy_install', None) if easy_install_script: if "ENSUREPIP_OPTIONS" not in os.environ: spec = 'easy_install = ' + easy_install_script generated.extend(maker.make(spec)) spec = 'easy_install-%s = %s' % (sys.version[:3], easy_install_script) generated.extend(maker.make(spec)) # Delete any other versioned easy_install entry points easy_install_ep = [ k for k in console if re.match(r'easy_install(-\d\.\d)?$', k) ] for k in easy_install_ep: del console[k] # Generate the console and GUI entry points specified in the wheel if len(console) > 0: generated_console_scripts = maker.make_multiple( ['%s = %s' % kv for kv in console.items()] ) generated.extend(generated_console_scripts) if warn_script_location: msg = message_about_scripts_not_on_PATH(generated_console_scripts) if msg is not None: logger.warning(msg) if len(gui) > 0: generated.extend( maker.make_multiple( ['%s = %s' % kv for kv in gui.items()], {'gui': True} ) ) # Record pip as the installer installer = os.path.join(info_dir[0], 'INSTALLER') temp_installer = os.path.join(info_dir[0], 'INSTALLER.pip') with open(temp_installer, 'wb') as installer_file: installer_file.write(b'pip\n') shutil.move(temp_installer, installer) generated.append(installer) # Record details of all files installed record = os.path.join(info_dir[0], 'RECORD') temp_record = os.path.join(info_dir[0], 'RECORD.pip') with open_for_csv(record, 'r') as record_in: with open_for_csv(temp_record, 'w+') as record_out: reader = csv.reader(record_in) outrows = get_csv_rows_for_installed( reader, installed=installed, changed=changed, generated=generated, lib_dir=lib_dir, ) writer = csv.writer(record_out) # Sort to simplify testing. for row in sorted_outrows(outrows): writer.writerow(row) shutil.move(temp_record, record)