def get_pip_version() -> str: pip_pkg_dir = os.path.join(os.path.dirname(__file__), "..", "..") pip_pkg_dir = os.path.abspath(pip_pkg_dir) return "pip {} from {} (python {})".format( __version__, pip_pkg_dir, get_major_minor_version(), )
def install_unpacked_wheel( name, # type: str wheeldir, # type: str wheel_zip, # type: ZipFile scheme, # type: Scheme req_description, # type: str pycompile=True, # type: bool warn_script_location=True # type: bool ): # type: (...) -> None """Install a wheel. :param name: Name of the project to install :param wheeldir: Base directory of the unpacked wheel :param wheel_zip: open ZipFile for wheel being installed :param scheme: Distutils scheme dictating the install directories :param req_description: String used in place of the requirement, for logging :param pycompile: Whether to byte-compile installed Python files :param warn_script_location: Whether to check that scripts are installed into a directory on PATH :raises UnsupportedWheel: * when the directory holds an unpacked wheel with incompatible Wheel-Version * when the .dist-info dir does not match the wheel """ # TODO: Investigate and break this up. # TODO: Look into moving this into a dedicated class for representing an # installation. source = wheeldir.rstrip(os.path.sep) + os.path.sep info_dir, metadata = parse_wheel(wheel_zip, name) if wheel_root_is_purelib(metadata): lib_dir = scheme.purelib else: lib_dir = scheme.platlib subdirs = os.listdir(source) data_dirs = [s for s in subdirs if s.endswith('.data')] # Record details of the files moved # installed = files copied from the wheel to the destination # changed = files changed while installing (scripts #! line typically) # generated = files newly generated during the install (script wrappers) installed = {} # type: Dict[str, str] changed = set() generated = [] # type: List[str] # Compile all of the pyc files that we're going to be installing if pycompile: with captured_stdout() as stdout: with warnings.catch_warnings(): warnings.filterwarnings('ignore') compileall.compile_dir(source, force=True, quiet=True) logger.debug(stdout.getvalue()) def record_installed(srcfile, destfile, modified=False): # type: (str, str, bool) -> None """Map archive RECORD paths to installation RECORD paths.""" oldpath = normpath(srcfile, wheeldir) newpath = normpath(destfile, lib_dir) installed[oldpath] = newpath if modified: changed.add(destfile) def clobber( source, # type: str dest, # type: str is_base, # type: bool fixer=None, # type: Optional[Callable[[str], Any]] filter=None # type: Optional[Callable[[str], bool]] ): # type: (...) -> None ensure_dir(dest) # common for the 'include' path for dir, subdirs, files in os.walk(source): basedir = dir[len(source):].lstrip(os.path.sep) destdir = os.path.join(dest, basedir) if is_base and basedir == '': subdirs[:] = [s for s in subdirs if not s.endswith('.data')] for f in files: # Skip unwanted files if filter and filter(f): continue srcfile = os.path.join(dir, f) destfile = os.path.join(dest, basedir, f) # directory creation is lazy and after the file filtering above # to ensure we don't install empty dirs; empty dirs can't be # uninstalled. ensure_dir(destdir) # copyfile (called below) truncates the destination if it # exists and then writes the new contents. This is fine in most # cases, but can cause a segfault if pip has loaded a shared # object (e.g. from pyopenssl through its vendored urllib3) # Since the shared object is mmap'd an attempt to call a # symbol in it will then cause a segfault. Unlinking the file # allows writing of new contents while allowing the process to # continue to use the old copy. if os.path.exists(destfile): os.unlink(destfile) # We use copyfile (not move, copy, or copy2) to be extra sure # that we are not moving directories over (copyfile fails for # directories) as well as to ensure that we are not copying # over any metadata because we want more control over what # metadata we actually copy over. shutil.copyfile(srcfile, destfile) # Copy over the metadata for the file, currently this only # includes the atime and mtime. st = os.stat(srcfile) if hasattr(os, "utime"): os.utime(destfile, (st.st_atime, st.st_mtime)) # If our file is executable, then make our destination file # executable. if os.access(srcfile, os.X_OK): st = os.stat(srcfile) permissions = (st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) os.chmod(destfile, permissions) changed = False if fixer: changed = fixer(destfile) record_installed(srcfile, destfile, changed) clobber(source, lib_dir, True) dest_info_dir = os.path.join(lib_dir, info_dir) # Get the defined entry points ep_file = os.path.join(dest_info_dir, 'entry_points.txt') console, gui = get_entrypoints(ep_file) def is_entrypoint_wrapper(name): # type: (str) -> bool # EP, EP.exe and EP-script.py are scripts generated for # entry point EP by setuptools if name.lower().endswith('.exe'): matchname = name[:-4] elif name.lower().endswith('-script.py'): matchname = name[:-10] elif name.lower().endswith(".pya"): matchname = name[:-4] else: matchname = name # Ignore setuptools-generated scripts return (matchname in console or matchname in gui) for datadir in data_dirs: fixer = None filter = None for subdir in os.listdir(os.path.join(wheeldir, datadir)): fixer = None if subdir == 'scripts': fixer = fix_script filter = is_entrypoint_wrapper source = os.path.join(wheeldir, datadir, subdir) dest = getattr(scheme, subdir) clobber(source, dest, False, fixer=fixer, filter=filter) maker = PipScriptMaker(None, scheme.scripts) # Ensure old scripts are overwritten. # See https://github.com/pypa/pip/issues/1800 maker.clobber = True # Ensure we don't generate any variants for scripts because this is almost # never what somebody wants. # See https://bitbucket.org/pypa/distlib/issue/35/ maker.variants = {''} # This is required because otherwise distlib creates scripts that are not # executable. # See https://bitbucket.org/pypa/distlib/issue/32/ maker.set_mode = True scripts_to_generate = [] # Special case pip and setuptools to generate versioned wrappers # # The issue is that some projects (specifically, pip and setuptools) use # code in setup.py to create "versioned" entry points - pip2.7 on Python # 2.7, pip3.3 on Python 3.3, etc. But these entry points are baked into # the wheel metadata at build time, and so if the wheel is installed with # a *different* version of Python the entry points will be wrong. The # correct fix for this is to enhance the metadata to be able to describe # such versioned entry points, but that won't happen till Metadata 2.0 is # available. # In the meantime, projects using versioned entry points will either have # incorrect versioned entry points, or they will not be able to distribute # "universal" wheels (i.e., they will need a wheel per Python version). # # Because setuptools and pip are bundled with _ensurepip and virtualenv, # we need to use universal wheels. So, as a stopgap until Metadata 2.0, we # override the versioned entry points in the wheel and generate the # correct ones. This code is purely a short-term measure until Metadata 2.0 # is available. # # To add the level of hack in this section of code, in order to support # ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment # variable which will control which version scripts get installed. # # ENSUREPIP_OPTIONS=altinstall # - Only pipX.Y and easy_install-X.Y will be generated and installed # ENSUREPIP_OPTIONS=install # - pipX.Y, pipX, easy_install-X.Y will be generated and installed. Note # that this option is technically if ENSUREPIP_OPTIONS is set and is # not altinstall # DEFAULT # - The default behavior is to install pip, pipX, pipX.Y, easy_install # and easy_install-X.Y. pip_script = console.pop('pip', None) if pip_script: if "ENSUREPIP_OPTIONS" not in os.environ: scripts_to_generate.append('pip = ' + pip_script) if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall": scripts_to_generate.append('pip%s = %s' % (sys.version_info[0], pip_script)) scripts_to_generate.append('pip%s = %s' % (get_major_minor_version(), pip_script)) # Delete any other versioned pip entry points pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)] for k in pip_ep: del console[k] easy_install_script = console.pop('easy_install', None) if easy_install_script: if "ENSUREPIP_OPTIONS" not in os.environ: scripts_to_generate.append('easy_install = ' + easy_install_script) scripts_to_generate.append( 'easy_install-%s = %s' % (get_major_minor_version(), easy_install_script)) # Delete any other versioned easy_install entry points easy_install_ep = [ k for k in console if re.match(r'easy_install(-\d\.\d)?$', k) ] for k in easy_install_ep: del console[k] # Generate the console and GUI entry points specified in the wheel scripts_to_generate.extend('%s = %s' % kv for kv in console.items()) gui_scripts_to_generate = ['%s = %s' % kv for kv in gui.items()] generated_console_scripts = [] # type: List[str] try: generated_console_scripts = maker.make_multiple(scripts_to_generate) generated.extend(generated_console_scripts) generated.extend( maker.make_multiple(gui_scripts_to_generate, {'gui': True})) except MissingCallableSuffix as e: entry = e.args[0] raise InstallationError( "Invalid script entry point: {} for req: {} - A callable " "suffix is required. Cf https://packaging.python.org/" "specifications/entry-points/#use-for-scripts for more " "information.".format(entry, req_description)) if warn_script_location: msg = message_about_scripts_not_on_PATH(generated_console_scripts) if msg is not None: logger.warning(msg) # Record pip as the installer installer = os.path.join(dest_info_dir, 'INSTALLER') temp_installer = os.path.join(dest_info_dir, 'INSTALLER.pip') with open(temp_installer, 'wb') as installer_file: installer_file.write(b'pip\n') shutil.move(temp_installer, installer) generated.append(installer) # Record details of all files installed record = os.path.join(dest_info_dir, 'RECORD') temp_record = os.path.join(dest_info_dir, 'RECORD.pip') with open_for_csv(record, 'r') as record_in: with open_for_csv(temp_record, 'w+') as record_out: reader = csv.reader(record_in) outrows = get_csv_rows_for_installed( reader, installed=installed, changed=changed, generated=generated, lib_dir=lib_dir, ) writer = csv.writer(record_out) # Sort to simplify testing. for row in sorted_outrows(outrows): writer.writerow(row) shutil.move(temp_record, record)
def get_console_script_specs(console: Dict[str, str]) -> List[str]: """ Given the mapping from entrypoint name to callable, return the relevant console script specs. """ # Don't mutate caller's version console = console.copy() scripts_to_generate = [] # Special case pip and setuptools to generate versioned wrappers # # The issue is that some projects (specifically, pip and setuptools) use # code in setup.py to create "versioned" entry points - pip2.7 on Python # 2.7, pip3.3 on Python 3.3, etc. But these entry points are baked into # the wheel metadata at build time, and so if the wheel is installed with # a *different* version of Python the entry points will be wrong. The # correct fix for this is to enhance the metadata to be able to describe # such versioned entry points, but that won't happen till Metadata 2.0 is # available. # In the meantime, projects using versioned entry points will either have # incorrect versioned entry points, or they will not be able to distribute # "universal" wheels (i.e., they will need a wheel per Python version). # # Because setuptools and pip are bundled with _ensurepip and virtualenv, # we need to use universal wheels. So, as a stopgap until Metadata 2.0, we # override the versioned entry points in the wheel and generate the # correct ones. This code is purely a short-term measure until Metadata 2.0 # is available. # # To add the level of hack in this section of code, in order to support # ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment # variable which will control which version scripts get installed. # # ENSUREPIP_OPTIONS=altinstall # - Only pipX.Y and easy_install-X.Y will be generated and installed # ENSUREPIP_OPTIONS=install # - pipX.Y, pipX, easy_install-X.Y will be generated and installed. Note # that this option is technically if ENSUREPIP_OPTIONS is set and is # not altinstall # DEFAULT # - The default behavior is to install pip, pipX, pipX.Y, easy_install # and easy_install-X.Y. pip_script = console.pop("pip", None) if pip_script: if "ENSUREPIP_OPTIONS" not in os.environ: scripts_to_generate.append("pip = " + pip_script) if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall": scripts_to_generate.append("pip{} = {}".format( sys.version_info[0], pip_script)) scripts_to_generate.append( f"pip{get_major_minor_version()} = {pip_script}") # Delete any other versioned pip entry points pip_ep = [k for k in console if re.match(r"pip(\d(\.\d)?)?$", k)] for k in pip_ep: del console[k] easy_install_script = console.pop("easy_install", None) if easy_install_script: if "ENSUREPIP_OPTIONS" not in os.environ: scripts_to_generate.append("easy_install = " + easy_install_script) scripts_to_generate.append("easy_install-{} = {}".format( get_major_minor_version(), easy_install_script)) # Delete any other versioned easy_install entry points easy_install_ep = [ k for k in console if re.match(r"easy_install(-\d\.\d)?$", k) ] for k in easy_install_ep: del console[k] # Generate the console entry points specified in the wheel scripts_to_generate.extend(starmap("{} = {}".format, console.items())) return scripts_to_generate