def run_egg_info(self): # type: () -> None if self.name: logger.debug( 'Running setup.py (path:%s) egg_info for package %s', self.setup_py, self.name, ) else: logger.debug( 'Running setup.py (path:%s) egg_info for package from %s', self.setup_py, self.link, ) script = SETUPTOOLS_SHIM % self.setup_py base_cmd = [sys.executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] # We can't put the .egg-info files at the root, because then the # source code will be mistaken for an installed egg, causing # problems if self.editable: egg_base_option = [] # type: List[str] else: egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info') ensure_dir(egg_info_dir) egg_base_option = ['--egg-base', 'pip-egg-info'] with self.build_env: call_subprocess( egg_info_cmd + egg_base_option, cwd=self.setup_py_dir, command_desc='python setup.py egg_info')
def save(self, pypi_version, current_time): # type: (str, datetime.datetime) -> None # If we do not have a path to cache in, don't bother saving. if not self.statefile_path: return # Check to make sure that we own the directory if not check_path_owner(os.path.dirname(self.statefile_path)): return # Now that we've ensured the directory is owned by this user, we'll go # ahead and make sure that all our directories are created. ensure_dir(os.path.dirname(self.statefile_path)) # Attempt to write out our version check file with lockfile.LockFile(self.statefile_path): if os.path.exists(self.statefile_path): with open(self.statefile_path) as statefile: state = json.load(statefile) else: state = {} state[sys.prefix] = { "last_check": current_time.strftime(SELFCHECK_DATE_FMT), "pypi_version": pypi_version, } with open(self.statefile_path, "w") as statefile: json.dump(state, statefile, sort_keys=True, separators=(",", ":"))
def run_egg_info(self): assert self.source_dir if self.name: logger.debug( 'Running setup.py (path:%s) egg_info for package %s', self.setup_py, self.name, ) else: logger.debug( 'Running setup.py (path:%s) egg_info for package from %s', self.setup_py, self.link, ) with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py base_cmd = [sys.executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] # We can't put the .egg-info files at the root, because then the # source code will be mistaken for an installed egg, causing # problems if self.editable: egg_base_option = [] else: egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info') ensure_dir(egg_info_dir) egg_base_option = ['--egg-base', 'pip-egg-info'] with self.build_env: call_subprocess( egg_info_cmd + egg_base_option, cwd=self.setup_py_dir, show_stdout=False, command_desc='python setup.py egg_info') if not self.req: if isinstance(parse_version(self.pkg_info()["Version"]), Version): op = "==" else: op = "===" self.req = Requirement( "".join([ self.pkg_info()["Name"], op, self.pkg_info()["Version"], ]) ) self._correct_build_location() else: metadata_name = canonicalize_name(self.pkg_info()["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( 'Running setup.py (path:%s) egg_info for package %s ' 'produced metadata for project name %s. Fix your ' '#egg=%s fragments.', self.setup_py, self.name, metadata_name, self.name ) self.req = Requirement(metadata_name)
def clobber(source, dest, is_base, fixer=None, filter=None): ensure_dir(dest) # common for the 'include' path for dir, subdirs, files in os.walk(source): basedir = dir[len(source):].lstrip(os.path.sep) destdir = os.path.join(dest, basedir) if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'): continue for s in subdirs: destsubdir = os.path.join(dest, basedir, s) if is_base and basedir == '' and destsubdir.endswith('.data'): data_dirs.append(s) continue elif (is_base and s.endswith('.dist-info') and canonicalize_name(s).startswith( canonicalize_name(req.name))): assert not info_dir, ('Multiple .dist-info directories: ' + destsubdir + ', ' + ', '.join(info_dir)) info_dir.append(destsubdir) for f in files: # Skip unwanted files if filter and filter(f): continue srcfile = os.path.join(dir, f) destfile = os.path.join(dest, basedir, f) # directory creation is lazy and after the file filtering above # to ensure we don't install empty dirs; empty dirs can't be # uninstalled. ensure_dir(destdir) # We use copyfile (not move, copy, or copy2) to be extra sure # that we are not moving directories over (copyfile fails for # directories) as well as to ensure that we are not copying # over any metadata because we want more control over what # metadata we actually copy over. shutil.copyfile(srcfile, destfile) # Copy over the metadata for the file, currently this only # includes the atime and mtime. st = os.stat(srcfile) if hasattr(os, "utime"): os.utime(destfile, (st.st_atime, st.st_mtime)) # If our file is executable, then make our destination file # executable. if os.access(srcfile, os.X_OK): st = os.stat(srcfile) permissions = ( st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH ) os.chmod(destfile, permissions) changed = False if fixer: changed = fixer(destfile) record_installed(srcfile, destfile, changed)
def _handle_target_dir(self, target_dir, target_temp_dir, upgrade): ensure_dir(target_dir) # Checking both purelib and platlib directories for installed # packages to be moved to target directory lib_dir_list = [] with target_temp_dir: # Checking both purelib and platlib directories for installed # packages to be moved to target directory scheme = distutils_scheme('', home=target_temp_dir.path) purelib_dir = scheme['purelib'] platlib_dir = scheme['platlib'] data_dir = scheme['data'] if os.path.exists(purelib_dir): lib_dir_list.append(purelib_dir) if os.path.exists(platlib_dir) and platlib_dir != purelib_dir: lib_dir_list.append(platlib_dir) if os.path.exists(data_dir): lib_dir_list.append(data_dir) for lib_dir in lib_dir_list: for item in os.listdir(lib_dir): if lib_dir == data_dir: ddir = os.path.join(data_dir, item) if any(s.startswith(ddir) for s in lib_dir_list[:-1]): continue target_item_dir = os.path.join(target_dir, item) if os.path.exists(target_item_dir): if not upgrade: logger.warning( 'Target directory %s already exists. Specify ' '--upgrade to force replacement.', target_item_dir ) continue if os.path.islink(target_item_dir): logger.warning( 'Target directory %s already exists and is ' 'a link. Pip will not automatically replace ' 'links, please remove if replacement is ' 'desired.', target_item_dir ) continue if os.path.isdir(target_item_dir): shutil.rmtree(target_item_dir) else: os.remove(target_item_dir) shutil.move( os.path.join(lib_dir, item), target_item_dir )
def create_file(path, contents=None): """Create a file on the path, with the given contents """ from pip._internal.utils.misc import ensure_dir ensure_dir(os.path.dirname(path)) with open(path, "w") as f: if contents is not None: f.write(contents) else: f.write("\n")
def test_adjacent_directory_exists(self, name, tmpdir): block_name, expect_name = itertools.islice( AdjacentTempDirectory._generate_names(name), 2) original = os.path.join(tmpdir, name) blocker = os.path.join(tmpdir, block_name) ensure_dir(original) ensure_dir(blocker) with AdjacentTempDirectory(original) as atmp_dir: assert expect_name == os.path.split(atmp_dir.path)[1]
def save(self): # type: () -> None """Save the currentin-memory state. """ self._ensure_have_load_only() for fname, parser in self._modified_parsers: logger.info("Writing to %s", fname) # Ensure directory exists. ensure_dir(os.path.dirname(fname)) with open(fname, "w") as f: parser.write(f) # type: ignore
def resolve(self, requirement_set): # type: (RequirementSet) -> None """Resolve what operations need to be done As a side-effect of this method, the packages (and their dependencies) are downloaded, unpacked and prepared for installation. This preparation is done by ``pip.operations.prepare``. Once PyPI has static dependency metadata available, it would be possible to move the preparation to become a step separated from dependency resolution. """ # make the wheelhouse if self.preparer.wheel_download_dir: ensure_dir(self.preparer.wheel_download_dir) # If any top-level requirement has a hash specified, enter # hash-checking mode, which requires hashes from all. root_reqs = ( requirement_set.unnamed_requirements + list(requirement_set.requirements.values()) ) self.require_hashes = ( requirement_set.require_hashes or any(req.has_hash_options for req in root_reqs) ) # Display where finder is looking for packages locations = self.finder.get_formatted_locations() if locations: logger.info(locations) # Actually prepare the files, and collect any exceptions. Most hash # exceptions cannot be checked ahead of time, because # req.populate_link() needs to be called before we can make decisions # based on link type. discovered_reqs = [] # type: List[InstallRequirement] hash_errors = HashErrors() for req in chain(root_reqs, discovered_reqs): try: discovered_reqs.extend( self._resolve_one(requirement_set, req) ) except HashError as exc: exc.req = req hash_errors.append(exc) if hash_errors: raise hash_errors
def test_adjacent_directory_permission_error(self, monkeypatch): name = "ABC" def raising_mkdir(*args, **kwargs): raise OSError("Unknown OSError") with TempDirectory() as tmp_dir: original = os.path.join(tmp_dir.path, name) ensure_dir(original) monkeypatch.setattr("os.mkdir", raising_mkdir) with pytest.raises(OSError): with AdjacentTempDirectory(original): pass
def tmpfile(self, contents): # Create a temporary file fd, path = tempfile.mkstemp( prefix="pip_", suffix="_config.ini", text=True ) os.close(fd) contents = textwrap.dedent(contents).lstrip() ensure_dir(os.path.dirname(path)) with open(path, "w") as f: f.write(contents) yield path os.remove(path)
def prepare_pep517_metadata(self): # type: () -> None assert self.pep517_backend is not None metadata_dir = os.path.join( self.setup_py_dir, 'pip-wheel-metadata' ) ensure_dir(metadata_dir) with self.build_env: # Note that Pep517HookCaller implements a fallback for # prepare_metadata_for_build_wheel, so we don't have to # consider the possibility that this hook doesn't exist. backend = self.pep517_backend self.spin_message = "Preparing wheel metadata" distinfo_dir = backend.prepare_metadata_for_build_wheel( metadata_dir ) self.metadata_directory = os.path.join(metadata_dir, distinfo_dir)
def build(self, requirements, session, autobuilding=False): """Build wheels. :param unpack: If True, replace the sdist we built from with the newly built wheel, in preparation for installation. :return: True if all the wheels built correctly. """ from pip._internal import index building_is_possible = self._wheel_dir or ( autobuilding and self.wheel_cache.cache_dir ) assert building_is_possible buildset = [] for req in requirements: if req.constraint: continue if req.is_wheel: if not autobuilding: logger.info( 'Skipping %s, due to already being wheel.', req.name, ) elif autobuilding and req.editable: pass elif autobuilding and not req.source_dir: pass elif autobuilding and req.link and not req.link.is_artifact: # VCS checkout. Build wheel just for this run. buildset.append((req, True)) else: ephem_cache = False if autobuilding: link = req.link base, ext = link.splitext() if index.egg_info_matches(base, None, link) is None: # E.g. local directory. Build wheel just for this run. ephem_cache = True if "binary" not in index.fmt_ctl_formats( self.finder.format_control, canonicalize_name(req.name)): logger.info( "Skipping bdist_wheel for %s, due to binaries " "being disabled for it.", req.name, ) continue buildset.append((req, ephem_cache)) if not buildset: return True # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for (req, _) in buildset]), ) _cache = self.wheel_cache # shorter name with indent_log(): build_success, build_failure = [], [] for req, ephem in buildset: python_tag = None if autobuilding: python_tag = pep425tags.implementation_tag if ephem: output_dir = _cache.get_ephem_path_for_link(req.link) else: output_dir = _cache.get_path_for_link(req.link) try: ensure_dir(output_dir) except OSError as e: logger.warning("Building wheel for %s failed: %s", req.name, e) build_failure.append(req) continue else: output_dir = self._wheel_dir wheel_file = self._build_one( req, output_dir, python_tag=python_tag, ) if wheel_file: build_success.append(req) if autobuilding: # XXX: This is mildly duplicative with prepare_files, # but not close enough to pull out to a single common # method. # The code below assumes temporary source dirs - # prevent it doing bad things. if req.source_dir and not os.path.exists(os.path.join( req.source_dir, PIP_DELETE_MARKER_FILENAME)): raise AssertionError( "bad source dir - missing marker") # Delete the source we built the wheel from req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. req.source_dir = req.build_location( self.preparer.build_dir ) # Update the link for this. req.link = index.Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_url( req.link, req.source_dir, None, False, session=session, ) else: build_failure.append(req) # notify success/failure if build_success: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_success]), ) if build_failure: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) # Return True if all builds were successful return len(build_failure) == 0
def run(self, options, args): # type: (Values, List[str]) -> int options.ignore_installed = True # editable doesn't really make sense for `pip download`, but the bowels # of the RequirementSet code require that property. options.editables = [] cmdoptions.check_dist_restriction(options) options.download_dir = normalize_path(options.download_dir) <<<<<<< HEAD ======= >>>>>>> 74c061954d5e927be4caafbd793e96a50563c265 ensure_dir(options.download_dir) session = self.get_default_session(options) target_python = make_target_python(options) finder = self._build_package_finder( options=options, session=session, target_python=target_python, ) <<<<<<< HEAD ======= build_delete = (not (options.no_clean or options.build_dir)) >>>>>>> 74c061954d5e927be4caafbd793e96a50563c265 req_tracker = self.enter_context(get_requirement_tracker())
def _open(self) -> TextIOWrapper: ensure_dir(os.path.dirname(self.baseFilename)) return super()._open()
def clobber(source, dest, is_base, fixer=None, filter=None): ensure_dir(dest) # common for the 'include' path for dir, subdirs, files in os.walk(source): basedir = dir[len(source):].lstrip(os.path.sep) destdir = os.path.join(dest, basedir) if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'): continue for s in subdirs: destsubdir = os.path.join(dest, basedir, s) if is_base and basedir == '' and destsubdir.endswith('.data'): data_dirs.append(s) continue elif (is_base and s.endswith('.dist-info') and canonicalize_name(s).startswith( canonicalize_name(req.name))): assert not info_dir, ('Multiple .dist-info directories: ' + destsubdir + ', ' + ', '.join(info_dir)) info_dir.append(destsubdir) for f in files: # Skip unwanted files if filter and filter(f): continue srcfile = os.path.join(dir, f) destfile = os.path.join(dest, basedir, f) # directory creation is lazy and after the file filtering above # to ensure we don't install empty dirs; empty dirs can't be # uninstalled. ensure_dir(destdir) # copyfile (called below) truncates the destination if it # exists and then writes the new contents. This is fine in most # cases, but can cause a segfault if pip has loaded a shared # object (e.g. from pyopenssl through its vendored urllib3) # Since the shared object is mmap'd an attempt to call a # symbol in it will then cause a segfault. Unlinking the file # allows writing of new contents while allowing the process to # continue to use the old copy. if os.path.exists(destfile): os.unlink(destfile) # We use copyfile (not move, copy, or copy2) to be extra sure # that we are not moving directories over (copyfile fails for # directories) as well as to ensure that we are not copying # over any metadata because we want more control over what # metadata we actually copy over. shutil.copyfile(srcfile, destfile) # Copy over the metadata for the file, currently this only # includes the atime and mtime. st = os.stat(srcfile) if hasattr(os, "utime"): os.utime(destfile, (st.st_atime, st.st_mtime)) # If our file is executable, then make our destination file # executable. if os.access(srcfile, os.X_OK): st = os.stat(srcfile) permissions = ( st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH ) os.chmod(destfile, permissions) changed = False if fixer: changed = fixer(destfile) record_installed(srcfile, destfile, changed)
def build( self, requirements, # type: Iterable[InstallRequirement] session, # type: PipSession autobuilding=False # type: bool ): # type: (...) -> List[InstallRequirement] """Build wheels. :param unpack: If True, replace the sdist we built from with the newly built wheel, in preparation for installation. :return: True if all the wheels built correctly. """ buildset = [] format_control = self.finder.format_control # Whether a cache directory is available for autobuilding=True. cache_available = bool(self._wheel_dir or self.wheel_cache.cache_dir) for req in requirements: ephem_cache = should_use_ephemeral_cache( req, format_control=format_control, autobuilding=autobuilding, cache_available=cache_available, ) if ephem_cache is None: continue buildset.append((req, ephem_cache)) if not buildset: return [] # Is any wheel build not using the ephemeral cache? if any(not ephem_cache for _, ephem_cache in buildset): have_directory_for_build = self._wheel_dir or ( autobuilding and self.wheel_cache.cache_dir ) assert have_directory_for_build # TODO by @pradyunsg # Should break up this method into 2 separate methods. # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for (req, _) in buildset]), ) _cache = self.wheel_cache # shorter name with indent_log(): build_success, build_failure = [], [] for req, ephem in buildset: python_tag = None if autobuilding: python_tag = pep425tags.implementation_tag if ephem: output_dir = _cache.get_ephem_path_for_link(req.link) else: output_dir = _cache.get_path_for_link(req.link) try: ensure_dir(output_dir) except OSError as e: logger.warning("Building wheel for %s failed: %s", req.name, e) build_failure.append(req) continue else: output_dir = self._wheel_dir wheel_file = self._build_one( req, output_dir, python_tag=python_tag, ) if wheel_file: build_success.append(req) if autobuilding: # XXX: This is mildly duplicative with prepare_files, # but not close enough to pull out to a single common # method. # The code below assumes temporary source dirs - # prevent it doing bad things. if req.source_dir and not os.path.exists(os.path.join( req.source_dir, PIP_DELETE_MARKER_FILENAME)): raise AssertionError( "bad source dir - missing marker") # Delete the source we built the wheel from req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. req.source_dir = req.build_location( self.preparer.build_dir ) # Update the link for this. req.link = Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_url( req.link, req.source_dir, None, False, session=session, ) else: build_failure.append(req) # notify success/failure if build_success: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_success]), ) if build_failure: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) # Return a list of requirements that failed to build return build_failure
def untar_file(filename, location): # type: (str, str) -> None """ Untar the file (with path `filename`) to the destination `location`. All files are written based on system defaults and umask (i.e. permissions are not preserved), except that regular file members with any execute permissions (users, group, or world) have "chmod +x" applied after being written. Note that for windows, any execute changes using os.chmod are no-ops per the python docs. """ ensure_dir(location) if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): mode = 'r:gz' elif filename.lower().endswith(BZ2_EXTENSIONS): mode = 'r:bz2' elif filename.lower().endswith(XZ_EXTENSIONS): mode = 'r:xz' elif filename.lower().endswith('.tar'): mode = 'r' else: logger.warning( 'Cannot determine compression type for file %s', filename, ) mode = 'r:*' tar = tarfile.open(filename, mode) try: leading = has_leading_dir([member.name for member in tar.getmembers()]) for member in tar.getmembers(): fn = member.name if leading: # https://github.com/python/mypy/issues/1174 fn = split_leading_dir(fn)[1] # type: ignore path = os.path.join(location, fn) if not is_within_directory(location, path): message = ( 'The tar file ({}) has a file ({}) trying to install ' 'outside target directory ({})') raise InstallationError( message.format(filename, path, location)) if member.isdir(): ensure_dir(path) elif member.issym(): try: # https://github.com/python/typeshed/issues/2673 tar._extract_member(member, path) # type: ignore except Exception as exc: # Some corrupt tar files seem to produce this # (specifically bad symlinks) logger.warning( 'In the tar file %s the member %s is invalid: %s', filename, member.name, exc, ) continue else: try: fp = tar.extractfile(member) except (KeyError, AttributeError) as exc: # Some corrupt tar files seem to produce this # (specifically bad symlinks) logger.warning( 'In the tar file %s the member %s is invalid: %s', filename, member.name, exc, ) continue ensure_dir(os.path.dirname(path)) assert fp is not None with open(path, 'wb') as destfp: shutil.copyfileobj(fp, destfp) fp.close() # Update the timestamp (useful for cython compiled files) # https://github.com/python/typeshed/issues/2673 tar.utime(member, path) # type: ignore # member have any execute permissions for users/group/world? if member.mode & 0o111: set_extracted_file_to_default_mode_plus_executable(path) finally: tar.close()
def install( self, install_options, # type: List[str] global_options=None, # type: Optional[Sequence[str]] root=None, # type: Optional[str] home=None, # type: Optional[str] prefix=None, # type: Optional[str] warn_script_location=True, # type: bool use_user_site=False, # type: bool pycompile=True # type: bool ): # type: (...) -> None scheme = get_scheme( self.name, user=use_user_site, home=home, root=root, isolated=self.isolated, prefix=prefix, ) global_options = global_options if global_options is not None else [] if self.editable: self.install_editable( install_options, global_options, prefix=prefix, home=home, use_user_site=use_user_site, ) return if self.is_wheel: self.move_wheel_files( self.source_dir, scheme=scheme, warn_script_location=warn_script_location, pycompile=pycompile, ) self.install_succeeded = True return # Extend the list of global and install options passed on to # the setup.py call with the ones from the requirements file. # Options specified in requirements file override those # specified on the command line, since the last option given # to setup.py is the one that is used. global_options = list(global_options) + \ self.options.get('global_options', []) install_options = list(install_options) + \ self.options.get('install_options', []) header_dir = scheme.headers with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') install_args = make_setuptools_install_args( self.setup_py_path, global_options=global_options, install_options=install_options, record_filename=record_filename, root=root, prefix=prefix, header_dir=header_dir, home=home, use_user_site=use_user_site, no_user_config=self.isolated, pycompile=pycompile, ) runner = runner_with_spinner_message( "Running setup.py install for {}".format(self.name) ) with indent_log(), self.build_env: runner( cmd=install_args, cwd=self.unpacked_source_directory, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True def prepend_root(path): # type: (str) -> str if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: deprecated( reason=( "{} did not indicate that it installed an " ".egg-info directory. Only setup.py projects " "generating .egg-info directories are supported." ).format(self), replacement=( "for maintainers: updating the setup.py of {0}. " "For users: contact the maintainers of {0} to let " "them know to update their setup.py.".format( self.name ) ), gone_in="20.2", issue=6998, ) # FIXME: put the record somewhere return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( os.path.relpath(prepend_root(filename), egg_info_dir) ) new_lines.sort() ensure_dir(egg_info_dir) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n')
def run(self, options, args): options.ignore_installed = True # editable doesn't really make sense for `pip download`, but the bowels # of the RequirementSet code require that property. options.editables = [] cmdoptions.check_dist_restriction(options) options.download_dir = normalize_path(options.download_dir) ensure_dir(options.download_dir) session = self.get_default_session(options) target_python = make_target_python(options) finder = self._build_package_finder( options=options, session=session, target_python=target_python, ) build_delete = (not (options.no_clean or options.build_dir)) with get_requirement_tracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="download" ) as directory: requirement_set = RequirementSet() self.populate_requirement_set( requirement_set, args, options, finder, session, None ) preparer = self.make_requirement_preparer( temp_build_dir=directory, options=options, req_tracker=req_tracker, session=session, finder=finder, download_dir=options.download_dir, use_user_site=False, ) resolver = self.make_resolver( preparer=preparer, finder=finder, options=options, py_version_info=options.python_version, ) self.trace_basic_info(finder) resolver.resolve(requirement_set) downloaded = ' '.join([ req.name for req in requirement_set.successfully_downloaded ]) if downloaded: write_output('Successfully downloaded %s', downloaded) # Clean up if not options.no_clean: requirement_set.cleanup_files() return requirement_set
def run(self, options, args): # type: (Values, List[str]) -> int options.ignore_installed = True # editable doesn't really make sense for `pip download`, but the bowels # of the RequirementSet code require that property. options.editables = [] cmdoptions.check_dist_restriction(options) options.download_dir = normalize_path(options.download_dir) ensure_dir(options.download_dir) session = self.get_default_session(options) target_python = make_target_python(options) finder = self._build_package_finder( options=options, session=session, target_python=target_python, ignore_requires_python=options.ignore_requires_python, ) req_tracker = self.enter_context(get_requirement_tracker()) directory = TempDirectory( delete=not options.no_clean, kind="download", globally_managed=True, ) reqs = self.get_requirements(args, options, finder, session) preparer = self.make_requirement_preparer( temp_build_dir=directory, options=options, req_tracker=req_tracker, session=session, finder=finder, download_dir=options.download_dir, use_user_site=False, ) resolver = self.make_resolver( preparer=preparer, finder=finder, options=options, ignore_requires_python=options.ignore_requires_python, py_version_info=options.python_version, ) self.trace_basic_info(finder) requirement_set = resolver.resolve(reqs, check_supported_wheels=True) downloaded = [] # type: List[str] for req in requirement_set.requirements.values(): if req.satisfied_by is None: assert req.name is not None preparer.save_linked_requirement(req) downloaded.append(req.name) if downloaded: write_output("Successfully downloaded %s", " ".join(downloaded)) return SUCCESS
def clobber( source, # type: str dest, # type: str is_base, # type: bool fixer=None, # type: Optional[Callable[[str], Any]] filter=None, # type: Optional[Callable[[str], bool]] ): # type: (...) -> None ensure_dir(dest) # common for the 'include' path for dir, subdirs, files in os.walk(source): basedir = dir[len(source):].lstrip(os.path.sep) destdir = os.path.join(dest, basedir) if is_base and basedir == "": subdirs[:] = [s for s in subdirs if not s.endswith(".data")] for f in files: # Skip unwanted files if filter and filter(f): continue srcfile = os.path.join(dir, f) destfile = os.path.join(dest, basedir, f) # directory creation is lazy and after the file filtering above # to ensure we don't install empty dirs; empty dirs can't be # uninstalled. ensure_dir(destdir) # copyfile (called below) truncates the destination if it # exists and then writes the new contents. This is fine in most # cases, but can cause a segfault if pip has loaded a shared # object (e.g. from pyopenssl through its vendored urllib3) # Since the shared object is mmap'd an attempt to call a # symbol in it will then cause a segfault. Unlinking the file # allows writing of new contents while allowing the process to # continue to use the old copy. if os.path.exists(destfile): os.unlink(destfile) # We use copyfile (not move, copy, or copy2) to be extra sure # that we are not moving directories over (copyfile fails for # directories) as well as to ensure that we are not copying # over any metadata because we want more control over what # metadata we actually copy over. shutil.copyfile(srcfile, destfile) # Copy over the metadata for the file, currently this only # includes the atime and mtime. st = os.stat(srcfile) if hasattr(os, "utime"): os.utime(destfile, (st.st_atime, st.st_mtime)) # If our file is executable, then make our destination file # executable. if os.access(srcfile, os.X_OK): st = os.stat(srcfile) permissions = st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH os.chmod(destfile, permissions) changed = False if fixer: changed = fixer(destfile) record_installed(srcfile, destfile, changed)
def run(self, options, args): options.ignore_installed = True # editable doesn't really make sense for `pip download`, but the bowels # of the RequirementSet code require that property. options.editables = [] cmdoptions.check_dist_restriction(options) options.download_dir = normalize_path(options.download_dir) ensure_dir(options.download_dir) session = self.get_default_session(options) target_python = make_target_python(options) finder = self._build_package_finder( options=options, session=session, target_python=target_python, ) build_delete = (not (options.no_clean or options.build_dir)) req_tracker = self.enter_context(get_requirement_tracker()) directory = TempDirectory( options.build_dir, delete=build_delete, kind="download", globally_managed=True, ) reqs = self.get_requirements(args, options, finder, session, None) preparer = self.make_requirement_preparer( temp_build_dir=directory, options=options, req_tracker=req_tracker, session=session, finder=finder, download_dir=options.download_dir, use_user_site=False, ) resolver = self.make_resolver( preparer=preparer, finder=finder, options=options, py_version_info=options.python_version, ) self.trace_basic_info(finder) requirement_set = resolver.resolve(reqs, check_supported_wheels=True) downloaded = ' '.join([ req.name for req in requirement_set.requirements.values() if req.successfully_downloaded ]) if downloaded: write_output('Successfully downloaded %s', downloaded) return requirement_set
def run(self, options, args): options.ignore_installed = True # editable doesn't really make sense for `pip download`, but the bowels # of the RequirementSet code require that property. options.editables = [] if options.python_version: python_versions = [options.python_version] else: python_versions = None dist_restriction_set = any([ options.python_version, options.platform, options.abi, options.implementation, ]) binary_only = FormatControl(set(), {':all:'}) no_sdist_dependencies = ( options.format_control != binary_only and not options.ignore_dependencies ) if dist_restriction_set and no_sdist_dependencies: raise CommandError( "When restricting platform and interpreter constraints using " "--python-version, --platform, --abi, or --implementation, " "either --no-deps must be set, or --only-binary=:all: must be " "set and --no-binary must not be set (or must be set to " ":none:)." ) options.src_dir = os.path.abspath(options.src_dir) options.download_dir = normalize_path(options.download_dir) ensure_dir(options.download_dir) with self._build_session(options) as session: finder = self._build_package_finder( options=options, session=session, platform=options.platform, python_versions=python_versions, abi=options.abi, implementation=options.implementation, ) build_delete = (not (options.no_clean or options.build_dir)) if options.cache_dir and not check_path_owner(options.cache_dir): logger.warning( "The directory '%s' or its parent directory is not owned " "by the current user and caching wheels has been " "disabled. check the permissions and owner of that " "directory. If executing pip with sudo, you may want " "sudo's -H flag.", options.cache_dir, ) options.cache_dir = None with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="download" ) as directory: requirement_set = RequirementSet( require_hashes=options.require_hashes, ) self.populate_requirement_set( requirement_set, args, options, finder, session, self.name, None ) preparer = RequirementPreparer( build_dir=directory.path, src_dir=options.src_dir, download_dir=options.download_dir, wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, req_tracker=req_tracker, ) resolver = Resolver( preparer=preparer, finder=finder, session=session, wheel_cache=None, use_user_site=False, upgrade_strategy="to-satisfy-only", force_reinstall=False, ignore_dependencies=options.ignore_dependencies, ignore_requires_python=False, ignore_installed=True, isolated=options.isolated_mode, ) resolver.resolve(requirement_set) downloaded = ' '.join([ req.name for req in requirement_set.successfully_downloaded ]) if downloaded: logger.info('Successfully downloaded %s', downloaded) # Clean up if not options.no_clean: requirement_set.cleanup_files() return requirement_set
def unzip_file(filename, location, flatten=True): # type: (str, str, bool) -> None """ Unzip the file (with path `filename`) to the destination `location`. All files are written based on system defaults and umask (i.e. permissions are not preserved), except that regular file members with any execute permissions (user, group, or world) have "chmod +x" applied after being written. Note that for windows, any execute changes using os.chmod are no-ops per the python docs. """ ensure_dir(location) zipfp = open(filename, 'rb') try: zip = zipfile.ZipFile(zipfp, allowZip64=True) leading = has_leading_dir(zip.namelist()) and flatten for info in zip.infolist(): name = info.filename fn = name if leading: fn = split_leading_dir(name)[1] fn = os.path.join(location, fn) dir = os.path.dirname(fn) if not is_within_directory(location, fn): message = ( 'The zip file ({}) has a file ({}) trying to install ' 'outside target directory ({})') raise InstallationError(message.format(filename, fn, location)) if fn.endswith('/') or fn.endswith('\\'): # A directory ensure_dir(fn) else: ensure_dir(dir) # Don't use read() to avoid allocating an arbitrarily large # chunk of memory for the file's content fp = zip.open(name) try: with open(fn, 'wb') as destfp: shutil.copyfileobj(fp, destfp) finally: fp.close() if zip_item_is_executable(info): set_extracted_file_to_default_mode_plus_executable(fn) finally: zipfp.close()
def _open(self): # type: () -> IO[Any] ensure_dir(os.path.dirname(self.baseFilename)) return super()._open()
def install( self, install_options, # type: List[str] global_options=None, # type: Optional[Sequence[str]] root=None, # type: Optional[str] home=None, # type: Optional[str] prefix=None, # type: Optional[str] warn_script_location=True, # type: bool use_user_site=False, # type: bool pycompile=True # type: bool ): # type: (...) -> None global_options = global_options if global_options is not None else [] if self.editable: self.install_editable( install_options, global_options, prefix=prefix, ) return if self.is_wheel: version = wheel.wheel_version(self.source_dir) wheel.check_compatibility(version, self.name) self.move_wheel_files( self.source_dir, root=root, prefix=prefix, home=home, warn_script_location=warn_script_location, use_user_site=use_user_site, pycompile=pycompile, ) self.install_succeeded = True return # Extend the list of global and install options passed on to # the setup.py call with the ones from the requirements file. # Options specified in requirements file override those # specified on the command line, since the last option given # to setup.py is the one that is used. global_options = list(global_options) + \ self.options.get('global_options', []) install_options = list(install_options) + \ self.options.get('install_options', []) if self.isolated: # https://github.com/python/mypy/issues/1174 global_options = global_options + ["--no-user-cfg"] # type: ignore with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') install_args = self.get_install_args( global_options, record_filename, root, prefix, pycompile, ) msg = 'Running setup.py install for %s' % (self.name, ) with open_spinner(msg) as spinner: with indent_log(): with self.build_env: call_subprocess( install_args + install_options, cwd=self.setup_py_dir, show_stdout=False, spinner=spinner, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True def prepend_root(path): if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: logger.warning( 'Could not find .egg-info directory in install record' ' for %s', self, ) # FIXME: put the record somewhere # FIXME: should this be an error? return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( os.path.relpath(prepend_root(filename), egg_info_dir)) new_lines.sort() ensure_dir(egg_info_dir) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n')
@with_cleanup def run(self, options, args): # type: (Values, List[str]) -> int cmdoptions.check_install_build_global(options) session = self.get_default_session(options) finder = self._build_package_finder(options, session) <<<<<<< HEAD ======= build_delete = (not (options.no_clean or options.build_dir)) >>>>>>> 74c061954d5e927be4caafbd793e96a50563c265 wheel_cache = WheelCache(options.cache_dir, options.format_control) options.wheel_dir = normalize_path(options.wheel_dir) ensure_dir(options.wheel_dir) req_tracker = self.enter_context(get_requirement_tracker()) directory = TempDirectory( <<<<<<< HEAD delete=not options.no_clean, ======= options.build_dir, delete=build_delete, >>>>>>> 74c061954d5e927be4caafbd793e96a50563c265 kind="wheel", globally_managed=True, ) reqs = self.get_requirements(args, options, finder, session)
def run(self, options, args): options.ignore_installed = True # editable doesn't really make sense for `pip download`, but the bowels # of the RequirementSet code require that property. options.editables = [] cmdoptions.check_dist_restriction(options) options.src_dir = os.path.abspath(options.src_dir) options.download_dir = normalize_path(options.download_dir) ensure_dir(options.download_dir) session = self.get_default_session(options) target_python = make_target_python(options) finder = self._build_package_finder( options=options, session=session, target_python=target_python, ) build_delete = (not (options.no_clean or options.build_dir)) if options.cache_dir and not check_path_owner(options.cache_dir): logger.warning( "The directory '%s' or its parent directory is not owned " "by the current user and caching wheels has been " "disabled. check the permissions and owner of that " "directory. If executing pip with sudo, you may want " "sudo's -H flag.", options.cache_dir, ) options.cache_dir = None with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="download") as directory: requirement_set = RequirementSet() self.populate_requirement_set(requirement_set, args, options, finder, session, None) preparer = self.make_requirement_preparer( temp_build_dir=directory, options=options, req_tracker=req_tracker, download_dir=options.download_dir, ) resolver = self.make_resolver( preparer=preparer, finder=finder, session=session, options=options, py_version_info=options.python_version, ) resolver.resolve(requirement_set) downloaded = ' '.join( [req.name for req in requirement_set.successfully_downloaded]) if downloaded: write_output('Successfully downloaded %s', downloaded) # Clean up if not options.no_clean: requirement_set.cleanup_files() return requirement_set
def run(self, options, args): options.ignore_installed = True # editable doesn't really make sense for `pip download`, but the bowels # of the RequirementSet code require that property. options.editables = [] if options.python_version: python_versions = [options.python_version] else: python_versions = None dist_restriction_set = any([ options.python_version, options.platform, options.abi, options.implementation, ]) binary_only = FormatControl(set(), {':all:'}) no_sdist_dependencies = (options.format_control != binary_only and not options.ignore_dependencies) if dist_restriction_set and no_sdist_dependencies: raise CommandError( "When restricting platform and interpreter constraints using " "--python-version, --platform, --abi, or --implementation, " "either --no-deps must be set, or --only-binary=:all: must be " "set and --no-binary must not be set (or must be set to " ":none:).") options.src_dir = os.path.abspath(options.src_dir) options.download_dir = normalize_path(options.download_dir) ensure_dir(options.download_dir) with self._build_session(options) as session: finder = self._build_package_finder( options=options, session=session, platform=options.platform, python_versions=python_versions, abi=options.abi, implementation=options.implementation, ) build_delete = (not (options.no_clean or options.build_dir)) if options.cache_dir and not check_path_owner(options.cache_dir): logger.warning( "The directory '%s' or its parent directory is not owned " "by the current user and caching wheels has been " "disabled. check the permissions and owner of that " "directory. If executing pip with sudo, you may want " "sudo's -H flag.", options.cache_dir, ) options.cache_dir = None with TempDirectory(options.build_dir, delete=build_delete, kind="download") as directory: requirement_set = RequirementSet( require_hashes=options.require_hashes, ) self.populate_requirement_set(requirement_set, args, options, finder, session, self.name, None) preparer = RequirementPreparer( build_dir=directory.path, src_dir=options.src_dir, download_dir=options.download_dir, wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, ) resolver = Resolver( preparer=preparer, finder=finder, session=session, wheel_cache=None, use_user_site=False, upgrade_strategy="to-satisfy-only", force_reinstall=False, ignore_dependencies=options.ignore_dependencies, ignore_requires_python=False, ignore_installed=True, isolated=options.isolated_mode, ) resolver.resolve(requirement_set) downloaded = SPACE.join([ req.name for req in requirement_set.successfully_downloaded ]) if downloaded: logger.info('Successfully downloaded %s', downloaded) # Clean up if not options.no_clean: requirement_set.cleanup_files() return requirement_set
def create_file(*args): fpath = os.path.join(*args) ensure_dir(os.path.dirname(fpath)) with open(fpath, "w") as f: f.write("Holla!")
def install( self, install_options, # type: List[str] global_options=None, # type: Optional[Sequence[str]] root=None, # type: Optional[str] home=None, # type: Optional[str] prefix=None, # type: Optional[str] warn_script_location=True, # type: bool use_user_site=False, # type: bool pycompile=True # type: bool ): # type: (...) -> None global_options = global_options if global_options is not None else [] if self.editable: self.install_editable( install_options, global_options, prefix=prefix, ) return if self.is_wheel: version = wheel.wheel_version(self.source_dir) wheel.check_compatibility(version, self.name) scheme = distutils_scheme( self.name, user=use_user_site, home=home, root=root, isolated=self.isolated, prefix=prefix, ) self.move_wheel_files( self.source_dir, scheme=scheme, warn_script_location=warn_script_location, pycompile=pycompile, ) self.install_succeeded = True return # Extend the list of global and install options passed on to # the setup.py call with the ones from the requirements file. # Options specified in requirements file override those # specified on the command line, since the last option given # to setup.py is the one that is used. global_options = list(global_options) + \ self.options.get('global_options', []) install_options = list(install_options) + \ self.options.get('install_options', []) header_dir = None # type: Optional[str] if running_under_virtualenv(): py_ver_str = 'python' + sysconfig.get_python_version() header_dir = os.path.join(sys.prefix, 'include', 'site', py_ver_str, self.name) with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') install_args = make_setuptools_install_args( self.setup_py_path, global_options=global_options, install_options=install_options, record_filename=record_filename, root=root, prefix=prefix, header_dir=header_dir, no_user_config=self.isolated, pycompile=pycompile, ) runner = runner_with_spinner_message( "Running setup.py install for {}".format(self.name)) with indent_log(), self.build_env: runner( cmd=install_args, cwd=self.unpacked_source_directory, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True def prepend_root(path): # type: (str) -> str if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: logger.warning( 'Could not find .egg-info directory in install record' ' for %s', self, ) # FIXME: put the record somewhere return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( os.path.relpath(prepend_root(filename), egg_info_dir)) new_lines.sort() ensure_dir(egg_info_dir) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n')
def run(self, options, args): # type: (Values, List[Any]) -> None cmdoptions.check_install_build_global(options) session = self.get_default_session(options) finder = self._build_package_finder(options, session) build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) options.wheel_dir = normalize_path(options.wheel_dir) ensure_dir(options.wheel_dir) req_tracker = self.enter_context(get_requirement_tracker()) directory = TempDirectory( options.build_dir, delete=build_delete, kind="wheel", globally_managed=True, ) reqs = self.get_requirements(args, options, finder, session) preparer = self.make_requirement_preparer( temp_build_dir=directory, options=options, req_tracker=req_tracker, session=session, finder=finder, wheel_download_dir=options.wheel_dir, use_user_site=False, ) resolver = self.make_resolver( preparer=preparer, finder=finder, options=options, wheel_cache=wheel_cache, ignore_requires_python=options.ignore_requires_python, use_pep517=options.use_pep517, ) self.trace_basic_info(finder) requirement_set = resolver.resolve( reqs, check_supported_wheels=True ) reqs_to_build = [ r for r in requirement_set.requirements.values() if should_build_for_wheel_command(r) ] # build wheels build_successes, build_failures = build( reqs_to_build, wheel_cache=wheel_cache, build_options=options.build_options or [], global_options=options.global_options or [], ) for req in build_successes: assert req.link and req.link.is_wheel assert req.local_file_path # copy from cache to target directory try: shutil.copy(req.local_file_path, options.wheel_dir) except OSError as e: logger.warning( "Building wheel for %s failed: %s", req.name, e, ) build_failures.append(req) if len(build_failures) != 0: raise CommandError( "Failed to build one or more wheels" )
def install( install_req, # type: InstallRequirement install_options, # type: List[str] global_options, # type: Sequence[str] root, # type: Optional[str] home, # type: Optional[str] prefix, # type: Optional[str] use_user_site, # type: bool pycompile, # type: bool scheme, # type: Scheme ): # type: (...) -> None # Extend the list of global and install options passed on to # the setup.py call with the ones from the requirements file. # Options specified in requirements file override those # specified on the command line, since the last option given # to setup.py is the one that is used. global_options = list(global_options) + install_req.global_options install_options = list(install_options) + install_req.install_options header_dir = scheme.headers with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') install_args = make_setuptools_install_args( install_req.setup_py_path, global_options=global_options, install_options=install_options, record_filename=record_filename, root=root, prefix=prefix, header_dir=header_dir, home=home, use_user_site=use_user_site, no_user_config=install_req.isolated, pycompile=pycompile, ) runner = runner_with_spinner_message( "Running setup.py install for {}".format(install_req.name) ) with indent_log(), install_req.build_env: runner( cmd=install_args, cwd=install_req.unpacked_source_directory, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return install_req.install_succeeded = True # We intentionally do not use any encoding to read the file because # setuptools writes the file using distutils.file_util.write_file, # which does not specify an encoding. with open(record_filename) as f: record_lines = f.read().splitlines() def prepend_root(path): # type: (str) -> str if root is None or not os.path.isabs(path): return path else: return change_root(root, path) for line in record_lines: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: deprecated( reason=( "{} did not indicate that it installed an " ".egg-info directory. Only setup.py projects " "generating .egg-info directories are supported." ).format(install_req), replacement=( "for maintainers: updating the setup.py of {0}. " "For users: contact the maintainers of {0} to let " "them know to update their setup.py.".format( install_req.name ) ), gone_in="20.2", issue=6998, ) # FIXME: put the record somewhere return new_lines = [] for line in record_lines: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( os.path.relpath(prepend_root(filename), egg_info_dir) ) new_lines.sort() ensure_dir(egg_info_dir) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n')
def run(self, options, args): # type: (Values, List[Any]) -> None cmdoptions.check_install_build_global(options) session = self.get_default_session(options) finder = self._build_package_finder(options, session) build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) options.wheel_dir = normalize_path(options.wheel_dir) ensure_dir(options.wheel_dir) with get_requirement_tracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="wheel") as directory: requirement_set = RequirementSet() try: self.populate_requirement_set(requirement_set, args, options, finder, session, wheel_cache) preparer = self.make_requirement_preparer( temp_build_dir=directory, options=options, req_tracker=req_tracker, session=session, finder=finder, wheel_download_dir=options.wheel_dir, use_user_site=False, ) resolver = self.make_resolver( preparer=preparer, finder=finder, options=options, wheel_cache=wheel_cache, ignore_requires_python=options.ignore_requires_python, use_pep517=options.use_pep517, ) self.trace_basic_info(finder) resolver.resolve(requirement_set) # build wheels wb = WheelBuilder(preparer) build_successes, build_failures = wb.build( requirement_set.requirements.values(), should_unpack=False, wheel_cache=wheel_cache, build_options=options.build_options or [], global_options=options.global_options or [], ) for req in build_successes: assert req.link and req.link.is_wheel assert req.local_file_path # copy from cache to target directory try: shutil.copy(req.local_file_path, options.wheel_dir) except OSError as e: logger.warning( "Building wheel for %s failed: %s", req.name, e, ) build_failures.append(req) if len(build_failures) != 0: raise CommandError("Failed to build one or more wheels") except PreviousBuildDirError: options.no_clean = True raise finally: if not options.no_clean: requirement_set.cleanup_files() wheel_cache.cleanup()
def _handle_target_dir(self, target_dir, target_temp_dir, upgrade): # type: (str, TempDirectory, bool) -> None ensure_dir(target_dir) # Checking both purelib and platlib directories for installed # packages to be moved to target directory lib_dir_list = [] # Checking both purelib and platlib directories for installed # packages to be moved to target directory scheme = distutils_scheme('', home=target_temp_dir.path) purelib_dir = scheme['purelib'] platlib_dir = scheme['platlib'] data_dir = scheme['data'] if os.path.exists(purelib_dir): lib_dir_list.append(purelib_dir) if os.path.exists(platlib_dir) and platlib_dir != purelib_dir: lib_dir_list.append(platlib_dir) if os.path.exists(data_dir): lib_dir_list.append(data_dir) for lib_dir in lib_dir_list: for item in os.listdir(lib_dir): if lib_dir == data_dir: ddir = os.path.join(data_dir, item) if any(s.startswith(ddir) for s in lib_dir_list[:-1]): continue target_item_dir = os.path.join(target_dir, item) if os.path.exists(target_item_dir): if not upgrade: logger.warning( 'Target directory %s already exists. Specify ' '--upgrade to force replacement.', target_item_dir) continue if os.path.islink(target_item_dir): logger.warning( 'Target directory %s already exists and is ' 'a link. pip will not automatically replace ' 'links, please remove if replacement is ' 'desired.', target_item_dir) continue if os.path.isdir(target_item_dir): shutil.rmtree(target_item_dir) else: os.remove(target_item_dir) shutil.move(os.path.join(lib_dir, item), target_item_dir)
def install( self, install_options, # type: List[str] global_options=None, # type: Optional[Sequence[str]] root=None, # type: Optional[str] home=None, # type: Optional[str] prefix=None, # type: Optional[str] warn_script_location=True, # type: bool use_user_site=False, # type: bool pycompile=True # type: bool ): # type: (...) -> None global_options = global_options if global_options is not None else [] if self.editable: self.install_editable( install_options, global_options, prefix=prefix, ) return if self.is_wheel: version = wheel.wheel_version(self.source_dir) wheel.check_compatibility(version, self.name) self.move_wheel_files( self.source_dir, root=root, prefix=prefix, home=home, warn_script_location=warn_script_location, use_user_site=use_user_site, pycompile=pycompile, ) self.install_succeeded = True return # Extend the list of global and install options passed on to # the setup.py call with the ones from the requirements file. # Options specified in requirements file override those # specified on the command line, since the last option given # to setup.py is the one that is used. global_options = list(global_options) + \ self.options.get('global_options', []) install_options = list(install_options) + \ self.options.get('install_options', []) if self.isolated: # https://github.com/python/mypy/issues/1174 global_options = global_options + ["--no-user-cfg"] # type: ignore with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') install_args = self.get_install_args( global_options, record_filename, root, prefix, pycompile, ) msg = 'Running setup.py install for %s' % (self.name,) with open_spinner(msg) as spinner: with indent_log(): with self.build_env: call_subprocess( install_args + install_options, cwd=self.setup_py_dir, spinner=spinner, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True def prepend_root(path): if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: logger.warning( 'Could not find .egg-info directory in install record' ' for %s', self, ) # FIXME: put the record somewhere # FIXME: should this be an error? return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( os.path.relpath(prepend_root(filename), egg_info_dir) ) new_lines.sort() ensure_dir(egg_info_dir) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n')
def build(self, requirements, session, autobuilding=False): """Build wheels. :param unpack: If True, replace the sdist we built from with the newly built wheel, in preparation for installation. :return: True if all the wheels built correctly. """ from pip._internal import index building_is_possible = self._wheel_dir or (autobuilding and self.wheel_cache.cache_dir) assert building_is_possible buildset = [] for req in requirements: if req.constraint: continue if req.is_wheel: if not autobuilding: logger.info( 'Skipping %s, due to already being wheel.', req.name, ) elif autobuilding and req.editable: pass elif autobuilding and not req.source_dir: pass elif autobuilding and req.link and not req.link.is_artifact: # VCS checkout. Build wheel just for this run. buildset.append((req, True)) else: ephem_cache = False if autobuilding: link = req.link base, ext = link.splitext() if index.egg_info_matches(base, None, link) is None: # E.g. local directory. Build wheel just for this run. ephem_cache = True if "binary" not in index.fmt_ctl_formats( self.finder.format_control, canonicalize_name(req.name)): logger.info( "Skipping bdist_wheel for %s, due to binaries " "being disabled for it.", req.name, ) continue buildset.append((req, ephem_cache)) if not buildset: return True # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for (req, _) in buildset]), ) _cache = self.wheel_cache # shorter name with indent_log(): build_success, build_failure = [], [] for req, ephem in buildset: python_tag = None if autobuilding: python_tag = pep425tags.implementation_tag if ephem: output_dir = _cache.get_ephem_path_for_link(req.link) else: output_dir = _cache.get_path_for_link(req.link) try: ensure_dir(output_dir) except OSError as e: logger.warning("Building wheel for %s failed: %s", req.name, e) build_failure.append(req) continue else: output_dir = self._wheel_dir wheel_file = self._build_one( req, output_dir, python_tag=python_tag, ) if wheel_file: build_success.append(req) if autobuilding: # XXX: This is mildly duplicative with prepare_files, # but not close enough to pull out to a single common # method. # The code below assumes temporary source dirs - # prevent it doing bad things. if req.source_dir and not os.path.exists( os.path.join(req.source_dir, PIP_DELETE_MARKER_FILENAME)): raise AssertionError( "bad source dir - missing marker") # Delete the source we built the wheel from req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. req.source_dir = req.build_location( self.preparer.build_dir) # Update the link for this. req.link = index.Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_url( req.link, req.source_dir, None, False, session=session, ) else: build_failure.append(req) # notify success/failure if build_success: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_success]), ) if build_failure: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) # Return True if all builds were successful return len(build_failure) == 0
Returns the generated metadata directory. """ logger.debug( 'Running setup.py (path:%s) egg_info for package %s', setup_py_path, details, ) egg_info_dir = None # type: Optional[str] # For non-editable installs, don't put the .egg-info files at the root, # to avoid confusion due to the source code being considered an installed # egg. if not editable: egg_info_dir = os.path.join(source_dir, 'pip-egg-info') # setuptools complains if the target directory does not exist. ensure_dir(egg_info_dir) args = make_setuptools_egg_info_args( setup_py_path, egg_info_dir=egg_info_dir, no_user_config=isolated, ) with build_env: call_subprocess( args, cwd=source_dir, command_desc='python setup.py egg_info', ) # Return the .egg-info directory.
def run(self, options, args): options.ignore_installed = True # editable doesn't really make sense for `pip download`, but the bowels # of the RequirementSet code require that property. options.editables = [] cmdoptions.check_dist_restriction(options) options.src_dir = os.path.abspath(options.src_dir) options.download_dir = normalize_path(options.download_dir) ensure_dir(options.download_dir) with self._build_session(options) as session: target_python = make_target_python(options) finder = self._build_package_finder( options=options, session=session, target_python=target_python, ) build_delete = (not (options.no_clean or options.build_dir)) if options.cache_dir and not check_path_owner(options.cache_dir): logger.warning( "The directory '%s' or its parent directory is not owned " "by the current user and caching wheels has been " "disabled. check the permissions and owner of that " "directory. If executing pip with sudo, you may want " "sudo's -H flag.", options.cache_dir, ) options.cache_dir = None with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="download" ) as directory: requirement_set = RequirementSet( require_hashes=options.require_hashes, ) self.populate_requirement_set( requirement_set, args, options, finder, session, self.name, None ) preparer = RequirementPreparer( build_dir=directory.path, src_dir=options.src_dir, download_dir=options.download_dir, wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, req_tracker=req_tracker, ) resolver = Resolver( preparer=preparer, finder=finder, session=session, wheel_cache=None, use_user_site=False, upgrade_strategy="to-satisfy-only", force_reinstall=False, ignore_dependencies=options.ignore_dependencies, py_version_info=options.python_version, ignore_requires_python=False, ignore_installed=True, isolated=options.isolated_mode, ) resolver.resolve(requirement_set) downloaded = ' '.join([ req.name for req in requirement_set.successfully_downloaded ]) if downloaded: logger.info('Successfully downloaded %s', downloaded) # Clean up if not options.no_clean: requirement_set.cleanup_files() return requirement_set
def _open(self): ensure_dir(os.path.dirname(self.baseFilename)) return logging.handlers.RotatingFileHandler._open(self)
def build( self, requirements, # type: Iterable[InstallRequirement] should_unpack=False # type: bool ): # type: (...) -> List[InstallRequirement] """Build wheels. :param should_unpack: If True, after building the wheel, unpack it and replace the sdist with the unpacked version in preparation for installation. :return: True if all the wheels built correctly. """ # pip install uses should_unpack=True. # pip install never provides a _wheel_dir. # pip wheel uses should_unpack=False. # pip wheel always provides a _wheel_dir (via the preparer). assert ( (should_unpack and not self._wheel_dir) or (not should_unpack and self._wheel_dir) ) buildset = [] cache_available = bool(self.wheel_cache.cache_dir) for req in requirements: ephem_cache = should_use_ephemeral_cache( req, should_unpack=should_unpack, cache_available=cache_available, check_binary_allowed=self.check_binary_allowed, ) if ephem_cache is None: continue # Determine where the wheel should go. if should_unpack: if ephem_cache: output_dir = self.wheel_cache.get_ephem_path_for_link( req.link ) else: output_dir = self.wheel_cache.get_path_for_link(req.link) else: output_dir = self._wheel_dir buildset.append((req, output_dir)) if not buildset: return [] # TODO by @pradyunsg # Should break up this method into 2 separate methods. # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for (req, _) in buildset]), ) python_tag = None if should_unpack: python_tag = pep425tags.implementation_tag with indent_log(): build_success, build_failure = [], [] for req, output_dir in buildset: try: ensure_dir(output_dir) except OSError as e: logger.warning( "Building wheel for %s failed: %s", req.name, e, ) build_failure.append(req) continue wheel_file = self._build_one( req, output_dir, python_tag=python_tag, ) if wheel_file: build_success.append(req) if should_unpack: # XXX: This is mildly duplicative with prepare_files, # but not close enough to pull out to a single common # method. # The code below assumes temporary source dirs - # prevent it doing bad things. if ( req.source_dir and not has_delete_marker_file(req.source_dir) ): raise AssertionError( "bad source dir - missing marker") # Delete the source we built the wheel from req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. req.source_dir = req.ensure_build_location( self.preparer.build_dir ) # Update the link for this. req.link = Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_file(req.link.file_path, req.source_dir) else: build_failure.append(req) # notify success/failure if build_success: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_success]), ) if build_failure: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) # Return a list of requirements that failed to build return build_failure
def clobber(source, dest, is_base, fixer=None, filter=None): ensure_dir(dest) # common for the 'include' path for dir, subdirs, files in os.walk(source): basedir = dir[len(source):].lstrip(os.path.sep) destdir = os.path.join(dest, basedir) if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'): continue for s in subdirs: destsubdir = os.path.join(dest, basedir, s) if is_base and basedir == '' and destsubdir.endswith('.data'): data_dirs.append(s) continue elif (is_base and s.endswith('.dist-info') and canonicalize_name(s).startswith( canonicalize_name(req.name))): assert not info_dir, ('Multiple .dist-info directories: ' + destsubdir + ', ' + ', '.join(info_dir)) info_dir.append(destsubdir) for f in files: # Skip unwanted files if filter and filter(f): continue srcfile = os.path.join(dir, f) destfile = os.path.join(dest, basedir, f) # directory creation is lazy and after the file filtering above # to ensure we don't install empty dirs; empty dirs can't be # uninstalled. ensure_dir(destdir) # Ensure that the destination file does not exist. This is # required so that we break a hard link instead of replacing # the contents in place # xref: https://github.com/conda/conda/issues/6861 if os.path.lexists(destfile): os.unlink(destfile) # We use copyfile (not move, copy, or copy2) to be extra sure # that we are not moving directories over (copyfile fails for # directories) as well as to ensure that we are not copying # over any metadata because we want more control over what # metadata we actually copy over. shutil.copyfile(srcfile, destfile) # Copy over the metadata for the file, currently this only # includes the atime and mtime. st = os.stat(srcfile) if hasattr(os, "utime"): os.utime(destfile, (st.st_atime, st.st_mtime)) # If our file is executable, then make our destination file # executable. if os.access(srcfile, os.X_OK): st = os.stat(srcfile) permissions = (st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) os.chmod(destfile, permissions) changed = False if fixer: changed = fixer(destfile) record_installed(srcfile, destfile, changed)
def install( install_options, # type: List[str] global_options, # type: Sequence[str] root, # type: Optional[str] home, # type: Optional[str] prefix, # type: Optional[str] use_user_site, # type: bool pycompile, # type: bool scheme, # type: Scheme setup_py_path, # type: str isolated, # type: bool req_name, # type: str build_env, # type: BuildEnvironment unpacked_source_directory, # type: str req_description, # type: str ): # type: (...) -> bool header_dir = scheme.headers with TempDirectory(kind="record") as temp_dir: try: record_filename = os.path.join(temp_dir.path, 'install-record.txt') install_args = make_setuptools_install_args( setup_py_path, global_options=global_options, install_options=install_options, record_filename=record_filename, root=root, prefix=prefix, header_dir=header_dir, home=home, use_user_site=use_user_site, no_user_config=isolated, pycompile=pycompile, ) runner = runner_with_spinner_message( "Running setup.py install for {}".format(req_name)) with indent_log(), build_env: runner( cmd=install_args, cwd=unpacked_source_directory, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) # Signal to the caller that we didn't install the new package return False except Exception: # Signal to the caller that we didn't install the new package raise LegacyInstallFailure # At this point, we have successfully installed the requirement. # We intentionally do not use any encoding to read the file because # setuptools writes the file using distutils.file_util.write_file, # which does not specify an encoding. with open(record_filename) as f: record_lines = f.read().splitlines() def prepend_root(path): # type: (str) -> str if root is None or not os.path.isabs(path): return path else: return change_root(root, path) for line in record_lines: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: deprecated( reason=("{} did not indicate that it installed an " ".egg-info directory. Only setup.py projects " "generating .egg-info directories are supported." ).format(req_description), replacement=( "for maintainers: updating the setup.py of {0}. " "For users: contact the maintainers of {0} to let " "them know to update their setup.py.".format(req_name)), gone_in="20.2", issue=6998, ) # FIXME: put the record somewhere return True new_lines = [] for line in record_lines: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append(os.path.relpath(prepend_root(filename), egg_info_dir)) new_lines.sort() ensure_dir(egg_info_dir) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n') return True
def run_egg_info(self): assert self.source_dir if self.name: logger.debug( "Running setup.py (path:%s) egg_info for package %s", self.setup_py, self.name, ) else: logger.debug( "Running setup.py (path:%s) egg_info for package from %s", self.setup_py, self.link, ) with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py base_cmd = [sys.executable, "-c", script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ["egg_info"] # We can't put the .egg-info files at the root, because then the # source code will be mistaken for an installed egg, causing # problems if self.editable: egg_base_option = [] else: egg_info_dir = os.path.join(self.setup_py_dir, "pip-egg-info") ensure_dir(egg_info_dir) egg_base_option = ["--egg-base", "pip-egg-info"] with self.build_env: call_subprocess( egg_info_cmd + egg_base_option, cwd=self.setup_py_dir, show_stdout=False, command_desc="python setup.py egg_info", ) if not self.req: if isinstance(parse_version(self.pkg_info()["Version"]), Version): op = "==" else: op = "===" self.req = Requirement("".join([ self.pkg_info()["Name"], op, self.pkg_info()["Version"], ])) self._correct_build_location() else: metadata_name = canonicalize_name(self.pkg_info()["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( "Running setup.py (path:%s) egg_info for package %s " "produced metadata for project name %s. Fix your " "#egg=%s fragments.", self.setup_py, self.name, metadata_name, self.name, ) self.req = Requirement(metadata_name)