def install_editable( self, install_options, # type: List[str] global_options=(), # type: Sequence[str] prefix=None # type: Optional[str] ): # type: (...) -> None logger.info('Running setup.py develop for %s', self.name) if prefix: prefix_param = ['--prefix={}'.format(prefix)] install_options = list(install_options) + prefix_param base_cmd = make_setuptools_shim_args( self.setup_py_path, global_options=global_options, no_user_config=self.isolated ) with indent_log(): with self.build_env: call_subprocess( base_cmd + ['develop', '--no-deps'] + list(install_options), cwd=self.unpacked_source_directory, ) self.install_succeeded = True
def prepare_editable_requirement(self, req, require_hashes, use_user_site, finder): """Prepare an editable requirement """ assert req.editable, "cannot prepare a non-editable req as editable" logger.info('Obtaining %s', req) with indent_log(): if require_hashes: raise InstallationError( 'The editable requirement %s cannot be installed when ' 'requiring hashes, because there is no single file to ' 'hash.' % req ) req.ensure_has_source_dir(self.src_dir) req.update_editable(not self._download_should_save) abstract_dist = make_abstract_dist(req) abstract_dist.prep_for_dist(finder, self.build_isolation) if self._download_should_save: req.archive(self.download_dir) req.check_if_exists(use_user_site) return abstract_dist
def print_env_var_values(self) -> None: """Get key-values pairs present as environment variables""" write_output("%s:", "env_var") with indent_log(): for key, value in sorted(self.configuration.get_environ_vars()): env_var = f"PIP_{key.upper()}" write_output("%s=%r", env_var, value)
def install_editable( install_options, # type: List[str] global_options, # type: Sequence[str] prefix, # type: Optional[str] home, # type: Optional[str] use_user_site, # type: bool name, # type: str setup_py_path, # type: str isolated, # type: bool build_env, # type: BuildEnvironment unpacked_source_directory, # type: str ): # type: (...) -> None """Install a package in editable mode. Most arguments are pass-through to setuptools. """ logger.info('Running setup.py develop for %s', name) args = make_setuptools_develop_args( setup_py_path, global_options=global_options, install_options=install_options, no_user_config=isolated, prefix=prefix, home=home, use_user_site=use_user_site, ) with indent_log(): with build_env: call_subprocess( args, cwd=unpacked_source_directory, )
def prepare_editable_requirement( self, req, # type: InstallRequirement ): # type: (...) -> AbstractDistribution """Prepare an editable requirement """ assert req.editable, "cannot prepare a non-editable req as editable" logger.info('Obtaining %s', req) with indent_log(): if self.require_hashes: raise InstallationError( 'The editable requirement {} cannot be installed when ' 'requiring hashes, because there is no single file to ' 'hash.'.format(req) ) req.ensure_has_source_dir(self.src_dir) req.update_editable(not self._download_should_save) abstract_dist = _get_prepared_distribution( req, self.req_tracker, self.finder, self.build_isolation, ) if self._download_should_save: req.archive(self.download_dir) req.check_if_exists(self.use_user_site) return abstract_dist
def remove(self, auto_confirm=False, verbose=False): """Remove paths in ``self.paths`` with confirmation (unless ``auto_confirm`` is True).""" if not self.paths: logger.info( "Can't uninstall '%s'. No files were found to uninstall.", self.dist.project_name, ) return dist_name_version = (self.dist.project_name + "-" + self.dist.version) logger.info('Uninstalling %s:', dist_name_version) with indent_log(): if auto_confirm or self._allowed_to_proceed(verbose): moved = self._moved_paths for_rename = compress_for_rename(self.paths) for path in sorted(compact(for_rename)): moved.stash(path) logger.debug('Removing file or directory %s', path) for pth in self.pth.values(): pth.remove() logger.info('Successfully uninstalled %s', dist_name_version)
def _copy_dist_from_dir(link_path, location): """Copy distribution files in `link_path` to `location`. Invoked when user requests to install a local directory. E.g.: pip install . pip install ~/dev/git-repos/python-prompt-toolkit """ # Note: This is currently VERY SLOW if you have a lot of data in the # directory, because it copies everything with `shutil.copytree`. # What it should really do is build an sdist and install that. # See https://github.com/pypa/pip/issues/2195 if os.path.isdir(location): rmtree(location) # build an sdist setup_py = 'setup.py' sdist_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable)] sdist_args.append('-c') sdist_args.append(SETUPTOOLS_SHIM % setup_py) sdist_args.append('sdist') sdist_args += ['--dist-dir', location] logger.info('Running setup.py sdist for %s', link_path) with indent_log(): call_subprocess(sdist_args, cwd=link_path, show_stdout=False) # unpack sdist into `location` sdist = os.path.join(location, os.listdir(location)[0]) logger.info('Unpacking sdist %s into %s', sdist, location) unpack_file(sdist, location, content_type=None, link=None)
def prepare_installed_requirement( self, req, # type: InstallRequirement skip_reason # type: str ): # type: (...) -> AbstractDistribution """Prepare an already-installed requirement """ assert req.satisfied_by, "req should have been satisfied but isn't" assert skip_reason is not None, ( "did not get skip reason skipped but req.satisfied_by " "is set to {}".format(req.satisfied_by) ) logger.info( 'Requirement %s: %s (%s)', skip_reason, req, req.satisfied_by.version ) with indent_log(): if self.require_hashes: logger.debug( 'Since it is already installed, we are trusting this ' 'package without checking its hash. To ensure a ' 'completely repeatable environment, install into an ' 'empty virtualenv.' ) abstract_dist = InstalledDistribution(req) return abstract_dist
def cleanup_files(self): # type: () -> None """Clean up files, remove builds.""" logger.debug('Cleaning up...') with indent_log(): for req in self.reqs_to_cleanup: req.remove_temporary_source()
def prepare_linked_requirement(self, req, parallel_builds=False): # type: (InstallRequirement, bool) -> Distribution """Prepare a requirement to be obtained from req.link.""" assert req.link link = req.link self._log_preparing_link(req) with indent_log(): # Check if the relevant file is already available # in the download directory file_path = None if self.download_dir is not None and link.is_wheel: hashes = self._get_linked_req_hashes(req) file_path = _check_download_dir(req.link, self.download_dir, hashes) if file_path is not None: # The file is already available, so mark it as downloaded self._downloaded[req.link.url] = file_path, None else: # The file is not available, attempt to fetch only metadata wheel_dist = self._fetch_metadata_using_lazy_wheel(link) if wheel_dist is not None: req.needs_more_preparation = True return wheel_dist # None of the optimizations worked, fully prepare the requirement return self._prepare_linked_requirement(req, parallel_builds)
def show_tags(options): # type: (Values) -> None tag_limit = 10 target_python = make_target_python(options) tags = target_python.get_tags() # Display the target options that were explicitly provided. formatted_target = target_python.format_given() suffix = '' if formatted_target: suffix = ' (target: {})'.format(formatted_target) msg = 'Compatible tags: {}{}'.format(len(tags), suffix) logger.info(msg) if options.verbose < 1 and len(tags) > tag_limit: tags_limited = True tags = tags[:tag_limit] else: tags_limited = False with indent_log(): for tag in tags: logger.info(str(tag)) if tags_limited: msg = ( '...\n' '[First {tag_limit} tags shown. Pass --verbose to show all.]' ).format(tag_limit=tag_limit) logger.info(msg)
def prepare_editable_requirement(self, req, require_hashes, use_user_site, finder): """Prepare an editable requirement """ assert req.editable, "cannot prepare a non-editable req as editable" logger.info('Obtaining %s', req) with indent_log(): if require_hashes: raise InstallationError( 'The editable requirement %s cannot be installed when ' 'requiring hashes, because there is no single file to ' 'hash.' % req) req.ensure_has_source_dir(self.src_dir) req.update_editable(not self._download_should_save) abstract_dist = make_abstract_dist(req) with self.req_tracker.track(req): abstract_dist.prep_for_dist(finder, self.build_isolation) if self._download_should_save: req.archive(self.download_dir) req.check_if_exists(use_user_site) return abstract_dist
def remove(self, auto_confirm: bool = False, verbose: bool = False) -> None: """Remove paths in ``self._paths`` with confirmation (unless ``auto_confirm`` is True).""" if not self._paths: logger.info( "Can't uninstall '%s'. No files were found to uninstall.", self._dist.raw_name, ) return dist_name_version = f"{self._dist.raw_name}-{self._dist.version}" logger.info("Uninstalling %s:", dist_name_version) with indent_log(): if auto_confirm or self._allowed_to_proceed(verbose): moved = self._moved_paths for_rename = compress_for_rename(self._paths) for path in sorted(compact(for_rename)): moved.stash(path) logger.verbose("Removing file or directory %s", path) for pth in self._pth.values(): pth.remove() logger.info("Successfully uninstalled %s", dist_name_version)
def remove(self, auto_confirm=False, verbose=False): """Remove paths in ``self.paths`` with confirmation (unless ``auto_confirm`` is True).""" if not self.paths: logger.info( "Can't uninstall '%s'. No files were found to uninstall.", self.dist.project_name, ) return dist_name_version = ( self.dist.project_name + "-" + self.dist.version ) logger.info('Uninstalling %s:', dist_name_version) with indent_log(): if auto_confirm or self._allowed_to_proceed(verbose): self.save_dir.create() for path in sorted(compact(self.paths)): new_path = self._stash(path) logger.debug('Removing file or directory %s', path) self._moved_paths.append(path) renames(path, new_path) for pth in self.pth.values(): pth.remove() logger.info('Successfully uninstalled %s', dist_name_version)
def _copy_dist_from_dir(link_path, location): """Copy distribution files in `link_path` to `location`. Invoked when user requests to install a local directory. E.g.: pip install . pip install ~/dev/git-repos/python-prompt-toolkit """ # Note: This is currently VERY SLOW if you have a lot of data in the # directory, because it copies everything with `shutil.copytree`. # What it should really do is build an sdist and install that. # See https://github.com/pypa/pip/issues/2195 if os.path.isdir(location): rmtree(location) # build an sdist setup_py = 'setup.py' sdist_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable)] sdist_args.append('-c') sdist_args.append(SETUPTOOLS_SHIM % setup_py) sdist_args.append('sdist') sdist_args += ['--dist-dir', location] logger.info('Running setup.py sdist for %s', link_path) with indent_log(): call_subprocess(sdist_args, cwd=link_path, show_stdout=False) # unpack sdist into `location` sdist = os.path.join(location, os.listdir(location)[0]) logger.info('Unpacking sdist %s into %s', sdist, location) unpack_file(sdist, location, content_type=None, link=None)
def install_editable(self, install_options, global_options=(), prefix=None): logger.info('Running setup.py develop for %s', self.name) if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] if prefix: prefix_param = ['--prefix={}'.format(prefix)] install_options = list(install_options) + prefix_param with indent_log(): # FIXME: should we do --install-headers here too? with self.build_env: call_subprocess( [ os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', SETUPTOOLS_SHIM % self.setup_py ] + list(global_options) + ['develop', '--no-deps'] + list(install_options), cwd=self.setup_py_dir, show_stdout=False, ) self.install_succeeded = True
def install_editable( install_options: List[str], global_options: Sequence[str], prefix: Optional[str], home: Optional[str], use_user_site: bool, name: str, setup_py_path: str, isolated: bool, build_env: BuildEnvironment, unpacked_source_directory: str, ) -> None: """Install a package in editable mode. Most arguments are pass-through to setuptools. """ logger.info("Running setup.py develop for %s", name) args = make_setuptools_develop_args( setup_py_path, global_options=global_options, install_options=install_options, no_user_config=isolated, prefix=prefix, home=home, use_user_site=use_user_site, ) with indent_log(): with build_env: call_subprocess( args, command_desc="python setup.py develop", cwd=unpacked_source_directory, )
def print_results(hits, name_column_width=None, terminal_width=None): if not hits: return if name_column_width is None: name_column_width = max([ len(hit['name']) + len(highest_version(hit.get('versions', ['-']))) for hit in hits ]) + 4 installed_packages = [p.project_name for p in pkg_resources.working_set] for hit in hits: name = hit['name'] summary = hit['summary'] or '' latest = highest_version(hit.get('versions', ['-'])) if terminal_width is not None: target_width = terminal_width - name_column_width - 5 if target_width > 10: # wrap and indent summary to fit terminal summary = textwrap.wrap(summary, target_width) summary = ('\n' + ' ' * (name_column_width + 3)).join(summary) line = '%-*s - %s' % (name_column_width, '%s (%s)' % (name, latest), summary) try: logger.info(line) if name in installed_packages: dist = pkg_resources.get_distribution(name) with indent_log(): if dist.version == latest: logger.info('INSTALLED: %s (latest)', dist.version) else: logger.info('INSTALLED: %s', dist.version) logger.info('LATEST: %s', latest) except UnicodeEncodeError: pass
def install_given_reqs( to_install, # type: List[InstallRequirement] install_options, # type: List[str] global_options=(), # type: Sequence[str] *args, **kwargs): # type: (...) -> List[InstallRequirement] """ Install everything in the given list. (to be called after having downloaded and unpacked the packages) """ if to_install: logger.info( 'Installing collected packages: %s', ', '.join([req.name for req in to_install]), ) with indent_log(): for requirement in to_install: if requirement.conflicts_with: logger.info( 'Found existing installation: %s', requirement.conflicts_with, ) with indent_log(): uninstalled_pathset = requirement.uninstall( auto_confirm=True) try: requirement.install(install_options, global_options, *args, **kwargs) except Exception: should_rollback = (requirement.conflicts_with and not requirement.install_succeeded) # if install did not succeed, rollback previous uninstall if should_rollback: uninstalled_pathset.rollback() raise else: should_commit = (requirement.conflicts_with and requirement.install_succeeded) if should_commit: uninstalled_pathset.commit() requirement.remove_temporary_source() return to_install
def _display(msg: str, paths: Iterable[str]) -> None: if not paths: return logger.info(msg) with indent_log(): for path in sorted(compact(paths)): logger.info(path)
def _display(msg, paths): if not paths: return logger.info(msg) with indent_log(): for path in sorted(compact(paths)): logger.info(path)
def _display(msg, paths): if not paths: return logger.info(msg) with indent_log(): for path in sorted(compact(paths)): logger.info(path)
def run_egg_info(self): assert self.source_dir if self.name: logger.debug( 'Running setup.py (path:%s) egg_info for package %s', self.setup_py, self.name, ) else: logger.debug( 'Running setup.py (path:%s) egg_info for package from %s', self.setup_py, self.link, ) with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py base_cmd = [os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] # We can't put the .egg-info files at the root, because then the # source code will be mistaken for an installed egg, causing # problems if self.editable: egg_base_option = [] else: egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info') ensure_dir(egg_info_dir) egg_base_option = ['--egg-base', 'pip-egg-info'] with self.build_env: call_subprocess( egg_info_cmd + egg_base_option, cwd=self.setup_py_dir, show_stdout=False, command_desc='python setup.py egg_info') if not self.req: if isinstance(parse_version(self.pkg_info()["Version"]), Version): op = "==" else: op = "===" self.req = Requirement( "".join([ self.pkg_info()["Name"], op, self.pkg_info()["Version"], ]) ) self._correct_build_location() else: metadata_name = canonicalize_name(self.pkg_info()["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( 'Running setup.py (path:%s) egg_info for package %s ' 'produced metadata for project name %s. Fix your ' '#egg=%s fragments.', self.setup_py, self.name, metadata_name, self.name ) self.req = Requirement(metadata_name)
def build( requirements: Iterable[InstallRequirement], wheel_cache: WheelCache, verify: bool, build_options: List[str], global_options: List[str], ) -> BuildResult: """Build wheels. :return: The list of InstallRequirement that succeeded to build and the list of InstallRequirement that failed to build. """ if not requirements: return [], [] # Build the wheels. logger.info( "Building wheels for collected packages: %s", ", ".join(req.name for req in requirements), # type: ignore ) with indent_log(): build_successes, build_failures = [], [] for req in requirements: assert req.name cache_dir = _get_cache_dir(req, wheel_cache) wheel_file = _build_one( req, cache_dir, verify, build_options, global_options, req.editable and req.permit_editable_wheels, ) if wheel_file: # Update the link for this. req.link = Link(path_to_url(wheel_file)) req.local_file_path = req.link.file_path assert req.link.is_wheel build_successes.append(req) else: build_failures.append(req) # notify success/failure if build_successes: logger.info( "Successfully built %s", " ".join([req.name for req in build_successes]), # type: ignore ) if build_failures: logger.info( "Failed to build %s", " ".join([req.name for req in build_failures]), # type: ignore ) # Return a list of requirements that failed to build return build_successes, build_failures
def show_sys_implementation(): # type: () -> None logger.info('sys.implementation:') if hasattr(sys, 'implementation'): implementation = sys.implementation # type: ignore implementation_name = implementation.name else: implementation_name = '' with indent_log(): show_value('name', implementation_name)
def export(self, location): """Export the svn repository at the url to the destination location""" url, rev_options = self.get_url_rev_options(self.url) logger.info('Exporting svn repository %s to %s', url, location) with indent_log(): if os.path.exists(location): # Subversion doesn't like to check out over an existing # directory --force fixes this, but was only added in svn 1.5 rmtree(location) cmd_args = ['export'] + rev_options.to_args() + [url, location] self.run_command(cmd_args, show_stdout=False)
def export(self, location): """Export the svn repository at the url to the destination location""" url, rev = self.get_url_rev() rev_options = get_rev_options(self, url, rev) url = self.remove_auth_from_url(url) logger.info('Exporting svn repository %s to %s', url, location) with indent_log(): if os.path.exists(location): # Subversion doesn't like to check out over an existing # directory --force fixes this, but was only added in svn 1.5 rmtree(location) cmd_args = ['export'] + rev_options.to_args() + [url, location] self.run_command(cmd_args, show_stdout=False)
def build( requirements, # type: Iterable[InstallRequirement] wheel_cache, # type: WheelCache build_options, # type: List[str] global_options, # type: List[str] ): # type: (...) -> BuildResult """Build wheels. :return: The list of InstallRequirement that succeeded to build and the list of InstallRequirement that failed to build. """ if not requirements: return [], [] # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join(req.name for req in requirements), ) with indent_log(): build_successes, build_failures = [], [] for req in requirements: cache_dir = _get_cache_dir(req, wheel_cache) wheel_file = _build_one(req, cache_dir, build_options, global_options) if wheel_file: # Update the link for this. req.link = Link(path_to_url(wheel_file)) req.local_file_path = req.link.file_path assert req.link.is_wheel build_successes.append(req) else: build_failures.append(req) # notify success/failure if build_successes: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_successes]), ) if build_failures: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failures]), ) # Return a list of requirements that failed to build return build_successes, build_failures
def print_dist_installation_info(name: str, latest: str) -> None: env = get_default_environment() dist = env.get_distribution(name) if dist is not None: with indent_log(): if dist.version == latest: write_output('INSTALLED: %s (latest)', dist.version) else: write_output('INSTALLED: %s', dist.version) if parse_version(latest).pre: write_output( 'LATEST: %s (pre-release; install' ' with "pip install --pre")', latest) else: write_output('LATEST: %s', latest)
def list_config_values(self, options: Values, args: List[str]) -> None: """List config key-value pairs across different config files""" self._get_n_args(args, "debug", n=0) self.print_env_var_values() # Iterate over config files and print if they exist, and the # key-value pairs present in them if they do for variant, files in sorted(self.configuration.iter_config_files()): write_output("%s:", variant) for fname in files: with indent_log(): file_exists = os.path.exists(fname) write_output("%s, exists: %r", fname, file_exists) if file_exists: self.print_config_file_values(variant)
def _log_preparing_link(self, req: InstallRequirement) -> None: """Provide context for the requirement being prepared.""" if req.link.is_file and not req.original_link_is_in_wheel_cache: message = "Processing %s" information = str(display_path(req.link.file_path)) else: message = "Collecting %s" information = str(req.req or req) if (message, information) != self._previous_requirement_header: self._previous_requirement_header = (message, information) logger.info(message, information) if req.original_link_is_in_wheel_cache: with indent_log(): logger.info("Using cached %s", req.link.filename)
def find_all_candidates(self, project_name): # type: (str) -> List[InstallationCandidate] """Find all available InstallationCandidate for project_name This checks index_urls and find_links. All versions found are returned as an InstallationCandidate list. See LinkEvaluator.evaluate_link() for details on which files are accepted. """ collected_links = self._link_collector.collect_links(project_name) link_evaluator = self.make_link_evaluator(project_name) find_links_versions = self.evaluate_links( link_evaluator, links=collected_links.find_links, ) page_versions = [] for page_url, page_links in collected_links.pages.items(): logger.debug('Analyzing links from page %s', page_url) with indent_log(): new_versions = self.evaluate_links( link_evaluator, links=page_links, ) page_versions.extend(new_versions) file_versions = self.evaluate_links( link_evaluator, links=collected_links.files, ) if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.link.url) for candidate in file_versions ]) ) # This is an intentional priority ordering return file_versions + find_links_versions + page_versions
def prepare_metadata(self) -> None: """Ensure that project metadata is available. Under PEP 517, call the backend hook to prepare the metadata. Under legacy processing, call setup.py egg-info. """ assert self.source_dir with indent_log(): self.metadata_directory = self._generate_metadata() # Act on the newly generated metadata, based on the name and version. if not self.name: self._set_requirement() else: self.warn_on_mismatching_name() self.assert_source_matches_version()
def process_project_url(self, project_url, link_evaluator): # type: (Link, LinkEvaluator) -> List[InstallationCandidate] logger.debug( 'Fetching project page and analyzing links: %s', project_url, ) html_page = self._link_collector.fetch_page(project_url) if html_page is None: return [] page_links = list(parse_links(html_page)) with indent_log(): package_links = self.evaluate_links( link_evaluator, links=page_links, ) return package_links
def export(self, location, url): # type: (str, HiddenText) -> None """Export the svn repository at the url to the destination location""" url, rev_options = self.get_url_rev_options(url) logger.info('Exporting svn repository %s to %s', url, location) with indent_log(): if os.path.exists(location): # Subversion doesn't like to check out over an existing # directory --force fixes this, but was only added in svn 1.5 rmtree(location) cmd_args = make_command( 'export', self.get_remote_call_options(), rev_options.to_args(), url, location, ) self.run_command(cmd_args, show_stdout=False)
def prepare_installed_requirement(self, req, require_hashes, skip_reason): """Prepare an already-installed requirement """ assert req.satisfied_by, "req should have been satisfied but isn't" assert skip_reason is not None, ( "did not get skip reason skipped but req.satisfied_by " "is set to %r" % (req.satisfied_by,) ) logger.info( 'Requirement %s: %s (%s)', skip_reason, req, req.satisfied_by.version ) with indent_log(): if require_hashes: logger.debug( 'Since it is already installed, we are trusting this ' 'package without checking its hash. To ensure a ' 'completely repeatable environment, install into an ' 'empty virtualenv.' ) abstract_dist = Installed(req) return abstract_dist
def _resolve_one(self, requirement_set, req_to_install, ignore_requires_python=False): """Prepare a single requirements file. :return: A list of additional InstallRequirements to also install. """ # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) if ignore_requires_python or self.ignore_requires_python: self.ignore_compatibility = True if req_to_install.constraint or req_to_install.prepared: return [] req_to_install.prepared = True # register tmp src for cleanup in case something goes wrong requirement_set.reqs_to_cleanup.append(req_to_install) abstract_dist = self._get_abstract_dist_for(req_to_install) # Parse and return dependencies dist = abstract_dist.dist(self.finder) try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: if self.ignore_compatibility: logger.warning(err.args[0]) else: raise # A huge hack, by Kenneth Reitz. try: self.requires_python = check_dist_requires_python(dist, absorb=False) except TypeError: self.requires_python = None more_reqs = [] def add_req(subreq, extras_requested): sub_install_req = InstallRequirement.from_req( str(subreq), req_to_install, isolated=self.isolated, wheel_cache=self.wheel_cache, ) parent_req_name = req_to_install.name to_scan_again, add_to_parent = requirement_set.add_requirement( sub_install_req, parent_req_name=parent_req_name, extras_requested=extras_requested, ) if parent_req_name and add_to_parent: self._discovered_dependencies[parent_req_name].append( add_to_parent ) more_reqs.extend(to_scan_again) with indent_log(): # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): available_requested = sorted( set(dist.extras) & set(req_to_install.extras) ) # 'unnamed' requirements will get added here req_to_install.is_direct = True requirement_set.add_requirement( req_to_install, parent_req_name=None, extras_requested=available_requested, ) if not self.ignore_dependencies: if req_to_install.extras: logger.debug( "Installing extra requirements: %r", ','.join(req_to_install.extras), ) missing_requested = sorted( set(req_to_install.extras) - set(dist.extras) ) for missing in missing_requested: logger.warning( '%s does not provide the extra \'%s\'', dist, missing ) available_requested = sorted( set(dist.extras) & set(req_to_install.extras) ) for subreq in dist.requires(available_requested): add_req(subreq, extras_requested=available_requested) # Hack for deep-resolving extras. for available in available_requested: if hasattr(dist, '_DistInfoDistribution__dep_map'): for req in dist._DistInfoDistribution__dep_map[available]: req = InstallRequirement.from_req( str(req), req_to_install, isolated=self.isolated, wheel_cache=self.wheel_cache, ) more_reqs.append(req) if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took # action on them. requirement_set.successfully_downloaded.append(req_to_install) return more_reqs
def prepare_linked_requirement(self, req, session, finder, upgrade_allowed, require_hashes): """Prepare a requirement that would be obtained from req.link """ # TODO: Breakup into smaller functions if req.link and req.link.scheme == 'file': path = url_to_path(req.link.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req) with indent_log(): # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory req.ensure_has_source_dir(self.build_dir) # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req.source_dir` # package unpacked in `req.source_dir` # if os.path.exists(os.path.join(req.source_dir, 'setup.py')): # raise PreviousBuildDirError( # "pip can't proceed with requirements '%s' due to a" # " pre-existing build directory (%s). This is " # "likely due to a previous installation that failed" # ". pip is being responsible and not assuming it " # "can delete this. Please delete it and try again." # % (req, req.source_dir) # ) req.populate_link(finder, upgrade_allowed, require_hashes) # We can't hit this spot and have populate_link return None. # req.satisfied_by is None here (because we're # guarded) and upgrade has no impact except when satisfied_by # is not None. # Then inside find_requirement existing_applicable -> False # If no new versions are found, DistributionNotFound is raised, # otherwise a result is guaranteed. assert req.link link = req.link # Now that we have the real link, we can tell what kind of # requirements we have and raise some more informative errors # than otherwise. (For example, we can raise VcsHashUnsupported # for a VCS URL rather than HashMissing.) if require_hashes: # We could check these first 2 conditions inside # unpack_url and save repetition of conditions, but then # we would report less-useful error messages for # unhashable requirements, complaining that there's no # hash provided. if is_vcs_url(link): raise VcsHashUnsupported() elif is_file_url(link) and is_dir_url(link): raise DirectoryUrlHashUnsupported() if not req.original_link and not req.is_pinned: # Unpinned packages are asking for trouble when a new # version is uploaded. This isn't a security check, but # it saves users a surprising hash mismatch in the # future. # # file:/// URLs aren't pinnable, so don't complain # about them not being pinned. raise HashUnpinned() hashes = req.hashes(trust_internet=not require_hashes) if require_hashes and not hashes: # Known-good hashes are missing for this requirement, so # shim it with a facade object that will provoke hash # computation and then raise a HashMissing exception # showing the user what the hash should be. hashes = MissingHashes() try: download_dir = self.download_dir # We always delete unpacked sdists after pip ran. autodelete_unpacked = True if req.link.is_wheel and self.wheel_download_dir: # when doing 'pip wheel` we download wheels to a # dedicated dir. download_dir = self.wheel_download_dir if req.link.is_wheel: if download_dir: # When downloading, we only unpack wheels to get # metadata. autodelete_unpacked = True else: # When installing a wheel, we use the unpacked # wheel. autodelete_unpacked = False unpack_url( req.link, req.source_dir, download_dir, autodelete_unpacked, session=session, hashes=hashes, progress_bar=self.progress_bar ) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because of error %s', req, exc, ) raise InstallationError( 'Could not install requirement %s because of HTTP ' 'error %s for URL %s' % (req, exc, req.link) ) abstract_dist = make_abstract_dist(req) abstract_dist.prep_for_dist(finder, self.build_isolation) if self._download_should_save: # Make a .zip of the source_dir we already created. if req.link.scheme in vcs.all_schemes: req.archive(self.download_dir) return abstract_dist
def find_all_candidates(self, project_name): """Find all available InstallationCandidate for project_name This checks index_urls, find_links and dependency_links. All versions found are returned as an InstallationCandidate list. See _link_package_versions for details on which files are accepted """ index_locations = self._get_index_urls_locations(project_name) index_file_loc, index_url_loc = self._sort_locations(index_locations) fl_file_loc, fl_url_loc = self._sort_locations( self.find_links, expand_dir=True, ) dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links) file_locations = (Link(url) for url in itertools.chain( index_file_loc, fl_file_loc, dep_file_loc, )) # We trust every url that the user has given us whether it was given # via --index-url or --find-links # We explicitly do not trust links that came from dependency_links # We want to filter out any thing which does not have a secure origin. url_locations = [ link for link in itertools.chain( (Link(url) for url in index_url_loc), (Link(url) for url in fl_url_loc), (Link(url) for url in dep_url_loc), ) if self._validate_secure_origin(logger, link) ] logger.debug('%d location(s) to search for versions of %s:', len(url_locations), project_name) for location in url_locations: logger.debug('* %s', location) canonical_name = canonicalize_name(project_name) formats = fmt_ctl_formats(self.format_control, canonical_name) search = Search(project_name, canonical_name, formats) find_links_versions = self._package_versions( # We trust every directly linked archive in find_links (Link(url, '-f') for url in self.find_links), search ) page_versions = [] for page in self._get_pages(url_locations, project_name): logger.debug('Analyzing links from page %s', page.url) with indent_log(): page_versions.extend( self._package_versions(page.links, search) ) dependency_versions = self._package_versions( (Link(url) for url in self.dependency_links), search ) if dependency_versions: logger.debug( 'dependency_links found: %s', ', '.join([ version.location.url for version in dependency_versions ]) ) file_versions = self._package_versions(file_locations, search) if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.location.url) for candidate in file_versions ]) ) # This is an intentional priority ordering return ( file_versions + find_links_versions + page_versions + dependency_versions )
def build(self, requirements, session, autobuilding=False): """Build wheels. :param unpack: If True, replace the sdist we built from with the newly built wheel, in preparation for installation. :return: True if all the wheels built correctly. """ from pipenv.patched.notpip._internal import index building_is_possible = self._wheel_dir or ( autobuilding and self.wheel_cache.cache_dir ) assert building_is_possible buildset = [] for req in requirements: if req.constraint: continue if req.is_wheel: if not autobuilding: logger.info( 'Skipping %s, due to already being wheel.', req.name, ) elif autobuilding and req.editable: pass elif autobuilding and not req.source_dir: pass elif autobuilding and req.link and not req.link.is_artifact: # VCS checkout. Build wheel just for this run. buildset.append((req, True)) else: ephem_cache = False if autobuilding: link = req.link base, ext = link.splitext() if index.egg_info_matches(base, None, link) is None: # E.g. local directory. Build wheel just for this run. ephem_cache = True if "binary" not in index.fmt_ctl_formats( self.finder.format_control, canonicalize_name(req.name)): logger.info( "Skipping bdist_wheel for %s, due to binaries " "being disabled for it.", req.name, ) continue buildset.append((req, ephem_cache)) if not buildset: return True # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for (req, _) in buildset]), ) _cache = self.wheel_cache # shorter name with indent_log(): build_success, build_failure = [], [] for req, ephem in buildset: python_tag = None if autobuilding: python_tag = pep425tags.implementation_tag if ephem: output_dir = _cache.get_ephem_path_for_link(req.link) else: output_dir = _cache.get_path_for_link(req.link) try: ensure_dir(output_dir) except OSError as e: logger.warning("Building wheel for %s failed: %s", req.name, e) build_failure.append(req) continue else: output_dir = self._wheel_dir wheel_file = self._build_one( req, output_dir, python_tag=python_tag, ) if wheel_file: build_success.append(req) if autobuilding: # XXX: This is mildly duplicative with prepare_files, # but not close enough to pull out to a single common # method. # The code below assumes temporary source dirs - # prevent it doing bad things. if req.source_dir and not os.path.exists(os.path.join( req.source_dir, PIP_DELETE_MARKER_FILENAME)): raise AssertionError( "bad source dir - missing marker") # Delete the source we built the wheel from req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. req.source_dir = req.build_location( self.preparer.build_dir ) # Update the link for this. req.link = index.Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_url( req.link, req.source_dir, None, False, session=session, ) else: build_failure.append(req) # notify success/failure if build_success: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_success]), ) if build_failure: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) # Return True if all builds were successful return len(build_failure) == 0