def get_formatted_locations(self): # type: () -> str lines = [] if self.index_urls and self.index_urls != [PyPI.simple_url]: lines.append( "Looking in indexes: {}".format( ", ".join(redact_auth_from_url(url) for url in self.index_urls) ) ) if self.find_links: lines.append( "Looking in links: {}".format( ", ".join(redact_auth_from_url(url) for url in self.find_links) ) ) return "\n".join(lines)
def create(cls, session, options, suppress_no_index=False): # type: (PipSession, Values, bool) -> LinkCollector """ :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. """ index_urls = [options.index_url] + options.extra_index_urls if options.no_index and not suppress_no_index: logger.debug( 'Ignoring indexes: %s', ','.join(redact_auth_from_url(url) for url in index_urls), ) index_urls = [] # Make sure find_links is a list before passing to create(). find_links = options.find_links or [] search_scope = SearchScope.create( find_links=find_links, index_urls=index_urls, ) link_collector = LinkCollector( session=session, search_scope=search_scope, ) return link_collector
def _prepare_download( resp, # type: Response link, # type: Link progress_bar # type: str ): # type: (...) -> Iterable[bytes] total_length = _get_http_response_size(resp) if link.netloc == PyPI.file_storage_domain: url = link.show_url else: url = link.url_without_fragment logged_url = redact_auth_from_url(url) if total_length: logged_url = '{} ({})'.format(logged_url, format_size(total_length)) if is_from_cache(resp): logger.info("Using cached %s", logged_url) else: logger.info("Downloading %s", logged_url) if logger.getEffectiveLevel() > logging.INFO: show_progress = False elif is_from_cache(resp): show_progress = False elif not total_length: show_progress = True elif total_length > (40 * 1000): show_progress = True else: show_progress = False chunks = response_chunks(resp, CONTENT_CHUNK_SIZE) if not show_progress: return chunks return DownloadProgressProvider(progress_bar, max=total_length)(chunks)
def get_formatted_locations(self): # type: () -> str lines = [] redacted_index_urls = [] if self.index_urls and self.index_urls != [PyPI.simple_url]: for url in self.index_urls: redacted_index_url = redact_auth_from_url(url) # Parse the URL purl = urllib.parse.urlsplit(redacted_index_url) # URL is generally invalid if scheme and netloc is missing # there are issues with Python and URL parsing, so this test # is a bit crude. See bpo-20271, bpo-23505. Python doesn't # always parse invalid URLs correctly - it should raise # exceptions for malformed URLs if not purl.scheme and not purl.netloc: logger.warning( 'The index url "%s" seems invalid, ' 'please provide a scheme.', redacted_index_url) redacted_index_urls.append(redacted_index_url) lines.append('Looking in indexes: {}'.format( ', '.join(redacted_index_urls))) if self.find_links: lines.append('Looking in links: {}'.format(', '.join( redact_auth_from_url(url) for url in self.find_links))) return '\n'.join(lines)
def _download_url( resp, # type: Response link, # type: Link content_file, # type: IO[Any] hashes, # type: Optional[Hashes] progress_bar # type: str ): # type: (...) -> None try: total_length = int(resp.headers['content-length']) except (ValueError, KeyError, TypeError): total_length = 0 if link.netloc == PyPI.file_storage_domain: url = link.show_url else: url = link.url_without_fragment redacted_url = redact_auth_from_url(url) if total_length: logger.info("Downloading %s (%s)", redacted_url, format_size(total_length)) elif is_from_cache(resp): logger.info("Using cached %s", redacted_url) else: logger.info("Downloading %s", redacted_url) if logger.getEffectiveLevel() > logging.INFO: show_progress = False elif is_from_cache(resp): show_progress = False elif total_length > (40 * 1000): show_progress = True elif not total_length: show_progress = True else: show_progress = False def written_chunks(chunks): for chunk in chunks: content_file.write(chunk) yield chunk progress_indicator = _progress_indicator if show_progress: # We don't show progress on cached responses progress_indicator = DownloadProgressProvider(progress_bar, max=total_length) downloaded_chunks = written_chunks( progress_indicator(response_chunks(resp, CONTENT_CHUNK_SIZE), CONTENT_CHUNK_SIZE)) if hashes: hashes.check_against_chunks(downloaded_chunks) else: consume(downloaded_chunks)
def __str__(self) -> str: if self.req: s = str(self.req) if self.link: s += " from {}".format(redact_auth_from_url(self.link.url)) elif self.link: s = redact_auth_from_url(self.link.url) else: s = "<InstallRequirement>" if self.satisfied_by is not None: s += " in {}".format(display_path(self.satisfied_by.location)) if self.comes_from: if isinstance(self.comes_from, str): comes_from: Optional[str] = self.comes_from else: comes_from = self.comes_from.from_path() if comes_from: s += f" (from {comes_from})" return s
def __str__(self): # type: () -> str if self.req: s = str(self.req) if self.link: s += ' from %s' % redact_auth_from_url(self.link.url) elif self.link: s = redact_auth_from_url(self.link.url) else: s = '<InstallRequirement>' if self.satisfied_by is not None: s += ' in %s' % display_path(self.satisfied_by.location) if self.comes_from: if isinstance(self.comes_from, six.string_types): comes_from = self.comes_from # type: Optional[str] else: comes_from = self.comes_from.from_path() if comes_from: s += ' (from %s)' % comes_from return s
def __str__(self): # type: () -> str if self.requires_python: rp = f' (requires-python:{self.requires_python})' else: rp = '' if self.comes_from: return '{} (from {}){}'.format(redact_auth_from_url(self._url), self.comes_from, rp) else: return redact_auth_from_url(str(self._url))
def __str__(self): # type: () -> str if req: s = str(req) if link: s += ' from {}'.format(redact_auth_from_url(link.url)) elif link: s = redact_auth_from_url(link.url) else: s = '<InstallRequirement>' if satisfied_by is not None: s += ' in {}'.format(display_path(satisfied_by.location)) if comes_from: if isinstance(comes_from, six.string_types): comes_from = comes_from # type: Optional[str] else: comes_from = comes_from.from_path() if comes_from: s += ' (from {})'.format(comes_from) return s
def __str__(self): # type: () -> str if self.req: s = str(self.req) if self.link: s += " from {}".format(redact_auth_from_url(self.link.url)) elif self.link: s = redact_auth_from_url(self.link.url) else: s = "<InstallRequirement>" if self.satisfied_by is not None: s += " in {}".format(display_path(self.satisfied_by.location)) if self.comes_from: if isinstance(self.comes_from, six.string_types): comes_from = self.comes_from # type: Optional[str] else: comes_from = self.comes_from.from_path() if comes_from: s += " (from {})".format(comes_from) return s
def __str__(self): # type: () -> str if self.req: s = str(self.req) if self.link: s += ' from {}'.format(redact_auth_from_url(self.link.url)) elif self.link: s = redact_auth_from_url(self.link.url) else: s = '<InstallRequirement>' if self.satisfied_by is not None: s += ' in {}'.format(display_path(self.satisfied_by.location)) if self.comes_from: if isinstance(self.comes_from, str): comes_from = self.comes_from # type: Optional[str] else: comes_from = self.comes_from.from_path() if comes_from: s += f' (from {comes_from})' return s
def expand_default(self, option): default_value = None if self.parser is not None: self.parser._update_defaults(self.parser.defaults) default_value = self.parser.defaults.get(option.dest) help_text = optparse.IndentedHelpFormatter.expand_default(self, option) if default_value and option.metavar == 'URL': help_text = help_text.replace(default_value, redact_auth_from_url(default_value)) return help_text
def _get_html_response(url, session): # type: (str, PipSession) -> Response """Access an HTML page with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotHTML` if it is not HTML. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got HTML, and raise `_NotHTML` otherwise. """ if _is_url_like_archive(url): _ensure_html_response(url, session=session) logger.debug('Getting page %s', redact_auth_from_url(url)) resp = session.get( url, headers={ "Accept": "text/html", # We don't want to blindly returned cached data for # /simple/, because authors generally expecting that # twine upload && pip install will function, but if # they've done a pip install in the last ~10 minutes # it won't. Thus by setting this to zero we will not # blindly use any cached data, however the benefit of # using max-age=0 instead of no-cache, is that we will # still support conditional requests, so we will still # minimize traffic sent in cases where the page hasn't # changed at all, we will just always incur the round # trip for the conditional GET now instead of only # once per 10 minutes. # For more information, please see pypa/pip#5670. # However if we want to override Cache-Control, e.g. via CLI, # we can still do so. "Cache-Control": session.headers.get('Cache-Control', 'max-age=0'), }, ) resp.raise_for_status() # The check for archives above only works if the url ends with # something that looks like an archive. However that is not a # requirement of an url. Unless we issue a HEAD request on every # url we cannot know ahead of time for sure if something is HTML # or not. However we can check after we've downloaded it. _ensure_html_header(resp) return resp
def __str__(self) -> str: versions = [] pre_versions = [] for candidate in sorted(self.candidates_tried): version = str(candidate.version) if candidate.version.is_prerelease: pre_versions.append(version) else: versions.append(version) lines = [f"Could not find a version that matches {self.ireq}"] if versions: lines.append(f"Tried: {', '.join(versions)}") if pre_versions: if self.finder.allow_all_prereleases: line = "Tried" else: line = "Skipped" line += f" pre-versions: {', '.join(pre_versions)}" lines.append(line) if versions or pre_versions: lines.append( "There are incompatible versions in the resolved dependencies:" ) source_ireqs = getattr(self.ireq, "_source_ireqs", []) lines.extend(f" {ireq}" for ireq in source_ireqs) else: redacted_urls = tuple( redact_auth_from_url(url) for url in self.finder.index_urls ) lines.append("No versions found") lines.append( "{} {} reachable?".format( "Were" if len(redacted_urls) > 1 else "Was", " or ".join(redacted_urls), ) ) return "\n".join(lines)
def expand_default(self, option): default_values = None if self.parser is not None: self.parser._update_defaults(self.parser.defaults) default_values = self.parser.defaults.get(option.dest) help_text = super().expand_default(option) if default_values and option.metavar == "URL": if isinstance(default_values, str): default_values = [default_values] # If its not a list, we should abort and just return the help text if not isinstance(default_values, list): default_values = [] for val in default_values: help_text = help_text.replace(val, redact_auth_from_url(val)) return help_text
def _prepare_download( resp: Response, link: Link, progress_bar: str, ) -> Iterable[bytes]: total_length = _get_http_response_size(resp) if link.netloc == PyPI.file_storage_domain: url = link.show_url else: url = link.url_without_fragment logged_url = redact_auth_from_url(url) if total_length: logged_url = "{} ({})".format(logged_url, format_size(total_length)) if is_from_cache(resp): logger.info("Using cached %s", logged_url) else: logger.info("Downloading %s", logged_url) if logger.getEffectiveLevel() > logging.INFO: show_progress = False elif is_from_cache(resp): show_progress = False elif not total_length: show_progress = True elif total_length > (40 * 1000): show_progress = True else: show_progress = False chunks = response_chunks(resp, CONTENT_CHUNK_SIZE) if not show_progress: return chunks renderer = get_download_progress_renderer(bar_type=progress_bar, size=total_length) return renderer(chunks)
def get_formatted_locations(self): # type: () -> str lines = [] redacted_index_urls = [] if self.index_urls and self.index_urls != [PyPI.simple_url]: for url in self.index_urls: redacted_index_url = redact_auth_from_url(url) # Parse the URL purl = urllib_parse.urlsplit(redacted_index_url) # URL is generally invalid if scheme and netloc is missing # there are issues with Python and URL parsing, so this test # is a bit crude. See bpo-20271, bpo-23505. Python doesn't # always parse invalid URLs correctly - it should raise # exceptions for malformed URLs if not purl.scheme and not purl.netloc: logger.warning( <<<<<<< HEAD 'The index url "{}" seems invalid, ' 'please provide a scheme.'.format(redacted_index_url))
def make_search_scope(options, suppress_no_index=False): # type: (Values, bool) -> SearchScope """ :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. """ index_urls = [options.index_url] + options.extra_index_urls if options.no_index and not suppress_no_index: logger.debug( 'Ignoring indexes: %s', ','.join(redact_auth_from_url(url) for url in index_urls), ) index_urls = [] # Make sure find_links is a list before passing to create(). find_links = options.find_links or [] search_scope = SearchScope.create( find_links=find_links, index_urls=index_urls, ) return search_scope
def cli( ctx: click.Context, verbose: int, quiet: int, dry_run: bool, pre: bool, rebuild: bool, find_links: Tuple[str], index_url: str, extra_index_url: Tuple[str], cert: Optional[str], client_cert: Optional[str], trusted_host: Tuple[str], header: bool, emit_trusted_host: bool, annotate: bool, upgrade: bool, upgrade_packages: Tuple[str], output_file: Optional[LazyFile], allow_unsafe: bool, generate_hashes: bool, reuse_hashes: bool, src_files: Tuple[str], max_rounds: int, build_isolation: bool, emit_find_links: bool, cache_dir: str, pip_args_str: Optional[str], emit_index_url: bool, ) -> None: """Compiles requirements.txt from requirements.in specs.""" log.verbosity = verbose - quiet if len(src_files) == 0: if os.path.exists(DEFAULT_REQUIREMENTS_FILE): src_files = (DEFAULT_REQUIREMENTS_FILE, ) elif os.path.exists("setup.py"): src_files = ("setup.py", ) else: raise click.BadParameter(("If you do not specify an input file, " "the default is {} or setup.py" ).format(DEFAULT_REQUIREMENTS_FILE)) if not output_file: # An output file must be provided for stdin if src_files == ("-", ): raise click.BadParameter( "--output-file is required if input is from stdin") # Use default requirements output file if there is a setup.py the source file elif os.path.basename(src_files[0]) in METADATA_FILENAMES: file_name = os.path.join(os.path.dirname(src_files[0]), DEFAULT_REQUIREMENTS_OUTPUT_FILE) # An output file must be provided if there are multiple source files elif len(src_files) > 1: raise click.BadParameter( "--output-file is required if two or more input files are given." ) # Otherwise derive the output file from the source file else: base_name = src_files[0].rsplit(".", 1)[0] file_name = base_name + ".txt" output_file = click.open_file(file_name, "w+b", atomic=True, lazy=True) # Close the file at the end of the context execution assert output_file is not None ctx.call_on_close(safecall(output_file.close_intelligently)) ### # Setup ### right_args = shlex.split(pip_args_str or "") pip_args = [] for link in find_links: pip_args.extend(["-f", link]) if index_url: pip_args.extend(["-i", index_url]) for extra_index in extra_index_url: pip_args.extend(["--extra-index-url", extra_index]) if cert: pip_args.extend(["--cert", cert]) if client_cert: pip_args.extend(["--client-cert", client_cert]) if pre: pip_args.extend(["--pre"]) for host in trusted_host: pip_args.extend(["--trusted-host", host]) if not build_isolation: pip_args.append("--no-build-isolation") pip_args.extend(right_args) repository: BaseRepository repository = PyPIRepository(pip_args, cache_dir=cache_dir) # Parse all constraints coming from --upgrade-package/-P upgrade_reqs_gen = (install_req_from_line(pkg) for pkg in upgrade_packages) upgrade_install_reqs = { key_from_ireq(install_req): install_req for install_req in upgrade_reqs_gen } existing_pins_to_upgrade = set() # Proxy with a LocalRequirementsRepository if --upgrade is not specified # (= default invocation) if not upgrade and os.path.exists(output_file.name): # Use a temporary repository to ensure outdated(removed) options from # existing requirements.txt wouldn't get into the current repository. tmp_repository = PyPIRepository(pip_args, cache_dir=cache_dir) ireqs = parse_requirements( output_file.name, finder=tmp_repository.finder, session=tmp_repository.session, options=tmp_repository.options, ) # Exclude packages from --upgrade-package/-P from the existing # constraints, and separately gather pins to be upgraded existing_pins = {} for ireq in filter(is_pinned_requirement, ireqs): key = key_from_ireq(ireq) if key in upgrade_install_reqs: existing_pins_to_upgrade.add(key) else: existing_pins[key] = ireq repository = LocalRequirementsRepository(existing_pins, repository, reuse_hashes=reuse_hashes) ### # Parsing/collecting initial requirements ### constraints = [] for src_file in src_files: is_setup_file = os.path.basename(src_file) in METADATA_FILENAMES if src_file == "-": # pip requires filenames and not files. Since we want to support # piping from stdin, we need to briefly save the input from stdin # to a temporary file and have pip read that. also used for # reading requirements from install_requires in setup.py. tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False) tmpfile.write(sys.stdin.read()) comes_from = "-r -" tmpfile.flush() reqs = list( parse_requirements( tmpfile.name, finder=repository.finder, session=repository.session, options=repository.options, )) for req in reqs: req.comes_from = comes_from constraints.extend(reqs) elif is_setup_file: dist = meta.load(os.path.dirname(os.path.abspath(src_file))) comes_from = f"{dist.metadata.get_all('Name')[0]} ({src_file})" constraints.extend([ install_req_from_line(req, comes_from=comes_from) for req in dist.requires or [] ]) else: constraints.extend( parse_requirements( src_file, finder=repository.finder, session=repository.session, options=repository.options, )) primary_packages = { key_from_ireq(ireq) for ireq in constraints if not ireq.constraint } allowed_upgrades = primary_packages | existing_pins_to_upgrade constraints.extend(ireq for key, ireq in upgrade_install_reqs.items() if key in allowed_upgrades) # Filter out pip environment markers which do not match (PEP496) constraints = [ req for req in constraints if req.markers is None # We explicitly set extra=None to filter out optional requirements # since evaluating an extra marker with no environment raises UndefinedEnvironmentName # (see https://packaging.pypa.io/en/latest/markers.html#usage) or req.markers.evaluate({"extra": None}) ] log.debug("Using indexes:") with log.indentation(): for index_url in dedup(repository.finder.index_urls): log.debug(redact_auth_from_url(index_url)) if repository.finder.find_links: log.debug("") log.debug("Using links:") with log.indentation(): for find_link in dedup(repository.finder.find_links): log.debug(redact_auth_from_url(find_link)) try: resolver = Resolver( constraints, repository, prereleases=repository.finder.allow_all_prereleases or pre, cache=DependencyCache(cache_dir), clear_caches=rebuild, allow_unsafe=allow_unsafe, ) results = resolver.resolve(max_rounds=max_rounds) hashes = resolver.resolve_hashes(results) if generate_hashes else None except PipToolsError as e: log.error(str(e)) sys.exit(2) log.debug("") ## # Output ## writer = OutputWriter( cast(BinaryIO, output_file), click_ctx=ctx, dry_run=dry_run, emit_header=header, emit_index_url=emit_index_url, emit_trusted_host=emit_trusted_host, annotate=annotate, generate_hashes=generate_hashes, default_index_url=repository.DEFAULT_INDEX_URL, index_urls=repository.finder.index_urls, trusted_hosts=repository.finder.trusted_hosts, format_control=repository.finder.format_control, allow_unsafe=allow_unsafe, find_links=repository.finder.find_links, emit_find_links=emit_find_links, ) writer.write( results=results, unsafe_requirements=resolver.unsafe_constraints, markers={ key_from_ireq(ireq): ireq.markers for ireq in constraints if ireq.markers }, hashes=hashes, ) if dry_run: log.info("Dry-run, so nothing updated.")
def get_compile_command(click_ctx: click.Context) -> str: """ Returns a normalized compile command depending on cli context. The command will be normalized by: - expanding options short to long - removing values that are already default - sorting the arguments - removing one-off arguments like '--upgrade' - removing arguments that don't change build behaviour like '--verbose' """ from piptools.scripts.compile import cli # Map of the compile cli options (option name -> click.Option) compile_options = {option.name: option for option in cli.params} left_args = [] right_args = [] for option_name, value in click_ctx.params.items(): option = compile_options[option_name] # Collect variadic args separately, they will be added # at the end of the command later if option.nargs < 0: # These will necessarily be src_files # Re-add click-stripped '--' if any start with '-' if any(val.startswith("-") and val != "-" for val in value): right_args.append("--") right_args.extend([shlex.quote(val) for val in value]) continue assert isinstance(option, click.Option) # Get the latest option name (usually it'll be a long name) option_long_name = option.opts[-1] # Exclude one-off options (--upgrade/--upgrade-package/--rebuild/...) # or options that don't change compile behaviour (--verbose/--dry-run/...) if option_long_name in COMPILE_EXCLUDE_OPTIONS: continue # Skip options without a value if option.default is None and not value: continue # Skip options with a default value if option.default == value: continue # Use a file name for file-like objects if isinstance(value, LazyFile): value = value.name # Convert value to the list if not isinstance(value, (tuple, list)): value = [value] for val in value: # Flags don't have a value, thus add to args true or false option long name if option.is_flag: # If there are false-options, choose an option name depending on a value if option.secondary_opts: # Get the latest false-option secondary_option_long_name = option.secondary_opts[-1] arg = option_long_name if val else secondary_option_long_name # There are no false-options, use true-option else: arg = option_long_name left_args.append(shlex.quote(arg)) # Append to args the option with a value else: if isinstance(val, str) and is_url(val): val = redact_auth_from_url(val) if option.name == "pip_args_str": # shlex.quote() would produce functional but noisily quoted results, # e.g. --pip-args='--cache-dir='"'"'/tmp/with spaces'"'"'' # Instead, we try to get more legible quoting via repr: left_args.append(f"{option_long_name}={repr(val)}") else: left_args.append( f"{option_long_name}={shlex.quote(str(val))}") return " ".join(["pip-compile", *sorted(left_args), *sorted(right_args)])
def __str__(self): # type: () -> str return redact_auth_from_url(self.url)
def test_redact_auth_from_url(auth_url: str, expected_url: str) -> None: url = redact_auth_from_url(auth_url) assert url == expected_url
def cli( ctx, verbose, quiet, dry_run, pre, rebuild, find_links, index_url, extra_index_url, cert, client_cert, trusted_host, header, index, emit_trusted_host, annotate, upgrade, upgrade_packages, output_file, allow_unsafe, generate_hashes, reuse_hashes, src_files, max_rounds, build_isolation, emit_find_links, cache_dir, pip_args, emit_index_url, ): """Compiles requirements.txt from requirements.in specs.""" log.verbosity = verbose - quiet if len(src_files) == 0: if os.path.exists(DEFAULT_REQUIREMENTS_FILE): src_files = (DEFAULT_REQUIREMENTS_FILE, ) elif os.path.exists("setup.py"): src_files = ("setup.py", ) else: raise click.BadParameter(("If you do not specify an input file, " "the default is {} or setup.py" ).format(DEFAULT_REQUIREMENTS_FILE)) if not output_file: # An output file must be provided for stdin if src_files == ("-", ): raise click.BadParameter( "--output-file is required if input is from stdin") # Use default requirements output file if there is a setup.py the source file elif src_files == ("setup.py", ): file_name = DEFAULT_REQUIREMENTS_OUTPUT_FILE # An output file must be provided if there are multiple source files elif len(src_files) > 1: raise click.BadParameter( "--output-file is required if two or more input files are given." ) # Otherwise derive the output file from the source file else: base_name = src_files[0].rsplit(".", 1)[0] file_name = base_name + ".txt" output_file = click.open_file(file_name, "w+b", atomic=True, lazy=True) # Close the file at the end of the context execution ctx.call_on_close(safecall(output_file.close_intelligently)) if cli.has_arg("index") and cli.has_arg("emit_index_url"): raise click.BadParameter( "--index/--no-index and --emit-index-url/--no-emit-index-url " "are mutually exclusive.") elif cli.has_arg("index"): warnings.warn( "--index and --no-index are deprecated and will be removed " "in future versions. Use --emit-index-url/--no-emit-index-url instead.", category=FutureWarning, ) emit_index_url = index ### # Setup ### right_args = shlex.split(pip_args or "") pip_args = [] for link in find_links: pip_args.extend(["-f", link]) if index_url: pip_args.extend(["-i", index_url]) for extra_index in extra_index_url: pip_args.extend(["--extra-index-url", extra_index]) if cert: pip_args.extend(["--cert", cert]) if client_cert: pip_args.extend(["--client-cert", client_cert]) if pre: pip_args.extend(["--pre"]) for host in trusted_host: pip_args.extend(["--trusted-host", host]) if not build_isolation: pip_args.append("--no-build-isolation") pip_args.extend(right_args) repository = PyPIRepository(pip_args, cache_dir=cache_dir) # Parse all constraints coming from --upgrade-package/-P upgrade_reqs_gen = (install_req_from_line(pkg) for pkg in upgrade_packages) upgrade_install_reqs = { key_from_ireq(install_req): install_req for install_req in upgrade_reqs_gen } existing_pins_to_upgrade = set() # Proxy with a LocalRequirementsRepository if --upgrade is not specified # (= default invocation) if not upgrade and os.path.exists(output_file.name): # Use a temporary repository to ensure outdated(removed) options from # existing requirements.txt wouldn't get into the current repository. tmp_repository = PyPIRepository(pip_args, cache_dir=cache_dir) ireqs = parse_requirements( output_file.name, finder=tmp_repository.finder, session=tmp_repository.session, options=tmp_repository.options, ) # Exclude packages from --upgrade-package/-P from the existing # constraints, and separately gather pins to be upgraded existing_pins = {} for ireq in filter(is_pinned_requirement, ireqs): key = key_from_ireq(ireq) if key in upgrade_install_reqs: existing_pins_to_upgrade.add(key) else: existing_pins[key] = ireq repository = LocalRequirementsRepository(existing_pins, repository, reuse_hashes=reuse_hashes) ### # Parsing/collecting initial requirements ### constraints = [] for src_file in src_files: is_setup_file = os.path.basename(src_file) == "setup.py" if is_setup_file or src_file == "-": # pip requires filenames and not files. Since we want to support # piping from stdin, we need to briefly save the input from stdin # to a temporary file and have pip read that. also used for # reading requirements from install_requires in setup.py. tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False) if is_setup_file: from distutils.core import run_setup dist = run_setup(src_file) tmpfile.write("\n".join(dist.install_requires)) comes_from = f"{dist.get_name()} ({src_file})" else: tmpfile.write(sys.stdin.read()) comes_from = "-r -" tmpfile.flush() reqs = list( parse_requirements( tmpfile.name, finder=repository.finder, session=repository.session, options=repository.options, )) for req in reqs: req.comes_from = comes_from constraints.extend(reqs) else: constraints.extend( parse_requirements( src_file, finder=repository.finder, session=repository.session, options=repository.options, )) primary_packages = { key_from_ireq(ireq) for ireq in constraints if not ireq.constraint } allowed_upgrades = primary_packages | existing_pins_to_upgrade constraints.extend(ireq for key, ireq in upgrade_install_reqs.items() if key in allowed_upgrades) # Filter out pip environment markers which do not match (PEP496) constraints = [ req for req in constraints if req.markers is None or req.markers.evaluate() ] log.debug("Using indexes:") with log.indentation(): for index_url in dedup(repository.finder.index_urls): log.debug(redact_auth_from_url(index_url)) if repository.finder.find_links: log.debug("") log.debug("Using links:") with log.indentation(): for find_link in dedup(repository.finder.find_links): log.debug(redact_auth_from_url(find_link)) try: resolver = Resolver( constraints, repository, prereleases=repository.finder.allow_all_prereleases or pre, cache=DependencyCache(cache_dir), clear_caches=rebuild, allow_unsafe=allow_unsafe, ) results = resolver.resolve(max_rounds=max_rounds) if generate_hashes: hashes = resolver.resolve_hashes(results) else: hashes = None except PipToolsError as e: log.error(str(e)) sys.exit(2) log.debug("") ## # Output ## writer = OutputWriter( output_file, click_ctx=ctx, dry_run=dry_run, emit_header=header, emit_index_url=emit_index_url, emit_trusted_host=emit_trusted_host, annotate=annotate, generate_hashes=generate_hashes, default_index_url=repository.DEFAULT_INDEX_URL, index_urls=repository.finder.index_urls, trusted_hosts=repository.finder.trusted_hosts, format_control=repository.finder.format_control, allow_unsafe=allow_unsafe, find_links=repository.finder.find_links, emit_find_links=emit_find_links, ) writer.write( results=results, unsafe_requirements=resolver.unsafe_constraints, markers={ key_from_ireq(ireq): ireq.markers for ireq in constraints if ireq.markers }, hashes=hashes, ) if dry_run: log.info("Dry-run, so nothing updated.")
def test_redact_auth_from_url(auth_url, expected_url): url = redact_auth_from_url(auth_url) assert url == expected_url
<<<<<<< HEAD if self.index_urls and self.index_urls != [PyPI.simple_url]: lines.append( 'Looking in indexes: {}'.format(', '.join( redact_password_from_url(url) for url in self.index_urls)) ) if self.find_links: lines.append( 'Looking in links: {}'.format(', '.join( redact_password_from_url(url) for url in self.find_links)) ======= redacted_index_urls = [] if self.index_urls and self.index_urls != [PyPI.simple_url]: for url in self.index_urls: redacted_index_url = redact_auth_from_url(url) # Parse the URL purl = urllib_parse.urlsplit(redacted_index_url) # URL is generally invalid if scheme and netloc is missing # there are issues with Python and URL parsing, so this test # is a bit crude. See bpo-20271, bpo-23505. Python doesn't # always parse invalid URLs correctly - it should raise # exceptions for malformed URLs if not purl.scheme and not purl.netloc: logger.warning( 'The index url "{}" seems invalid, ' 'please provide a scheme.'.format(redacted_index_url)) redacted_index_urls.append(redacted_index_url)
def __str__(self): return redact_auth_from_url(self.url)
def _get_simple_response(url: str, session: PipSession) -> Response: """Access an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. """ if is_archive_file(Link(url).filename): _ensure_api_response(url, session=session) logger.debug("Getting page %s", redact_auth_from_url(url)) resp = session.get( url, headers={ "Accept": ", ".join([ "application/vnd.pypi.simple.v1+json", "application/vnd.pypi.simple.v1+html; q=0.1", "text/html; q=0.01", ]), # We don't want to blindly returned cached data for # /simple/, because authors generally expecting that # twine upload && pip install will function, but if # they've done a pip install in the last ~10 minutes # it won't. Thus by setting this to zero we will not # blindly use any cached data, however the benefit of # using max-age=0 instead of no-cache, is that we will # still support conditional requests, so we will still # minimize traffic sent in cases where the page hasn't # changed at all, we will just always incur the round # trip for the conditional GET now instead of only # once per 10 minutes. # For more information, please see pypa/pip#5670. "Cache-Control": "max-age=0", }, ) raise_for_status(resp) # The check for archives above only works if the url ends with # something that looks like an archive. However that is not a # requirement of an url. Unless we issue a HEAD request on every # url we cannot know ahead of time for sure if something is a # Simple API response or not. However we can check after we've # downloaded it. _ensure_api_header(resp) logger.debug( "Fetched page %s as %s", redact_auth_from_url(url), resp.headers.get("Content-Type", "Unknown"), ) return resp
"-r", "--rebuild", is_flag=True, help="Clear any caches upfront, rebuild from scratch", ) @click.option( "-f", "--find-links", multiple=True, help="Look for archives in this directory or on this HTML page", ) @click.option( "-i", "--index-url", help="Change index URL (defaults to {index_url})".format( index_url=redact_auth_from_url(_get_default_option("index_url"))), ) @click.option("--extra-index-url", multiple=True, help="Add additional index URL to search") @click.option("--cert", help="Path to alternate CA bundle.") @click.option( "--client-cert", help="Path to SSL client certificate, a single file containing " "the private key and the certificate in PEM format.", ) @click.option( "--trusted-host", multiple=True, help="Mark this host as trusted, even though it does not have " "valid or any HTTPS.",
'The index url "%s" seems invalid, ' 'please provide a scheme.', redacted_index_url) ======= 'The index url "{}" seems invalid, ' 'please provide a scheme.'.format(redacted_index_url)) >>>>>>> b66a76afa15ab74019740676a52a071b85ed8f71 redacted_index_urls.append(redacted_index_url) lines.append('Looking in indexes: {}'.format( ', '.join(redacted_index_urls))) if self.find_links: lines.append( 'Looking in links: {}'.format(', '.join( redact_auth_from_url(url) for url in self.find_links)) ) return '\n'.join(lines) def get_index_urls_locations(self, project_name): # type: (str) -> List[str] """Returns the locations found via self.index_urls Checks the url_name on the main (first in the list) index and use this url_name to produce all locations """ def mkurl_pypi_url(url): # type: (str) -> str loc = posixpath.join( url,
def _prepare_download( resp, # type: Response link, # type: Link progress_bar # type: str ): # type: (...) -> Iterable[bytes] total_length = _get_http_response_size(resp) if link.netloc == PyPI.file_storage_domain: url = link.show_url else: url = link.url_without_fragment logged_url = redact_auth_from_url(url) if total_length: logged_url = '{} ({})'.format(logged_url, format_size(total_length)) if is_from_cache(resp): logger.info("Using cached %s", logged_url) else: logger.info("Downloading %s", logged_url) if logger.getEffectiveLevel() > logging.INFO: show_progress = False elif is_from_cache(resp): show_progress = False elif not total_length: show_progress = True