def populate_requirement_set(requirement_set, args, options, finder, session, name, wheel_cache): """ Marshal cmd line args into a requirement set. """ # NOTE: As a side-effect, options.require_hashes and # requirement_set.require_hashes may be updated for filename in options.constraints: for req_to_add in parse_requirements(filename, constraint=True, finder=finder, options=options, session=session, wheel_cache=wheel_cache): req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for req in args: req_to_add = install_req_from_line(req, None, isolated=options.isolated_mode, wheel_cache=wheel_cache) req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for req in options.editables: req_to_add = install_req_from_editable( req, isolated=options.isolated_mode, wheel_cache=wheel_cache) req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for filename in options.requirements: for req_to_add in parse_requirements(filename, finder=finder, options=options, session=session, wheel_cache=wheel_cache): req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) # If --require-hashes was a line in a requirements file, tell # RequirementSet about it: requirement_set.require_hashes = options.require_hashes if not (args or options.editables or options.requirements): opts = {'name': name} if options.find_links: raise CommandError( 'You must give at least one requirement to %(name)s ' '(maybe you meant "pip %(name)s %(links)s"?)' % dict(opts, links=' '.join(options.find_links))) else: raise CommandError( 'You must give at least one requirement to %(name)s ' '(see "pip help %(name)s")' % opts)
def make_install_req_from_editable( link: Link, template: InstallRequirement) -> InstallRequirement: assert template.editable, "template not editable" return install_req_from_editable( link.url, user_supplied=template.user_supplied, comes_from=template.comes_from, use_pep517=template.use_pep517, isolated=template.isolated, constraint=template.constraint, options=dict( install_options=template.install_options, global_options=template.global_options, hashes=template.hash_options, ), )
def populate_requirement_set( self, requirement_set, # type: RequirementSet args, # type: List[str] options, # type: Values finder, # type: PackageFinder session, # type: PipSession wheel_cache, # type: Optional[WheelCache] ): # type: (...) -> None """ Marshal cmd line args into a requirement set. """ for filename in options.constraints: for req_to_add in parse_requirements(filename, constraint=True, finder=finder, options=options, session=session, wheel_cache=wheel_cache): req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for req in args: req_to_add = install_req_from_line(req, None, isolated=options.isolated_mode, use_pep517=options.use_pep517, wheel_cache=wheel_cache) req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) for req in options.editables: req_to_add = install_req_from_editable( req, isolated=options.isolated_mode, use_pep517=options.use_pep517, wheel_cache=wheel_cache) req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) # NOTE: options.require_hashes may be set if --require-hashes is True for filename in options.requirements: for req_to_add in parse_requirements( filename, finder=finder, options=options, session=session, wheel_cache=wheel_cache, use_pep517=options.use_pep517): req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) # If any requirement has hash options, enable hash checking. requirements = (requirement_set.unnamed_requirements + list(requirement_set.requirements.values())) if any(req.has_hash_options for req in requirements): options.require_hashes = True if not (args or options.editables or options.requirements): opts = {'name': self.name} if options.find_links: raise CommandError( 'You must give at least one requirement to %(name)s ' '(maybe you meant "pip %(name)s %(links)s"?)' % dict(opts, links=' '.join(options.find_links))) else: raise CommandError( 'You must give at least one requirement to %(name)s ' '(see "pip help %(name)s")' % opts)
def process_line( line, # type: Text filename, # type: str line_number, # type: int finder=None, # type: Optional[PackageFinder] comes_from=None, # type: Optional[str] options=None, # type: Optional[optparse.Values] session=None, # type: Optional[PipSession] wheel_cache=None, # type: Optional[WheelCache] use_pep517=None, # type: Optional[bool] constraint=False # type: bool ): # type: (...) -> Iterator[InstallRequirement] """Process a single requirements line; This can result in creating/yielding requirements, or updating the finder. For lines that contain requirements, the only options that have an effect are from SUPPORTED_OPTIONS_REQ, and they are scoped to the requirement. Other options from SUPPORTED_OPTIONS may be present, but are ignored. For lines that do not contain requirements, the only options that have an effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may be present, but are ignored. These lines may contain multiple options (although our docs imply only one is supported), and all our parsed and affect the finder. :param constraint: If True, parsing a constraints file. :param options: OptionParser options that we may update """ parser = build_parser(line) defaults = parser.get_default_values() defaults.index_url = None if finder: defaults.format_control = finder.format_control args_str, options_str = break_args_options(line) # Prior to 2.7.3, shlex cannot deal with unicode entries if sys.version_info < (2, 7, 3): # https://github.com/python/mypy/issues/1174 options_str = options_str.encode('utf8') # type: ignore # https://github.com/python/mypy/issues/1174 opts, _ = parser.parse_args( shlex.split(options_str), defaults) # type: ignore # preserve for the nested code path line_comes_from = '%s %s (line %s)' % ( '-c' if constraint else '-r', filename, line_number, ) # yield a line requirement if args_str: isolated = options.isolated_mode if options else False if options: cmdoptions.check_install_build_global(options, opts) # get the options that apply to requirements req_options = {} for dest in SUPPORTED_OPTIONS_REQ_DEST: if dest in opts.__dict__ and opts.__dict__[dest]: req_options[dest] = opts.__dict__[dest] yield install_req_from_line( args_str, line_comes_from, constraint=constraint, use_pep517=use_pep517, isolated=isolated, options=req_options, wheel_cache=wheel_cache ) # yield an editable requirement elif opts.editables: isolated = options.isolated_mode if options else False yield install_req_from_editable( opts.editables[0], comes_from=line_comes_from, use_pep517=use_pep517, constraint=constraint, isolated=isolated, wheel_cache=wheel_cache ) # parse a nested requirements file elif opts.requirements or opts.constraints: if opts.requirements: req_path = opts.requirements[0] nested_constraint = False else: req_path = opts.constraints[0] nested_constraint = True # original file is over http if SCHEME_RE.search(filename): # do a url join so relative paths work req_path = urllib_parse.urljoin(filename, req_path) # original file and nested file are paths elif not SCHEME_RE.search(req_path): # do a join so relative paths work req_path = os.path.join(os.path.dirname(filename), req_path) # TODO: Why not use `comes_from='-r {} (line {})'` here as well? parsed_reqs = parse_requirements( req_path, finder, comes_from, options, session, constraint=nested_constraint, wheel_cache=wheel_cache ) for req in parsed_reqs: yield req # percolate hash-checking option upward elif opts.require_hashes: options.require_hashes = opts.require_hashes # set finder options elif finder: if opts.index_url: finder.index_urls = [opts.index_url] if opts.no_index is True: finder.index_urls = [] if opts.extra_index_urls: finder.index_urls.extend(opts.extra_index_urls) if opts.find_links: # FIXME: it would be nice to keep track of the source # of the find_links: support a find-links local path # relative to a requirements file. value = opts.find_links[0] req_dir = os.path.dirname(os.path.abspath(filename)) relative_to_reqs_file = os.path.join(req_dir, value) if os.path.exists(relative_to_reqs_file): value = relative_to_reqs_file finder.find_links.append(value) if opts.pre: finder.allow_all_prereleases = True if opts.trusted_hosts: finder.secure_origins.extend( ("*", host, "*") for host in opts.trusted_hosts)
def freeze( requirement=None, # type: Optional[List[str]] find_links=None, # type: Optional[List[str]] local_only=None, # type: Optional[bool] user_only=None, # type: Optional[bool] paths=None, # type: Optional[List[str]] skip_regex=None, # type: Optional[str] isolated=False, # type: bool wheel_cache=None, # type: Optional[WheelCache] exclude_editable=False, # type: bool skip=() # type: Container[str] ): # type: (...) -> Iterator[str] find_links = find_links or [] skip_match = None if skip_regex: skip_match = re.compile(skip_regex).search for link in find_links: yield '-f %s' % link installations = {} # type: Dict[str, FrozenRequirement] for dist in get_installed_distributions(local_only=local_only, skip=(), user_only=user_only, paths=paths): try: req = FrozenRequirement.from_dist(dist) except RequirementParseError as exc: # We include dist rather than dist.project_name because the # dist string includes more information, like the version and # location. We also include the exception message to aid # troubleshooting. logger.warning( 'Could not generate requirement for distribution %r: %s', dist, exc) continue if exclude_editable and req.editable: continue installations[req.canonical_name] = req if requirement: # the options that don't get turned into an InstallRequirement # should only be emitted once, even if the same option is in multiple # requirements files, so we need to keep track of what has been emitted # so that we don't emit it again if it's seen again emitted_options = set() # type: Set[str] # keep track of which files a requirement is in so that we can # give an accurate warning if a requirement appears multiple times. req_files = collections.defaultdict(list) # type: Dict[str, List[str]] for req_file_path in requirement: with open(req_file_path) as req_file: for line in req_file: if (not line.strip() or line.strip().startswith('#') or (skip_match and skip_match(line)) or line.startswith( ('-r', '--requirement', '-Z', '--always-unzip', '-f', '--find-links', '-i', '--index-url', '--pre', '--trusted-host', '--process-dependency-links', '--extra-index-url'))): line = line.rstrip() if line not in emitted_options: emitted_options.add(line) yield line continue if line.startswith('-e') or line.startswith('--editable'): if line.startswith('-e'): line = line[2:].strip() else: line = line[len('--editable'):].strip().lstrip('=') line_req = install_req_from_editable( line, isolated=isolated, wheel_cache=wheel_cache, ) else: line_req = install_req_from_line( COMMENT_RE.sub('', line).strip(), isolated=isolated, wheel_cache=wheel_cache, ) if not line_req.name: logger.info( "Skipping line in requirement file [%s] because " "it's not clear what it would install: %s", req_file_path, line.strip(), ) logger.info( " (add #egg=PackageName to the URL to avoid" " this warning)") else: line_req_canonical_name = canonicalize_name( line_req.name) if line_req_canonical_name not in installations: # either it's not installed, or it is installed # but has been processed already if not req_files[line_req.name]: logger.warning( "Requirement file [%s] contains %s, but " "package %r is not installed", req_file_path, COMMENT_RE.sub('', line).strip(), line_req.name) else: req_files[line_req.name].append(req_file_path) else: yield str(installations[line_req_canonical_name] ).rstrip() del installations[line_req_canonical_name] req_files[line_req.name].append(req_file_path) # Warn about requirements that were included multiple times (in a # single requirements file or in different requirements files). for name, files in six.iteritems(req_files): if len(files) > 1: logger.warning("Requirement %s included multiple times [%s]", name, ', '.join(sorted(set(files)))) yield ('## The following requirements were added by ' 'pip freeze:') for installation in sorted(installations.values(), key=lambda x: x.name.lower()): if installation.canonical_name not in skip: yield str(installation).rstrip()
def freeze( requirement: Optional[List[str]] = None, local_only: bool = False, user_only: bool = False, paths: Optional[List[str]] = None, isolated: bool = False, exclude_editable: bool = False, skip: Container[str] = (), ) -> Iterator[str]: installations: Dict[str, FrozenRequirement] = {} dists = get_environment(paths).iter_installed_distributions( local_only=local_only, skip=(), user_only=user_only, ) for dist in dists: req = FrozenRequirement.from_dist(dist) if exclude_editable and req.editable: continue installations[req.canonical_name] = req if requirement: # the options that don't get turned into an InstallRequirement # should only be emitted once, even if the same option is in multiple # requirements files, so we need to keep track of what has been emitted # so that we don't emit it again if it's seen again emitted_options: Set[str] = set() # keep track of which files a requirement is in so that we can # give an accurate warning if a requirement appears multiple times. req_files: Dict[str, List[str]] = collections.defaultdict(list) for req_file_path in requirement: with open(req_file_path) as req_file: for line in req_file: if (not line.strip() or line.strip().startswith("#") or line.startswith(( "-r", "--requirement", "-f", "--find-links", "-i", "--index-url", "--pre", "--trusted-host", "--process-dependency-links", "--extra-index-url", "--use-feature", ))): line = line.rstrip() if line not in emitted_options: emitted_options.add(line) yield line continue if line.startswith("-e") or line.startswith("--editable"): if line.startswith("-e"): line = line[2:].strip() else: line = line[len("--editable"):].strip().lstrip("=") line_req = install_req_from_editable( line, isolated=isolated, ) else: line_req = install_req_from_line( COMMENT_RE.sub("", line).strip(), isolated=isolated, ) if not line_req.name: logger.info( "Skipping line in requirement file [%s] because " "it's not clear what it would install: %s", req_file_path, line.strip(), ) logger.info( " (add #egg=PackageName to the URL to avoid" " this warning)") else: line_req_canonical_name = canonicalize_name( line_req.name) if line_req_canonical_name not in installations: # either it's not installed, or it is installed # but has been processed already if not req_files[line_req.name]: logger.warning( "Requirement file [%s] contains %s, but " "package %r is not installed", req_file_path, COMMENT_RE.sub("", line).strip(), line_req.name, ) else: req_files[line_req.name].append(req_file_path) else: yield str(installations[line_req_canonical_name] ).rstrip() del installations[line_req_canonical_name] req_files[line_req.name].append(req_file_path) # Warn about requirements that were included multiple times (in a # single requirements file or in different requirements files). for name, files in req_files.items(): if len(files) > 1: logger.warning( "Requirement %s included multiple times [%s]", name, ", ".join(sorted(set(files))), ) yield ("## The following requirements were added by pip freeze:") for installation in sorted(installations.values(), key=lambda x: x.name.lower()): if installation.canonical_name not in skip: yield str(installation).rstrip()
def handle_line( line, # type: ParsedLine finder=None, # type: Optional[PackageFinder] options=None, # type: Optional[optparse.Values] session=None, # type: Optional[PipSession] wheel_cache=None, # type: Optional[WheelCache] use_pep517=None, # type: Optional[bool] ): # type: (...) -> Optional[InstallRequirement] """Handle a single parsed requirements line; This can result in creating/yielding requirements, or updating the finder. For lines that contain requirements, the only options that have an effect are from SUPPORTED_OPTIONS_REQ, and they are scoped to the requirement. Other options from SUPPORTED_OPTIONS may be present, but are ignored. For lines that do not contain requirements, the only options that have an effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may be present, but are ignored. These lines may contain multiple options (although our docs imply only one is supported), and all our parsed and affect the finder. """ # preserve for the nested code path line_comes_from = '%s %s (line %s)' % ( '-c' if line.constraint else '-r', line.filename, line.lineno, ) # return a line requirement if line.args: isolated = options.isolated_mode if options else False if options: cmdoptions.check_install_build_global(options, line.opts) # get the options that apply to requirements req_options = {} for dest in SUPPORTED_OPTIONS_REQ_DEST: if dest in line.opts.__dict__ and line.opts.__dict__[dest]: req_options[dest] = line.opts.__dict__[dest] line_source = 'line {} of {}'.format(line.lineno, line.filename) return install_req_from_line( line.args, comes_from=line_comes_from, use_pep517=use_pep517, isolated=isolated, options=req_options, wheel_cache=wheel_cache, constraint=line.constraint, line_source=line_source, ) # return an editable requirement elif line.opts.editables: isolated = options.isolated_mode if options else False return install_req_from_editable(line.opts.editables[0], comes_from=line_comes_from, use_pep517=use_pep517, constraint=line.constraint, isolated=isolated, wheel_cache=wheel_cache) # percolate hash-checking option upward elif line.opts.require_hashes: options.require_hashes = line.opts.require_hashes # set finder options elif finder: find_links = finder.find_links index_urls = finder.index_urls if line.opts.index_url: index_urls = [line.opts.index_url] if line.opts.no_index is True: index_urls = [] if line.opts.extra_index_urls: index_urls.extend(line.opts.extra_index_urls) if line.opts.find_links: # FIXME: it would be nice to keep track of the source # of the find_links: support a find-links local path # relative to a requirements file. value = line.opts.find_links[0] req_dir = os.path.dirname(os.path.abspath(line.filename)) relative_to_reqs_file = os.path.join(req_dir, value) if os.path.exists(relative_to_reqs_file): value = relative_to_reqs_file find_links.append(value) search_scope = SearchScope( find_links=find_links, index_urls=index_urls, ) finder.search_scope = search_scope if line.opts.pre: finder.set_allow_all_prereleases() if session: for host in line.opts.trusted_hosts or []: source = 'line {} of {}'.format(line.lineno, line.filename) session.add_trusted_host(host, source=source) return None
def freeze(requirement=None, find_links=None, local_only=None, user_only=None, skip_regex=None, isolated=False, wheel_cache=None, exclude_editable=False, skip=()): find_links = find_links or [] skip_match = None if skip_regex: skip_match = re.compile(skip_regex).search dependency_links = [] for dist in pkg_resources.working_set: if dist.has_metadata('dependency_links.txt'): dependency_links.extend( dist.get_metadata_lines('dependency_links.txt')) for link in find_links: if '#egg=' in link: dependency_links.append(link) for link in find_links: yield '-f %s' % link installations = {} for dist in get_installed_distributions(local_only=local_only, skip=(), user_only=user_only): try: req = FrozenRequirement.from_dist(dist, dependency_links) except RequirementParseError: logger.warning("Could not parse requirement: %s", dist.project_name) continue if exclude_editable and req.editable: continue installations[req.name] = req if requirement: # the options that don't get turned into an InstallRequirement # should only be emitted once, even if the same option is in multiple # requirements files, so we need to keep track of what has been emitted # so that we don't emit it again if it's seen again emitted_options = set() # keep track of which files a requirement is in so that we can # give an accurate warning if a requirement appears multiple times. req_files = collections.defaultdict(list) for req_file_path in requirement: with open(req_file_path) as req_file: for line in req_file: if (not line.strip() or line.strip().startswith('#') or (skip_match and skip_match(line)) or line.startswith( ('-r', '--requirement', '-Z', '--always-unzip', '-f', '--find-links', '-i', '--index-url', '--pre', '--trusted-host', '--process-dependency-links', '--extra-index-url'))): line = line.rstrip() if line not in emitted_options: emitted_options.add(line) yield line continue if line.startswith('-e') or line.startswith('--editable'): if line.startswith('-e'): line = line[2:].strip() else: line = line[len('--editable'):].strip().lstrip('=') line_req = install_req_from_editable( line, isolated=isolated, wheel_cache=wheel_cache, ) else: line_req = install_req_from_line( COMMENT_RE.sub('', line).strip(), isolated=isolated, wheel_cache=wheel_cache, ) if not line_req.name: logger.info( "Skipping line in requirement file [%s] because " "it's not clear what it would install: %s", req_file_path, line.strip(), ) logger.info( " (add #egg=PackageName to the URL to avoid" " this warning)") elif line_req.name not in installations: # either it's not installed, or it is installed # but has been processed already if not req_files[line_req.name]: logger.warning( "Requirement file [%s] contains %s, but that " "package is not installed", req_file_path, COMMENT_RE.sub('', line).strip(), ) else: req_files[line_req.name].append(req_file_path) else: yield str(installations[line_req.name]).rstrip() del installations[line_req.name] req_files[line_req.name].append(req_file_path) # Warn about requirements that were included multiple times (in a # single requirements file or in different requirements files). for name, files in six.iteritems(req_files): if len(files) > 1: logger.warning("Requirement %s included multiple times [%s]", name, ', '.join(sorted(set(files)))) yield ('## The following requirements were added by ' 'pip freeze:') for installation in sorted(installations.values(), key=lambda x: x.name.lower()): if canonicalize_name(installation.name) not in skip: yield str(installation).rstrip()
def get_requirements( self, args: List[str], options: Values, finder: PackageFinder, session: PipSession, ) -> List[InstallRequirement]: """ Parse command-line arguments into the corresponding requirements. """ requirements: List[InstallRequirement] = [] for filename in options.constraints: for parsed_req in parse_requirements( filename, constraint=True, finder=finder, options=options, session=session, ): req_to_add = install_req_from_parsed_requirement( parsed_req, isolated=options.isolated_mode, user_supplied=False, ) requirements.append(req_to_add) for req in args: req_to_add = install_req_from_line( req, None, isolated=options.isolated_mode, use_pep517=options.use_pep517, user_supplied=True, ) requirements.append(req_to_add) for req in options.editables: req_to_add = install_req_from_editable( req, user_supplied=True, isolated=options.isolated_mode, use_pep517=options.use_pep517, ) requirements.append(req_to_add) # NOTE: options.require_hashes may be set if --require-hashes is True for filename in options.requirements: for parsed_req in parse_requirements(filename, finder=finder, options=options, session=session): req_to_add = install_req_from_parsed_requirement( parsed_req, isolated=options.isolated_mode, use_pep517=options.use_pep517, user_supplied=True, ) requirements.append(req_to_add) # If any requirement has hash options, enable hash checking. if any(req.has_hash_options for req in requirements): options.require_hashes = True if not (args or options.editables or options.requirements): opts = {"name": self.name} if options.find_links: raise CommandError( "You must give at least one requirement to {name} " '(maybe you meant "pip {name} {links}"?)'.format( **dict(opts, links=" ".join(options.find_links)))) else: raise CommandError( "You must give at least one requirement to {name} " '(see "pip help {name}")'.format(**opts)) return requirements