def _add_requires(entry, base, extras): """Append all requirements in an entry to the list. This combines the `environment` key's content into the requirement's existing markers, and add them to the appropriate list(s). """ requires = entry.get('requires') if not requires: return environment = entry.get('environment') e_extra = entry.get('extra') for s in requires: r, r_extra = Requirement.parse(s) if environment: if r.marker: m = Marker('({}) and ({})'.format(environment, r.marker)) else: m = Marker(environment) r.marker = m if not e_extra and not r_extra: base.add(r) elif e_extra: extras[e_extra].add(r) elif r_extra: extras[r_extra].add(r)
def split_marker_extras(marker: str) -> Tuple[Set[str], str]: """An element can be stripped from the marker only if all parts are connected with `and` operator. The rest part are returned as a string or `None` if all are stripped. """ def extract_extras(submarker: Union[tuple, list]) -> Set[str]: if isinstance(submarker, tuple): if submarker[0].value == "extra": if submarker[1].value == "==": return {submarker[2].value} elif submarker[1].value == "in": return {v.strip() for v in submarker[2].value.split(",")} else: return set() else: return set() else: if "and" in submarker: return set() pure_extras = [extract_extras(m) for m in submarker if m != "or"] if all(pure_extras): return set(itertools.chain.from_iterable(pure_extras)) return set() if not marker: return set(), marker new_marker = PackageMarker(marker) submarkers = PackageMarker(marker)._markers if "or" in submarkers: extras = extract_extras(submarkers) if extras: return extras, "" return set(), marker extras = set() submarkers_no_extras: List[Union[tuple, list]] = [] # Below this point the submarkers are connected with 'and' for submarker in submarkers: if submarker == "and": continue new_extras = extract_extras(submarker) if new_extras: if extras: # extras are not allowed to appear in more than one parts return set(), marker extras.update(new_extras) else: submarkers_no_extras.append(submarker) if not submarkers_no_extras: return extras, "" new_marker._markers = join_list_with(submarkers_no_extras, "and") return extras, str(new_marker)
def _upraw(self, pp, extra="", version_req="", depth=20, path=[]): """build a nested list of user packages with given extra and depth""" envi = {"extra": extra, **self.environment} p = normalize(pp) ret_all = [] if p in path: print("cycle!", "->".join(path + [p])) elif p in self.distro and len(path) <= depth: if extra == "": ret_all = [f'{p}=={self.distro[p]["version"]} {version_req}'] else: ret_all = [ f'{p}[{extra}]=={self.distro[p]["version"]} {version_req}' ] ret = [] for r in self.distro[p]["wanted_per"]: if r["req_key"] in self.distro and r["req_key"] not in path: if "req_marker" not in r or Marker( r["req_marker"]).evaluate(environment=envi): ret += self._upraw( r["req_key"], "", f"[requires: {p}" + ("[" + r["req_extra"] + "]" if r["req_extra"] != "" else "") + f'{r["req_version"]}]', depth, path + [p], ) if not ret == []: ret_all += [ret] return ret_all
def _downraw(self, pp, extra="", version_req="", depth=20, path=[]): """build a nested list of needed packages with given extra and depth""" envi = {"extra": extra, **self.environment} p = normalize(pp) ret_all = [] if p in path: print("cycle!", "->".join(path + [p])) elif p in self.distro and len(path) <= depth: if extra == "": ret = [f'{p}=={self.distro[p]["version"]} {version_req}'] else: ret = [ f'{p}[{extra}]=={self.distro[p]["version"]} {version_req}' ] for r in self.distro[p]["requires_dist"]: if r["req_key"] in self.distro: if "req_marker" not in r or Marker( r["req_marker"]).evaluate(environment=envi): ret += self._downraw( r["req_key"], r["req_extra"], r["req_version"], depth, path + [p], ) ret_all += [ret] return ret_all
def split_marker_extras( marker: PackageMarker, ) -> Tuple[Sequence[str], Optional[Marker]]: """An element can be stripped from the marker only if all parts are connected with `and` operater. The rest part are returned as a string or `None` if all are stripped. :param marker: the input marker string :returns: an iterable of (op, value) pairs together with the stripped part. """ if "or" in marker._markers: if "and" in marker._markers or any( not isinstance(p, tuple) or p[0].value != "extra" for p in marker._markers if p != "or" ): return [], marker result = [] bare_markers = [m for m in marker._markers if m not in ("and", "or")] for m in bare_markers[:]: if not isinstance(m, tuple): continue if m[0].value == "extra": if m[1].value == "==": result.append(m[2].value) elif m[1].value == "in": result.extend(v.strip() for v in m[2].value.split(",")) bare_markers.remove(m) new_markers = join_list_with(bare_markers, "and") if not new_markers: return result, None marker._markers = new_markers return result, marker
def test_markers_match(self): # match for markers in ( 'python_version >= "1.0"', 'sys_platform == %r' % sys.platform, ): line = 'name; ' + markers req = install_req_from_line(line, comes_from='') assert str(req.markers) == str(Marker(markers)) assert req.match_markers() # don't match for markers in ( 'python_version >= "5.0"', 'sys_platform != %r' % sys.platform, ): line = 'name; ' + markers req = install_req_from_line(line, comes_from='') assert str(req.markers) == str(Marker(markers)) assert not req.match_markers()
def test_markers_match_from_line(self): # match for markers in ( 'python_version >= "1.0"', 'sys_platform == {sys.platform!r}'.format(**globals()), ): line = 'name; ' + markers req = install_req_from_line(line) assert str(req.markers) == str(Marker(markers)) assert req.match_markers() # don't match for markers in ( 'python_version >= "5.0"', 'sys_platform != {sys.platform!r}'.format(**globals()), ): line = 'name; ' + markers req = install_req_from_line(line) assert str(req.markers) == str(Marker(markers)) assert not req.match_markers()
def test_markers_match_from_line(self): # match for markers in ( 'python_version >= "1.0"', 'sys_platform == %r' % sys.platform, ): line = 'name; ' + markers req = InstallRequirement.from_line(line) assert str(req.markers) == str(Marker(markers)) assert req.match_markers() # don't match for markers in ( 'python_version >= "5.0"', 'sys_platform != %r' % sys.platform, ): line = 'name; ' + markers req = InstallRequirement.from_line(line) assert str(req.markers) == str(Marker(markers)) assert not req.match_markers()
def test_markers_match(self) -> None: # match for markers in ( 'python_version >= "1.0"', f"sys_platform == {sys.platform!r}", ): line = "name; " + markers req = install_req_from_line(line, comes_from="") assert str(req.markers) == str(Marker(markers)) assert req.match_markers() # don't match for markers in ( 'python_version >= "5.0"', f"sys_platform != {sys.platform!r}", ): line = "name; " + markers req = install_req_from_line(line, comes_from="") assert str(req.markers) == str(Marker(markers)) assert not req.match_markers()
def split_marker_extras( marker: PackageMarker, ) -> Tuple[List[str], Optional[PackageMarker]]: """An element can be stripped from the marker only if all parts are connected with `and` operater. The rest part are returned as a string or `None` if all are stripped. :param marker: the input marker string :returns: an iterable of (op, value) pairs together with the stripped part. """ def extract_extras(submarker: Union[tuple, list]) -> List[str]: if isinstance(submarker, tuple): if submarker[0].value == "extra": if submarker[1].value == "==": return [submarker[2].value] elif submarker[1].value == "in": return [v.strip() for v in submarker[2].value.split(",")] else: return [] else: return [] else: if "and" in submarker: return [] pure_extras = [extract_extras(m) for m in submarker if m != "or"] if all(pure_extras): return list(itertools.chain.from_iterable(pure_extras)) return [] submarkers = marker._markers if "or" in submarkers: extras = extract_extras(submarkers) if extras: return extras, None return [], marker extras = [] submarkers_no_extras: List[Union[tuple, list]] = [] for submarker in submarkers: if submarker == "and": continue new_extras = extract_extras(submarker) if new_extras: if extras: return [], marker extras.extend(new_extras) else: submarkers_no_extras.append(submarker) if not submarkers_no_extras: return extras, None marker._markers = join_list_with(submarkers_no_extras, "and") return extras, marker
def parse_req_from_line(name, line_source): # type: (str, Optional[str]) -> RequirementParts if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: name, markers_as_string = name.split(marker_sep, 1) markers_as_string = markers_as_string.strip() if not markers_as_string: markers = None else: markers = Marker(markers_as_string) else: markers = None name = name.strip() req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None extras_as_string = None if is_url(name): link = Link(name) else: p, extras_as_string = _strip_extras(path) url = _get_url_from_path(p, name) if url is not None: link = Link(url) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == 'file' and re.search(r'\.\./', link.url): link = Link( path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req_as_string = f"{wheel.name}=={wheel.version}" else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req_as_string = link.egg_fragment # a requirement specifier else: req_as_string = name extras = convert_extras(extras_as_string) def with_source(text): # type: (str) -> str if not line_source: return text return f'{text} (from {line_source})' if req_as_string is not None: try: req = Requirement(req_as_string) except InvalidRequirement: if os.path.sep in req_as_string: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req_as_string) elif ('=' in req_as_string and not any(op in req_as_string for op in operators)): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = '' msg = with_source(f'Invalid requirement: {req_as_string!r}') if add_msg: msg += f'\nHint: {add_msg}' raise InstallationError(msg) else: # Deprecate extras after specifiers: "name>=1.0[extras]" # This currently works by accident because _strip_extras() parses # any extras in the end of the string and those are saved in # RequirementParts for spec in req.specifier: spec_str = str(spec) if spec_str.endswith(']'): msg = f"Extras after version '{spec_str}'." raise InstallationError(msg) else: req = None return RequirementParts(req, link, markers, extras)
def from_line( cls, name, comes_from=None, isolated=False, options=None, wheel_cache=None, constraint=False): """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ from pip._internal.index import Link if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: name, markers = name.split(marker_sep, 1) markers = markers.strip() if not markers: markers = None else: markers = Marker(markers) else: markers = None name = name.strip() req = None path = os.path.normpath(os.path.abspath(name)) link = None extras = None if is_url(name): link = Link(name) else: p, extras = _strip_extras(path) looks_like_dir = os.path.isdir(p) and ( os.path.sep in name or (os.path.altsep is not None and os.path.altsep in name) or name.startswith('.') ) if looks_like_dir: if not is_installable_dir(p): raise InstallationError( "Directory %r is not installable. File 'setup.py' " "not found." % name ) link = Link(path_to_url(p)) elif is_archive_file(p): if not os.path.isfile(p): logger.warning( 'Requirement %r looks like a filename, but the ' 'file does not exist', name ) link = Link(path_to_url(p)) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == 'file' and re.search(r'\.\./', link.url): link = Link( path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req = link.egg_fragment # a requirement specifier else: req = name if extras: extras = Requirement("placeholder" + extras.lower()).extras else: extras = () if req is not None: try: req = Requirement(req) except InvalidRequirement: if os.path.sep in req: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req) elif '=' in req and not any(op in req for op in operators): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = traceback.format_exc() raise InstallationError( "Invalid requirement: '%s'\n%s" % (req, add_msg)) return cls( req, comes_from, link=link, markers=markers, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, constraint=constraint, extras=extras, )
def parse_req_from_line(name, line_source): # type: (str, Optional[str]) -> RequirementParts if is_url(name): marker_sep = "; " else: marker_sep = ";" if marker_sep in name: name, markers_as_string = name.split(marker_sep, 1) markers_as_string = markers_as_string.strip() if not markers_as_string: markers = None else: markers = Marker(markers_as_string) else: markers = None name = name.strip() req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None extras_as_string = None if is_url(name): link = Link(name) else: p, extras_as_string = _strip_extras(path) url = _get_url_from_path(p, name) if url is not None: link = Link(url) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == "file" and re.search(r"\.\./", link.url): link = Link(path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req_as_string = "{wheel.name}=={wheel.version}".format(**locals()) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req_as_string = link.egg_fragment # a requirement specifier else: req_as_string = name extras = convert_extras(extras_as_string) def with_source(text): # type: (str) -> str if not line_source: return text return "{} (from {})".format(text, line_source) if req_as_string is not None: try: req = Requirement(req_as_string) except InvalidRequirement: if os.path.sep in req_as_string: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req_as_string) elif "=" in req_as_string and not any( op in req_as_string for op in operators ): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = "" msg = with_source("Invalid requirement: {!r}".format(req_as_string)) if add_msg: msg += "\nHint: {}".format(add_msg) raise InstallationError(msg) else: # Deprecate extras after specifiers: "name>=1.0[extras]" # This currently works by accident because _strip_extras() parses # any extras in the end of the string and those are saved in # RequirementParts for spec in req.specifier: spec_str = str(spec) if spec_str.endswith("]"): msg = "Extras after version '{}'.".format(spec_str) replace = "moving the extras before version specifiers" deprecated(msg, replacement=replace, gone_in="21.0") else: req = None return RequirementParts(req, link, markers, extras)
def install_req_from_line( name, # type: str comes_from=None, # type: Optional[Union[str, InstallRequirement]] use_pep517=None, # type: Optional[bool] isolated=False, # type: bool options=None, # type: Optional[Dict[str, Any]] wheel_cache=None, # type: Optional[WheelCache] constraint=False, # type: bool line_source=None, # type: Optional[str] ): # type: (...) -> InstallRequirement """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. :param line_source: An optional string describing where the line is from, for logging purposes in case of an error. """ if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: name, markers_as_string = name.split(marker_sep, 1) markers_as_string = markers_as_string.strip() if not markers_as_string: markers = None else: markers = Marker(markers_as_string) else: markers = None name = name.strip() req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None extras_as_string = None if is_url(name): link = Link(name) else: p, extras_as_string = _strip_extras(path) looks_like_dir = os.path.isdir(p) and (os.path.sep in name or (os.path.altsep is not None and os.path.altsep in name) or name.startswith('.')) if looks_like_dir: if not is_installable_dir(p): raise InstallationError( "Directory %r is not installable. Neither 'setup.py' " "nor 'pyproject.toml' found." % name) link = Link(path_to_url(p)) elif is_archive_file(p): if not os.path.isfile(p): logger.warning( 'Requirement %r looks like a filename, but the ' 'file does not exist', name) link = Link(path_to_url(p)) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == 'file' and re.search(r'\.\./', link.url): link = Link( path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req_as_string = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req_as_string = link.egg_fragment # a requirement specifier else: req_as_string = name if extras_as_string: extras = Requirement("placeholder" + extras_as_string.lower()).extras else: extras = () if req_as_string is not None: try: req = Requirement(req_as_string) except InvalidRequirement: if os.path.sep in req_as_string: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req_as_string) elif ('=' in req_as_string and not any(op in req_as_string for op in operators)): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = '' if line_source is None: source = '' else: source = ' (from {})'.format(line_source) msg = ('Invalid requirement: {!r}{}'.format(req_as_string, source)) if add_msg: msg += '\nHint: {}'.format(add_msg) raise InstallationError(msg) else: req = None return InstallRequirement( req, comes_from, link=link, markers=markers, use_pep517=use_pep517, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, constraint=constraint, extras=extras, )
def parse_req_from_line(name, line_source): # type: (str, Optional[str]) -> RequirementParts if is_url(name): marker_sep = "; " else: marker_sep = ";" if marker_sep in name: name, markers_as_string = name.split(marker_sep, 1) markers_as_string = markers_as_string.strip() if not markers_as_string: markers = None else: markers = Marker(markers_as_string) else: markers = None name = name.strip() req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None extras_as_string = None if is_url(name): link = Link(name) else: p, extras_as_string = _strip_extras(path) url = _get_url_from_path(p, name) if url is not None: link = Link(url) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == "file" and re.search(r"\.\./", link.url): link = Link( path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req_as_string = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req_as_string = link.egg_fragment # a requirement specifier else: req_as_string = name extras = convert_extras(extras_as_string) def with_source(text): if not line_source: return text return "{} (from {})".format(text, line_source) if req_as_string is not None: try: req = Requirement(req_as_string) except InvalidRequirement: if os.path.sep in req_as_string: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req_as_string) elif "=" in req_as_string and not any(op in req_as_string for op in operators): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = "" msg = with_source( "Invalid requirement: {!r}".format(req_as_string)) if add_msg: msg += "\nHint: {}".format(add_msg) raise InstallationError(msg) else: req = None return RequirementParts(req, link, markers, extras)
def from_line(cls, name, comes_from=None, isolated=False, options=None, wheel_cache=None, constraint=False): """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ from pip.index import Link if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: name, markers = name.split(marker_sep, 1) markers = markers.strip() if not markers: markers = None else: markers = Marker(markers) else: markers = None name = name.strip() req = None path = os.path.normpath(os.path.abspath(name)) link = None extras = None if is_url(name): link = Link(name) else: p, extras = _strip_extras(path) if (os.path.isdir(p) and (os.path.sep in name or name.startswith('.'))): if not is_installable_dir(p): raise InstallationError( "Directory %r is not installable. File 'setup.py' " "not found." % name) link = Link(path_to_url(p)) elif is_archive_file(p): if not os.path.isfile(p): logger.warning( 'Requirement %r looks like a filename, but the ' 'file does not exist', name) link = Link(path_to_url(p)) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == 'file' and re.search(r'\.\./', link.url): link = Link( path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req = link.egg_fragment # a requirement specifier else: req = name options = options if options else {} res = cls(req, comes_from, link=link, markers=markers, isolated=isolated, options=options, wheel_cache=wheel_cache, constraint=constraint) if extras: res.extras = _safe_extras( Requirement('placeholder' + extras).extras) return res
def install_req_from_line( name, # type: str comes_from=None, # type: Optional[Union[str, InstallRequirement]] use_pep517=None, # type: Optional[bool] isolated=False, # type: bool options=None, # type: Optional[Dict[str, Any]] wheel_cache=None, # type: Optional[WheelCache] constraint=False, # type: bool line_source=None, # type: Optional[str] ): # type: (...) -> InstallRequirement """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. :param line_source: An optional string describing where the line is from, for logging purposes in case of an error. """ if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: name, markers_as_string = name.split(marker_sep, 1) markers_as_string = markers_as_string.strip() if not markers_as_string: markers = None else: markers = Marker(markers_as_string) else: markers = None name = name.strip() req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None extras_as_string = None if is_url(name): link = Link(name) else: p, extras_as_string = _strip_extras(path) url = _get_url_from_path(p, name) if url is not None: link = Link(url) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == 'file' and re.search(r'\.\./', link.url): link = Link( path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req_as_string = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req_as_string = link.egg_fragment # a requirement specifier else: req_as_string = name extras = convert_extras(extras_as_string) def with_source(text): if not line_source: return text return '{} (from {})'.format(text, line_source) if req_as_string is not None: try: req = Requirement(req_as_string) except InvalidRequirement: if os.path.sep in req_as_string: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req_as_string) elif ('=' in req_as_string and not any(op in req_as_string for op in operators)): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = '' msg = with_source( 'Invalid requirement: {!r}'.format(req_as_string)) if add_msg: msg += '\nHint: {}'.format(add_msg) raise InstallationError(msg) else: req = None return InstallRequirement( req, comes_from, link=link, markers=markers, use_pep517=use_pep517, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, constraint=constraint, extras=extras, )
def match_markers(self): if self.markers is not None: return Marker(self.markers).evaluate() else: return True
# type: (...) -> InstallRequirement """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ >>>>>>> 71358189c5e72ee2ac9883b408a2f540a7f5745e if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: name, markers_as_string = name.split(marker_sep, 1) markers_as_string = markers_as_string.strip() if not markers_as_string: markers = None else: markers = Marker(markers_as_string) else: markers = None name = name.strip() req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None extras_as_string = None if is_url(name): link = Link(name) else: p, extras_as_string = _strip_extras(path) <<<<<<< HEAD url = _get_url_from_path(p, name) if url is not None: